diff --git a/Makefile.in b/Makefile.in index d8f5695e5d..9076ffc085 100644 --- a/Makefile.in +++ b/Makefile.in @@ -30,7 +30,8 @@ BCC = @BUILD_CC@ @BUILD_CFLAGS@ # CC = @CC@ CFLAGS = @CPPFLAGS@ @CFLAGS@ -TCC = $(CC) $(CFLAGS) -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/fts3 +TCC = ${CC} ${CFLAGS} -I. -I${TOP}/src -I${TOP}/ext/rtree -I${TOP}/ext/icu +TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session # Define this for the autoconf-based build, so that the code knows it can # include the generated config.h @@ -179,7 +180,8 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -341,14 +343,15 @@ SRC += \ SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c +SRC += \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c SRC += \ $(TOP)/ext/misc/json1.c - - # Generated source code files # SRC += \ @@ -379,6 +382,7 @@ TESTSRC = \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ + $(TOP)/src/test_delete.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ @@ -405,14 +409,17 @@ TESTSRC = \ $(TOP)/src/test_windirent.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ - $(TOP)/ext/fts3/fts3_test.c \ + $(TOP)/ext/fts3/fts3_test.c \ + $(TOP)/ext/session/test_session.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ @@ -423,6 +430,7 @@ TESTSRC += \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ @@ -474,7 +482,8 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c \ + $(TOP)/ext/session/sqlite3session.c # Header files used by all library source files. # @@ -533,7 +542,8 @@ TESTPROGS = \ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ - sqldiff$(TEXE) + sqldiff$(TEXE) \ + dbhash$(TEXE) # Databases containing fuzzer test cases # @@ -541,7 +551,8 @@ FUZZDATA = \ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db \ - $(TOP)/test/fuzzdata4.db + $(TOP)/test/fuzzdata4.db \ + $(TOP)/test/fuzzdata5.db # Standard options to testfixture # @@ -550,9 +561,14 @@ TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 +# SHELL_OPT += -DSQLITE_ENABLE_FTS5 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c +DBFUZZ_OPT = # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -581,8 +597,15 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c $(TOP)/src/shell.c sqlite3.c \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" -sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) +sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS) + +dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) + +scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo + $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ + $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) srcck1$(BEXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c @@ -594,11 +617,18 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) -fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h - $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) +fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) -mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c - $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ +ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ + $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) + +dbfuzz$(TEXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c $(TLIBS) + +mptester$(TEXE): sqlite3.lo $(TOP)/mptest/mptest.c + $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \ $(TLIBS) -rpath "$(libdir)" MPTEST1=./mptester$(TEXE) mptest.db $(TOP)/mptest/crash01.test --repeat 20 @@ -634,6 +664,7 @@ mptest: mptester$(TEXE) sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . + cp $(TOP)/ext/session/sqlite3session.h . sqlite3ext.h: .target_source cp tsrc/sqlite3ext.h . @@ -997,6 +1028,9 @@ fts3_write.lo: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c +sqlite3session.lo: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c + json1.lo: $(TOP)/ext/misc/json1.c $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c @@ -1022,7 +1056,7 @@ FTS5_SRC = \ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h - ./lemon $(OPTS) fts5parse.y + ./lemon$(BEXE) $(OPTS) fts5parse.y fts5parse.h: fts5parse.c @@ -1081,6 +1115,11 @@ fastfuzztest: fuzzcheck$(TEXE) $(FUZZDATA) valgrindfuzz: fuzzcheck$(TEXT) $(FUZZDATA) valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA) +# The veryquick.test TCL tests. +# +tcltest: ./testfixture$(TEXE) + ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) + # Minimal testing that runs in less than 3 minutes # quicktest: ./testfixture$(TEXE) @@ -1089,8 +1128,7 @@ quicktest: ./testfixture$(TEXE) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # -test: $(TESTPROGS) sourcetest fastfuzztest - ./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS) +test: fastfuzztest sourcetest $(TESTPROGS) tcltest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. @@ -1117,6 +1155,10 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo + $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ + $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) + showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) @@ -1129,17 +1171,25 @@ showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) +changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) + rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c -wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c - $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) +wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo - $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c + $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) + +KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ + +kvtest$(TEXE): $(TOP)/test/kvtest.c sqlite3.c + $(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) @@ -1151,8 +1201,9 @@ loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # +VALIDIDS=' sqlite3(changeset|changegroup|session)?_' checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + nm -g --defined-only sqlite3.lo | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' # Build the amalgamation-autoconf package. The amalamgation-tarball target builds @@ -1215,13 +1266,12 @@ clean: rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz rm -f mkkeywordhash$(BEXE) keywordhash.h rm -f *.da *.bb *.bbg gmon.out - rm -rf quota2a quota2b quota2c rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) - rm -f wordcount$(TEXE) + rm -f wordcount$(TEXE) changeset$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h @@ -1234,6 +1284,7 @@ clean: rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe + rm -f dbhash dbhash.exe rm -f fts5.* fts5parse.* distclean: clean diff --git a/Makefile.msc b/Makefile.msc index 15fbf6c65d..7e700afea2 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -24,6 +24,27 @@ USE_AMALGAMATION = 1 USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This +# has no effect if (any) optimizations are enabled. +# +!IFNDEF USE_RUNTIME_CHECKS +USE_RUNTIME_CHECKS = 0 +!ENDIF + +# Set this non-0 to create a SQLite amalgamation file that excludes the +# various built-in extensions. +# +!IFNDEF MINIMAL_AMALGAMATION +MINIMAL_AMALGAMATION = 0 +!ENDIF + # Set this non-0 to use "stdcall" calling convention for the core library # and shell executable. # @@ -198,6 +219,12 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# Set this to non-0 to enable support for the session extension. +# +!IFNDEF SESSION +SESSION = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -257,15 +284,50 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF !ENDIF +# <> +# These are the names of the customized Tcl header files used by various parts +# of this makefile when the stdcall calling convention is in use. It is not +# used for any other purpose. +# +!IFNDEF SQLITETCLH +SQLITETCLH = sqlite_tcl.h +!ENDIF + +!IFNDEF SQLITETCLDECLSH +SQLITETCLDECLSH = sqlite_tclDecls.h +!ENDIF + +# These are the additional targets that the targets that integrate with the +# Tcl library should depend on when compiling, etc. +# +!IFNDEF SQLITE_TCL_DEP +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +SQLITE_TCL_DEP = $(SQLITETCLDECLSH) $(SQLITETCLH) +!ELSE +SQLITE_TCL_DEP = +!ENDIF +!ENDIF +# <> + # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +!IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +!ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Should the session extension be enabled? If so, add compilation options +# to enable it. +# +!IF $(SESSION)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -438,6 +500,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) @@ -450,20 +518,32 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +# <> +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +# <> !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +# <> +TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +# <> !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = +# <> +TEST_CCONV_OPTS = +# <> !ENDIF !ENDIF !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = +# <> +TEST_CCONV_OPTS = +# <> !ENDIF # These are additional compiler options used for the core library. @@ -607,6 +687,8 @@ TCC = $(TCC) -I$(TOP)\ext\fts3 RCC = $(RCC) -I$(TOP)\ext\fts3 TCC = $(TCC) -I$(TOP)\ext\rtree RCC = $(RCC) -I$(TOP)\ext\rtree +TCC = $(TCC) -I$(TOP)\ext\session +RCC = $(RCC) -I$(TOP)\ext\session !ENDIF # The mksqlite3c.tcl script accepts some options on the command @@ -614,12 +696,35 @@ RCC = $(RCC) -I$(TOP)\ext\rtree # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # +!IFNDEF MKSQLITE3C_TOOL +!IF $(MINIMAL_AMALGAMATION)!=0 +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c-noext.tcl +!ELSE +MKSQLITE3C_TOOL = $(TOP)\tool\mksqlite3c.tcl +!ENDIF +!ENDIF + !IFNDEF MKSQLITE3C_ARGS !IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = !ENDIF +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +MKSQLITE3C_ARGS = $(MKSQLITE3C_ARGS) --useapicall +!ENDIF +!ENDIF + +# The mksqlite3h.tcl script accepts some options on the command line. +# When compiling with stdcall support, some of these options are +# necessary. +# +!IFNDEF MKSQLITE3H_ARGS +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +MKSQLITE3H_ARGS = --useapicall +!ELSE +MKSQLITE3H_ARGS = +!ENDIF !ENDIF # <> @@ -641,6 +746,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 RCC = $(RCC) -DSQLITE_DEBUG=1 +!IF $(DYNAMIC_SHELL)==0 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE +!ENDIF !ENDIF !IF $(DEBUG)>4 || $(OSTRACE)!=0 @@ -702,11 +811,11 @@ TCLLIBDIR = c:\tcl\lib !ENDIF !IFNDEF LIBTCL -LIBTCL = tcl85.lib +LIBTCL = tcl86.lib !ENDIF !IFNDEF LIBTCLSTUB -LIBTCLSTUB = tclstub85.lib +LIBTCLSTUB = tclstub86.lib !ENDIF !IFNDEF LIBTCLPATH @@ -736,7 +845,7 @@ LIBICU = icuuc.lib icuin.lib # specific Tcl shell to use. # !IFNDEF TCLSH_CMD -TCLSH_CMD = tclsh85 +TCLSH_CMD = tclsh !ENDIF # <> @@ -819,6 +928,10 @@ RCC = $(RCC) -D_DEBUG !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od +!IF $(USE_RUNTIME_CHECKS)!=0 +TCC = $(TCC) -RTC1 +BCC = $(BCC) -RTC1 +!ENDIF !ELSEIF $(OPTIMIZATIONS)>=3 TCC = $(TCC) -Ox BCC = $(BCC) -Ox @@ -991,7 +1104,8 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ - random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ + random.lo resolve.lo rowset.lo rtree.lo \ + sqlite3session.lo select.lo sqlite3rbu.lo status.lo \ table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ @@ -1178,6 +1292,7 @@ SRC07 = \ $(TOP)\ext\fts3\fts3_write.c \ $(TOP)\ext\icu\icu.c \ $(TOP)\ext\rtree\rtree.c \ + $(TOP)\ext\session\sqlite3session.c \ $(TOP)\ext\rbu\sqlite3rbu.c \ $(TOP)\ext\misc\json1.c @@ -1200,7 +1315,8 @@ SRC09 = \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\icu\sqliteicu.h \ $(TOP)\ext\rtree\rtree.h \ - $(TOP)\ext\rbu\sqlite3rbu.h + $(TOP)\ext\rbu\sqlite3rbu.h \ + $(TOP)\ext\session\sqlite3session.h # Generated source code files # @@ -1216,6 +1332,16 @@ SRC11 = \ parse.h \ $(SQLITE3H) +# Generated Tcl header files +# +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +SRC12 = \ + $(SQLITETCLH) \ + $(SQLITETCLDECLSH) +!ELSE +SRC12 = +!ENDIF + # All source code files. # SRC = $(SRC00) $(SRC01) $(SRC02) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) @@ -1239,6 +1365,7 @@ TESTSRC = \ $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ + $(TOP)\src\test_delete.c \ $(TOP)\src\test_demovfs.c \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ @@ -1266,13 +1393,16 @@ TESTSRC = \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ - $(TOP)\ext\rbu\test_rbu.c + $(TOP)\ext\rbu\test_rbu.c \ + $(TOP)\ext\session\test_session.c # Statically linked extensions. # TESTEXT = \ $(TOP)\ext\misc\amatch.c \ + $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\closure.c \ + $(TOP)\ext\misc\csv.c \ $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ @@ -1283,12 +1413,14 @@ TESTEXT = \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\remember.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\totype.c \ $(TOP)\ext\misc\wholenumber.c # Source code to the library files needed by the test fixture +# (non-amalgamation) # TESTSRC2 = \ $(SRC00) \ @@ -1318,7 +1450,7 @@ HDR = \ parse.h \ $(TOP)\src\pragma.h \ $(SQLITE3H) \ - $(TOP)\src\sqlite3ext.h \ + sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ @@ -1347,6 +1479,8 @@ EXTHDR = $(EXTHDR) \ $(TOP)\ext\icu\sqliteicu.h EXTHDR = $(EXTHDR) \ $(TOP)\ext\rtree\sqlite3rtree.h +EXTHDR = $(EXTHDR) \ + $(TOP)\ext\session\sqlite3session.h # executables needed for testing # @@ -1354,7 +1488,8 @@ TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ - sqldiff.exe + sqldiff.exe \ + dbhash.exe # Databases containing fuzzer test cases # @@ -1362,7 +1497,8 @@ FUZZDATA = \ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ - $(TOP)\test\fuzzdata4.db + $(TOP)\test\fuzzdata4.db \ + $(TOP)\test\fuzzdata5.db # <> # Additional compiler options for the shell. These are only effective @@ -1377,7 +1513,13 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_ # MPTESTER_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c +OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c +DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION +KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +DBSELFTEST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +ST_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 # Standard options to testfixture. # @@ -1420,7 +1562,7 @@ $(SQLITE3DLL): $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_[^@]*)(?:@\d+)?$$" \1 \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3(?:session|changeset)?_[^@]*)(?:@\d+)?$$" \1 \ | sort >> sqlite3.def # <> @@ -1432,6 +1574,12 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + srcck1.exe: $(TOP)\tool\srcck1.c $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c @@ -1441,8 +1589,14 @@ sourcetest: srcck1.exe sqlite3.c fuzzershell.exe: $(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -fuzzcheck.exe: $(TOP)\test\fuzzcheck.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) mptester.exe: $(TOP)\mptest\mptest.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(MPTESTER_COMPILE_OPTS) $(TOP)\mptest\mptest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1467,7 +1621,7 @@ mptest: mptester.exe # files are automatically generated. This target takes care of # all that automatic generation. # -.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c $(SQLITE_TCL_DEP) -rmdir /Q/S tsrc 2>NUL -mkdir tsrc for %i in ($(SRC00)) do copy /Y %i tsrc @@ -1482,6 +1636,7 @@ mptest: mptester.exe for %i in ($(SRC09)) do copy /Y %i tsrc for %i in ($(SRC10)) do copy /Y %i tsrc for %i in ($(SRC11)) do copy /Y %i tsrc + for %i in ($(SRC12)) do copy /Y %i tsrc copy /Y fts5.c tsrc copy /Y fts5.h tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL @@ -1489,9 +1644,10 @@ mptest: mptester.exe move vdbe.new tsrc\vdbe.c echo > .target_source -sqlite3.c: .target_source sqlite3ext.h $(TOP)\tool\mksqlite3c.tcl - $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl $(MKSQLITE3C_ARGS) +sqlite3.c: .target_source sqlite3ext.h $(MKSQLITE3C_TOOL) + $(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) copy tsrc\shell.c . + copy $(TOP)\ext\session\sqlite3session.h . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl @@ -1762,10 +1918,10 @@ wherecode.lo: $(TOP)\src\wherecode.c $(HDR) whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\whereexpr.c -tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) @@ -1791,10 +1947,16 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > $(SQLITE3H) $(MKSQLITE3H_ARGS) sqlite3ext.h: .target_source - copy tsrc\sqlite3ext.h . +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 + type tsrc\sqlite3ext.h | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*\)" "(SQLITE_CALLBACK *)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_APICALL *" > sqlite3ext.h + copy /Y sqlite3ext.h tsrc\sqlite3ext.h +!ELSE + copy /Y tsrc\sqlite3ext.h sqlite3ext.h +!ENDIF mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) \ @@ -1870,6 +2032,9 @@ fts3_write.lo: $(TOP)\ext\fts3\fts3_write.c $(HDR) $(EXTHDR) rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\rtree\rtree.c +sqlite3session.lo: $(TOP)\ext\session\sqlite3session.c $(HDR) $(EXTHDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\session\sqlite3session.c + # FTS5 things # FTS5_SRC = \ @@ -1924,6 +2089,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CORE $(NO_WARN) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) @@ -1933,7 +2099,27 @@ TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF -testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) +!IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 +sqlite_tclDecls.h: + echo #ifndef SQLITE_TCLAPI > $(SQLITETCLDECLSH) + echo # define SQLITE_TCLAPI >> $(SQLITETCLDECLSH) + echo #endif >> $(SQLITETCLDECLSH) + type "$(TCLINCDIR)\tclDecls.h" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN(?: CONST\d+?)?\s+?[^\(]*?\s+?)Tcl_" "\1 SQLITE_TCLAPI Tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "^(EXTERN\s+?(?:void|VOID)\s+?)TclFreeObj" "\1 SQLITE_TCLAPI TclFreeObj" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tcl_" "(SQLITE_TCLAPI *tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*tclFreeObj" "(SQLITE_TCLAPI *tclFreeObj" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "\(\*" "(SQLITE_TCLAPI *" >> $(SQLITETCLDECLSH) + +sqlite_tcl.h: + type "$(TCLINCDIR)\tcl.h" | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact tclDecls.h sqlite_tclDecls.h \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl regsub "typedef (.*?)\(Tcl_" "typedef \1 (SQLITE_TCLAPI Tcl_" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "void (*freeProc)" "void (SQLITE_TCLAPI *freeProc)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*findProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *findProc)" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact "Tcl_HashEntry *(*createProc)" "Tcl_HashEntry *(SQLITE_TCLAPI *createProc)" >> $(SQLITETCLH) +!ENDIF + +testfixture.exe: $(TESTFIXTURE_SRC) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ @@ -1982,7 +2168,7 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(SQLITE_TCL_DEP) echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ @@ -1995,6 +2181,10 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) + testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c @@ -2002,42 +2192,54 @@ testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ + $(TOP)\ext\session\changeset.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c $(SQLITE3H) - $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) + $(LTLINK) $(NO_WARN) $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \ + $(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +dbselftest.exe: $(TOP)\test\dbselftest.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(DBSELFTEST_COMPILE_OPTS) $(TOP)\test\dbselftest.c $(SQLITE3C) + rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU -Fe$@ \ + $(LTLINK) $(NO_WARN) -DSQLITE_ENABLE_RBU \ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) moreclean: clean @@ -2056,22 +2258,20 @@ clean: del /Q notasharedlib.* 2>NUL -rmdir /Q/S .deps 2>NUL -rmdir /Q/S .libs 2>NUL - -rmdir /Q/S quota2a 2>NUL - -rmdir /Q/S quota2b 2>NUL - -rmdir /Q/S quota2c 2>NUL -rmdir /Q/S tsrc 2>NUL del /Q .target_source 2>NUL - del /Q tclsqlite3.exe 2>NUL + del /Q tclsqlite3.exe $(SQLITETCLH) $(SQLITETCLDECLSH) 2>NUL del /Q testloadext.dll 2>NUL del /Q testfixture.exe test.db 2>NUL - del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe 2>NUL + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL + del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL del /Q sqlite3.c sqlite3-*.c 2>NUL del /Q sqlite3rc.h 2>NUL - del /Q shell.c sqlite3ext.h 2>NUL + del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL - del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL + del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # <> diff --git a/README.md b/README.md index dbc020574e..7b860c211e 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,48 @@ If you are reading this on a Git mirror someplace, you are doing it wrong. The [official repository](https://www.sqlite.org/src/) is better. Go there now. +## Obtaining The Code + +SQLite sources are managed using the +[Fossil](https://www.fossil-scm.org/), a distributed version control system +that was specifically designed to support SQLite development. +If you do not want to use Fossil, you can download tarballs or ZIP +archives as follows: + + * Lastest trunk check-in: + or + . + + * Latest release: + or + . + + * For other check-ins, substitute an appropriate branch name or + tag or hash prefix for "release" in the URLs of the previous + bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) + to locate the check-in desired, click on its information page link, + then click on the "Tarball" or "ZIP Archive" links on the information + page. + +If you do want to use Fossil to check out the source tree, +first install Fossil version 2.0 or later. +(Source tarballs and precompiled binaries available +[here](https://www.fossil-scm.org/fossil/uv/download.html).) +Then run commands like this: + + mkdir ~/sqlite + cd ~/sqlite + fossil clone https://www.sqlite.org/src sqlite.fossil + fossil open sqlite.fossil + +After setting up a repository using the steps above, you can always +update to the lastest version using: + + fossil update trunk ;# latest trunk check-in + fossil update release ;# latest official release + +Or type "fossil ui" to get a web-based user interface. + ## Compiling First create a directory in which to place @@ -18,13 +60,13 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make ;# Run the makefile. - make sqlite3.c ;# Build the "amalgamation" source file - make test ;# Run some tests (requires Tcl) + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) See the makefile for additional targets. @@ -43,13 +85,13 @@ with the provided "Makefile.msc" to build one of the supported targets. For example: - mkdir bld - cd bld - nmake /f Makefile.msc TOP=..\sqlite - nmake /f Makefile.msc sqlite3.c TOP=..\sqlite - nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite - nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite - nmake /f Makefile.msc test TOP=..\sqlite + mkdir bld + cd bld + nmake /f Makefile.msc TOP=..\sqlite + nmake /f Makefile.msc sqlite3.c TOP=..\sqlite + nmake /f Makefile.msc sqlite3.dll TOP=..\sqlite + nmake /f Makefile.msc sqlite3.exe TOP=..\sqlite + nmake /f Makefile.msc test TOP=..\sqlite There are several build options that can be set via the NMAKE command line. For example, to build for WinRT, simply add "FOR_WINRT=1" argument diff --git a/VERSION b/VERSION index 92536a9e48..419f300965 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.12.0 +3.19.0 diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index cca23ba67d..e8211596d9 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -1,5 +1,5 @@ -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ @SESSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE lib_LTLIBRARIES = libsqlite3.la libsqlite3_la_SOURCES = sqlite3.c diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index ad829bffa0..ad88504be8 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -24,6 +24,27 @@ TOP = . USE_FULLWARN = 0 !ENDIF +# Set this non-0 to enable treating warnings as errors (-WX, etc) when +# compiling. +# +!IFNDEF USE_FATAL_WARN +USE_FATAL_WARN = 0 +!ENDIF + +# Set this non-0 to enable full runtime error checks (-RTC1, etc). This +# has no effect if (any) optimizations are enabled. +# +!IFNDEF USE_RUNTIME_CHECKS +USE_RUNTIME_CHECKS = 0 +!ENDIF + +# Set this non-0 to create a SQLite amalgamation file that excludes the +# various built-in extensions. +# +!IFNDEF MINIMAL_AMALGAMATION +MINIMAL_AMALGAMATION = 0 +!ENDIF + # Set this non-0 to use "stdcall" calling convention for the core library # and shell executable. # @@ -183,6 +204,12 @@ DEBUG = 0 OPTIMIZATIONS = 2 !ENDIF +# Set this to non-0 to enable support for the session extension. +# +!IFNDEF SESSION +SESSION = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -242,15 +269,26 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF !ENDIF + # These are the "standard" SQLite compilation options used when compiling for # the Windows platform. # !IFNDEF OPT_FEATURE_FLAGS +!IF $(MINIMAL_AMALGAMATION)==0 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1 +!ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF +# Should the session extension be enabled? If so, add compilation options +# to enable it. +# +!IF $(SESSION)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SESSION=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 +!ENDIF + # These are the "extended" SQLite compilation options used when compiling for # the Windows 10 platform. # @@ -423,6 +461,12 @@ TCC = $(CC) -nologo -W4 -DINCLUDE_MSVC_H=1 $(CCOPTS) $(TCCOPTS) TCC = $(CC) -nologo -W3 $(CCOPTS) $(TCCOPTS) !ENDIF +# Check if warnings should be treated as errors when compiling. +# +!IF $(USE_FATAL_WARN)!=0 +TCC = $(TCC) -WX +!ENDIF + TCC = $(TCC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -fp:precise RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) @@ -435,12 +479,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_STDCALL=__stdcall +CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -601,6 +645,10 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG=1 RCC = $(RCC) -DSQLITE_DEBUG=1 +!IF $(DYNAMIC_SHELL)==0 +TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE +RCC = $(RCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE +!ENDIF !ENDIF !IF $(DEBUG)>4 || $(OSTRACE)!=0 @@ -726,6 +774,10 @@ RCC = $(RCC) -D_DEBUG !IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od +!IF $(USE_RUNTIME_CHECKS)!=0 +TCC = $(TCC) -RTC1 +BCC = $(BCC) -RTC1 +!ENDIF !ELSEIF $(OPTIMIZATIONS)>=3 TCC = $(TCC) -Ox BCC = $(BCC) -Ox @@ -902,7 +954,7 @@ Replace.exe: sqlite3.def: Replace.exe $(LIBOBJ) echo EXPORTS > sqlite3.def dumpbin /all $(LIBOBJ) \ - | .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ + | .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \ | sort >> sqlite3.def $(SQLITE3EXE): $(TOP)\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H) diff --git a/autoconf/configure.ac b/autoconf/configure.ac index f86a39c37b..5a607de054 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -30,39 +30,52 @@ AC_FUNC_STRERROR_R AC_CONFIG_FILES([Makefile sqlite3.pc]) AC_SUBST(BUILD_CFLAGS) -#----------------------------------------------------------------------- +#------------------------------------------------------------------------- +# Two options to enable readline compatible libraries: +# # --enable-editline # --enable-readline # -AC_ARG_ENABLE(editline, [AS_HELP_STRING( - [--enable-editline], - [use BSD libedit])], - [], [enable_editline=yes]) -AC_ARG_ENABLE(readline, [AS_HELP_STRING( - [--enable-readline], - [use readline])], - [], [enable_readline=no]) -if test x"$enable_editline" != xno ; then - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS([readline],[edit],[enable_readline=no],[enable_editline=no]) - READLINE_LIBS=$LIBS - if test x"$LIBS" != "x"; then - AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) - else - unset ac_cv_search_readline - fi - LIBS=$sLIBS -fi -if test x"$enable_readline" != xno ; then - sLIBS=$LIBS - LIBS="" - AC_SEARCH_LIBS(tgetent, curses ncurses ncursesw, [], []) - AC_SEARCH_LIBS(readline, readline, [], [enable_readline=no]) - AC_CHECK_FUNCS(readline, [], []) - READLINE_LIBS=$LIBS - LIBS=$sLIBS -fi +# Both are enabled by default. If, after command line processing both are +# still enabled, the script searches for editline first and automatically +# disables readline if it is found. So, to use readline explicitly, the +# user must pass "--disable-editline". To disable command line editing +# support altogether, "--disable-editline --disable-readline". +# +# When searching for either library, check for headers before libraries +# as some distros supply packages that contain libraries but not header +# files, which come as a separate development package. +# +AC_ARG_ENABLE(editline, [AS_HELP_STRING([--enable-editline],[use BSD libedit])]) +AC_ARG_ENABLE(readline, [AS_HELP_STRING([--enable-readline],[use readline])]) + +AS_IF([ test x"$enable_editline" != xno ],[ + AC_CHECK_HEADERS([editline/readline.h],[ + sLIBS=$LIBS + LIBS="" + AC_SEARCH_LIBS([readline],[edit],[ + AC_DEFINE([HAVE_EDITLINE],1,Define to use BSD editline) + READLINE_LIBS="$LIBS -ltinfo" + enable_readline=no + ],[],[-ltinfo]) + AS_UNSET(ac_cv_search_readline) + LIBS=$sLIBS + ]) +]) + +AS_IF([ test x"$enable_readline" != xno ],[ + AC_CHECK_HEADERS([readline/readline.h],[ + sLIBS=$LIBS + LIBS="" + AC_SEARCH_LIBS(tgetent, termcap curses ncurses ncursesw, [], []) + AC_SEARCH_LIBS(readline,[readline edit], [ + AC_DEFINE([HAVE_READLINE],1,Define to use readline or wrapper) + READLINE_LIBS=$LIBS + ]) + LIBS=$sLIBS + ]) +]) + AC_SUBST(READLINE_LIBS) #----------------------------------------------------------------------- @@ -103,7 +116,7 @@ AC_SUBST(DYNAMIC_EXTENSION_FLAGS) AC_ARG_ENABLE(fts5, [AS_HELP_STRING( [--enable-fts5], [include fts5 support [default=no]])], [], [enable_fts5=no]) -if test x"$enable_fts5" == "xyes"; then +if test x"$enable_fts5" = "xyes"; then AC_SEARCH_LIBS(log, m) FTS5_FLAGS=-DSQLITE_ENABLE_FTS5 fi @@ -116,12 +129,24 @@ AC_SUBST(FTS5_FLAGS) AC_ARG_ENABLE(json1, [AS_HELP_STRING( [--enable-json1], [include json1 support [default=no]])], [], [enable_json1=no]) -if test x"$enable_json1" == "xyes"; then +if test x"$enable_json1" = "xyes"; then JSON1_FLAGS=-DSQLITE_ENABLE_JSON1 fi AC_SUBST(JSON1_FLAGS) #----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# --enable-session +# +AC_ARG_ENABLE(session, [AS_HELP_STRING( + [--enable-session], [enable the session extension [default=no]])], + [], [enable_session=no]) +if test x"$enable_session" = "xyes"; then + SESSION_FLAGS="-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi +AC_SUBST(SESSION_FLAGS) +#----------------------------------------------------------------------- + #----------------------------------------------------------------------- # --enable-static-shell # @@ -129,7 +154,7 @@ AC_ARG_ENABLE(static-shell, [AS_HELP_STRING( [--enable-static-shell], [statically link libsqlite3 into shell tool [default=yes]])], [], [enable_static_shell=yes]) -if test x"$enable_static_shell" == "xyes"; then +if test x"$enable_static_shell" = "xyes"; then EXTRA_SHELL_OBJ=sqlite3-sqlite3.$OBJEXT else EXTRA_SHELL_OBJ=libsqlite3.la diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index 8df0af6195..7fca05b2e7 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -78,7 +78,6 @@ TEA_ADD_LIBS([]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) -TEA_ADD_CFLAGS([-DSQLITE_OMIT_DEPRECATED=1]) TEA_ADD_STUB_SOURCES([]) TEA_ADD_TCL_SOURCES([]) diff --git a/configure b/configure index 8363818c6f..84422d3343 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.12.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.19.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.12.0' -PACKAGE_STRING='sqlite 3.12.0' +PACKAGE_VERSION='3.19.0' +PACKAGE_STRING='sqlite 3.19.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -903,11 +903,14 @@ with_readline_inc enable_debug enable_amalgamation enable_load_extension +enable_memsys5 +enable_memsys3 enable_fts3 enable_fts4 enable_fts5 enable_json1 enable_rtree +enable_session enable_gcov ' ac_precious_vars='build_alias @@ -1460,7 +1463,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.12.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.19.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1525,7 +1528,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.12.0:";; + short | recursive ) echo "Configuration of sqlite 3.19.0:";; esac cat <<\_ACEOF @@ -1551,11 +1554,14 @@ Optional Features: separately --disable-load-extension Disable loading of external extensions + --enable-memsys5 Enable MEMSYS5 + --enable-memsys3 Enable MEMSYS3 --enable-fts3 Enable the FTS3 extension --enable-fts4 Enable the FTS4 extension --enable-fts5 Enable the FTS5 extension --enable-json1 Enable the JSON1 extension --enable-rtree Enable the RTREE extension + --enable-session Enable the SESSION extension --enable-gcov Enable coverage testing using gcov Optional Packages: @@ -1646,7 +1652,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.12.0 +sqlite configure 3.19.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2065,7 +2071,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.12.0, which was +It was created by sqlite $as_me 3.19.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3923,13 +3929,13 @@ if ${lt_cv_nm_interface+:} false; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3926: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3932: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3929: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3935: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3932: output\"" >&5) + (eval echo "\"\$as_me:3938: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5135,7 +5141,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5138 "configure"' > conftest.$ac_ext + echo '#line 5144 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -6660,11 +6666,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6663: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6669: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6667: \$? = $ac_status" >&5 + echo "$as_me:6673: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -6999,11 +7005,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7002: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7008: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7006: \$? = $ac_status" >&5 + echo "$as_me:7012: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7104,11 +7110,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7107: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7113: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7111: \$? = $ac_status" >&5 + echo "$as_me:7117: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7159,11 +7165,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7162: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7168: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7166: \$? = $ac_status" >&5 + echo "$as_me:7172: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -9539,7 +9545,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9542 "configure" +#line 9548 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -9635,7 +9641,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9638 "configure" +#line 9644 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10755,6 +10761,20 @@ else fi fi + # Recent versions of Xcode on Macs hid the tclConfig.sh file + # in a strange place. + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi + fi + # then check for a private Tcl installation if test x"${ac_cv_c_tclconfig}" = x ; then for i in \ @@ -11232,7 +11252,7 @@ else fi if test "${use_debug}" = "yes" ; then - TARGET_DEBUG="-DSQLITE_DEBUG=1" + TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE" else TARGET_DEBUG="-DNDEBUG" fi @@ -11323,6 +11343,44 @@ else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi +########## +# Do we want to support memsys3 and/or memsys5 +# +# Check whether --enable-memsys5 was given. +if test "${enable_memsys5+set}" = set; then : + enableval=$enable_memsys5; enable_memsys5=yes +else + enable_memsys5=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS5" >&5 +$as_echo_n "checking whether to support MEMSYS5... " >&6; } +if test "${enable_memsys5}" = "yes"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +# Check whether --enable-memsys3 was given. +if test "${enable_memsys3+set}" = set; then : + enableval=$enable_memsys3; enable_memsys3=yes +else + enable_memsys3=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to support MEMSYS3" >&5 +$as_echo_n "checking whether to support MEMSYS3... " >&6; } +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + ######### # See whether we should enable Full Text Search extensions # Check whether --enable-fts3 was given. @@ -11494,6 +11552,20 @@ if test "${enable_rtree}" = "yes" ; then OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" fi +######### +# See whether we should enable the SESSION extension +# Check whether --enable-session was given. +if test "${enable_session+set}" = set; then : + enableval=$enable_session; enable_session=yes +else + enable_session=no +fi + +if test "${enable_session}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi + ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS @@ -12079,7 +12151,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.12.0, which was +This file was extended by sqlite $as_me 3.19.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12145,7 +12217,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.12.0 +sqlite config.status 3.19.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 08c8867f06..095db6bf81 100644 --- a/configure.ac +++ b/configure.ac @@ -334,6 +334,20 @@ if test "${use_tcl}" = "yes" ; then fi fi + # Recent versions of Xcode on Macs hid the tclConfig.sh file + # in a strange place. + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi + fi + # then check for a private Tcl installation if test x"${ac_cv_c_tclconfig}" = x ; then for i in \ @@ -546,7 +560,7 @@ AC_SEARCH_LIBS(fdatasync, [rt]) AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]), [use_debug=$enableval],[use_debug=no]) if test "${use_debug}" = "yes" ; then - TARGET_DEBUG="-DSQLITE_DEBUG=1" + TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE" else TARGET_DEBUG="-DNDEBUG" fi @@ -574,6 +588,30 @@ else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi +########## +# Do we want to support memsys3 and/or memsys5 +# +AC_ARG_ENABLE(memsys5, + AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]), + [enable_memsys5=yes],[enable_memsys5=no]) +AC_MSG_CHECKING([whether to support MEMSYS5]) +if test "${enable_memsys5}" = "yes"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS5" + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi +AC_ARG_ENABLE(memsys3, + AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]), + [enable_memsys3=yes],[enable_memsys3=no]) +AC_MSG_CHECKING([whether to support MEMSYS3]) +if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_MEMSYS3" + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi + ######### # See whether we should enable Full Text Search extensions AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3], @@ -615,6 +653,16 @@ if test "${enable_rtree}" = "yes" ; then OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" fi +######### +# See whether we should enable the SESSION extension +AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session], + [Enable the SESSION extension]), + [enable_session=yes],[enable_session=no]) +if test "${enable_session}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_SESSION" + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_PREUPDATE_HOOK" +fi + ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS diff --git a/doc/lemon.html b/doc/lemon.html index 2b8c613640..114526f372 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -5,14 +5,17 @@

The Lemon Parser Generator

-

Lemon is an LALR(1) parser generator for C or C++. -It does the same job as ``bison'' and ``yacc''. -But lemon is not another bison or yacc clone. It +

Lemon is an LALR(1) parser generator for C. +It does the same job as "bison" and "yacc". +But lemon is not a bison or yacc clone. Lemon uses a different grammar syntax which is designed to -reduce the number of coding errors. Lemon also uses a more -sophisticated parsing engine that is faster than yacc and -bison and which is both reentrant and thread-safe. -Furthermore, Lemon implements features that can be used +reduce the number of coding errors. Lemon also uses a +parsing engine that is faster than yacc and +bison and which is both reentrant and threadsafe. +(Update: Since the previous sentence was written, bison +has also been updated so that it too can generate a +reentrant and threadsafe parser.) +Lemon also implements features that can be used to eliminate resource leaks, making is suitable for use in long-running programs such as graphical user interfaces or embedded controllers.

@@ -44,18 +47,18 @@ one and three files of outputs. automaton. By default, all three of these output files are generated. -The header file is suppressed if the ``-m'' command-line option is -used and the report file is omitted when ``-q'' is selected.

+The header file is suppressed if the "-m" command-line option is +used and the report file is omitted when "-q" is selected.

-

The grammar specification file uses a ``.y'' suffix, by convention. +

The grammar specification file uses a ".y" suffix, by convention. In the examples used in this document, we'll assume the name of the -grammar file is ``gram.y''. A typical use of Lemon would be the +grammar file is "gram.y". A typical use of Lemon would be the following command: 

    lemon gram.y
-This command will generate three output files named ``gram.c'', -``gram.h'' and ``gram.out''. +This command will generate three output files named "gram.c", +"gram.h" and "gram.out". The first is C code to implement the parser. The second is the header file that defines numerical values for all terminal symbols, and the last is the report that explains @@ -71,39 +74,35 @@ with a brief explanation of what each does by typing As of this writing, the following command-line options are supported:
    -
  • -b -
  • -c -
  • -g -
  • -m -
  • -q -
  • -s -
  • -x +
  • -b +Show only the basis for each parser state in the report file. +
  • -c +Do not compress the generated action tables. +
  • -Dname +Define C preprocessor macro name. This macro is useable by +"%ifdef" lines in the grammar file. +
  • -g +Do not generate a parser. Instead write the input grammar to standard +output with all comments, actions, and other extraneous text removed. +
  • -l +Omit "#line" directives in the generated parser C code. +
  • -m +Cause the output C source code to be compatible with the "makeheaders" +program. +
  • -p +Display all conflicts that are resolved by +precedence rules. +
  • -q +Suppress generation of the report file. +
  • -r +Do not sort or renumber the parser states as part of optimization. +
  • -s +Show parser statistics before existing. +
  • -Tfile +Use file as the template for the generated C-code parser implementation. +
  • -x +Print the Lemon version number.
-The ``-b'' option reduces the amount of text in the report file by -printing only the basis of each parser state, rather than the full -configuration. -The ``-c'' option suppresses action table compression. Using -c -will make the parser a little larger and slower but it will detect -syntax errors sooner. -The ``-g'' option causes no output files to be generated at all. -Instead, the input grammar file is printed on standard output but -with all comments, actions and other extraneous text deleted. This -is a useful way to get a quick summary of a grammar. -The ``-m'' option causes the output C source file to be compatible -with the ``makeheaders'' program. -Makeheaders is a program that automatically generates header files -from C source code. When the ``-m'' option is used, the header -file is not output since the makeheaders program will take care -of generated all header files automatically. -The ``-q'' option suppresses the report file. -Using ``-s'' causes a brief summary of parser statistics to be -printed. Like this: -
-   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
-                      340 states, 2026 parser table entries, 0 conflicts
-
-Finally, the ``-x'' option causes Lemon to print its version number -and then stops without attempting to read the grammar or generate a parser.

The Parser Interface

@@ -121,12 +120,12 @@ A new parser is created as follows: The ParseAlloc() routine allocates and initializes a new parser and returns a pointer to it. -The actual data structure used to represent a parser is opaque -- +The actual data structure used to represent a parser is opaque — its internal structure is not visible or usable by the calling routine. For this reason, the ParseAlloc() routine returns a pointer to void rather than a pointer to some particular structure. The sole argument to the ParseAlloc() routine is a pointer to the -subroutine used to allocate memory. Typically this means ``malloc()''.

+subroutine used to allocate memory. Typically this means malloc().

After a program is finished using a parser, it can reclaim all memory allocated by that parser by calling @@ -151,13 +150,13 @@ type of the next token in the data stream. There is one token type for each terminal symbol in the grammar. The gram.h file generated by Lemon contains #define statements that map symbolic terminal symbol names into appropriate integer values. -(A value of 0 for the second argument is a special flag to the -parser to indicate that the end of input has been reached.) +A value of 0 for the second argument is a special flag to the +parser to indicate that the end of input has been reached. The third argument is the value of the given token. By default, the type of the third argument is integer, but the grammar will usually redefine this type to be some kind of structure. Typically the second argument will be a broad category of tokens -such as ``identifier'' or ``number'' and the third argument will +such as "identifier" or "number" and the third argument will be the name of the identifier or the value of the number.

The Parse() function may have either three or four arguments, @@ -193,7 +192,7 @@ following: This example shows a user-written routine that parses a file of text and returns a pointer to the parse tree. -(We've omitted all error-handling from this example to keep it +(All error-handling code is omitted from this example to keep it simple.) We assume the existence of some kind of tokenizer which is created using TokenizerCreate() on line 8 and deleted by TokenizerFree() @@ -287,7 +286,7 @@ tokens) and it honors the same commenting conventions as C and C++.

Terminals and Nonterminals

A terminal symbol (token) is any string of alphanumeric -and underscore characters +and/or underscore characters that begins with an upper case letter. A terminal can contain lowercase letters after the first character, but the usual convention is to make terminals all upper case. @@ -314,7 +313,7 @@ must have alphanumeric names.

The main component of a Lemon grammar file is a sequence of grammar rules. Each grammar rule consists of a nonterminal symbol followed by -the special symbol ``::='' and then a list of terminals and/or nonterminals. +the special symbol "::=" and then a list of terminals and/or nonterminals. The rule is terminated by a period. The list of terminals and nonterminals on the right-hand side of the rule can be empty. @@ -330,9 +329,9 @@ A typical sequence of grammar rules might look something like this:

-

There is one non-terminal in this example, ``expr'', and five -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'', -``RPAREN'' and ``VALUE''.

+

There is one non-terminal in this example, "expr", and five +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN", +"RPAREN" and "VALUE".

Like yacc and bison, Lemon allows the grammar to specify a block of C code that will be executed whenever a grammar rule is reduced @@ -348,15 +347,15 @@ For example:

In order to be useful, grammar actions must normally be linked to their associated grammar rules. -In yacc and bison, this is accomplished by embedding a ``$$'' in the +In yacc and bison, this is accomplished by embedding a "$$" in the action to stand for the value of the left-hand side of the rule and -symbols ``$1'', ``$2'', and so forth to stand for the value of +symbols "$1", "$2", and so forth to stand for the value of the terminal or nonterminal at position 1, 2 and so forth on the right-hand side of the rule. This idea is very powerful, but it is also very error-prone. The single most common source of errors in a yacc or bison grammar is to miscount the number of symbols on the right-hand side of a grammar -rule and say ``$7'' when you really mean ``$8''.

+rule and say "$7" when you really mean "$8".

Lemon avoids the need to count grammar symbols by assigning symbolic names to each symbol in a grammar rule and then using those symbolic @@ -386,7 +385,7 @@ For example, the rule 

   expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
-will generate an error because the linking symbol ``C'' is used +will generate an error because the linking symbol "C" is used in the grammar rule but not in the reduce action.

The Lemon notation for linking grammar rules to reduce actions @@ -394,6 +393,7 @@ also facilitates the use of destructors for reclaiming memory allocated by the values of terminals and nonterminals on the right-hand side of a rule.

+

Precedence Rules

Lemon resolves parsing ambiguities in exactly the same way as @@ -405,7 +405,10 @@ whichever rule comes first in the grammar file.

yacc and bison, Lemon allows a measure of control over the resolution of paring conflicts using precedence rules. A precedence value can be assigned to any terminal symbol -using the %left, %right or %nonassoc directives. Terminal symbols +using the +%left, +%right or +%nonassoc directives. Terminal symbols mentioned in earlier directives have a lower precedence that terminal symbols mentioned in later directives. For example:

@@ -525,7 +528,11 @@ other than that, the order of directives in Lemon is arbitrary.

  • %default_destructor
  • %default_type
  • %destructor +
  • %endif
  • %extra_argument +
  • %fallback +
  • %ifdef +
  • %ifndef
  • %include
  • %left
  • %name @@ -537,49 +544,57 @@ other than that, the order of directives in Lemon is arbitrary.

  • %stack_size
  • %start_symbol
  • %syntax_error +
  • %token_class
  • %token_destructor
  • %token_prefix
  • %token_type
  • %type +
  • %wildcard Each of these directives will be described separately in the following sections:

    +

    The %code directive

    -

    The %code directive is used to specify addition C/C++ code that +

    The %code directive is used to specify addition C code that is added to the end of the main output file. This is similar to -the %include directive except that %include is inserted at the -beginning of the main output file.

    +the %include directive except that %include +is inserted at the beginning of the main output file.

    %code is typically used to include some action routines or perhaps -a tokenizer as part of the output file.

    +a tokenizer or even the "main()" function +as part of the output file.

    +

    The %default_destructor directive

    The %default_destructor directive specifies a destructor to use for non-terminals that do not have their own destructor specified by a separate %destructor directive. See the documentation -on the %destructor directive below for additional information.

    +on the %destructor directive below for +additional information.

    In some grammers, many different non-terminal symbols have the same datatype and hence the same destructor. This directive is a convenience way to specify the same destructor for all those non-terminals using a single statement.

    +

    The %default_type directive

    The %default_type directive specifies the datatype of non-terminal symbols that do no have their own datatype defined using a separate -%type directive. See the documentation on %type below for addition -information.

    +%type directive. +

    +

    The %destructor directive

    The %destructor directive is used to specify a destructor for a non-terminal symbol. -(See also the %token_destructor directive which is used to -specify a destructor for terminal symbols.)

    +(See also the %token_destructor +directive which is used to specify a destructor for terminal symbols.)

    A non-terminal's destructor is called to dispose of the non-terminal's value whenever the non-terminal is popped from @@ -602,26 +617,25 @@ or other resources held by that non-terminal.

    This example is a bit contrived but it serves to illustrate how destructors work. The example shows a non-terminal named -``nt'' that holds values of type ``void*''. When the rule for -an ``nt'' reduces, it sets the value of the non-terminal to +"nt" that holds values of type "void*". When the rule for +an "nt" reduces, it sets the value of the non-terminal to space obtained from malloc(). Later, when the nt non-terminal is popped from the stack, the destructor will fire and call free() on this malloced space, thus avoiding a memory leak. -(Note that the symbol ``$$'' in the destructor code is replaced +(Note that the symbol "$$" in the destructor code is replaced by the value of the non-terminal.)

    It is important to note that the value of a non-terminal is passed to the destructor whenever the non-terminal is removed from the stack, unless the non-terminal is used in a C-code action. If the non-terminal is used by C-code, then it is assumed that the -C-code will take care of destroying it if it should really -be destroyed. More commonly, the value is used to build some +C-code will take care of destroying it. +More commonly, the value is used to build some larger structure and we don't want to destroy it, which is why the destructor is not called in this circumstance.

    -

    By appropriate use of destructors, it is possible to -build a parser using Lemon that can be used within a long-running -program, such as a GUI, that will not leak memory or other resources. +

    Destructors help avoid memory leaks by automatically freeing +allocated objects when they go out of scope. To do the same using yacc or bison is much more difficult.

    @@ -638,17 +652,66 @@ and so forth. For example, if the grammar file contains:

    Then the Parse() function generated will have an 4th parameter -of type ``MyStruct*'' and all action routines will have access to -a variable named ``pAbc'' that is the value of the 4th parameter +of type "MyStruct*" and all action routines will have access to +a variable named "pAbc" that is the value of the 4th parameter in the most recent call to Parse().

    + +

    The %fallback directive

    + +

    The %fallback directive specifies an alternative meaning for one +or more tokens. The alternative meaning is tried if the original token +would have generated a syntax error. + +

    The %fallback directive was added to support robust parsing of SQL +syntax in SQLite. +The SQL language contains a large assortment of keywords, each of which +appears as a different token to the language parser. SQL contains so +many keywords, that it can be difficult for programmers to keep up with +them all. Programmers will, therefore, sometimes mistakenly use an +obscure language keyword for an identifier. The %fallback directive +provides a mechanism to tell the parser: "If you are unable to parse +this keyword, try treating it as an identifier instead." + +

    The syntax of %fallback is as follows: + +

    +%fallback ID TOKEN... . +
    + +

    In words, the %fallback directive is followed by a list of token names +terminated by a period. The first token name is the fallback token - the +token to which all the other tokens fall back to. The second and subsequent +arguments are tokens which fall back to the token identified by the first +argument. + + +

    The %ifdef, %ifndef, and %endif directives.

    + +

    The %ifdef, %ifndef, and %endif directives are similar to +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general. +Each of these directives must begin at the left margin. No whitespace +is allowed between the "%" and the directive name. + +

    Grammar text in between "%ifdef MACRO" and the next nested "%endif" is +ignored unless the "-DMACRO" command-line option is used. Grammar text +betwen "%ifndef MACRO" and the next nested "%endif" is included except when +the "-DMACRO" command-line option is used. + +

    Note that the argument to %ifdef and %ifndef must be a single +preprocessor symbol name, not a general expression. There is no "%else" +directive. + + +

    The %include directive

    The %include directive specifies C code that is included at the top of the generated parser. You can include any text you want -- the Lemon parser generator copies it blindly. If you have multiple -%include directives in your grammar file the value of the last -%include directive overwrites all the others.

    The %include directive is very handy for getting some extra #include preprocessor statements at the beginning of the generated parser. @@ -661,12 +724,13 @@ For example:

    This might be needed, for example, if some of the C actions in the grammar call functions that are prototyed in unistd.h.

    +

    The %left directive

    -The %left directive is used (along with the %right and -%nonassoc directives) to declare precedences of terminal -symbols. Every terminal symbol whose name appears after -a %left directive but before the next period (``.'') is +The %left directive is used (along with the %right and +%nonassoc directives) to declare precedences of +terminal symbols. Every terminal symbol whose name appears after +a %left directive but before the next period (".") is given the same left-associative precedence value. Subsequent %left directives have higher precedence. For example:

    @@ -687,10 +751,11 @@ a large amount of stack space if you make heavy use or right-associative operators. For this reason, it is recommended that you use %left rather than %right whenever possible.

    +

    The %name directive

    By default, the functions generated by Lemon all begin with the -five-character string ``Parse''. You can change this string to something +five-character string "Parse". You can change this string to something different using the %name directive. For instance:

    @@ -709,16 +774,19 @@ The %name directive allows you to generator two or more different
 parsers and link them all into the same executable.
 
    
 
+
 
    The %nonassoc directive
    
 
 
    This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on precedence rules
-or on the %left directive for additional information.
    
+one or more terminal symbols.  See the section on 
+precedence rules
+or on the %left directive for additional information.
    
 
+
 
    The %parse_accept directive
    
 
 
    The %parse_accept directive specifies a block of C code that is
-executed whenever the parser accepts its input string.  To ``accept''
+executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.
    
 
@@ -730,7 +798,7 @@ without error.
    
    }

    - +

    The %parse_failure directive

    The %parse_failure directive specifies a block of C code that @@ -745,12 +813,15 @@ only invoked when parsing is unable to continue.

    }

    +

    The %right directive

    This directive is used to assign right-associative precedence to -one or more terminal symbols. See the section on precedence rules -or on the %left directive for additional information.

    +one or more terminal symbols. See the section on +precedence rules +or on the %left directive for additional information.

    +

    The %stack_overflow directive

    The %stack_overflow directive specifies a block of C code that @@ -779,6 +850,7 @@ Not like this: list ::= . +

    The %stack_size directive

    If stack overflow is a problem and you can't resolve the trouble @@ -791,6 +863,7 @@ with a stack of the requested size. The default value is 100.

    %stack_size 2000

    +

    The %start_symbol directive

    By default, the start-symbol for the grammar that Lemon generates @@ -801,6 +874,7 @@ can choose a different start-symbol using the %start_symbol directive.

    %start_symbol prog

    +

    The %token_destructor directive

    The %destructor directive assigns a destructor to a non-terminal @@ -813,6 +887,7 @@ the %token_type directive) and so they use a common destructor. Other than that, the token destructor works just like the non-terminal destructors.

    +

    The %token_prefix directive

    Lemon generates #defines that assign small integer constants @@ -838,6 +913,7 @@ to cause Lemon to produce these symbols instead: #define TOKEN_PLUS 4 +

    The %token_type and %type directives

    These directives are used to specify the data types for values @@ -853,7 +929,7 @@ token structure. Like this:

    If the data type of terminals is not specified, the default value -is ``int''.

    +is "void*".

    Non-terminal symbols can each have their own data types. Typically the data type of a non-terminal is a pointer to the root of a parse-tree @@ -874,6 +950,17 @@ non-terminal whose data type requires 1K of storage, then your 100 entry parser stack will require 100K of heap space. If you are willing and able to pay that price, fine. You just need to know.

    + +

    The %wildcard directive

    + +

    The %wildcard directive is followed by a single token name and a +period. This directive specifies that the identified token should +match any input token. + +

    When the generated parser has the choice of matching an input against +the wildcard token and some other token, the other token is always used. +The wildcard token is only matched if there are no other alternatives. +

    Error Processing

    After extensive experimentation over several years, it has been @@ -885,7 +972,7 @@ first invokes the code specified by the %syntax_error directive, if any. It then enters its error recovery strategy. The error recovery strategy is to begin popping the parsers stack until it enters a state where it is permitted to shift a special non-terminal symbol -named ``error''. It then shifts this non-terminal and continues +named "error". It then shifts this non-terminal and continues parsing. But the %syntax_error routine will not be called again until at least three new tokens have been successfully shifted.

    @@ -894,7 +981,7 @@ is unable to shift the error symbol, then the %parse_failed routine is invoked and the parser resets itself to its start state, ready to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the -``error'' non-terminal in your grammar.

    +"error" non-terminal in your grammar.

    diff --git a/ext/README.md b/ext/README.md new file mode 100644 index 0000000000..933a33d053 --- /dev/null +++ b/ext/README.md @@ -0,0 +1,8 @@ +## Loadable Extensions + +Various [loadable extensions](https://www.sqlite.org/loadext.html) for +SQLite are found in subfolders. + +Most subfolders are dedicated to a single loadable extension (for +example FTS5, or RTREE). But the misc/ subfolder contains a collection +of smaller single-file extensions. diff --git a/ext/README.txt b/ext/README.txt deleted file mode 100644 index 009495f59d..0000000000 --- a/ext/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -Version loadable extensions to SQLite are found in subfolders -of this folder. diff --git a/ext/fts2/fts2_tokenizer.c b/ext/fts2/fts2_tokenizer.c index a3d6a6312a..dda33a72d2 100644 --- a/ext/fts2/fts2_tokenizer.c +++ b/ext/fts2/fts2_tokenizer.c @@ -99,7 +99,11 @@ static void scalarFunc( #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif #include /* diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 748faefec5..6f059bc95d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -349,8 +349,9 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ -int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; u32 a; u64 b; int shift; @@ -492,6 +493,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ assert( p->pSegments==0 ); /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } @@ -1363,9 +1365,9 @@ static int fts3InitVtab( p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; @@ -1396,7 +1398,9 @@ static int fts3InitVtab( char *z; int n = 0; z = (char *)sqlite3Fts3NextToken(aCol[iCol], &n); - memcpy(zCsr, z, n); + if( n>0 ){ + memcpy(zCsr, z, n); + } zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; @@ -1680,6 +1684,26 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return SQLITE_OK; } +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -1687,7 +1711,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); @@ -1705,20 +1729,23 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ ** ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to ** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->pStmt==0 ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; } - *ppStmt = pCsr->pStmt; return rc; } @@ -1730,9 +1757,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); pCsr->isRequireSeek = 0; @@ -3190,7 +3215,7 @@ static int fts3FilterMethod( assert( iIdx==nVal ); /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); + fts3CursorFinalizeStmt(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); sqlite3Fts3ExprFree(pCsr->pExpr); @@ -3258,7 +3283,7 @@ static int fts3FilterMethod( rc = SQLITE_NOMEM; } }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } @@ -3386,8 +3411,10 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK && p->nLeafAdd>(nMinMerge/16) && p->nAutoincrmerge && p->nAutoincrmerge!=0xff @@ -3402,6 +3429,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); return rc; } @@ -3422,7 +3450,7 @@ static int fts3SetHasStat(Fts3Table *p){ if( rc==SQLITE_OK ){ int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + if( rc==SQLITE_OK ) p->bHasStat = (u8)bHasStat; } sqlite3_free(zSql); }else{ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0c86c4217e..c3cab9d821 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -230,6 +230,7 @@ struct Fts3Table { ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ char *zReadExprlist; char *zWriteExprlist; @@ -299,6 +300,7 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int iLangid; /* Language being queried for */ diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index dec977b916..a48a556c98 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -18,7 +18,14 @@ ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. */ -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #include #include @@ -143,7 +150,7 @@ static int nm_match_count( /* ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS? */ -static int fts3_near_match_cmd( +static int SQLITE_TCLAPI fts3_near_match_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -278,7 +285,7 @@ static int fts3_near_match_cmd( ** # Restore initial incr-load settings: ** eval fts3_configure_incr_load $cfg */ -static int fts3_configure_incr_load_cmd( +static int SQLITE_TCLAPI fts3_configure_incr_load_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -458,7 +465,7 @@ static int testTokenizerNext( if( pCsr->iLangid & 0x00000001 ){ for(i=0; iaBuffer[i] = pToken[i]; }else{ - for(i=0; iaBuffer[i] = testTolower(pToken[i]); + for(i=0; iaBuffer[i] = (char)testTolower(pToken[i]); } pCsr->iToken++; pCsr->iInput = (int)(p - pCsr->aInput); @@ -488,7 +495,7 @@ static int testTokenizerLanguage( } #endif -static int fts3_test_tokenizer_cmd( +static int SQLITE_TCLAPI fts3_test_tokenizer_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -517,7 +524,7 @@ static int fts3_test_tokenizer_cmd( return TCL_OK; } -static int fts3_test_varint_cmd( +static int SQLITE_TCLAPI fts3_test_varint_cmd( ClientData clientData, Tcl_Interp *interp, int objc, diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index b7d9d2b3b2..bfc36af3e3 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -224,7 +224,11 @@ int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif #include /* diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 94fc27b5b4..dfb2680c50 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -136,16 +136,16 @@ static int unicodeAddExceptions( ){ const unsigned char *z = (const unsigned char *)zIn; const unsigned char *zTerm = &z[nIn]; - int iCode; + unsigned int iCode; int nEntry = 0; assert( bAlnum==0 || bAlnum==1 ); while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = iCode; + aNew[i] = (int)iCode; nNew++; } } @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode = 0; + unsigned int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -330,7 +330,7 @@ static int unicodeNext( ** the input. */ while( z=zTerm ) return SQLITE_DONE; @@ -350,7 +350,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } @@ -358,8 +358,8 @@ static int unicodeNext( /* If the cursor is not at EOF, read the next character */ if( z>=zTerm ) break; READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, iCode) - || sqlite3FtsUnicodeIsdiacritic(iCode) + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) ); /* Set the output variables and return. */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index 20b7a25dbf..da7251ed0c 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -127,9 +127,9 @@ int sqlite3FtsUnicodeIsalnum(int c){ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, }; - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; @@ -322,16 +322,17 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ int ret = c; - assert( c>=0 ); assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); if( c<128 ){ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); }else if( c<65536 ){ + const struct TableEntry *p; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; int iRes = -1; + assert( c>aEntry[0].iCode ); while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; int cmp = (c - aEntry[iTest].iCode); @@ -342,14 +343,12 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ iHi = iTest-1; } } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } if( bRemoveDiacritic ) ret = remove_diacritic(ret); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3ff481b0b0..d700c8403c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -4956,11 +4956,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** Convert the text beginning at *pz into an integer and return ** its value. Advance *pz to point to the first character past ** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. */ static int fts3Getint(const char **pz){ const char *z = *pz; int i = 0; - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; *pz = z; return i; } diff --git a/ext/fts3/tool/fts3cov.sh b/ext/fts3/tool/fts3cov.sh new file mode 100644 index 0000000000..b1f34dce76 --- /dev/null +++ b/ext/fts3/tool/fts3cov.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +srcdir=`dirname $(dirname $(dirname $(dirname $0)))` +./testfixture $srcdir/test/fts3.test --output=fts3cov-out.txt + +echo "" + +for f in `ls $srcdir/ext/fts3/*.c` +do + f=`basename $f` + echo -ne "$f: " + gcov -b $f | grep Taken | sed 's/Taken at least once://' +done + diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index a8d7981af0..37f9b7396d 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -398,8 +398,8 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){ if( sqlite3_step(pStmt)==SQLITE_ROW && (nLeaf = sqlite3_column_int(pStmt, 0))>0 ){ - nIdx = sqlite3_column_int(pStmt, 5); sqlite3_int64 sz; + nIdx = sqlite3_column_int(pStmt, 5); printf("For level %d:\n", i); printf(" Number of indexes...................... %9d\n", nIdx); printf(" Number of leaf segments................ %9d\n", nLeaf); diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index aafb4e9f9b..de89099122 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -227,7 +227,7 @@ proc print_isalnum {zFunc lRange} { an_print_ascii_bitmap $lRange puts { if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 96ecb38e33..a45c145d38 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -143,11 +143,13 @@ struct Fts5PhraseIter { ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid ** ** with $p set to a phrase equivalent to the phrase iPhrase of the -** current query is executed. For each row visited, the callback function -** passed as the fourth argument is invoked. The context and API objects -** passed to the callback function may be used to access the properties of -** each matched row. Invoking Api.xUserData() returns a copy of the pointer -** passed as the third argument to pUserData. +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. ** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. @@ -316,7 +318,7 @@ struct Fts5ExtensionApi { ** behaviour. The structure methods are expected to function as follows: ** ** xCreate: -** This function is used to allocate and inititalize a tokenizer instance. +** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) @@ -575,4 +577,3 @@ struct fts5_api { #endif #endif /* _FTS5_H */ - diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 35f15abbaf..d350856573 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -30,7 +30,9 @@ typedef short i16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif #define testcase(x) #define ALWAYS(x) 1 @@ -47,6 +49,10 @@ typedef sqlite3_uint64 u64; #endif +/* Truncate very long tokens to this many bytes. Hard limit is +** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset +** field that occurs at the start of each leaf page (see fts5_index.c). */ +#define FTS5_MAX_TOKEN_SIZE 32768 /* ** Maximum number of prefix indexes on single FTS5 table. This must be @@ -480,6 +486,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p); int sqlite3Fts5IndexReinit(Fts5Index *p); int sqlite3Fts5IndexOptimize(Fts5Index *p); int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); +int sqlite3Fts5IndexReset(Fts5Index *p); int sqlite3Fts5IndexLoadConfig(Fts5Index *p); @@ -622,6 +629,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); int sqlite3Fts5StorageOptimize(Fts5Storage *p); int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); +int sqlite3Fts5StorageReset(Fts5Storage *p); /* ** End of interface to code in fts5_storage.c. @@ -680,7 +688,6 @@ int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); -void sqlite3Fts5ExprClearEof(Fts5Expr*); int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); @@ -732,6 +739,7 @@ void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 836de8a0f0..219ea6fff8 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -189,7 +189,7 @@ static int fts5HighlightCb( if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; - if( iPositer.iEnd ){ + if( iPos>=p->iter.iStart && iPositer.iEnd ){ fts5HighlightAppend(&rc, p, p->zClose, -1); } } @@ -246,6 +246,118 @@ static void fts5HighlightFunction( ** End of highlight() implementation. **************************************************************************/ +/* +** Context object passed to the fts5SentenceFinderCb() function. +*/ +typedef struct Fts5SFinder Fts5SFinder; +struct Fts5SFinder { + int iPos; /* Current token position */ + int nFirstAlloc; /* Allocated size of aFirst[] */ + int nFirst; /* Number of entries in aFirst[] */ + int *aFirst; /* Array of first token in each sentence */ + const char *zDoc; /* Document being tokenized */ +}; + +/* +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an +** error occurs. +*/ +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ + if( p->nFirstAlloc==p->nFirst ){ + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; + int *aNew; + + aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + p->aFirst = aNew; + p->nFirstAlloc = nNew; + } + p->aFirst[p->nFirst++] = iAdd; + return SQLITE_OK; +} + +/* +** This function is an xTokenize() callback used by the auxiliary snippet() +** function. Its job is to identify tokens that are the first in a sentence. +** For each such token, an entry is added to the SFinder.aFirst[] array. +*/ +static int fts5SentenceFinderCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + int rc = SQLITE_OK; + + UNUSED_PARAM2(pToken, nToken); + UNUSED_PARAM(iEndOff); + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + Fts5SFinder *p = (Fts5SFinder*)pContext; + if( p->iPos>0 ){ + int i; + char c = 0; + for(i=iStartOff-1; i>=0; i--){ + c = p->zDoc[i]; + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; + } + if( i!=iStartOff-1 && (c=='.' || c==':') ){ + rc = fts5SentenceFinderAdd(p, p->iPos); + } + }else{ + rc = fts5SentenceFinderAdd(p, 0); + } + p->iPos++; + } + return rc; +} + +static int fts5SnippetScore( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + int nDocsize, /* Size of column in tokens */ + unsigned char *aSeen, /* Array with one element per query phrase */ + int iCol, /* Column to score */ + int iPos, /* Starting offset to score */ + int nToken, /* Max tokens per snippet */ + int *pnScore, /* OUT: Score */ + int *piPos /* OUT: Adjusted offset */ +){ + int rc; + int i; + int ip = 0; + int ic = 0; + int iOff = 0; + int iFirst = -1; + int nInst; + int nScore = 0; + int iLast = 0; + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; ixInst(pFts, i, &ip, &ic, &iOff); + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){ + nScore += (aSeen[ip] ? 1 : 1000); + aSeen[ip] = 1; + if( iFirst<0 ) iFirst = iOff; + iLast = iOff + pApi->xPhraseSize(pFts, ip); + } + } + + *pnScore = nScore; + if( piPos ){ + int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; + if( iAdj<0 ) iAdj = 0; + *piPos = iAdj; + } + + return rc; +} + /* ** Implementation of snippet() function. */ @@ -267,9 +379,10 @@ static void fts5SnippetFunction( unsigned char *aSeen; /* Array of "seen instance" flags */ int iBestCol; /* Column containing best snippet */ int iBestStart = 0; /* First token of best snippet */ - int iBestLast; /* Last token of best snippet */ int nBestScore = 0; /* Score of best snippet */ int nColSize = 0; /* Total size of iBestCol in tokens */ + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ + int nCol; if( nVal!=5 ){ const char *zErr = "wrong number of arguments to function snippet()"; @@ -277,13 +390,13 @@ static void fts5SnippetFunction( return; } + nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); zEllips = (const char*)sqlite3_value_text(apVal[3]); nToken = sqlite3_value_int(apVal[4]); - iBestLast = nToken-1; iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); @@ -291,65 +404,94 @@ static void fts5SnippetFunction( if( aSeen==0 ){ rc = SQLITE_NOMEM; } - if( rc==SQLITE_OK ){ rc = pApi->xInstCount(pFts, &nInst); } - for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &ip, &iSnippetCol, &iStart); - if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ - int nScore = 1000; - int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); - int j; - aSeen[ip] = 1; - for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); - iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; - if( rc==SQLITE_OK && ic==iSnippetCol && iLastiLast ) iLast = iFinal; + memset(&sFinder, 0, sizeof(Fts5SFinder)); + for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize(pFts, + sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb + ); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnSize(pFts, i, &nDocsize); + if( rc!=SQLITE_OK ) break; + + for(ii=0; rc==SQLITE_OK && iixInst(pFts, ii, &ip, &ic, &io); + if( ic!=i || rc!=SQLITE_OK ) continue; + memset(aSeen, 0, nPhrase); + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, + io, nToken, &nScore, &iAdj + ); + if( rc==SQLITE_OK && nScore>nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = iAdj; + nColSize = nDocsize; } - } - if( rc==SQLITE_OK && nScore>nBestScore ){ - iBestCol = iSnippetCol; - iBestStart = iStart; - iBestLast = iLast; - nBestScore = nScore; + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ + for(jj=0; jj<(sFinder.nFirst-1); jj++){ + if( sFinder.aFirst[jj+1]>io ) break; + } + + if( sFinder.aFirst[jj]nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = sFinder.aFirst[jj]; + nColSize = nDocsize; + } + } + } } } } - if( rc==SQLITE_OK ){ - rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); - } if( rc==SQLITE_OK ){ rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); } + if( rc==SQLITE_OK && nColSize==0 ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } if( ctx.zIn ){ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } - if( (iBestStart+nToken-1)>iBestLast ){ - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; - } - if( iBestStart+nToken>nColSize ){ - iBestStart = nColSize - nToken; - } - if( iBestStart<0 ) iBestStart = 0; - ctx.iRangeStart = iBestStart; ctx.iRangeEnd = iBestStart + nToken - 1; if( iBestStart>0 ){ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } + + /* Advance iterator ctx.iter so that it points to the first coalesced + ** phrase instance at or following position iBestStart. */ + while( ctx.iter.iStart>=0 && ctx.iter.iStartxTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); } @@ -358,15 +500,15 @@ static void fts5SnippetFunction( }else{ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } - - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - sqlite3_free(ctx.zOut); } + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); sqlite3_free(aSeen); + sqlite3_free(sFinder.aFirst); } /************************************************************************/ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 1e9be8117d..e18691ad0b 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -167,6 +167,7 @@ static int fts5ExprGetToken( case ',': tok = FTS5_COMMA; break; case '+': tok = FTS5_PLUS; break; case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; case '\0': tok = FTS5_EOF; break; case '"': { @@ -745,49 +746,61 @@ static int fts5ExprNearTest( ** Initialize all term iterators in the pNear object. If any term is found ** to match no documents at all, return immediately without initializing any ** further iterators. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. It is not considered an error if some term matches zero +** documents. */ static int fts5ExprNearInitAll( Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; - int i, j; - int rc = SQLITE_OK; + int i; assert( pNode->bNomatch==0 ); - for(i=0; rc==SQLITE_OK && inPhrase; i++){ + for(i=0; inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - for(j=0; jnTerm; j++){ - Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; - Fts5ExprTerm *p; - int bEof = 1; + if( pPhrase->nTerm==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + }else{ + int j; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bHit = 0; - for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ - if( p->pIter ){ - sqlite3Fts5IterClose(p->pIter); - p->pIter = 0; + for(p=pTerm; p; p=p->pSynonym){ + int rc; + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, + &p->pIter + ); + assert( (rc==SQLITE_OK)==(p->pIter!=0) ); + if( rc!=SQLITE_OK ) return rc; + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + bHit = 1; + } } - rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), - pNear->pColset, - &p->pIter - ); - assert( rc==SQLITE_OK || p->pIter==0 ); - if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ - bEof = 0; - } - } - if( bEof ){ - pNode->bEof = 1; - return rc; + if( bHit==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + } } } } - return rc; + pNode->bEof = 0; + return SQLITE_OK; } /* @@ -920,7 +933,7 @@ static int fts5ExprNodeTest_STRING( } }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - if( pIter->iRowid==iLast ) continue; + if( pIter->iRowid==iLast || pIter->bEof ) continue; bMatch = 0; if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ return rc; @@ -1097,7 +1110,10 @@ static int fts5ExprNodeNext_OR( || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } } } } @@ -1128,7 +1144,10 @@ static int fts5ExprNodeTest_AND( if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } } /* If the child node is now at EOF, so is the parent AND node. Otherwise, @@ -1167,6 +1186,8 @@ static int fts5ExprNodeNext_AND( int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; } return rc; } @@ -1209,6 +1230,9 @@ static int fts5ExprNodeNext_NOT( if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_NOT(pExpr, pNode); } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } return rc; } @@ -1331,7 +1355,10 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ /* If not at EOF but the current rowid occurs earlier than iFirst in ** the iteration order, move to document iFirst or later. */ - if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ + if( rc==SQLITE_OK + && 0==pRoot->bEof + && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 + ){ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); } @@ -1493,6 +1520,7 @@ static int fts5ParseTokenize( /* If an error has already occurred, this is a no-op */ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; @@ -1584,7 +1612,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ - int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); int n; sqlite3Fts5Dequote(z); n = (int)strlen(z); @@ -1636,7 +1664,6 @@ int sqlite3Fts5ExprClonePhrase( ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - int i; /* Used to iterate through phrase terms */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ @@ -1654,19 +1681,37 @@ int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); } + if( rc==SQLITE_OK ){ + Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; + if( pColsetOrig ){ + int nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); + if( pColset ){ + memcpy(pColset, pColsetOrig, nByte); + } + pNew->pRoot->pNear->pColset = pColset; + } + } - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), + 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } if( rc==SQLITE_OK ){ @@ -1781,6 +1826,34 @@ static Fts5Colset *fts5ParseColset( return pNew; } +/* +** Allocate and return an Fts5Colset object specifying the inverse of +** the colset passed as the second argument. Free the colset passed +** as the second argument before returning. +*/ +Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ + Fts5Colset *pRet; + int nCol = pParse->pConfig->nCol; + + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet ){ + int i; + int iOld = 0; + for(i=0; i=p->nCol || p->aiCol[iOld]!=i ){ + pRet->aiCol[pRet->nCol++] = i; + }else{ + iOld++; + } + } + } + + sqlite3_free(p); + return pRet; +} + Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ Fts5Colset *pColset, /* Existing colset object */ @@ -2495,12 +2568,13 @@ static int fts5ExprPopulatePoslistsCb( UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = strlen(pTerm->zTerm); + int nTerm = (int)strlen(pTerm->zTerm); if( (nTerm==nToken || (nTermbPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ @@ -2604,17 +2678,6 @@ void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } -static void fts5ExprClearEof(Fts5ExprNode *pNode){ - int i; - for(i=0; inChild; i++){ - fts5ExprClearEof(pNode->apChild[i]); - } - pNode->bEof = 0; -} -void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ - fts5ExprClearEof(pExpr->pRoot); -} - /* ** This function is only called for detail=columns tables. */ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index ada8bb16cb..afa2a30739 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -345,11 +345,11 @@ int sqlite3Fts5HashWrite( if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); - p->iCol = iCol; + p->iCol = (i16)iCol; p->iPos = 0; }else{ bNew = 1; - p->iCol = iPos = iCol; + p->iCol = (i16)(iPos = iCol); } } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 323e6cefdc..750e0ca82d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -304,6 +304,10 @@ struct Fts5Index { sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + + sqlite3_stmt *pDataVersion; + i64 iStructVersion; /* data_version when pStruct read */ + Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; struct Fts5DoclistIter { @@ -698,7 +702,6 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ return pRet; } - /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). @@ -707,6 +710,18 @@ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = fts5DataRead(p, iRowid); + if( pRet ){ + if( pRet->szLeaf>pRet->nn ){ + p->rc = FTS5_CORRUPT; + fts5DataRelease(pRet); + pRet = 0; + } + } + return pRet; +} + static int fts5IndexPrepareStmt( Fts5Index *p, sqlite3_stmt **ppStmt, @@ -959,6 +974,50 @@ static void fts5StructureExtendLevel( } } +static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ + Fts5Structure *pRet = 0; + Fts5Config *pConfig = p->pConfig; + int iCookie; /* Configuration cookie */ + Fts5Data *pData; + + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc==SQLITE_OK ){ + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); + if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + fts5DataRelease(pData); + if( p->rc!=SQLITE_OK ){ + fts5StructureRelease(pRet); + pRet = 0; + } + } + + return pRet; +} + +static i64 fts5IndexDataVersion(Fts5Index *p){ + i64 iVersion = 0; + + if( p->rc==SQLITE_OK ){ + if( p->pDataVersion==0 ){ + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) + ); + if( p->rc ) return 0; + } + + if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){ + iVersion = sqlite3_column_int64(p->pDataVersion, 0); + } + p->rc = sqlite3_reset(p->pDataVersion); + } + + return iVersion; +} + /* ** Read, deserialize and return the structure record. ** @@ -971,26 +1030,49 @@ static void fts5StructureExtendLevel( ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p){ - Fts5Config *pConfig = p->pConfig; - Fts5Structure *pRet = 0; /* Object to return */ - int iCookie; /* Configuration cookie */ - Fts5Data *pData; - pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); - if( p->rc ) return 0; - /* TODO: Do we need this if the leaf-index is appended? Probably... */ - memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); - p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->pStruct==0 ){ + p->iStructVersion = fts5IndexDataVersion(p); + if( p->rc==SQLITE_OK ){ + p->pStruct = fts5StructureReadUncached(p); + } } - fts5DataRelease(pData); - if( p->rc!=SQLITE_OK ){ - fts5StructureRelease(pRet); - pRet = 0; +#if 0 + else{ + Fts5Structure *pTest = fts5StructureReadUncached(p); + if( pTest ){ + int i, j; + assert_nc( p->pStruct->nSegment==pTest->nSegment ); + assert_nc( p->pStruct->nLevel==pTest->nLevel ); + for(i=0; inLevel; i++){ + assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge ); + assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg ); + for(j=0; jaLevel[i].nSeg; j++){ + Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j]; + Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j]; + assert_nc( p1->iSegid==p2->iSegid ); + assert_nc( p1->pgnoFirst==p2->pgnoFirst ); + assert_nc( p1->pgnoLast==p2->pgnoLast ); + } + } + fts5StructureRelease(pTest); + } + } +#endif + + if( p->rc!=SQLITE_OK ) return 0; + assert( p->iStructVersion!=0 ); + assert( p->pStruct!=0 ); + fts5StructureRef(p->pStruct); + return p->pStruct; +} + +static void fts5StructureInvalidate(Fts5Index *p){ + if( p->pStruct ){ + fts5StructureRelease(p->pStruct); + p->pStruct = 0; } - return pRet; } /* @@ -1448,7 +1530,7 @@ static void fts5SegIterNextPage( pIter->pLeaf = pIter->pNextLeaf; pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ - pIter->pLeaf = fts5DataRead(p, + pIter->pLeaf = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) ); }else{ @@ -1951,14 +2033,13 @@ static void fts5SegIterNext( if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist - ); + ); } - } else if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], iOff - ); + ); pIter->iLeafOffset = iOff; pIter->iEndofDoclist = iOff; bNewTerm = 1; @@ -1992,6 +2073,7 @@ static void fts5SegIterNext( */ int nSz; assert( p->rc==SQLITE_OK ); + assert( pIter->iLeafOffset<=pIter->pLeaf->nn ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; @@ -2198,6 +2280,11 @@ static void fts5LeafSeek( iTermOff += nKeep; iOff = iTermOff; + if( iOff>=n ){ + p->rc = FTS5_CORRUPT; + return; + } + /* Read the nKeep field of the next term. */ fts5FastGetVarint32(a, iOff, nKeep); } @@ -2250,6 +2337,18 @@ static void fts5LeafSeek( fts5SegIterLoadNPos(p, pIter); } +static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + } + return p->pIdxSelect; +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. @@ -2267,6 +2366,7 @@ static void fts5SegIterSeekInit( int iPg = 1; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ + sqlite3_stmt *pIdxSelect = 0; assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); @@ -2275,23 +2375,16 @@ static void fts5SegIterSeekInit( /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ - if( p->pIdxSelect==0 ){ - Fts5Config *pConfig = p->pConfig; - fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( - "SELECT pgno FROM '%q'.'%q_idx' WHERE " - "segid=? AND term<=? ORDER BY term DESC LIMIT 1", - pConfig->zDb, pConfig->zName - )); - } + pIdxSelect = fts5IdxSelectStmt(p); if( p->rc ) return; - sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); - sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ - i64 val = sqlite3_column_int(p->pIdxSelect, 0); + sqlite3_bind_int(pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pIdxSelect) ){ + i64 val = sqlite3_column_int(pIdxSelect, 0); iPg = (int)(val>>1); bDlidx = (val & 0x0001); } - p->rc = sqlite3_reset(p->pIdxSelect); + p->rc = sqlite3_reset(pIdxSelect); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -2748,6 +2841,7 @@ static void fts5MultiIterNext( i64 iFrom /* Advance at least as far as this */ ){ int bUseFrom = bFrom; + assert( pIter->base.bEof==0 ); while( p->rc==SQLITE_OK ){ int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; @@ -2974,7 +3068,7 @@ static void fts5ChunkIterate( break; }else{ pgno++; - pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); + pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); if( pData==0 ) break; pChunk = &pData->p[4]; nChunk = MIN(nRem, pData->szLeaf - 4); @@ -3118,6 +3212,15 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + UNUSED_PARAM(pSeg); + pIter->base.nData = 0; +} + /* ** xSetOutputs callback used by detail=col when there is a column filter ** and there are 100 or more columns. Also called as a fallback from @@ -3167,7 +3270,7 @@ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ if( aiCol==aiColEnd ) goto setoutputs_col_out; } if( *aiCol==iPrev ){ - *aOut++ = (iPrev - iPrevOut) + 2; + *aOut++ = (u8)((iPrev - iPrevOut) + 2); iPrevOut = iPrev; } } @@ -3223,6 +3326,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } @@ -3453,18 +3560,46 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ - while( iSegid==0 ){ - int iLvl, iSeg; - sqlite3_randomness(sizeof(u32), (void*)&iSegid); - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ - iSegid = 0; - } + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT ){ + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); } } } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & (1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + + { + sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); + if( p->rc==SQLITE_OK ){ + u8 aBlob[2] = {0xff, 0xff}; + sqlite3_bind_int(pIdxSelect, 1, iSegid); + sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); + assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + p->rc = sqlite3_reset(pIdxSelect); + } + } +#endif } } @@ -3710,6 +3845,9 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; +static int nCall = 0; +nCall++; + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ @@ -3909,7 +4047,9 @@ static void fts5WriteFinish( fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - fts5WriteFlushBtree(p, pWriter); + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); @@ -4328,6 +4468,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); + fts5StructureInvalidate(p); if( iSegid ){ const int pgsz = p->pConfig->pgsz; @@ -4512,7 +4653,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( if( pNew ){ Fts5StructureLevel *pLvl; - int nByte = nSeg * sizeof(Fts5StructureSegment); + nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; @@ -4547,6 +4688,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); pStruct = fts5StructureRead(p); + fts5StructureInvalidate(p); if( pStruct ){ pNew = fts5IndexOptimizeStruct(p, pStruct); @@ -4577,6 +4719,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; + fts5StructureInvalidate(p); if( nMerge<0 ){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); @@ -5004,6 +5147,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ int sqlite3Fts5IndexRollback(Fts5Index *p){ fts5CloseReader(p); fts5IndexDiscardData(p); + fts5StructureInvalidate(p); /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } @@ -5015,6 +5159,7 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; + fts5StructureInvalidate(p); memset(&s, 0, sizeof(Fts5Structure)); fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); @@ -5073,11 +5218,13 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); + fts5StructureInvalidate(p); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pDataVersion); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -5683,7 +5830,7 @@ static void fts5IndexIntegrityCheckSegment( ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iIdxLeafpgnoFirst ) continue; iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); - pLeaf = fts5DataRead(p, iRow); + pLeaf = fts5LeafRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal @@ -6333,3 +6480,12 @@ int sqlite3Fts5IndexInit(sqlite3 *db){ } return rc; } + + +int sqlite3Fts5IndexReset(Fts5Index *p){ + assert( p->pStruct==0 || p->iStructVersion!=0 ); + if( fts5IndexDataVersion(p)!=p->iStructVersion ){ + fts5StructureInvalidate(p); + } + return fts5IndexReturn(p); +} diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index dcd131c74a..384d3dd8f7 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -597,27 +597,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_OK; } +static int fts5NewTransaction(Fts5Table *pTab){ + Fts5Cursor *pCsr; + for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ + if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; + } + return sqlite3Fts5StorageReset(pTab->pStorage); +} + /* ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; - Fts5Cursor *pCsr; /* New cursor object */ + Fts5Cursor *pCsr = 0; /* New cursor object */ int nByte; /* Bytes of space to allocate */ - int rc = SQLITE_OK; /* Return code */ + int rc; /* Return code */ - nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); - pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); - if( pCsr ){ - Fts5Global *pGlobal = pTab->pGlobal; - memset(pCsr, 0, nByte); - pCsr->aColumnSize = (int*)&pCsr[1]; - pCsr->pNext = pGlobal->pCsr; - pGlobal->pCsr = pCsr; - pCsr->iCsrId = ++pGlobal->iNextId; - }else{ - rc = SQLITE_NOMEM; + rc = fts5NewTransaction(pTab); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); + pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); + if( pCsr ){ + Fts5Global *pGlobal = pTab->pGlobal; + memset(pCsr, 0, nByte); + pCsr->aColumnSize = (int*)&pCsr[1]; + pCsr->pNext = pGlobal->pCsr; + pGlobal->pCsr = pCsr; + pCsr->iCsrId = ++pGlobal->iNextId; + }else{ + rc = SQLITE_NOMEM; + } } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; @@ -1175,7 +1186,6 @@ static int fts5FilterMethod( pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); - sqlite3Fts5ExprClearEof(pCsr->pExpr); }else if( pMatch ){ const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); if( zExpr==0 ) zExpr = ""; @@ -1578,8 +1588,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){ ** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); + fts5NewTransaction((Fts5Table*)pVtab); return SQLITE_OK; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index e4f5dd8f51..c8c26bd1b6 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -145,6 +145,7 @@ static int fts5StorageGetStmt( } *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); return rc; } @@ -246,7 +247,11 @@ int sqlite3Fts5CreateTable( char *zErr = 0; rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", - pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" + pConfig->zDb, pConfig->zName, zPost, zDefn, +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID + bWithout?" WITHOUT ROWID": +#endif + "" ); if( zErr ){ *pzErr = sqlite3_mprintf( @@ -368,6 +373,7 @@ static int fts5StorageInsertCallback( Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } @@ -539,11 +545,6 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ } } - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -639,6 +640,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){ return sqlite3Fts5IndexMerge(p->pIndex, nMerge); } +int sqlite3Fts5StorageReset(Fts5Storage *p){ + return sqlite3Fts5IndexReset(p->pIndex); +} + /* ** Allocate a new rowid. This is used for "external content" tables when ** a NULL value is inserted into the rowid column. The new rowid is allocated @@ -743,11 +748,6 @@ int sqlite3Fts5StorageIndexInsert( } sqlite3_free(buf.p); - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -810,6 +810,7 @@ static int fts5StorageIntegrityCallback( int iCol; UNUSED_PARAM2(iUnused1, iUnused2); + if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; @@ -1081,12 +1082,17 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ ** Flush any data currently held in-memory to disk. */ int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ - if( bCommit && p->bTotalsValid ){ - int rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; - if( rc!=SQLITE_OK ) return rc; + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); + if( p->bTotalsValid ){ + rc = fts5StorageSaveTotals(p); + if( bCommit ) p->bTotalsValid = 0; } - return sqlite3Fts5IndexSync(p->pIndex, bCommit); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex, bCommit); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; } int sqlite3Fts5StorageRollback(Fts5Storage *p){ @@ -1121,5 +1127,3 @@ int sqlite3Fts5StorageConfigValue( } return rc; } - - diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 72db65777f..5fe690f6b7 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -14,7 +14,14 @@ #ifdef SQLITE_TEST -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #ifdef SQLITE_ENABLE_FTS5 @@ -78,7 +85,7 @@ static int f5tResultToErrorCode(const char *zRes){ return SQLITE_ERROR; } -static int f5tDbAndApi( +static int SQLITE_TCLAPI f5tDbAndApi( Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb, @@ -164,7 +171,7 @@ static int xTokenizeCb( return rc; } -static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); +static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); static int xQueryPhraseCb( const Fts5ExtensionApi *pApi, @@ -209,7 +216,7 @@ static void xSetAuxdataDestructor(void *p){ ** ** Description... */ -static int xF5tApi( +static int SQLITE_TCLAPI xF5tApi( void * clientData, Tcl_Interp *interp, int objc, @@ -602,7 +609,7 @@ static void xF5tDestroy(void *pCtx){ ** ** Description... */ -static int f5tCreateFunction( +static int SQLITE_TCLAPI f5tCreateFunction( void * clientData, Tcl_Interp *interp, int objc, @@ -672,7 +679,7 @@ static int xTokenizeCb2( ** ** Description... */ -static int f5tTokenize( +static int SQLITE_TCLAPI f5tTokenize( void * clientData, Tcl_Interp *interp, int objc, @@ -878,7 +885,7 @@ static int f5tTokenizerTokenize( /* ** sqlite3_fts5_token ?-colocated? TEXT START END */ -static int f5tTokenizerReturn( +static int SQLITE_TCLAPI f5tTokenizerReturn( void * clientData, Tcl_Interp *interp, int objc, @@ -949,7 +956,7 @@ static void f5tDelTokenizer(void *pCtx){ ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each ** token within the tokenized text. */ -static int f5tCreateTokenizer( +static int SQLITE_TCLAPI f5tCreateTokenizer( ClientData clientData, Tcl_Interp *interp, int objc, @@ -992,7 +999,7 @@ static int f5tCreateTokenizer( return TCL_OK; } -static void xF5tFree(ClientData clientData){ +static void SQLITE_TCLAPI xF5tFree(ClientData clientData){ ckfree(clientData); } @@ -1001,7 +1008,7 @@ static void xF5tFree(ClientData clientData){ ** ** Set or clear the global "may-be-corrupt" flag. Return the old value. */ -static int f5tMayBeCorrupt( +static int SQLITE_TCLAPI f5tMayBeCorrupt( void * clientData, Tcl_Interp *interp, int objc, @@ -1033,7 +1040,7 @@ static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){ return (h % nSlot); } -static int f5tTokenHash( +static int SQLITE_TCLAPI f5tTokenHash( void * clientData, Tcl_Interp *interp, int objc, @@ -1058,7 +1065,7 @@ static int f5tTokenHash( return TCL_OK; } -static int f5tRegisterMatchinfo( +static int SQLITE_TCLAPI f5tRegisterMatchinfo( void * clientData, Tcl_Interp *interp, int objc, @@ -1083,7 +1090,7 @@ static int f5tRegisterMatchinfo( return TCL_OK; } -static int f5tRegisterTok( +static int SQLITE_TCLAPI f5tRegisterTok( void * clientData, Tcl_Interp *interp, int objc, diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index f3a2381cc1..82c7dc9056 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -279,8 +279,19 @@ static int fts5VocabBestIndexMethod( } } - pInfo->idxNum = idxNum; + /* This virtual table always delivers results in ascending order of + ** the "term" column (column 0). So if the user has requested this + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the + ** sqlite3_index_info.orderByConsumed flag to tell the core the results + ** are already in sorted order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + pInfo->idxNum = idxNum; return SQLITE_OK; } diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y index 1607d3846a..1cc4b88e18 100644 --- a/ext/fts5/fts5parse.y +++ b/ext/fts5/fts5parse.y @@ -120,10 +120,17 @@ cnearset(A) ::= colset(X) COLON nearset(Y). { %type colsetlist {Fts5Colset*} %destructor colsetlist { sqlite3_free($$); } +colset(A) ::= MINUS LCP colsetlist(X) RCP. { + A = sqlite3Fts5ParseColsetInvert(pParse, X); +} colset(A) ::= LCP colsetlist(X) RCP. { A = X; } colset(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } +colset(A) ::= MINUS STRING(X). { + A = sqlite3Fts5ParseColset(pParse, 0, &X); + A = sqlite3Fts5ParseColsetInvert(pParse, A); +} colsetlist(A) ::= colsetlist(Y) STRING(X). { A = sqlite3Fts5ParseColset(pParse, Y, &X); } @@ -131,7 +138,6 @@ colsetlist(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } - %type nearset {Fts5ExprNearset*} %type nearphrases {Fts5ExprNearset*} %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index fdcf08398d..428ca6c1ea 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -432,9 +432,16 @@ proc funk {} { } db func funk funk +# This test case corrupts the structure record within the first invocation +# of function funk(). Which used to cause the bm25() function to throw an +# exception. But since bm25() can now used the cached structure record, +# it never sees the corruption introduced by funk() and so the following +# statement no longer fails. +# do_catchsql_test 16.2 { SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' -} {1 {SQL logic error or missing database}} +} {0 {{} -1e-06 {}}} +# {1 {SQL logic error or missing database}} #------------------------------------------------------------------------- # diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index d6b2241568..6aab55a557 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -72,42 +72,56 @@ foreach {tn doc res} { 2.2 {o X o o o o o o} {o [X] o o o o o...} 2.3 {o o X o o o o o} {o o [X] o o o o...} 2.4 {o o o X o o o o} {o o o [X] o o o...} - 2.5 {o o o o X o o o} {...o o o [X] o o o} - 2.6 {o o o o o X o o} {...o o o o [X] o o} - 2.7 {o o o o o o X o} {...o o o o o [X] o} + 2.5 {o o o o X o o o} {o o o o [X] o o...} + 2.6 {o o o o o X o o} {o o o o o [X] o...} + 2.7 {o o o o o o X o} {o o o o o o [X]...} 2.8 {o o o o o o o X} {...o o o o o o [X]} + 2.9 {o o o o o o o X o} {...o o o o o [X] o} + 2.10 {o o o o o o o X o o} {...o o o o [X] o o} + 2.11 {o o o o o o o X o o o} {...o o o [X] o o o} + 2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...} + + 3.1 {X o o o o o o o o} {[X] o o o o o o...} 3.2 {o X o o o o o o o} {o [X] o o o o o...} 3.3 {o o X o o o o o o} {o o [X] o o o o...} 3.4 {o o o X o o o o o} {o o o [X] o o o...} - 3.5 {o o o o X o o o o} {...o o o [X] o o o...} - 3.6 {o o o o o X o o o} {...o o o [X] o o o} - 3.7 {o o o o o o X o o} {...o o o o [X] o o} - 3.8 {o o o o o o o X o} {...o o o o o [X] o} - 3.9 {o o o o o o o o X} {...o o o o o o [X]} + + 3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...} + 3.6 {o o o o o o o o X o o o} {...o o o [X] o o o} + 3.7 {o o o o o o o o o X o o} {...o o o o [X] o o} + 3.8 {o o o o o o o o o o X o} {...o o o o o [X] o} + 3.9 {o o o o o o o o o o o X} {...o o o o o o [X]} 4.1 {X o o o o o X o o} {[X] o o o o o [X]...} - 4.2 {o X o o o o o X o} {...[X] o o o o o [X]...} - 4.3 {o o X o o o o o X} {...[X] o o o o o [X]} + 4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...} + 4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]} 5.1 {X o o o o X o o o} {[X] o o o o [X] o...} - 5.2 {o X o o o o X o o} {...[X] o o o o [X] o...} - 5.3 {o o X o o o o X o} {...[X] o o o o [X] o} - 5.4 {o o o X o o o o X} {...o [X] o o o o [X]} + 5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...} + 5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o} + 5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]} 6.1 {X o o o X o o o} {[X] o o o [X] o o...} 6.2 {o X o o o X o o o} {o [X] o o o [X] o...} - 6.3 {o o X o o o X o o} {...o [X] o o o [X] o...} - 6.4 {o o o X o o o X o} {...o [X] o o o [X] o} - 6.5 {o o o o X o o o X} {...o o [X] o o o [X]} + 6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...} + 6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o} + 6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]} 7.1 {X o o X o o o o o} {[X] o o [X] o o o...} 7.2 {o X o o X o o o o} {o [X] o o [X] o o...} - 7.3 {o o X o o X o o o} {...o [X] o o [X] o o...} - 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} - 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} - 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} + 7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...} + 7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o} + 7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o} + 7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]} + + 8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] [X] [X] o o...} + 8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} + {...o o [X] o o o o...} + 8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} + {o o o o [X] o o...} } { do_snippet_test 1.$tn $doc X $res } @@ -124,24 +138,43 @@ if {[detail_is_full]} { 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 2.3 {o o X Y o o o o} {o o [X Y] o o o...} - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} + 2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o} + 2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o} + 2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o} + 2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]} 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} + 3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...} + 3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o} + 3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o} + 3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o} + 3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]} } { do_snippet_test 2.$tn $doc "X + Y" $res } } +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(a, b); + INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13'); + SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz'); +} { + {1 2 3 4 5...} +} + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE p1 USING fts5(a, b); + INSERT INTO p1 VALUES( + 'x a a a a a a a a a a', + 'a a a a a a a a a a a a a a a a a a a x' + ); +} +do_execsql_test 5.1 { + SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); +} {{[x] a a a a a...}} + } ;# foreach_detail_mode finish_test diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test index 995fe85784..fa3167a4d1 100644 --- a/ext/fts5/test/fts5aux.test +++ b/ext/fts5/test/fts5aux.test @@ -246,5 +246,37 @@ foreach {tn lRow res} { } $res } +#------------------------------------------------------------------------- +# Test the built-in bm25() demo. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('a', NULL); -- 1 + INSERT INTO t1 VALUES('a', NULL); -- 2 + INSERT INTO t1 VALUES('a', NULL); -- 3 + INSERT INTO t1 VALUES('a', NULL); -- 4 + INSERT INTO t1 VALUES('a', NULL); -- 5 + INSERT INTO t1 VALUES('a', NULL); -- 6 + INSERT INTO t1 VALUES('a', NULL); -- 7 + INSERT INTO t1 VALUES('a', NULL); -- 8 + INSERT INTO t1 VALUES(NULL, 'a a b'); -- 9 + INSERT INTO t1 VALUES(NULL, 'b b a'); -- 10 +} + +do_execsql_test 9.2 { + SELECT rowid FROM t1('a AND b') ORDER BY rank; +} { + 10 9 +} + +do_execsql_test 9.3 { + SELECT rowid FROM t1('b:a AND b:b') ORDER BY rank; +} { + 9 10 +} + + + finish_test diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test new file mode 100644 index 0000000000..e3333241ee --- /dev/null +++ b/ext/fts5/test/fts5colset.test @@ -0,0 +1,59 @@ +# 2016 August 10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5colset + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $::testprefix { + if {[detail_is_none]} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d, detail=%DETAIL%); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 + } + + foreach {tn q res} { + 1 "a" {1 2 3 4} + 2 "{a} : a" {1} + 3 "-{a} : a" {2 3 4} + 4 "- {a c} : a" {2 4} + 5 " - {d d c} : a" {1 2} + 6 "- {d c b a} : a" {} + 7 "-{\"a\"} : b" {1 2 3} + 8 "- c : a" {1 2 4} + 9 "-c : a" {1 2 4} + 10 "-\"c\" : a" {1 2 4} + } { + breakpoint + do_execsql_test 1.$tn { + SELECT rowid FROM t1($q) + } $res + } + + +} + + +finish_test + + diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 3a4fcfaaed..c10017a266 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -37,7 +37,7 @@ do_execsql_test 1.0 { } set mask [expr 31 << 31] -if 1 { +if 0 { # Test 1: # @@ -84,6 +84,8 @@ foreach {tno stmt} { } } +} + # Using the same database as the 1.* tests. # # Run N-1 tests, where N is the number of bytes in the rightmost leaf page @@ -212,8 +214,6 @@ foreach {tn nCut} { # do_test 4.$tn.x { expr $nCorrupt>0 } 1 } -} - set doc [string repeat "A B C " 1000] do_execsql_test 5.0 { CREATE VIRTUAL TABLE x5 USING fts5(tt); diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index 7a8cb5c465..9653bcaac2 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} { if {$end<=$i} break lset var end [expr $end - $i] set struct [binary format c* $var] + + db close + sqlite3 db test.db + db eval { BEGIN; UPDATE t1_data SET block = $struct WHERE id=10; diff --git a/ext/fts5/test/fts5determin.test b/ext/fts5/test/fts5determin.test new file mode 100644 index 0000000000..e368c4704e --- /dev/null +++ b/ext/fts5/test/fts5determin.test @@ -0,0 +1,67 @@ +# 2016 March 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# +# Specifically, that the fts5 module is deterministic. At one point, when +# segment ids were allocated using sqlite3_randomness(), this was not the +# case. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5aa +return_if_no_fts5 + +proc do_determin_test {tn} { + uplevel [list + do_execsql_test $tn { + SELECT (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t2_data), + (SELECT md5sum(id, block) FROM t1_data)== + (SELECT md5sum(id, block) FROM t3_data) + } {1 1} + ] +} + +foreach_detail_mode $::testprefix { + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + CREATE VIRTUAL TABLE t3 USING fts5(a, b, prefix="1 2", detail=%DETAIL%); + } + + do_test 1.1 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL VALUES('a b c', 'd e f'); + INSERT INTO TBL VALUES('c1 c2 c3', 'c1 c2 c3'); + INSERT INTO TBL VALUES('xyzxyzxyz', 'xyzxyzxyz'); + }] + } + } {} + + do_determin_test 1.2 + + do_test 1.3 { + foreach t {t1 t2 t3} { + execsql [string map [list TBL $t] { + INSERT INTO TBL(TBL) VALUES('optimize'); + }] + } + } {} + + do_determin_test 1.4 +} + + +finish_test + + diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 232b5021f1..655beb984b 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -178,7 +178,7 @@ do_execsql_test 3.2 { ORDER BY rowid DESC; } {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1} -do_execsql_test 3.2 { +do_execsql_test 3.3 { INSERT INTO abc(abc) VALUES('integrity-check'); INSERT INTO abc(abc) VALUES('optimize'); INSERT INTO abc(abc) VALUES('integrity-check'); @@ -187,7 +187,7 @@ do_execsql_test 3.2 { set v [lindex $vocab 0] set i 0 foreach v $vocab { - do_execsql_test 3.3.[incr i] { + do_execsql_test 3.4.[incr i] { SELECT rowid FROM abc WHERE abc MATCH $v } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} } diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 69418aae63..510a0d790c 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -63,6 +63,23 @@ do_catchsql_test 2.1 { SELECT fts5_expr_tcl() } {1 {wrong number of arguments to function fts5_expr_tcl}} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE e1 USING fts5(text, tokenize = 'porter unicode61'); + INSERT INTO e1 VALUES ("just a few words with a / inside"); +} +do_execsql_test 3.1 { + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank; +} {1 -1e-06} +do_execsql_test 3.2 { + SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"' +} 1 +do_execsql_test 3.3 { + SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank; +} {1 -1e-06} + + + finish_test diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index acc43ebfc6..bfa54a5b04 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -86,7 +86,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}] do_faultsim_test 4 -faults oom-* -body { db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'} } -test { - faultsim_test_result {0 {0 {} 4}} + faultsim_test_result {0 {0 {} 3}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test index 40df8b6043..2ce33f4bc4 100644 --- a/ext/fts5/test/fts5faultB.test +++ b/ext/fts5/test/fts5faultB.test @@ -78,6 +78,34 @@ do_faultsim_test 2.4 -faults oom* -body { faultsim_test_result {0 {{3 2} {2 3}}} } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x1 USING fts5(z); +} + +do_faultsim_test 3.1 -faults oom* -body { + execsql { + SELECT rowid FROM x1('c') WHERE rowid>1; + } +} -test { + faultsim_test_result {0 {}} +} + +do_execsql_test 3.2 { + INSERT INTO x1 VALUES('a b c'); + INSERT INTO x1 VALUES('b c d'); + INSERT INTO x1 VALUES('c d e'); + INSERT INTO x1 VALUES('d e f'); +} +do_faultsim_test 3.3 -faults oom* -body { + execsql { + SELECT rowid FROM x1('c') WHERE rowid>1; + } +} -test { + faultsim_test_result {0 {2 3}} +} finish_test diff --git a/ext/fts5/test/fts5faultD.test b/ext/fts5/test/fts5faultD.test new file mode 100644 index 0000000000..e259cbf610 --- /dev/null +++ b/ext/fts5/test/fts5faultD.test @@ -0,0 +1,87 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultA + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + if {"%DETAIL%"=="none"} continue + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, b, c, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'A', 'B', 'C' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'C', 'A', 'B' FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<60 ) + INSERT INTO o1 SELECT 'B', 'C', 'A' FROM s; + } + + do_faultsim_test 1 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('a') } + } -test { + faultsim_test_result {0 180} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 2 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('a:a AND {b c}:b') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 3 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('{b c}:b NOT a:a') ORDER BY rank } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 4 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT * FROM o1('b:b OR a:a') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } + + do_faultsim_test 5 -faults int* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT count(*) FROM o1('c:b') } + expr 1 + } -test { + faultsim_test_result {0 1} {1 {vtable constructor failed: o1}} + } +} + +finish_test diff --git a/ext/fts5/test/fts5lastrowid.test b/ext/fts5/test/fts5lastrowid.test new file mode 100644 index 0000000000..0f7628b233 --- /dev/null +++ b/ext/fts5/test/fts5lastrowid.test @@ -0,0 +1,73 @@ +# 2017 Feb 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the last_insert_rowid functionality with fts5. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5lastrowid + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(str); +} + +do_execsql_test 1.1 { + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + SELECT last_insert_rowid(); +} {3} + +do_execsql_test 1.2 { + BEGIN; + INSERT INTO t1 VALUES('one string'); + INSERT INTO t1 VALUES('two string'); + INSERT INTO t1 VALUES('three string'); + COMMIT; + SELECT last_insert_rowid(); +} {6} + +do_execsql_test 1.3 { + INSERT INTO t1(rowid, str) VALUES(-22, 'some more text'); + SELECT last_insert_rowid(); +} {-22} + +do_execsql_test 1.4 { + BEGIN; + INSERT INTO t1(rowid, str) VALUES(45, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(46, 'some more text'); + INSERT INTO t1(rowid, str) VALUES(222, 'some more text'); + SELECT last_insert_rowid(); + COMMIT; + SELECT last_insert_rowid(); +} {222 222} + +do_execsql_test 1.5 { + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo'); + INSERT INTO t1 SELECT x FROM x1; + SELECT last_insert_rowid(); +} {226} + +do_execsql_test 1.6 { + INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1; + SELECT last_insert_rowid(); +} {14} + + +finish_test + diff --git a/ext/fts5/test/fts5multiclient.test b/ext/fts5/test/fts5multiclient.test new file mode 100644 index 0000000000..a1e9aa0d30 --- /dev/null +++ b/ext/fts5/test/fts5multiclient.test @@ -0,0 +1,48 @@ +# 2016 March 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/lock_common.tcl + +set testprefix fts5multiclient +return_if_no_fts5 + +foreach_detail_mode $testprefix { + +do_multiclient_test tn { + + do_test 1.$tn.1 { + sql1 { CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%) } + sql1 { INSERT INTO t1 VALUES('a b c') } + sql2 { SELECT rowid FROM t1('b') } + } {1} + + do_test 1.$tn.2 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { SELECT rowid FROM t1('b') } + } {1 2} + + do_test 1.$tn.3 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { SELECT rowid FROM t1('b') } + } {1 2 3} + + do_test 1.$tn.4 { + sql2 { INSERT INTO t1 VALUES('a b c') } + sql1 { INSERT INTO t1 VALUES('a b c') } + sql3 { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + +};# do_multiclient_test +};# foreach_detail_mode +finish_test + diff --git a/ext/fts5/test/fts5prefix.test b/ext/fts5/test/fts5prefix.test index 8e0d5a2954..01d0fe5d14 100644 --- a/ext/fts5/test/fts5prefix.test +++ b/ext/fts5/test/fts5prefix.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# This file containst tests focused on prefix indexes. +# This file contains tests focused on prefix indexes. # source [file join [file dirname [info script]] fts5_common.tcl] diff --git a/ext/fts5/test/fts5rank.test b/ext/fts5/test/fts5rank.test index 4961b42605..a70c5d68e3 100644 --- a/ext/fts5/test/fts5rank.test +++ b/ext/fts5/test/fts5rank.test @@ -91,7 +91,61 @@ do_test 2.7 { } {1 3 2} +#-------------------------------------------------------------------------- +# At one point there was a problem with queries such as: +# +# ... MATCH 'x OR y' ORDER BY rank; +# +# if there were zero occurrences of token 'y' in the dataset. The +# following tests verify that that problem has been addressed. +# +foreach_detail_mode $::testprefix { + do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%); + INSERT INTO y1 VALUES('test xyz'); + INSERT INTO y1 VALUES('test test xyz test'); + INSERT INTO y1 VALUES('test test xyz'); + } + do_execsql_test 3.1.1 { + SELECT rowid FROM y1('test OR tset'); + } {1 2 3} + + do_execsql_test 3.1.2 { + SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1) + } {2 3 1} + + do_execsql_test 3.1.3 { + SELECT rowid FROM y1('test OR tset') ORDER BY +rank + } {2 3 1} + + do_execsql_test 3.1.4 { + SELECT rowid FROM y1('test OR tset') ORDER BY rank + } {2 3 1} + + do_execsql_test 3.1.5 { + SELECT rowid FROM y1('test OR xyz') ORDER BY rank + } {3 2 1} + + + do_execsql_test 3.2.1 { + CREATE VIRTUAL TABLE z1 USING fts5(a, detail=%DETAIL%); + INSERT INTO z1 VALUES('wrinkle in time'); + SELECT * FROM z1 WHERE z1 MATCH 'wrinkle in time OR a wrinkle in time'; + } {{wrinkle in time}} +} + +do_execsql_test 4.1 { + DROP TABLE IF EXISTS VTest; + CREATE virtual TABLE VTest USING FTS5( + Title, AUthor, tokenize ='porter unicode61 remove_diacritics 1', + columnsize='1', detail=full + ); + INSERT INTO VTest (Title, Author) VALUES ('wrinkle in time', 'Bill Smith'); + + SELECT * FROM VTest WHERE + VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank; +} {{wrinkle in time} {Bill Smith}} diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 464b601548..5ac413c885 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -265,7 +265,7 @@ do_test 11.0 { execsql " INSERT INTO t4 VALUES('a b c \x1A'); INSERT INTO t4 VALUES('a b c d\x1A'); - INSERT INTO t4 VALUES('a b c \x1Ad'); + INSERT INTO t4 VALUES('a b c \x1Ag'); INSERT INTO t4 VALUES('a b c d'); " } {} @@ -340,7 +340,7 @@ do_test 14.2 { #------------------------------------------------------------------------- db func rnddoc fts5_rnddoc -do_execsql_test 4.0 { +do_execsql_test 14.3 { CREATE VIRTUAL TABLE x1 USING fts5(x); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); @@ -348,9 +348,9 @@ do_execsql_test 4.0 { INSERT INTO x1 SELECT rnddoc(5) FROM ii; } -do_execsql_test 4.1 { +do_execsql_test 14.4 { SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' -} {0 {} 4} +} {0 {} 3} #------------------------------------------------------------------------- reset_db @@ -448,4 +448,24 @@ do_execsql_test 20.2 { INSERT INTO x1(x1) VALUES('integrity-check'); } {} +#------------------------------------------------------------------------- +reset_db +set doc "a b [string repeat x 100000]" +do_execsql_test 21.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(rowid, x) VALUES(11111, $doc); + INSERT INTO x1(rowid, x) VALUES(11112, $doc); +} +do_execsql_test 21.1 { + INSERT INTO x1(x1) VALUES('integrity-check'); +} +do_execsql_test 21.2 { + SELECT rowid FROM x1($doc); +} {11111 11112} +do_execsql_test 21.3 { + DELETE FROM x1 WHERE rowid=11111; + INSERT INTO x1(x1) VALUES('integrity-check'); + SELECT rowid FROM x1($doc); +} {11112} + finish_test diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index 186d771f76..a6c1789473 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -332,6 +332,41 @@ do_execsql_test 16.0 { DELETE FROM t2; } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 17.0 { + CREATE VIRTUAL TABLE t2 USING fts5(x, y); + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + COMMIT; +} +do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 } +do_execsql_test 17.2 { + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 ; +} +do_execsql_test 17.3 { + COMMIT +} + +reset_db +do_execsql_test 17.4 { + CREATE VIRTUAL TABLE t2 USING fts5(x, y); + BEGIN; + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); + SELECT * FROM t2('y:a*') WHERE rowid>66; +} +do_execsql_test 17.5 { SELECT * FROM t2('x:b* OR y:a*') } +do_execsql_test 17.5 { COMMIT ; SELECT * FROM t2('x:b* OR y:a*') } +do_execsql_test 17.6 { + SELECT * FROM t2('x:b* OR y:a*') WHERE rowid>55 +} + #db eval {SELECT rowid, fts5_decode_none(rowid, block) aS r FROM t2_data} {puts $r} finish_test diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index b6922f84dd..c755ea0567 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -80,6 +80,40 @@ do_execsql_test 3.0 { SELECT * FROM x3('x OR y OR z'); } +#------------------------------------------------------------------------- +# Test that a crash occuring when the second or subsequent tokens in a +# phrase matched zero rows has been fixed. +# +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1 VALUES('ab'); + INSERT INTO t1 VALUES('cd'); + INSERT INTO t1 VALUES('ab cd'); + INSERT INTO t1 VALUES('ab cdXXX'); + INSERT INTO t1 VALUES('abXXX cd'); +} +do_execsql_test 4.1 { + SELECT * FROM t1('"ab cd" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.2 { + SELECT * FROM t1('"xy zz" OR "ab cd" *'); +} {{ab cd} {ab cdXXX}} +do_execsql_test 4.3 { + SELECT * FROM t1('"xy zz" OR "xy zz" *'); +} +do_execsql_test 4.4 { + SELECT * FROM t1('"ab cd" OR "xy zz" *'); +} {{ab cd}} +do_execsql_test 4.5 { + CREATE VIRTUAL TABLE t2 USING fts5(x); + INSERT INTO t2 VALUES('ab'); + INSERT INTO t2 VALUES('cd'); + INSERT INTO t2 VALUES('ef'); +} +do_execsql_test 4.6 { + SELECT * FROM t2('ab + xyz'); +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 185dda3ff4..6243663123 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -152,7 +152,7 @@ foreach {tn expr res} { 1 {abc} {"abc"} 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} - 4 {3*} {"3"|"iii"|"three" *} + 4 {3*} {"3" *} } { do_execsql_test 4.1.$tn { SELECT fts5_expr($expr, 'tokenize=tclnum') diff --git a/ext/fts5/test/fts5tokenizer.test b/ext/fts5/test/fts5tokenizer.test index 9316d3c234..f68aa19a10 100644 --- a/ext/fts5/test/fts5tokenizer.test +++ b/ext/fts5/test/fts5tokenizer.test @@ -262,5 +262,44 @@ do_execsql_test 8.3 { brown dog fox jump lazi over quick the } +#------------------------------------------------------------------------- +# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer +# implementation. +# +reset_db +proc tcl_create {args} { return "tcl_tokenize" } +sqlite3_fts5_create_tokenizer db tcl tcl_create +set ::flags [list] +proc tcl_tokenize {tflags text} { + lappend ::flags $tflags + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 9.1.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('xyz'); +} {} +do_test 9.1.2 { set ::flags } {document document} + +set ::flags [list] +do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc} +do_test 9.2.2 { set ::flags } {query} + +set ::flags [list] +do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc} +do_test 9.3.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {} +do_test 9.4.2 { set ::flags } {prefixquery} + +set ::flags [list] +do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {} +do_test 9.5.2 { set ::flags } {query} + + finish_test diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test index d3ff5128da..8e5bb8e22b 100644 --- a/ext/fts5/test/fts5unicode2.test +++ b/ext/fts5/test/fts5unicode2.test @@ -160,12 +160,12 @@ foreach {tn query snippet} { the maximum x value. } 4 "rollback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... } 5 "rOllback" { - ...[ROLLBACK]. Instead, the pending statement - will return SQLITE_ABORT upon next access after the [ROLLBACK]. + Pending statements no longer block [ROLLBACK]. Instead, the pending + statement will return SQLITE_ABORT upon... } 6 "lang*" { Added support for the FTS4 [languageid] option. diff --git a/ext/fts5/test/fts5vocab.test b/ext/fts5/test/fts5vocab.test index f7278dd5c1..5e0499fbf9 100644 --- a/ext/fts5/test/fts5vocab.test +++ b/ext/fts5/test/fts5vocab.test @@ -442,8 +442,44 @@ if {[detail_is_none]} { } sqlite3_fts5_may_be_corrupt 0 - } +#------------------------------------------------------------------------- +# Test that both "ORDER BY term" and "ORDER BY term DESC" work. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1 VALUES('def ABC ghi'); + INSERT INTO x1 VALUES('DEF abc GHI'); +} + +do_execsql_test 9.2 { + CREATE VIRTUAL TABLE rrr USING fts5vocab(x1, row); + SELECT * FROM rrr +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.3 { + SELECT * FROM rrr ORDER BY term ASC +} { + abc 2 2 def 2 2 ghi 2 2 +} +do_execsql_test 9.4 { + SELECT * FROM rrr ORDER BY term DESC +} { + ghi 2 2 def 2 2 abc 2 2 +} +do_test 9.5 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term ASC }] + expr [lsearch $e2 SorterSort]<0 +} 1 +do_test 9.6 { + set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }] + expr [lsearch $e2 SorterSort]<0 +} 0 + + + finish_test diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index d5df971d4d..4766b00b06 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -17,6 +17,7 @@ proc process_cmdline {} { {detail "full" "Fts5 detail mode to use"} {repeat 1 "Load each file this many times"} {prefix "" "Fts prefix= option"} + {trans 1 "True to use a transaction"} database file... } { @@ -214,7 +215,7 @@ foreach c [lrange $cols 1 end] { } append sql ")" -db eval BEGIN +if {$A(trans)} { db eval BEGIN } while {$i < $N} { foreach c $cols s $A(colsize) { set R($c) [lrange $tokens $i [expr $i+$s-1]] @@ -222,7 +223,7 @@ db eval BEGIN } db eval $sql } -db eval COMMIT +if {$A(trans)} { db eval COMMIT } diff --git a/ext/icu/icu.c b/ext/icu/icu.c index 3f45646194..7c37812d86 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -349,20 +349,22 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ** of upper() or lower(). ** ** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** lower('I', 'tr_tr') -> '\u131' (small dotless i) ** ** http://www.icu-project.org/userguide/posix.html#case_mappings */ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ - const UChar *zInput; - UChar *zOutput = 0; - int nInput; - int nOut; + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ UErrorCode status; const char *zLocale = 0; assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); if( nArg==2 ){ zLocale = (const char *)sqlite3_value_text(apArg[1]); } @@ -386,19 +388,23 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ } zOutput = zNew; status = U_ZERO_ERROR; - if( sqlite3_user_data(p) ){ + if( bToUpper ){ nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); }else{ nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); } - if( !U_SUCCESS(status) ){ - if( status==U_BUFFER_OVERFLOW_ERROR ) continue; - icuFunctionError(p, - sqlite3_user_data(p) ? "u_strToUpper" : "u_strToLower", status); - return; + + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); } + return; } - sqlite3_result_text16(p, zOutput, nOut, xFree); + assert( 0 ); /* Unreachable */ } /* @@ -487,38 +493,36 @@ static void icuLoadCollation( ** Register the ICU extension functions with database db. */ int sqlite3IcuInit(sqlite3 *db){ - struct IcuScalar { + static const struct IcuScalar { const char *zName; /* Function name */ - int nArg; /* Number of arguments */ - int enc; /* Optimal text encoding */ - void *pContext; /* sqlite3_user_data() context */ + unsigned char nArg; /* Number of arguments */ + unsigned short enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } scalars[] = { - {"regexp", 2, SQLITE_ANY, 0, icuRegexpFunc}, - - {"lower", 1, SQLITE_UTF16, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF16, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF16, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF16, (void*)1, icuCaseFunc16}, - - {"lower", 1, SQLITE_UTF8, 0, icuCaseFunc16}, - {"lower", 2, SQLITE_UTF8, 0, icuCaseFunc16}, - {"upper", 1, SQLITE_UTF8, (void*)1, icuCaseFunc16}, - {"upper", 2, SQLITE_UTF8, (void*)1, icuCaseFunc16}, - - {"like", 2, SQLITE_UTF8, 0, icuLikeFunc}, - {"like", 3, SQLITE_UTF8, 0, icuLikeFunc}, - - {"icu_load_collation", 2, SQLITE_UTF8, (void*)db, icuLoadCollation}, + {"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation}, + {"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc}, }; - int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ - struct IcuScalar *p = &scalars[i]; + const struct IcuScalar *p = &scalars[i]; rc = sqlite3_create_function( - db, p->zName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 ); } diff --git a/ext/misc/README.md b/ext/misc/README.md new file mode 100644 index 0000000000..970a09c2e9 --- /dev/null +++ b/ext/misc/README.md @@ -0,0 +1,40 @@ +## Miscellaneous Extensions + +This folder contains a collection of smaller loadable extensions. +See for instructions on how +to compile and use loadable extensions. +Each extension in this folder is implemented in a single file of C code. + +Each source file contains a description in its header comment. See the +header comments for details about each extension. Additional notes are +as follows: + + * **carray.c** — This module implements the + [carray](https://www.sqlite.org/carray.html) table-valued function. + It is a good example of how to go about implementing a custom + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). + + * **dbdump.c** — This is not actually a loadable extension, but + rather a library that implements an approximate equivalent to the + ".dump" command of the + [command-line shell](https://www.sqlite.org/cli.html). + + * **memvfs.c** — This file implements a custom + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database + file in a single block of RAM. It serves as a good example of how + to implement a simple custom VFS. + + * **rot13.c** — This file implements the very simple rot13() + substitution function. This file makes a good template for implementing + new custom SQL functions for SQLite. + + * **series.c** — This is an implementation of the + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + It can make a good template for new custom virtual table implementations. + + * **shathree.c** — An implementation of the sha3() and + sha3_query() SQL functions. The file is named "shathree.c" instead + of "sha3.c" because the default entry point names in SQLite are based + on the source filename with digits removed, so if we used the name + "sha3.c" then the entry point would conflict with the prior "sha1.c" + extension. diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index 852491988a..efe76566ee 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -625,10 +625,10 @@ static int amatchLoadOneRule( }else{ memset(pRule, 0, sizeof(*pRule)); pRule->zFrom = &pRule->zTo[nTo+1]; - pRule->nFrom = nFrom; + pRule->nFrom = (amatch_len)nFrom; memcpy(pRule->zFrom, zFrom, nFrom+1); memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; + pRule->nTo = (amatch_len)nTo; pRule->rCost = rCost; pRule->iLang = (int)iLang; } @@ -1081,7 +1081,7 @@ static void amatchAddWord( pWord->rCost = rCost; pWord->iSeq = pCur->nWord++; amatchWriteCost(pWord); - pWord->nMatch = nMatch; + pWord->nMatch = (short)nMatch; pWord->pNext = pCur->pAllWords; pCur->pAllWords = pWord; pWord->sCost.zKey = pWord->zCost; @@ -1162,7 +1162,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ #endif nWord = (int)strlen(pWord->zWord+2); if( nWord+20>nBuf ){ - nBuf = nWord+100; + nBuf = (char)(nWord+100); zBuf = sqlite3_realloc(zBuf, nBuf); if( zBuf==0 ) return SQLITE_NOMEM; } diff --git a/ext/misc/carray.c b/ext/misc/carray.c new file mode 100644 index 0000000000..025eb5db2c --- /dev/null +++ b/ext/misc/carray.c @@ -0,0 +1,364 @@ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers that +** has been cast to an integer. +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Allowed datatypes +*/ +#define CARRAY_INT32 0 +#define CARRAY_INT64 1 +#define CARRAY_DOUBLE 2 +#define CARRAY_TEXT 3 + +/* +** Names of types +*/ +static const char *azType[] = { "int32", "int64", "double", "char*" }; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + sqlite3_int64 iPtr; /* Pointer to array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: x = pCur->iPtr; break; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->iPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->iPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->iPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->iPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + if( idxNum ){ + pCur->iPtr = sqlite3_value_int64(argv[0]); + pCur->iCnt = sqlite3_value_int64(argv[1]); + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i=sizeof(azType)/sizeof(azType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + }else{ + pCur->iPtr = 0; + pCur->iCnt = 0; + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is 2 if the pointer= and count= constraints exist, +** 3 if the ctype= constraint also exists, and is 0 otherwise. +** If idxNum is 0, then carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 && cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + }else{ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_carray_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "carray", &carrayModule, 0); +#endif + return rc; +} diff --git a/ext/misc/csv.c b/ext/misc/csv.c new file mode 100644 index 0000000000..f887784ab8 --- /dev/null +++ b/ext/misc/csv.c @@ -0,0 +1,869 @@ +/* +** 2016-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the implementation of an SQLite virtual table for +** reading CSV files. +** +** Usage: +** +** .load ./csv +** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME); +** SELECT * FROM csv; +** +** The columns are named "c1", "c2", "c3", ... by default. But the +** application can define its own CREATE TABLE statement as an additional +** parameter. For example: +** +** CREATE VIRTUAL TABLE temp.csv2 USING csv( +** filename = "../http.log", +** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" +** ); +** +** Instead of specifying a file, the text of the CSV can be loaded using +** the data= parameter. +** +** If the columns=N parameter is supplied, then the CSV file is assumed to have +** N columns. If the columns parameter is omitted, the CSV file is opened +** as soon as the virtual table is constructed and the first row of the CSV +** is read in order to count the tables. +** +** Some extra debugging features (used for testing virtual tables) are available +** if this module is compiled with -DSQLITE_TEST. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define CSV_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define CSV_NOINLINE __declspec(noinline) +#else +# define CSV_NOINLINE +#endif + + +/* Max size of the error message in a CsvReader */ +#define CSV_MXERR 200 + +/* Size of the CsvReader input buffer */ +#define CSV_INBUFSZ 1024 + +/* A context object used when read a CSV file. */ +typedef struct CsvReader CsvReader; +struct CsvReader { + FILE *in; /* Read the CSV text from this input stream */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + char cTerm; /* Character that terminated the most recent field */ + size_t iIn; /* Next unread character in the input buffer */ + size_t nIn; /* Number of characters in the input buffer */ + char *zIn; /* The input buffer */ + char zErr[CSV_MXERR]; /* Error message */ +}; + +/* Initialize a CsvReader object */ +static void csv_reader_init(CsvReader *p){ + p->in = 0; + p->z = 0; + p->n = 0; + p->nAlloc = 0; + p->nLine = 0; + p->nIn = 0; + p->zIn = 0; + p->zErr[0] = 0; +} + +/* Close and reset a CsvReader object */ +static void csv_reader_reset(CsvReader *p){ + if( p->in ){ + fclose(p->in); + sqlite3_free(p->zIn); + } + sqlite3_free(p->z); + csv_reader_init(p); +} + +/* Report an error on a CsvReader */ +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap); + va_end(ap); +} + +/* Open the file associated with a CsvReader +** Return the number of errors. +*/ +static int csv_reader_open( + CsvReader *p, /* The reader to open */ + const char *zFilename, /* Read from this filename */ + const char *zData /* ... or use this data */ +){ + if( zFilename ){ + p->zIn = sqlite3_malloc( CSV_INBUFSZ ); + if( p->zIn==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + p->in = fopen(zFilename, "rb"); + if( p->in==0 ){ + csv_reader_reset(p); + csv_errmsg(p, "cannot open '%s' for reading", zFilename); + return 1; + } + }else{ + assert( p->in==0 ); + p->zIn = (char*)zData; + p->nIn = strlen(zData); + } + return 0; +} + +/* The input buffer has overflowed. Refill the input buffer, then +** return the next character +*/ +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){ + size_t got; + + assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */ + assert( p->in!=0 ); /* Only called if reading froma file */ + + got = fread(p->zIn, 1, CSV_INBUFSZ, p->in); + if( got==0 ) return EOF; + p->nIn = got; + p->iIn = 1; + return p->zIn[0]; +} + +/* Return the next character of input. Return EOF at end of input. */ +static int csv_getc(CsvReader *p){ + if( p->iIn >= p->nIn ){ + if( p->in!=0 ) return csv_getc_refill(p); + return EOF; + } + return p->zIn[p->iIn++]; +} + +/* Increase the size of p->z and append character c to the end. +** Return 0 on success and non-zero if there is an OOM error */ +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ + char *zNew; + int nNew = p->nAlloc*2 + 100; + zNew = sqlite3_realloc64(p->z, nNew); + if( zNew ){ + p->z = zNew; + p->nAlloc = nNew; + p->z[p->n++] = c; + return 0; + }else{ + csv_errmsg(p, "out of memory"); + return 1; + } +} + +/* Append a single character to the CsvReader.z[] array. +** Return 0 on success and non-zero if there is an OOM error */ +static int csv_append(CsvReader *p, char c){ + if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c); + p->z[p->n++] = c; + return 0; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** +** Return "" at EOF. Return 0 on an OOM error. +*/ +static char *csv_read_one_field(CsvReader *p){ + int c; + p->n = 0; + c = csv_getc(p); + if( c==EOF ){ + p->cTerm = EOF; + return ""; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + pc = ppc = 0; + while( 1 ){ + c = csv_getc(p); + if( c<='"' || pc=='"' ){ + if( c=='\n' ) p->nLine++; + if( c=='"' ){ + if( pc=='"' ){ + pc = 0; + continue; + } + } + if( (c==',' && pc=='"') + || (c=='\n' && pc=='"') + || (c=='\n' && pc=='\r' && ppc=='"') + || (c==EOF && pc=='"') + ){ + do{ p->n--; }while( p->z[p->n]!='"' ); + p->cTerm = (char)c; + break; + } + if( pc=='"' && c!='\r' ){ + csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"'); + break; + } + if( c==EOF ){ + csv_errmsg(p, "line %d: unterminated %c-quoted field\n", + startLine, '"'); + p->cTerm = (char)c; + break; + } + } + if( csv_append(p, (char)c) ) return 0; + ppc = pc; + pc = c; + } + }else{ + while( c>',' || (c!=EOF && c!=',' && c!='\n') ){ + if( csv_append(p, (char)c) ) return 0; + c = csv_getc(p); + } + if( c=='\n' ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = (char)c; + } + if( p->z ) p->z[p->n] = 0; + return p->z; +} + + +/* Forward references to the various virtual table methods implemented +** in this file. */ +static int csvtabCreate(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int csvtabDisconnect(sqlite3_vtab*); +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int csvtabClose(sqlite3_vtab_cursor*); +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int csvtabNext(sqlite3_vtab_cursor*); +static int csvtabEof(sqlite3_vtab_cursor*); +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); + +/* An instance of the CSV virtual table */ +typedef struct CsvTable { + sqlite3_vtab base; /* Base class. Must be first */ + char *zFilename; /* Name of the CSV file */ + char *zData; /* Raw CSV data in lieu of zFilename */ + long iStart; /* Offset to start of data in zFilename */ + int nCol; /* Number of columns in the CSV file */ + unsigned int tstFlags; /* Bit values used for testing */ +} CsvTable; + +/* Allowed values for tstFlags */ +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ + +/* A cursor for the CSV virtual table */ +typedef struct CsvCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + CsvReader rdr; /* The CsvReader object */ + char **azVal; /* Value of the current row */ + int *aLen; /* Length of each entry */ + sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ +} CsvCursor; + +/* Transfer error message text from a reader into a CsvTable */ +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){ + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr); +} + +/* +** This method is the destructor fo a CsvTable object. +*/ +static int csvtabDisconnect(sqlite3_vtab *pVtab){ + CsvTable *p = (CsvTable*)pVtab; + sqlite3_free(p->zFilename); + sqlite3_free(p->zData); + sqlite3_free(p); + return SQLITE_OK; +} + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *csv_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void csv_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void csv_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; izErr. If there are no errors, p->zErr[0]==0. +*/ +static int csv_string_parameter( + CsvReader *p, /* Leave the error message here, if there is one */ + const char *zParam, /* Parameter we are checking for */ + const char *zArg, /* Raw text of the virtual table argment */ + char **pzVal /* Write the dequoted string value here */ +){ + const char *zValue; + zValue = csv_parameter(zParam,(int)strlen(zParam),zArg); + if( zValue==0 ) return 0; + p->zErr[0] = 0; + if( *pzVal ){ + csv_errmsg(p, "more than one '%s' parameter", zParam); + return 1; + } + *pzVal = sqlite3_mprintf("%s", zValue); + if( *pzVal==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + csv_trim_whitespace(*pzVal); + csv_dequote(*pzVal); + return 1; +} + + +/* Return 0 if the argument is false and 1 if it is true. Return -1 if +** we cannot really tell. +*/ +static int csv_boolean(const char *z){ + if( sqlite3_stricmp("yes",z)==0 + || sqlite3_stricmp("on",z)==0 + || sqlite3_stricmp("true",z)==0 + || (z[0]=='1' && z[1]==0) + ){ + return 1; + } + if( sqlite3_stricmp("no",z)==0 + || sqlite3_stricmp("off",z)==0 + || sqlite3_stricmp("false",z)==0 + || (z[0]=='0' && z[1]==0) + ){ + return 0; + } + return -1; +} + + +/* +** Parameters: +** filename=FILENAME Name of file containing CSV content +** data=TEXT Direct CSV content. +** schema=SCHEMA Alternative CSV schema. +** header=YES|NO First row of CSV defines the names of +** columns if "yes". Default "no". +** columns=N Assume the CSV file contains N columns. +** +** Only available if compiled with SQLITE_TEST: +** +** testflags=N Bitmask of test flags. Optional +** +** If schema= is omitted, then the columns are named "c0", "c1", "c2", +** and so forth. If columns=N is omitted, then the file is opened and +** the number of columns in the first row is counted to determine the +** column count. If header=YES, then the first row is skipped. +*/ +static int csvtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + CsvTable *pNew = 0; /* The CsvTable object to construct */ + int bHeader = -1; /* header= flags. -1 means not seen yet */ + int rc = SQLITE_OK; /* Result code from this routine */ + int i, j; /* Loop counters */ +#ifdef SQLITE_TEST + int tstFlags = 0; /* Value for testflags=N parameter */ +#endif + int nCol = -99; /* Value of the columns= parameter */ + CsvReader sRdr; /* A CSV file reader used to store an error + ** message and/or to count the number of columns */ + static const char *azParam[] = { + "filename", "data", "schema", + }; + char *azPValue[3]; /* Parameter values */ +# define CSV_FILENAME (azPValue[0]) +# define CSV_DATA (azPValue[1]) +# define CSV_SCHEMA (azPValue[2]) + + + assert( sizeof(azPValue)==sizeof(azParam) ); + memset(&sRdr, 0, sizeof(sRdr)); + memset(azPValue, 0, sizeof(azPValue)); + for(i=3; i=0 ){ + csv_errmsg(&sRdr, "more than one 'header' parameter"); + goto csvtab_connect_error; + } + x = csv_boolean(zValue); + if( x==1 ){ + bHeader = 1; + }else if( x==0 ){ + bHeader = 0; + }else{ + csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue); + goto csvtab_connect_error; + } + }else +#ifdef SQLITE_TEST + if( (zValue = csv_parameter("testflags",9,z))!=0 ){ + tstFlags = (unsigned int)atoi(zValue); + }else +#endif + if( (zValue = csv_parameter("columns",7,z))!=0 ){ + if( nCol>0 ){ + csv_errmsg(&sRdr, "more than one 'columns' parameter"); + goto csvtab_connect_error; + } + nCol = atoi(zValue); + if( nCol<=0 ){ + csv_errmsg(&sRdr, "must have at least one column"); + goto csvtab_connect_error; + } + }else + { + csv_errmsg(&sRdr, "unrecognized parameter '%s'", z); + goto csvtab_connect_error; + } + } + if( (CSV_FILENAME==0)==(CSV_DATA==0) ){ + csv_errmsg(&sRdr, "must either filename= or data= but not both"); + goto csvtab_connect_error; + } + if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){ + goto csvtab_connect_error; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) goto csvtab_connect_oom; + memset(pNew, 0, sizeof(*pNew)); + if( nCol>0 ){ + pNew->nCol = nCol; + }else{ + do{ + const char *z = csv_read_one_field(&sRdr); + if( z==0 ) goto csvtab_connect_oom; + pNew->nCol++; + }while( sRdr.cTerm==',' ); + } + pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0; + pNew->zData = CSV_DATA; CSV_DATA = 0; +#ifdef SQLITE_TEST + pNew->tstFlags = tstFlags; +#endif + pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0; + csv_reader_reset(&sRdr); + if( CSV_SCHEMA==0 ){ + char *zSep = ""; + CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x("); + if( CSV_SCHEMA==0 ) goto csvtab_connect_oom; + for(i=0; inCol; i++){ + CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i); + zSep = ","; + } + CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA); + } + rc = sqlite3_declare_vtab(db, CSV_SCHEMA); + if( rc ) goto csvtab_connect_error; + for(i=0; ibase); + for(i=0; ibase.pVtab; + int i; + for(i=0; inCol; i++){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + } +} + +/* +** The xConnect and xCreate methods do the same thing, but they must be +** different so that the virtual table is not an eponymous virtual table. +*/ +static int csvtabCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Destructor for a CsvCursor. +*/ +static int csvtabClose(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + csvtabCursorRowReset(pCur); + csv_reader_reset(&pCur->rdr); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Constructor for a new CsvTable cursor object. +*/ +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + CsvTable *pTab = (CsvTable*)p; + CsvCursor *pCur; + size_t nByte; + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; + pCur = sqlite3_malloc64( nByte ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, nByte); + pCur->azVal = (char**)&pCur[1]; + pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; + *ppCursor = &pCur->base; + if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ + csv_xfer_error(pTab, &pCur->rdr); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + + +/* +** Advance a CsvCursor to its next row of input. +** Set the EOF marker if we reach the end of input. +*/ +static int csvtabNext(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + int i = 0; + char *z; + do{ + z = csv_read_one_field(&pCur->rdr); + if( z==0 ){ + csv_xfer_error(pTab, &pCur->rdr); + break; + } + if( inCol ){ + if( pCur->aLen[i] < pCur->rdr.n+1 ){ + char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1); + if( zNew==0 ){ + csv_errmsg(&pCur->rdr, "out of memory"); + csv_xfer_error(pTab, &pCur->rdr); + break; + } + pCur->azVal[i] = zNew; + pCur->aLen[i] = pCur->rdr.n+1; + } + memcpy(pCur->azVal[i], z, pCur->rdr.n+1); + i++; + } + }while( pCur->rdr.cTerm==',' ); + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } + if( z==0 || pCur->rdr.cTerm==EOF ){ + pCur->iRowid = -1; + }else{ + pCur->iRowid++; + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the CsvCursor +** is currently pointing. +*/ +static int csvtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + if( i>=0 && inCol && pCur->azVal[i]!=0 ){ + sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_STATIC); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + CsvCursor *pCur = (CsvCursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int csvtabEof(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + return pCur->iRowid<0; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int csvtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + CsvCursor *pCur = (CsvCursor*)pVtabCursor; + CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; + pCur->iRowid = 0; + if( pCur->rdr.in==0 ){ + assert( pCur->rdr.zIn==pTab->zData ); + assert( pTab->iStart>=0 ); + assert( (size_t)pTab->iStart<=pCur->rdr.nIn ); + pCur->rdr.iIn = pTab->iStart; + }else{ + fseek(pCur->rdr.in, pTab->iStart, SEEK_SET); + pCur->rdr.iIn = 0; + pCur->rdr.nIn = 0; + } + return csvtabNext(pVtabCursor); +} + +/* +** Only a forward full table scan is supported. xBestIndex is mostly +** a no-op. If CSVTEST_FIDX is set, then the presence of equality +** constraints lowers the estimated cost, which is fiction, but is useful +** for testing certain kinds of virtual table behavior. +*/ +static int csvtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = 1000000; +#ifdef SQLITE_TEST + if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){ + /* The usual (and sensible) case is to always do a full table scan. + ** The code in this branch only runs when testflags=1. This code + ** generates an artifical and unrealistic plan which is useful + ** for testing virtual table logic but is not helpful to real applications. + ** + ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual + ** table (even though it is not) and the cost of running the virtual table + ** is reduced from 1 million to just 10. The constraints are *not* marked + ** as omittable, however, so the query planner should still generate a + ** plan that gives a correct answer, even if they plan is not optimal. + */ + int i; + int nConst = 0; + for(i=0; inConstraint; i++){ + unsigned char op; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + op = pIdxInfo->aConstraint[i].op; + if( op==SQLITE_INDEX_CONSTRAINT_EQ + || op==SQLITE_INDEX_CONSTRAINT_LIKE + || op==SQLITE_INDEX_CONSTRAINT_GLOB + ){ + pIdxInfo->estimatedCost = 10; + pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1; + nConst++; + } + } + } +#endif + return SQLITE_OK; +} + + +static sqlite3_module CsvModule = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +#ifdef SQLITE_TEST +/* +** For virtual table testing, make a version of the CSV virtual table +** available that has an xUpdate function. But the xUpdate always returns +** SQLITE_READONLY since the CSV file is not really writable. +*/ +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){ + return SQLITE_READONLY; +} +static sqlite3_module CsvModuleFauxWrite = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + csvtabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; +#endif /* SQLITE_TEST */ + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. The new +** CSV virtual table module is registered with the calling database +** connection. +*/ +int sqlite3_csv_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "csv", &CsvModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0); + } +#endif + return rc; +#else + return SQLITE_OK; +#endif +} diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c new file mode 100644 index 0000000000..db581a8325 --- /dev/null +++ b/ext/misc/dbdump.c @@ -0,0 +1,717 @@ +/* +** 2016-03-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a C-language subroutine that converts the content +** of an SQLite database into UTF-8 text SQL statements that can be used +** to exactly recreate the original database. ROWID values are preserved. +** +** A prototype of the implemented subroutine is this: +** +** int sqlite3_db_dump( +** sqlite3 *db, +** const char *zSchema, +** const char *zTable, +** void (*xCallback)(void*, const char*), +** void *pArg +** ); +** +** The db parameter is the database connection. zSchema is the schema within +** that database which is to be dumped. Usually the zSchema is "main" but +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then +** only the content of that one table is dumped. If zTable is NULL, then all +** tables are dumped. +** +** The generate text is passed to xCallback() in multiple calls. The second +** argument to xCallback() is a copy of the pArg parameter. The first +** argument is some of the output text that this routine generates. The +** signature to xCallback() is designed to make it compatible with fputs(). +** +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error +** code if it encounters a problem. +** +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine +** is included so that this routine becomes a command-line utility. The +** command-line utility takes two or three arguments which are the name +** of the database file, the schema, and optionally the table, forming the +** first three arguments of a single call to the library routine. +*/ +#include "sqlite3.h" +#include +#include +#include + +/* +** The state of the dump process. +*/ +typedef struct DState DState; +struct DState { + sqlite3 *db; /* The database connection */ + int nErr; /* Number of errors seen so far */ + int rc; /* Error code */ + int writableSchema; /* True if in writable_schema mode */ + int (*xCallback)(const char*,void*); /* Send output here */ + void *pArg; /* Argument to xCallback() */ +}; + +/* +** A variable length string to which one can append text. +*/ +typedef struct DText DText; +struct DText { + char *z; /* The text */ + int n; /* Number of bytes of content in z[] */ + int nAlloc; /* Number of bytes allocated to z[] */ +}; + +/* +** Initialize and destroy a DText object +*/ +static void initText(DText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(DText *p){ + sqlite3_free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(DText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + len + 20; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + freeText(p); + return; + } + p->z = zNew; + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} + + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(DState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt = 0; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = 1; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + char **azNew; + nAlloc = nAlloc*2 + nCol + 10; + azNew = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azNew==0 ) goto col_oom; + azCol = azNew; + azCol[0] = 0; + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( azCol[nCol]==0 ) goto col_oom; + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + if( zSql==0 ) goto col_oom; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + int rc; + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; + +col_oom: + sqlite3_finalize(pStmt); + freeColumnList(azCol); + p->nErr++; + p->rc = SQLITE_NOMEM; + return 0; +} + +/* +** Send mprintf-formatted content to the output callback. +*/ +static void output_formatted(DState *p, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + p->xCallback(z, p->pArg); + sqlite3_free(z); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. +*/ +static void output_quoted_string(DState *p, const unsigned char *z){ + int i; + char c; + int inQuote = 0; + int bStarted = 0; + + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + output_formatted(p, "'%s'", z); + return; + } + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) p->xCallback("||", p->pArg); + p->xCallback("'", p->pArg); + inQuote = 1; + } + output_formatted(p, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + p->xCallback("'", p->pArg); + continue; + } + if( inQuote ){ + p->xCallback("'", p->pArg); + inQuote = 0; + } + if( c==0 ){ + break; + } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) p->xCallback("||", p->pArg); + output_formatted(p, "char(%d)", c); + bStarted = 1; + } + z += i; + } + if( inQuote ) p->xCallback("'", p->pArg); +} + +/* +** This is an sqlite3_exec callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + DState *p = (DState*)pArg; + sqlite3_stmt *pStmt; + + (void)azCol; + if( nArg!=3 ) return 1; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + + if( strcmp(zTable, "sqlite_sequence")==0 ){ + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ + p->xCallback("ANALYZE sqlite_master;\n", p->pArg); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + if( !p->writableSchema ){ + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); + p->writableSchema = 1; + } + output_formatted(p, + "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + return 0; + }else{ + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); + p->xCallback(zSql+13, p->pArg); + }else{ + p->xCallback(zSql, p->pArg); + } + p->xCallback(";\n", p->pArg); + } + + if( strcmp(zType, "table")==0 ){ + DText sSelect; + DText sTable; + char **azCol; + int i; + int nCol; + + azCol = tableColumnList(p, zTable); + if( azCol==0 ) return 0; + + initText(&sTable); + appendText(&sTable, "INSERT INTO ", 0); + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + appendText(&sTable, zTable, quoteChar(zTable)); + + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); + } + appendText(&sTable, ")", 0); + } + appendText(&sTable, " VALUES(", 0); + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + nCol = i; + if( azCol[0]==0 ) nCol--; + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + p->nErr++; + if( p->rc==SQLITE_OK ) p->rc = rc; + }else{ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->xCallback(sTable.z, p->pArg); + for(i=0; ixCallback(",", p->pArg); + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_INTEGER: { + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_column_double(pStmt,i); + output_formatted(p, "%!.20g", r); + break; + } + case SQLITE_NULL: { + p->xCallback("NULL", p->pArg); + break; + } + case SQLITE_TEXT: { + output_quoted_string(p, sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + int nByte = sqlite3_column_bytes(pStmt,i); + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); + int j; + p->xCallback("x'", p->pArg); + for(j=0; j>4)&15]; + zWord[1] = "0123456789abcdef"[a[j]&15]; + zWord[2] = 0; + p->xCallback(zWord, p->pArg); + } + p->xCallback("'", p->pArg); + break; + } + } + } + p->xCallback(");\n", p->pArg); + } + } + sqlite3_finalize(pStmt); + freeText(&sTable); + freeText(&sSelect); + } + return 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static void output_sql_from_query( + DState *p, /* Query context */ + const char *zSelect, /* SELECT statement to extract content */ + ... +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + char *zSql; + va_list ap; + va_start(ap, zSelect); + zSql = sqlite3_vmprintf(zSelect, ap); + va_end(ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + p->nErr++; + return; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK || !pSelect ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + p->nErr++; + return; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + p->xCallback(z, p->pArg); + for(i=1; ixCallback(",", p->pArg); + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + p->xCallback("\n;\n", p->pArg); + }else{ + p->xCallback(";\n", p->pArg); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static void run_schema_dump_query( + DState *p, + const char *zQuery, + ... +){ + char *zErr = 0; + char *z; + va_list ap; + va_start(ap, zQuery); + z = sqlite3_vmprintf(zQuery, ap); + va_end(ap); + sqlite3_exec(p->db, z, dump_callback, p, &zErr); + sqlite3_free(z); + if( zErr ){ + output_formatted(p, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + p->nErr++; + zErr = 0; + } +} + +/* +** Convert an SQLite database into SQL statements that will recreate that +** database. +*/ +int sqlite3_db_dump( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which schema to dump. Usually "main". */ + const char *zTable, /* Which table to dump. NULL means everything. */ + int (*xCallback)(const char*,void*), /* Output sent to this callback */ + void *pArg /* Second argument of the callback */ +){ + DState x; + memset(&x, 0, sizeof(x)); + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( x.rc ) return x.rc; + x.db = db; + x.xCallback = xCallback; + x.pArg = pArg; + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); + if( zTable==0 ){ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", + zSchema + ); + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE name=='sqlite_sequence'", zSchema + ); + output_sql_from_query(&x, + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); + }else{ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_master " + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" + " AND sql NOT NULL", + zSchema, zTable + ); + output_sql_from_query(&x, + "SELECT sql FROM \"%w\".sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name=%Q COLLATE nocase", + zSchema, zTable + ); + } + if( x.writableSchema ){ + xCallback("PRAGMA writable_schema=OFF;\n", pArg); + } + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); + sqlite3_exec(db, "COMMIT", 0, 0, 0); + return x.rc; +} + + + +/* The generic subroutine is above. The code the follows implements +** the command-line interface. +*/ +#ifdef DBDUMP_STANDALONE +#include + +/* +** Command-line interface +*/ +int main(int argc, char **argv){ + sqlite3 *db; + const char *zDb; + const char *zSchema; + const char *zTable = 0; + int rc; + + if( argc<2 || argc>4 ){ + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); + return 1; + } + zDb = argv[1]; + zSchema = argc>=3 ? argv[2] : "main"; + zTable = argc==4 ? argv[3] : 0; + + rc = sqlite3_open(zDb, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + rc = sqlite3_db_dump(db, zSchema, zTable, + (int(*)(const char*,void*))fputs, (void*)stdout); + if( rc ){ + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); + } + sqlite3_close(db); + return rc!=SQLITE_OK; +} +#endif /* DBDUMP_STANDALONE */ diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 3ed4b0a977..77db56d47a 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -344,10 +344,10 @@ static int fuzzerLoadOneRule( memset(pRule, 0, sizeof(*pRule)); pRule->zFrom = pRule->zTo; pRule->zFrom += nTo + 1; - pRule->nFrom = nFrom; + pRule->nFrom = (fuzzer_len)nFrom; memcpy(pRule->zFrom, zFrom, nFrom+1); memcpy(pRule->zTo, zTo, nTo+1); - pRule->nTo = nTo; + pRule->nTo = (fuzzer_len)nTo; pRule->rCost = nCost; pRule->iRuleset = (int)iRuleset; } diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 99d299c29c..bad5cd9b53 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -22,7 +22,7 @@ ** how JSONB might improve on that.) */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(_SQLITEINT_H_) +#if !defined(SQLITEINT_H) #include "sqlite3ext.h" #endif SQLITE_EXTENSION_INIT1 @@ -49,13 +49,15 @@ SQLITE_EXTENSION_INIT1 #ifdef sqlite3Isdigit /* Use the SQLite core versions if this routine is part of the ** SQLite amalgamation */ -# define safe_isdigit(x) sqlite3Isdigit(x) -# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) #else /* Use the standard library for separate compilation */ #include /* amalgamator: keep */ -# define safe_isdigit(x) isdigit((unsigned char)(x)) -# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) #endif /* @@ -136,9 +138,10 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x20 /* Is a label of an object */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ /* A single node of parsed JSON @@ -146,12 +149,13 @@ static const char * const jsonType[] = { struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ - u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ union { const char *zJContent; /* Content for INT, REAL, and STRING */ u32 iAppend; /* More terms for ARRAY and OBJECT */ u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ } u; }; @@ -408,6 +412,13 @@ static void jsonRenderNode( JsonString *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } switch( pNode->eType ){ default: { assert( pNode->eType==JSON_NULL ); @@ -439,12 +450,7 @@ static void jsonRenderNode( jsonAppendChar(pOut, '['); for(;;){ while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ - jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); - } - }else{ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } @@ -466,11 +472,7 @@ static void jsonRenderNode( jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - }else{ - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } + jsonRenderNode(&pNode[j+1], pOut, aReplace); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -593,12 +595,13 @@ static void jsonReturn( c = z[++i]; if( c=='u' ){ u32 v = 0, k; - for(k=0; k<4 && i='0' && c<='9' ) v = v*16 + c - '0'; - else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; - else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; - else break; + assert( safe_isxdigit(c) ); + if( c<='9' ) v = v*16 + c - '0'; + else if( c<='F' ) v = v*16 + c - 'A' + 10; + else v = v*16 + c - 'a' + 10; } if( v==0 ) break; if( v<=0x7f ){ @@ -696,12 +699,20 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; - p->iVal = 0; p->n = n; p->u.zJContent = zContent; return pParse->nNode++; } +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; +} + /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. @@ -776,8 +787,13 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c==0 ) return -1; if( c=='\\' ){ c = pParse->zJson[++j]; - if( c==0 ) return -1; - jnFlags = JNODE_ESCAPE; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(pParse->zJson+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } }else if( c=='"' ){ break; } @@ -1148,6 +1164,25 @@ static void jsonWrongNumArgs( sqlite3_free(zMsg); } +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + /**************************************************************************** ** SQL functions used for testing and debugging @@ -1211,6 +1246,26 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ +/* +** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** corresponding to the SQL value input. Mostly this means putting +** double-quotes around strings and returning the unquoted string "null" +** when given a NULL input. +*/ +static void jsonQuoteFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString jx; + UNUSED_PARAM(argc); + + jsonInit(&jx, ctx); + jsonAppendValue(&jx, argv[0]); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + /* ** Implementation of the json_array(VALUE,...) function. Return a JSON ** array that contains all values given in arguments. Or if any argument @@ -1320,6 +1375,105 @@ static void jsonExtractFunc( jsonParseReset(&x); } +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + int iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + u32 nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ + + UNUSED_PARAM(argc); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } + jsonParseReset(&x); + jsonParseReset(&y); +} + + /* ** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON ** object that contains all name/value given in arguments. Or if any name @@ -1423,11 +1577,11 @@ static void jsonReplaceFunc( if( x.nErr ) goto replace_err; if( pNode ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -1477,11 +1631,11 @@ static void jsonSetFunc( goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); + pNode->u.iReplace = i + 1; } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ jsonReturnJson(x.aNode, ctx, argv); } @@ -1625,7 +1779,7 @@ static void jsonObjectFinal(sqlite3_context *ctx){ if( pStr ){ jsonAppendChar(pStr, '}'); if( pStr->bErr ){ - if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); }else{ sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, @@ -1903,9 +2057,9 @@ static int jsonEachColumn( /* For json_each() path and root are the same so fall through ** into the root case */ } - case JEACH_ROOT: { + default: { const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; + if( zRoot==0 ) zRoot = "$"; sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); break; } @@ -2124,6 +2278,8 @@ int sqlite3Json1Init(sqlite3 *db){ { "json_extract", -1, 0, jsonExtractFunc }, { "json_insert", -1, 0, jsonSetFunc }, { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, { "json_remove", -1, 0, jsonRemoveFunc }, { "json_replace", -1, 0, jsonReplaceFunc }, { "json_set", -1, 1, jsonSetFunc }, diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c new file mode 100644 index 0000000000..62a8a033d7 --- /dev/null +++ b/ext/misc/memvfs.c @@ -0,0 +1,491 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory read-only VFS implementation. The application +** supplies a block of memory which is the database file, and this VFS +** uses that block of memory. +** +** Because there is no place to store journals and no good way to lock +** the "file", this VFS is read-only. +** +** USAGE: +** +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db, +** SQLITE_OPEN_READONLY | SQLITE_OPEN_URI, +** "memvfs"); +** +** The ptr= and sz= query parameters are required or the open will fail. +** The ptr= parameter gives the memory address of the buffer holding the +** read-only database and sz= gives the size of the database. The parameter +** values may be in hexadecimal or decimal. The filename is ignored. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + unsigned char *aData; /* content of the file */ +}; + +/* +** Methods for MemFile +*/ +static int memClose(sqlite3_file*); +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memTruncate(sqlite3_file*, sqlite3_int64 size); +static int memSync(sqlite3_file*, int flags); +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memLock(sqlite3_file*, int); +static int memUnlock(sqlite3_file*, int); +static int memCheckReservedLock(sqlite3_file*, int *pResOut); +static int memFileControl(sqlite3_file*, int op, void *pArg); +static int memSectorSize(sqlite3_file*); +static int memDeviceCharacteristics(sqlite3_file*); +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int memShmLock(sqlite3_file*, int offset, int n, int flags); +static void memShmBarrier(sqlite3_file*); +static int memShmUnmap(sqlite3_file*, int deleteFlag); +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memDlOpen(sqlite3_vfs*, const char *zFilename); +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memDlClose(sqlite3_vfs*, void*); +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memSleep(sqlite3_vfs*, int microseconds); +static int memCurrentTime(sqlite3_vfs*, double*); +static int memGetLastError(sqlite3_vfs*, int, char *); +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs mem_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData (set when registered) */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + memAccess, /* xAccess */ + memFullPathname, /* xFullPathname */ + memDlOpen, /* xDlOpen */ + memDlError, /* xDlError */ + memDlSym, /* xDlSym */ + memDlClose, /* xDlClose */ + memRandomness, /* xRandomness */ + memSleep, /* xSleep */ + memCurrentTime, /* xCurrentTime */ + memGetLastError, /* xGetLastError */ + memCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods mem_io_methods = { + 3, /* iVersion */ + memClose, /* xClose */ + memRead, /* xRead */ + memWrite, /* xWrite */ + memTruncate, /* xTruncate */ + memSync, /* xSync */ + memFileSize, /* xFileSize */ + memLock, /* xLock */ + memUnlock, /* xUnlock */ + memCheckReservedLock, /* xCheckReservedLock */ + memFileControl, /* xFileControl */ + memSectorSize, /* xSectorSize */ + memDeviceCharacteristics, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap, /* xShmUnmap */ + memFetch, /* xFetch */ + memUnfetch /* xUnfetch */ +}; + + + +/* +** Close an mem-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memClose(sqlite3_file *pFile){ + return SQLITE_OK; +} + +/* +** Read data from an mem-file. +*/ +static int memRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Write data to an mem-file. +*/ +static int memWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + return SQLITE_READONLY; +} + +/* +** Truncate an mem-file. +*/ +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ + return SQLITE_READONLY; +} + +/* +** Sync an mem-file. +*/ +static int memSync(sqlite3_file *pFile, int flags){ + return SQLITE_READONLY; +} + +/* +** Return the current file-size of an mem-file. +*/ +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an mem-file. +*/ +static int memLock(sqlite3_file *pFile, int eLock){ + return SQLITE_READONLY; +} + +/* +** Unlock an mem-file. +*/ +static int memUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an mem-file. +*/ +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an mem-file. +*/ +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an mem-file. +*/ +static int memSectorSize(sqlite3_file *pFile){ + return 1024; +} + +/* +** Return the device characteristic flags supported by an mem-file. +*/ +static int memDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_IMMUTABLE; +} + +/* Create a shared memory file mapping */ +static int memShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_READONLY; +} + +/* Perform locking on a shared-memory segment */ +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_READONLY; +} + +/* Memory barrier operation on shared memory */ +static void memShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + *pp = (void*)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + memset(p, 0, sizeof(*p)); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); + if( p->aData==0 ) return SQLITE_CANTOPEN; + p->sz = sqlite3_uri_int64(zName,"sz",0); + if( p->sz<0 ) return SQLITE_CANTOPEN; + pFile->pMethods = &mem_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_READONLY; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + if( flags==SQLITE_ACCESS_READWRITE ){ + *pResOut = 0; + }else{ + *pResOut = 1; + } + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +#ifdef MEMVFS_TEST +/* +** memload(FILENAME) +** +** This an SQL function used to help in testing the memvfs VFS. The +** function reads the content of a file into memory and then returns +** a string that gives the locate and size of the in-memory buffer. +*/ +#include +static void memvfsMemloadFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *p; + sqlite3_int64 sz; + FILE *in; + const char *zFilename = (const char*)sqlite3_value_text(argv[0]); + char zReturn[100]; + + if( zFilename==0 ) return; + in = fopen(zFilename, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + sz = ftell(in); + rewind(in); + p = sqlite3_malloc( sz ); + if( p==0 ){ + fclose(in); + sqlite3_result_error_nomem(context); + return; + } + fread(p, sz, 1, in); + fclose(in); + sqlite3_snprintf(sizeof(zReturn),zReturn,"ptr=%lld&sz=%lld", + (sqlite3_int64)p, sz); + sqlite3_result_text(context, zReturn, -1, SQLITE_TRANSIENT); +} +/* Called for each new database connection */ +static int memvfsRegister( + sqlite3 *db, + const char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0, + memvfsMemloadFunc, 0, 0); +} +#endif /* MEMVFS_TEST */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_memvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + mem_vfs.pAppData = sqlite3_vfs_find(0); + mem_vfs.szOsFile = sizeof(MemFile); + rc = sqlite3_vfs_register(&mem_vfs, 1); +#ifdef MEMVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index 74a4c9d12b..a5d7e84913 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -167,7 +167,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ /* ** Compare to doubles for sorting using qsort() */ -static int doubleCmp(const void *pA, const void *pB){ +static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ double a = *(double*)pA; double b = *(double*)pB; if( a==b ) return 0; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 7244d52998..b4a8ab5c04 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -136,7 +136,7 @@ struct ReCompiled { static void re_add_state(ReStateSet *pSet, int newState){ unsigned i; for(i=0; inState; i++) if( pSet->aState[i]==newState ) return; - pSet->aState[pSet->nState++] = newState; + pSet->aState[pSet->nState++] = (ReStateNumber)newState; } /* Extract the next unicode character from *pzIn and return it. Advance @@ -358,7 +358,7 @@ static int re_insert(ReCompiled *p, int iBefore, int op, int arg){ p->aArg[i] = p->aArg[i-1]; } p->nState++; - p->aOp[iBefore] = op; + p->aOp[iBefore] = (char)op; p->aArg[iBefore] = arg; return iBefore; } @@ -677,12 +677,12 @@ const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ for(j=0, i=1; jzInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){ unsigned x = pRe->aArg[i]; if( x<=127 ){ - pRe->zInit[j++] = x; + pRe->zInit[j++] = (unsigned char)x; }else if( x<=0xfff ){ - pRe->zInit[j++] = 0xc0 | (x>>6); + pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6)); pRe->zInit[j++] = 0x80 | (x&0x3f); }else if( x<=0xffff ){ - pRe->zInit[j++] = 0xd0 | (x>>12); + pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12)); pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f); pRe->zInit[j++] = 0x80 | (x&0x3f); }else{ diff --git a/ext/misc/remember.c b/ext/misc/remember.c new file mode 100644 index 0000000000..aa3eff8a3f --- /dev/null +++ b/ext/misc/remember.c @@ -0,0 +1,68 @@ +/* +** 2016-08-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create an SQL function that is a pass-through +** for integer values (it returns a copy of its argument) but also saves the +** value that is passed through into a C-language variable. The address of +** the C-language variable is supplied as the second argument. +** +** This allows, for example, a counter to incremented and the original +** value retrieved, atomically, using a single statement: +** +** UPDATE counterTab SET cnt=remember(cnt,$PTR)+1 WHERE id=$ID +** +** Prepare the above statement once. Then to use it, bind the address +** of the output variable to $PTR and the id of the counter to $ID and +** run the prepared statement. +** +** One can imagine doing similar things with floating-point values and +** strings, but this demonstration extension will stick to using just +** integers. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** remember(V,PTR) +** +** Return the integer value V. Also save the value of V in a +** C-language variable whose address is PTR. +*/ +static void rememberFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3_int64 v; + sqlite3_int64 ptr; + assert( argc==2 ); + v = sqlite3_value_int64(argv[0]); + ptr = sqlite3_value_int64(argv[1]); + *((sqlite3_int64*)ptr) = v; + sqlite3_result_int64(pCtx, v); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_remember_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_function(db, "remember", 2, SQLITE_UTF8, 0, + rememberFunc, 0, 0); + return rc; +} diff --git a/ext/misc/scrub.c b/ext/misc/scrub.c new file mode 100644 index 0000000000..92718e23d1 --- /dev/null +++ b/ext/misc/scrub.c @@ -0,0 +1,610 @@ +/* +** 2016-05-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a utility function (and a utility program) that +** makes a copy of an SQLite database while simultaneously zeroing out all +** deleted content. +** +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite +** deletes content, it does not overwrite the deleted content but rather marks +** the region of the file that held that content as being reusable. This can +** cause deleted content to recoverable from the database file. This stale +** content is removed by the VACUUM command, but VACUUM can be expensive for +** large databases. When in PRAGMA secure_delete=ON mode, the deleted content +** is zeroed, but secure_delete=ON has overhead as well. +** +** This utility attempts to make a copy of a complete SQLite database where +** all of the deleted content is zeroed out in the copy, and it attempts to +** do so while being faster than running VACUUM. +** +** Usage: +** +** int sqlite3_scrub_backup( +** const char *zSourceFile, // Source database filename +** const char *zDestFile, // Destination database filename +** char **pzErrMsg // Write error message here +** ); +** +** Simply call the API above specifying the filename of the source database +** and the name of the backup copy. The source database must already exist +** and can be in active use. (A read lock is held during the backup.) The +** destination file should not previously exist. If the pzErrMsg parameter +** is non-NULL and if an error occurs, then an error message might be written +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to +** that error message. But if the error is an OOM, the error might not be +** reported. The routine always returns non-zero if there is an error. +** +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and +** this file becomes a standalone program that can be run as follows: +** +** ./sqlite3scrub SOURCE DEST +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include + +typedef struct ScrubState ScrubState; +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; + + +/* State information for a scrub-and-backup operation */ +struct ScrubState { + const char *zSrcFile; /* Name of the source file */ + const char *zDestFile; /* Name of the destination file */ + int rcErr; /* Error code */ + char *zErr; /* Error message text */ + sqlite3 *dbSrc; /* Source database connection */ + sqlite3_file *pSrc; /* Source file handle */ + sqlite3 *dbDest; /* Destination database connection */ + sqlite3_file *pDest; /* Destination file handle */ + u32 szPage; /* Page size */ + u32 szUsable; /* Usable bytes on each page */ + u32 nPage; /* Number of pages */ + u32 iLastPage; /* Page number of last page written so far*/ + u8 *page1; /* Content of page 1 */ +}; + +/* Store an error message */ +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ + va_list ap; + sqlite3_free(p->zErr); + va_start(ap, zFormat); + p->zErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR; +} + +/* Allocate memory to hold a single page of content */ +static u8 *scrubBackupAllocPage(ScrubState *p){ + u8 *pPage; + if( p->rcErr ) return 0; + pPage = sqlite3_malloc( p->szPage ); + if( pPage==0 ) p->rcErr = SQLITE_NOMEM; + return pPage; +} + +/* Read a page from the source database into memory. Use the memory +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL. +*/ +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){ + int rc; + sqlite3_int64 iOff; + u8 *pOut = pBuf; + if( p->rcErr ) return 0; + if( pOut==0 ){ + pOut = scrubBackupAllocPage(p); + if( pOut==0 ) return 0; + } + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + if( pBuf==0 ) sqlite3_free(pOut); + pOut = 0; + scrubBackupErr(p, "read failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } + return pOut; +} + +/* Write a page to the destination database */ +static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){ + int rc; + sqlite3_int64 iOff; + if( p->rcErr ) return; + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + scrubBackupErr(p, "write failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } + if( pgno>p->iLastPage ) p->iLastPage = pgno; +} + +/* Prepare a statement against the "db" database. */ +static sqlite3_stmt *scrubBackupPrepare( + ScrubState *p, /* Backup context */ + sqlite3 *db, /* Database to prepare against */ + const char *zSql /* SQL statement */ +){ + sqlite3_stmt *pStmt; + if( p->rcErr ) return 0; + p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( p->rcErr ){ + scrubBackupErr(p, "SQL error \"%s\" on \"%s\"", + sqlite3_errmsg(db), zSql); + sqlite3_finalize(pStmt); + return 0; + } + return pStmt; +} + + +/* Open the source database file */ +static void scrubBackupOpenSrc(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + /* Open the source database file */ + p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc, + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_master; BEGIN;", + 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a read transaction on the source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL, + 0, 0); + if( rc ){ + scrubBackupErr(p, "cannot checkpoint the source database"); + return; + } + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->szPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the page size"); + } + sqlite3_finalize(pStmt); + if( p->rcErr ) return; + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the size of the source database"); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc); + if( p->pSrc==0 || p->pSrc->pMethods==0 ){ + scrubBackupErr(p, "cannot get the source file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +/* Create and open the destination file */ +static void scrubBackupOpenDest(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + char *zSql; + if( p->rcErr ) return; + p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage); + if( zSql==0 ){ + p->rcErr = SQLITE_NOMEM; + return; + } + p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot set the page size on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0); + p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a write transaction on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + scrubBackupErr(p, "cannot measure the size of the destination"); + }else if( sqlite3_column_int(pStmt, 0)>1 ){ + scrubBackupErr(p, "destination database is not empty - holds %d pages", + sqlite3_column_int(pStmt, 0)); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest); + if( p->pDest==0 || p->pDest->pMethods==0 ){ + scrubBackupErr(p, "cannot get the destination file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +/* Read a 32-bit big-endian integer */ +static u32 scrubBackupInt32(const u8 *a){ + u32 v = a[3]; + v += ((u32)a[2])<<8; + v += ((u32)a[1])<<16; + v += ((u32)a[0])<<24; + return v; +} + +/* Read a 16-bit big-endian integer */ +static u32 scrubBackupInt16(const u8 *a){ + return (a[0]<<8) + a[1]; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Return the number of bytes in a varint. +*/ +static int scrubBackupVarintSize(const u8 *z){ + int i; + for(i=0; i<8; i++){ + if( (z[i]&0x80)==0 ){ return i+1; } + } + return 9; +} + +/* +** Copy the freelist trunk page given, and all its descendents, +** zeroing out as much as possible in the process. +*/ +static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){ + u8 *a, *aBuf; + u32 n, mx; + + if( p->rcErr ) return; + aBuf = scrubBackupAllocPage(p); + if( aBuf==0 ) return; + + while( pgno && nFree){ + a = scrubBackupRead(p, pgno, aBuf); + if( a==0 ) break; + n = scrubBackupInt32(&a[4]); + mx = p->szUsable/4 - 2; + if( nszPage); + } + scrubBackupWrite(p, iLeaf, aZero); + nFree--; + } +#endif + } + sqlite3_free(aBuf); +} + +/* +** Copy an overflow chain from source to destination. Zero out any +** unused tail at the end of the overflow chain. +*/ +static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){ + u8 *a, *aBuf; + + aBuf = scrubBackupAllocPage(p); + if( aBuf==0 ) return; + while( nByte>0 && pgno!=0 ){ + a = scrubBackupRead(p, pgno, aBuf); + if( a==0 ) break; + if( nByte >= (p->szUsable)-4 ){ + nByte -= (p->szUsable) - 4; + }else{ + u32 x = (p->szUsable - 4) - nByte; + u32 i = p->szUsable - x; + memset(&a[i], 0, x); + nByte = 0; + } + scrubBackupWrite(p, pgno, a); + pgno = scrubBackupInt32(a); + } + sqlite3_free(aBuf); +} + + +/* +** Copy B-Tree page pgno, and all of its children, from source to destination. +** Zero out deleted content during the copy. +*/ +static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){ + u8 *a; + u32 i, n, pc; + u32 nCell; + u32 nPrefix; + u32 szHdr; + u32 iChild; + u8 *aTop; + u8 *aCell; + u32 x, y; + int ln = 0; + + + if( p->rcErr ) return; + if( iDepth>50 ){ + scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno); + return; + } + if( pgno==1 ){ + a = p->page1; + }else{ + a = scrubBackupRead(p, pgno, 0); + if( a==0 ) return; + } + nPrefix = pgno==1 ? 100 : 0; + aTop = &a[nPrefix]; + szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05); + aCell = aTop + szHdr; + nCell = scrubBackupInt16(&aTop[3]); + + /* Zero out the gap between the cell index and the start of the + ** cell content area */ + x = scrubBackupInt16(&aTop[5]); /* First byte of cell content area */ + if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + y = szHdr + nPrefix + nCell*2; + if( y>x ){ ln=__LINE__; goto btree_corrupt; } + if( y0 && pc(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; } + n = scrubBackupInt16(&a[pc+2]); + if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; } + if( n>4 ) memset(&a[pc+4], 0, n-4); + x = scrubBackupInt16(&a[pc]); + if( x0 ){ ln=__LINE__; goto btree_corrupt; } + pc = x; + } + + /* Write this one page */ + scrubBackupWrite(p, pgno, a); + + /* Walk the tree and process child pages */ + for(i=0; i p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; } + if( aTop[0]==0x05 || aTop[0]==0x02 ){ + if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + iChild = scrubBackupInt32(&a[pc]); + pc += 4; + scrubBackupBtree(p, iChild, iDepth+1); + if( aTop[0]==0x05 ) continue; + } + pc += scrubBackupVarint(&a[pc], &P); + if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; } + if( aTop[0]==0x0d ){ + X = p->szUsable - 35; + }else{ + X = ((p->szUsable - 12)*64/255) - 23; + } + if( P<=X ){ + /* All content is local. No overflow */ + continue; + } + M = ((p->szUsable - 12)*32/255)-23; + K = M + ((P-M)%(p->szUsable-4)); + if( aTop[0]==0x0d ){ + pc += scrubBackupVarintSize(&a[pc]); + if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; } + } + nLocal = K<=X ? K : M; + if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; } + iChild = scrubBackupInt32(&a[pc+nLocal]); + scrubBackupOverflow(p, iChild, P-nLocal); + } + + /* Walk the right-most tree */ + if( aTop[0]==0x05 || aTop[0]==0x02 ){ + iChild = scrubBackupInt32(&aTop[8]); + scrubBackupBtree(p, iChild, iDepth+1); + } + + /* All done */ + if( pgno>1 ) sqlite3_free(a); + return; + +btree_corrupt: + scrubBackupErr(p, "corruption on page %d of source database (errid=%d)", + pgno, ln); + if( pgno>1 ) sqlite3_free(a); +} + +/* +** Copy all ptrmap pages from source to destination. +** This routine is only called if the source database is in autovacuum +** or incremental vacuum mode. +*/ +static void scrubBackupPtrmap(ScrubState *p){ + u32 pgno = 2; + u32 J = p->szUsable/5; + u32 iLock = (1073742335/p->szPage)+1; + u8 *a, *pBuf; + if( p->rcErr ) return; + pBuf = scrubBackupAllocPage(p); + if( pBuf==0 ) return; + while( pgno<=p->nPage ){ + a = scrubBackupRead(p, pgno, pBuf); + if( a==0 ) break; + scrubBackupWrite(p, pgno, a); + pgno += J+1; + if( pgno==iLock ) pgno++; + } + sqlite3_free(pBuf); +} + +int sqlite3_scrub_backup( + const char *zSrcFile, /* Source file */ + const char *zDestFile, /* Destination file */ + char **pzErr /* Write error here if non-NULL */ +){ + ScrubState s; + u32 n, i; + sqlite3_stmt *pStmt; + + memset(&s, 0, sizeof(s)); + s.zSrcFile = zSrcFile; + s.zDestFile = zDestFile; + + /* Open both source and destination databases */ + scrubBackupOpenSrc(&s); + scrubBackupOpenDest(&s); + + /* Read in page 1 */ + s.page1 = scrubBackupRead(&s, 1, 0); + if( s.page1==0 ) goto scrub_abort; + s.szUsable = s.szPage - s.page1[20]; + + /* Copy the freelist */ + n = scrubBackupInt32(&s.page1[36]); + i = scrubBackupInt32(&s.page1[32]); + if( n ) scrubBackupFreelist(&s, i, n); + + /* Copy ptrmap pages */ + n = scrubBackupInt32(&s.page1[52]); + if( n ) scrubBackupPtrmap(&s); + + /* Copy all of the btrees */ + scrubBackupBtree(&s, 1, 0); + pStmt = scrubBackupPrepare(&s, s.dbSrc, + "SELECT rootpage FROM sqlite_master WHERE coalesce(rootpage,0)>0"); + if( pStmt==0 ) goto scrub_abort; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + i = (u32)sqlite3_column_int(pStmt, 0); + scrubBackupBtree(&s, i, 0); + } + sqlite3_finalize(pStmt); + + /* If the last page of the input db file is a free-list leaf, then the + ** backup file on disk is still smaller than the size indicated within + ** the database header. In this case, write a page of zeroes to the + ** last page of the backup database so that SQLite does not mistakenly + ** think the db is corrupt. */ + if( s.iLastPage +#include +#include + +/****************************************************************************** +** The Hash Engine +*/ +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + + +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + + /* Copy p->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + +#undef a +#undef b +#undef c +#undef d +#undef e +} + + +/* Initialize a SHA1 context */ +static void hash_init(SHA1Context *p){ + /* SHA1 initialization constants */ + p->state[0] = 0x67452301; + p->state[1] = 0xEFCDAB89; + p->state[2] = 0x98BADCFE; + p->state[3] = 0x10325476; + p->state[4] = 0xC3D2E1F0; + p->count[0] = p->count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step( + SHA1Context *p, /* Add content to this context */ + const unsigned char *data, /* Data to be added */ + unsigned int len /* Number of bytes in data */ +){ + unsigned int i, j; + + j = p->count[0]; + if( (p->count[0] += len << 3) < j ){ + p->count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&p->buffer[j], data, (i = 64-j)); + SHA1Transform(p->state, p->buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(p->state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&p->buffer[j], &data[i], len - i); +} + +/* Compute a string using sqlite3_vsnprintf() and hash it */ +static void hash_step_vformat( + SHA1Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + hash_step(p, (unsigned char*)zBuf, n); +} + + +/* Add padding and compute the message digest. Render the +** message digest as lower-case hexadecimal and put it into +** zOut[]. zOut[] must be at least 41 bytes long. */ +static void hash_finish( + SHA1Context *p, /* The SHA1 context to finish and render */ + char *zOut /* Store hexadecimal hash here */ +){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((p->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step(p, (const unsigned char *)"\200", 1); + while ((p->count[0] & 504) != 448){ + hash_step(p, (const unsigned char *)"\0", 1); + } + hash_step(p, finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha1(X) function. +** +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other +** types of input, X is converted into a UTF-8 string and the string +** is hash without the trailing 0x00 terminator. The hash of a NULL +** value is NULL. +*/ +static void sha1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA1Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + char zOut[44]; + + assert( argc==1 ); + if( eType==SQLITE_NULL ) return; + hash_init(&cx); + if( eType==SQLITE_BLOB ){ + hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + +/* +** Implementation of the sha1_query(SQL) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using SHA1 and that hash is returned. +** +** The original SQL text is included as part of the hash. +** +** The hash is not just a concatenation of the outputs. Each query +** is delimited and each row and value within the query is delimited, +** with all values being marked with their datatypes. +*/ +static void sha1QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA1Context cx; + char zOut[44]; + + assert( argc==1 ); + if( zSql==0 ) return; + hash_init(&cx); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + hash_step(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + hash_step(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + hash_step(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + hash_step(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + hash_step(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + hash_step(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + hash_finish(&cx, zOut); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8, 0, + sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8, 0, + sha1QueryFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c new file mode 100644 index 0000000000..612e395ccb --- /dev/null +++ b/ext/misc/shathree.c @@ -0,0 +1,714 @@ +/* +** 2017-03-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a functions that compute SHA1 hashes. +** Two SQL functions are implemented: +** +** sha3(X,SIZE) +** sha3_query(Y,SIZE) +** +** The sha3(X) function computes the SHA3 hash of the input X, or NULL if +** X is NULL. +** +** The sha3_query(Y) function evalutes all queries in the SQL statements of Y +** and returns a hash of their results. +** +** The SIZE argument is optional. If omitted, the SHA3-256 hash algorithm +** is used. If SIZE is included it must be one of the integers 224, 256, +** 384, or 512, to determine SHA3 hash variant that is computed. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +typedef sqlite3_uint64 u64; + +/****************************************************************************** +** The Hash Engine +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} +/* End of the hashing logic +*****************************************************************************/ + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_shathree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index b5859ea2c8..1ac1712f4e 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -390,7 +390,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ /* Special processing if either string is empty */ if( nA==0 ){ - cBprev = dc; + cBprev = (char)dc; for(xB=res=0; (cB = zB[xB])!=0; xB++){ res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV; cBprev = cB; @@ -398,7 +398,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ return res; } if( nB==0 ){ - cAprev = dc; + cAprev = (char)dc; for(xA=res=0; (cA = zA[xA])!=0; xA++){ res += insertOrDeleteCost(cAprev, cA, zA[xA+1]); cAprev = cA; @@ -420,8 +420,8 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ /* Compute the Wagner edit distance */ m[0] = 0; - cx[0] = dc; - cBprev = dc; + cx[0] = (char)dc; + cBprev = (char)dc; for(xB=1; xB<=nB; xB++){ cBnext = zB[xB]; cB = zB[xB-1]; @@ -429,7 +429,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext); cBprev = cB; } - cAprev = dc; + cAprev = (char)dc; for(xA=1; xA<=nA; xA++){ int lastA = (xA==nA); cA = zA[xA-1]; @@ -476,7 +476,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ d = m[xB]; dc = cx[xB]; m[xB] = totalCost; - cx[xB] = ncx; + cx[xB] = (char)ncx; cBprev = cB; } cAprev = cA; @@ -711,9 +711,9 @@ static int editDist3ConfigLoad( if( nExtra<0 ) nExtra = 0; pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra ); if( pCost==0 ){ rc = SQLITE_NOMEM; break; } - pCost->nFrom = nFrom; - pCost->nTo = nTo; - pCost->iCost = iCost; + pCost->nFrom = (u8)nFrom; + pCost->nTo = (u8)nTo; + pCost->iCost = (u16)iCost; memcpy(pCost->a, zFrom, nFrom); memcpy(pCost->a + nFrom, zTo, nTo); pCost->pNext = pLang->pCost; @@ -1616,7 +1616,7 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ zIn += sz; nIn -= sz; if( c<=127 ){ - zOut[nOut++] = c; + zOut[nOut++] = (unsigned char)c; }else{ int xTop, xBtm, x; xTop = sizeof(translit)/sizeof(translit[0]) - 1; @@ -2231,7 +2231,7 @@ static int spellfix1Score(int iDistance, int iRank){ ** Compare two spellfix1_row objects for sorting purposes in qsort() such ** that they sort in order of increasing distance. */ -static int spellfix1RowCompare(const void *A, const void *B){ +static int SQLITE_CDECL spellfix1RowCompare(const void *A, const void *B){ const struct spellfix1_row *a = (const struct spellfix1_row*)A; const struct spellfix1_row *b = (const struct spellfix1_row*)B; return a->iScore - b->iScore; diff --git a/ext/misc/vfsstat.c b/ext/misc/vfsstat.c new file mode 100644 index 0000000000..14753d277d --- /dev/null +++ b/ext/misc/vfsstat.c @@ -0,0 +1,816 @@ +/* +** 2016-05-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the implementation of an SQLite vfs shim that +** tracks I/O. Access to the accumulated status counts is provided using +** an eponymous virtual table. +*/ +#include +SQLITE_EXTENSION_INIT1 + +/* +** This module contains code for a wrapper VFS that cause stats for +** most VFS calls to be recorded. +** +** To use this module, first compile it as a loadable extension. See +** https://www.sqlite.org/loadext.html#build for compilations instructions. +** +** After compliing, load this extension, then open database connections to be +** measured. Query usages status using the vfsstat virtual table: +** +** SELECT * FROM vfsstat; +** +** Reset counters using UPDATE statements against vfsstat: +** +** UPDATE vfsstat SET count=0; +** +** EXAMPLE SCRIPT: +** +** .load ./vfsstat +** .open test.db +** DROP TABLE IF EXISTS t1; +** CREATE TABLE t1(x,y); +** INSERT INTO t1 VALUES(123, randomblob(5000)); +** CREATE INDEX t1x ON t1(x); +** DROP TABLE t1; +** VACUUM; +** SELECT * FROM vfsstat WHERE count>0; +** +** LIMITATIONS: +** +** This module increments counters without using mutex protection. So if +** two or more threads try to use this module at the same time, race conditions +** may occur which mess up the counts. This is harmless, other than giving +** incorrect statistics. +*/ +#include +#include +#include + +/* +** File types +*/ +#define VFSSTAT_MAIN 0 /* Main database file */ +#define VFSSTAT_JOURNAL 1 /* Rollback journal */ +#define VFSSTAT_WAL 2 /* Write-ahead log file */ +#define VFSSTAT_MASTERJRNL 3 /* Master journal */ +#define VFSSTAT_SUBJRNL 4 /* Subjournal */ +#define VFSSTAT_TEMPDB 5 /* TEMP database */ +#define VFSSTAT_TEMPJRNL 6 /* Journal for TEMP database */ +#define VFSSTAT_TRANSIENT 7 /* Transient database */ +#define VFSSTAT_ANY 8 /* Unspecified file type */ +#define VFSSTAT_nFile 9 /* This many file types */ + +/* Names of the file types. These are allowed values for the +** first column of the vfsstat virtual table. +*/ +static const char *azFile[] = { + "database", "journal", "wal", "master-journal", "sub-journal", + "temp-database", "temp-journal", "transient-db", "*" +}; + +/* +** Stat types +*/ +#define VFSSTAT_BYTESIN 0 /* Bytes read in */ +#define VFSSTAT_BYTESOUT 1 /* Bytes written out */ +#define VFSSTAT_READ 2 /* Read requests */ +#define VFSSTAT_WRITE 3 /* Write requests */ +#define VFSSTAT_SYNC 4 /* Syncs */ +#define VFSSTAT_OPEN 5 /* File opens */ +#define VFSSTAT_LOCK 6 /* Lock requests */ +#define VFSSTAT_ACCESS 0 /* xAccess calls. filetype==ANY only */ +#define VFSSTAT_DELETE 1 /* xDelete calls. filetype==ANY only */ +#define VFSSTAT_FULLPATH 2 /* xFullPathname calls. ANY only */ +#define VFSSTAT_RANDOM 3 /* xRandomness calls. ANY only */ +#define VFSSTAT_SLEEP 4 /* xSleep calls. ANY only */ +#define VFSSTAT_CURTIME 5 /* xCurrentTime calls. ANY only */ +#define VFSSTAT_nStat 7 /* This many stat types */ + + +/* Names for the second column of the vfsstat virtual table for all +** cases except when the first column is "*" or VFSSTAT_ANY. */ +static const char *azStat[] = { + "bytes-in", "bytes-out", "read", "write", "sync", "open", "lock", +}; +static const char *azStatAny[] = { + "access", "delete", "fullpathname", "randomness", "sleep", "currenttimestamp", + "not-used" +}; + +/* Total number of counters */ +#define VFSSTAT_MXCNT (VFSSTAT_nStat*VFSSTAT_nFile) + +/* +** Performance stats are collected in an instance of the following +** global array. +*/ +static sqlite3_uint64 aVfsCnt[VFSSTAT_MXCNT]; + +/* +** Access to a specific counter +*/ +#define STATCNT(filetype,stat) (aVfsCnt[(filetype)*VFSSTAT_nStat+(stat)]) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct VStatVfs VStatVfs; +typedef struct VStatFile VStatFile; + +/* An instance of the VFS */ +struct VStatVfs { + sqlite3_vfs base; /* VFS methods */ + sqlite3_vfs *pVfs; /* Parent VFS */ +}; + +/* An open file */ +struct VStatFile { + sqlite3_file base; /* IO methods */ + sqlite3_file *pReal; /* Underlying file handle */ + unsigned char eFiletype; /* What type of file is this */ +}; + +#define REALVFS(p) (((VStatVfs*)(p))->pVfs) + +/* +** Methods for VStatFile +*/ +static int vstatClose(sqlite3_file*); +static int vstatRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vstatWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int vstatTruncate(sqlite3_file*, sqlite3_int64 size); +static int vstatSync(sqlite3_file*, int flags); +static int vstatFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vstatLock(sqlite3_file*, int); +static int vstatUnlock(sqlite3_file*, int); +static int vstatCheckReservedLock(sqlite3_file*, int *pResOut); +static int vstatFileControl(sqlite3_file*, int op, void *pArg); +static int vstatSectorSize(sqlite3_file*); +static int vstatDeviceCharacteristics(sqlite3_file*); +static int vstatShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int vstatShmLock(sqlite3_file*, int offset, int n, int flags); +static void vstatShmBarrier(sqlite3_file*); +static int vstatShmUnmap(sqlite3_file*, int deleteFlag); +static int vstatFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int vstatUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for VStatVfs +*/ +static int vstatOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int vstatDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int vstatAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int vstatFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *vstatDlOpen(sqlite3_vfs*, const char *zFilename); +static void vstatDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void vstatDlClose(sqlite3_vfs*, void*); +static int vstatRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int vstatSleep(sqlite3_vfs*, int microseconds); +static int vstatCurrentTime(sqlite3_vfs*, double*); +static int vstatGetLastError(sqlite3_vfs*, int, char *); +static int vstatCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static VStatVfs vstat_vfs = { + { + 2, /* iVersion */ + 0, /* szOsFile (set by register_vstat()) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "vfslog", /* zName */ + 0, /* pAppData */ + vstatOpen, /* xOpen */ + vstatDelete, /* xDelete */ + vstatAccess, /* xAccess */ + vstatFullPathname, /* xFullPathname */ + vstatDlOpen, /* xDlOpen */ + vstatDlError, /* xDlError */ + vstatDlSym, /* xDlSym */ + vstatDlClose, /* xDlClose */ + vstatRandomness, /* xRandomness */ + vstatSleep, /* xSleep */ + vstatCurrentTime, /* xCurrentTime */ + vstatGetLastError, /* xGetLastError */ + vstatCurrentTimeInt64 /* xCurrentTimeInt64 */ + }, + 0 +}; + +static const sqlite3_io_methods vstat_io_methods = { + 3, /* iVersion */ + vstatClose, /* xClose */ + vstatRead, /* xRead */ + vstatWrite, /* xWrite */ + vstatTruncate, /* xTruncate */ + vstatSync, /* xSync */ + vstatFileSize, /* xFileSize */ + vstatLock, /* xLock */ + vstatUnlock, /* xUnlock */ + vstatCheckReservedLock, /* xCheckReservedLock */ + vstatFileControl, /* xFileControl */ + vstatSectorSize, /* xSectorSize */ + vstatDeviceCharacteristics, /* xDeviceCharacteristics */ + vstatShmMap, /* xShmMap */ + vstatShmLock, /* xShmLock */ + vstatShmBarrier, /* xShmBarrier */ + vstatShmUnmap, /* xShmUnmap */ + vstatFetch, /* xFetch */ + vstatUnfetch /* xUnfetch */ +}; + + + +/* +** Close an vstat-file. +*/ +static int vstatClose(sqlite3_file *pFile){ + VStatFile *p = (VStatFile *)pFile; + int rc = SQLITE_OK; + + if( p->pReal->pMethods ){ + rc = p->pReal->pMethods->xClose(p->pReal); + } + return rc; +} + + +/* +** Read data from an vstat-file. +*/ +static int vstatRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + VStatFile *p = (VStatFile *)pFile; + + rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + STATCNT(p->eFiletype,VFSSTAT_READ)++; + if( rc==SQLITE_OK ){ + STATCNT(p->eFiletype,VFSSTAT_BYTESIN) += iAmt; + } + return rc; +} + +/* +** Write data to an vstat-file. +*/ +static int vstatWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + VStatFile *p = (VStatFile *)pFile; + + rc = p->pReal->pMethods->xWrite(p->pReal, z, iAmt, iOfst); + STATCNT(p->eFiletype,VFSSTAT_WRITE)++; + if( rc==SQLITE_OK ){ + STATCNT(p->eFiletype,VFSSTAT_BYTESOUT) += iAmt; + } + return rc; +} + +/* +** Truncate an vstat-file. +*/ +static int vstatTruncate(sqlite3_file *pFile, sqlite_int64 size){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xTruncate(p->pReal, size); + return rc; +} + +/* +** Sync an vstat-file. +*/ +static int vstatSync(sqlite3_file *pFile, int flags){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xSync(p->pReal, flags); + STATCNT(p->eFiletype,VFSSTAT_SYNC)++; + return rc; +} + +/* +** Return the current file-size of an vstat-file. +*/ +static int vstatFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + return rc; +} + +/* +** Lock an vstat-file. +*/ +static int vstatLock(sqlite3_file *pFile, int eLock){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** Unlock an vstat-file. +*/ +static int vstatUnlock(sqlite3_file *pFile, int eLock){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xUnlock(p->pReal, eLock); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** Check if another file-handle holds a RESERVED lock on an vstat-file. +*/ +static int vstatCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); + STATCNT(p->eFiletype,VFSSTAT_LOCK)++; + return rc; +} + +/* +** File control method. For custom operations on an vstat-file. +*/ +static int vstatFileControl(sqlite3_file *pFile, int op, void *pArg){ + VStatFile *p = (VStatFile *)pFile; + int rc; + rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); + if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("vstat/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an vstat-file. +*/ +static int vstatSectorSize(sqlite3_file *pFile){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xSectorSize(p->pReal); + return rc; +} + +/* +** Return the device characteristic flags supported by an vstat-file. +*/ +static int vstatDeviceCharacteristics(sqlite3_file *pFile){ + int rc; + VStatFile *p = (VStatFile *)pFile; + rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal); + return rc; +} + +/* Create a shared memory file mapping */ +static int vstatShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmMap(p->pReal, iPg, pgsz, bExtend, pp); +} + +/* Perform locking on a shared-memory segment */ +static int vstatShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmLock(p->pReal, offset, n, flags); +} + +/* Memory barrier operation on shared memory */ +static void vstatShmBarrier(sqlite3_file *pFile){ + VStatFile *p = (VStatFile *)pFile; + p->pReal->pMethods->xShmBarrier(p->pReal); +} + +/* Unmap a shared memory segment */ +static int vstatShmUnmap(sqlite3_file *pFile, int deleteFlag){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xShmUnmap(p->pReal, deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int vstatFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xFetch(p->pReal, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int vstatUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + VStatFile *p = (VStatFile *)pFile; + return p->pReal->pMethods->xUnfetch(p->pReal, iOfst, pPage); +} + +/* +** Open an vstat file handle. +*/ +static int vstatOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + int rc; + VStatFile *p = (VStatFile*)pFile; + + p->pReal = (sqlite3_file*)&p[1]; + rc = REALVFS(pVfs)->xOpen(REALVFS(pVfs), zName, p->pReal, flags, pOutFlags); + if( flags & SQLITE_OPEN_MAIN_DB ){ + p->eFiletype = VFSSTAT_MAIN; + }else if( flags & SQLITE_OPEN_MAIN_JOURNAL ){ + p->eFiletype = VFSSTAT_JOURNAL; + }else if( flags & SQLITE_OPEN_WAL ){ + p->eFiletype = VFSSTAT_WAL; + }else if( flags & SQLITE_OPEN_MASTER_JOURNAL ){ + p->eFiletype = VFSSTAT_MASTERJRNL; + }else if( flags & SQLITE_OPEN_SUBJOURNAL ){ + p->eFiletype = VFSSTAT_SUBJRNL; + }else if( flags & SQLITE_OPEN_TEMP_DB ){ + p->eFiletype = VFSSTAT_TEMPDB; + }else if( flags & SQLITE_OPEN_TEMP_JOURNAL ){ + p->eFiletype = VFSSTAT_TEMPJRNL; + }else{ + p->eFiletype = VFSSTAT_TRANSIENT; + } + STATCNT(p->eFiletype,VFSSTAT_OPEN)++; + pFile->pMethods = rc ? 0 : &vstat_io_methods; + return rc; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int vstatDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc; + rc = REALVFS(pVfs)->xDelete(REALVFS(pVfs), zPath, dirSync); + STATCNT(VFSSTAT_ANY,VFSSTAT_DELETE)++; + return rc; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int vstatAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + int rc; + rc = REALVFS(pVfs)->xAccess(REALVFS(pVfs), zPath, flags, pResOut); + STATCNT(VFSSTAT_ANY,VFSSTAT_ACCESS)++; + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int vstatFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + STATCNT(VFSSTAT_ANY,VFSSTAT_FULLPATH)++; + return REALVFS(pVfs)->xFullPathname(REALVFS(pVfs), zPath, nOut, zOut); +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *vstatDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return REALVFS(pVfs)->xDlOpen(REALVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void vstatDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + REALVFS(pVfs)->xDlError(REALVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*vstatDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return REALVFS(pVfs)->xDlSym(REALVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void vstatDlClose(sqlite3_vfs *pVfs, void *pHandle){ + REALVFS(pVfs)->xDlClose(REALVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int vstatRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + STATCNT(VFSSTAT_ANY,VFSSTAT_RANDOM)++; + return REALVFS(pVfs)->xRandomness(REALVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int vstatSleep(sqlite3_vfs *pVfs, int nMicro){ + STATCNT(VFSSTAT_ANY,VFSSTAT_SLEEP)++; + return REALVFS(pVfs)->xSleep(REALVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int vstatCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++; + return REALVFS(pVfs)->xCurrentTime(REALVFS(pVfs), pTimeOut); +} + +static int vstatGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return REALVFS(pVfs)->xGetLastError(REALVFS(pVfs), a, b); +} +static int vstatCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + STATCNT(VFSSTAT_ANY,VFSSTAT_CURTIME)++; + return REALVFS(pVfs)->xCurrentTimeInt64(REALVFS(pVfs), p); +} + +/* +** A virtual table for accessing the stats collected by this VFS shim +*/ +static int vstattabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int vstattabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int vstattabDisconnect(sqlite3_vtab*); +static int vstattabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int vstattabClose(sqlite3_vtab_cursor*); +static int vstattabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int vstattabNext(sqlite3_vtab_cursor*); +static int vstattabEof(sqlite3_vtab_cursor*); +static int vstattabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int vstattabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); +static int vstattabUpdate(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); + +/* A cursor for the vfsstat virtual table */ +typedef struct VfsStatCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int i; /* Pointing to this aVfsCnt[] value */ +} VfsStatCursor; + + +static int vstattabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define VSTAT_COLUMN_FILE 0 +#define VSTAT_COLUMN_STAT 1 +#define VSTAT_COLUMN_COUNT 2 + + rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for vstat table object. +*/ +static int vstattabDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new vstat table cursor object. +*/ +static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + VfsStatCursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + + +/* +** Destructor for a VfsStatCursor. +*/ +static int vstattabClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a VfsStatCursor to its next row of output. +*/ +static int vstattabNext(sqlite3_vtab_cursor *cur){ + ((VfsStatCursor*)cur)->i++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the VfsStatCursor +** is currently pointing. +*/ +static int vstattabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + switch( i ){ + case VSTAT_COLUMN_FILE: { + sqlite3_result_text(ctx, azFile[pCur->i/VFSSTAT_nStat], -1, SQLITE_STATIC); + break; + } + case VSTAT_COLUMN_STAT: { + const char **az; + az = (pCur->i/VFSSTAT_nStat)==VFSSTAT_ANY ? azStatAny : azStat; + sqlite3_result_text(ctx, az[pCur->i%VFSSTAT_nStat], -1, SQLITE_STATIC); + break; + } + case VSTAT_COLUMN_COUNT: { + sqlite3_result_int64(ctx, aVfsCnt[pCur->i]); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int vstattabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + *pRowid = pCur->i; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int vstattabEof(sqlite3_vtab_cursor *cur){ + VfsStatCursor *pCur = (VfsStatCursor*)cur; + return pCur->i >= VFSSTAT_MXCNT; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int vstattabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + VfsStatCursor *pCur = (VfsStatCursor*)pVtabCursor; + pCur->i = 0; + return SQLITE_OK; +} + +/* +** Only a forwards full table scan is supported. xBestIndex is a no-op. +*/ +static int vstattabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + return SQLITE_OK; +} + +/* +** Any VSTAT_COLUMN_COUNT can be changed to a positive integer. +** No deletions or insertions are allowed. No changes to other +** columns are allowed. +*/ +static int vstattabUpdate( + sqlite3_vtab *tab, + int argc, sqlite3_value **argv, + sqlite3_int64 *pRowid +){ + sqlite3_int64 iRowid, x; + if( argc==1 ) return SQLITE_ERROR; + if( sqlite3_value_type(argv[0])!=SQLITE_INTEGER ) return SQLITE_ERROR; + iRowid = sqlite3_value_int64(argv[0]); + if( iRowid!=sqlite3_value_int64(argv[1]) ) return SQLITE_ERROR; + if( iRowid<0 || iRowid>=VFSSTAT_MXCNT ) return SQLITE_ERROR; + if( sqlite3_value_type(argv[VSTAT_COLUMN_COUNT+2])!=SQLITE_INTEGER ){ + return SQLITE_ERROR; + } + x = sqlite3_value_int64(argv[VSTAT_COLUMN_COUNT+2]); + if( x<0 ) return SQLITE_ERROR; + aVfsCnt[iRowid] = x; + return SQLITE_OK; +} + +static sqlite3_module VfsStatModule = { + 0, /* iVersion */ + 0, /* xCreate */ + vstattabConnect, /* xConnect */ + vstattabBestIndex, /* xBestIndex */ + vstattabDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + vstattabOpen, /* xOpen - open a cursor */ + vstattabClose, /* xClose - close a cursor */ + vstattabFilter, /* xFilter - configure scan constraints */ + vstattabNext, /* xNext - advance a cursor */ + vstattabEof, /* xEof - check for end of scan */ + vstattabColumn, /* xColumn - read data */ + vstattabRowid, /* xRowid - read data */ + vstattabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/* +** This routine is an sqlite3_auto_extension() callback, invoked to register +** the vfsstat virtual table for all new database connections. +*/ +static int vstatRegister( + sqlite3 *db, + const char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + return sqlite3_create_module(db, "vfsstat", &VfsStatModule, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** +** Register the new VFS. Make arrangement to register the virtual table +** for each new database connection. +*/ +int sqlite3_vfsstat_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + vstat_vfs.pVfs = sqlite3_vfs_find(0); + vstat_vfs.base.szOsFile = sizeof(VStatFile) + vstat_vfs.pVfs->szOsFile; + rc = sqlite3_vfs_register(&vstat_vfs.base, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension(vstatRegister); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c index 01348e8d3e..0709a26a7f 100644 --- a/ext/misc/vtshim.c +++ b/ext/misc/vtshim.c @@ -95,6 +95,7 @@ static int vtshimCreate( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; @@ -133,6 +134,7 @@ static int vtshimConnect( if( rc ){ sqlite3_free(pNew); *ppVtab = 0; + return rc; } pNew->pAux = pAux; pNew->ppPrev = &pAux->pAllVtab; diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index 018296b8dd..a3efad0d44 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -24,12 +24,22 @@ */ void usage(const char *zArgv0){ fprintf(stderr, -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n" +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" "\n" -" Argument RBU-DB must be an RBU database containing an update suitable for\n" -" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" -" (the default value), this program attempts to apply the entire update to\n" -" the target database.\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -vacuum\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" "\n" " If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" " to sqlite3rbu_step(). If the RBU update has not been completely applied\n" @@ -69,29 +79,43 @@ int main(int argc, char **argv){ char *zErrmsg; /* Error message, if any */ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ + int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; + int nArg = argc-2; - /* Process command line arguments. Following this block local variables - ** zTarget, zRbu and nStep are all set. */ - if( argc==5 ){ - size_t nArg1 = strlen(argv[1]); - if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); - nStep = atoi(argv[2]); - }else if( argc!=3 ){ - usage(argv[0]); + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i0 && $state=="oal" && (!$oal_exists || $wal_exists)) + || ($state=="move" && (!$oal_exists || $wal_exists)) + || ($state=="checkpoint" && ($oal_exists || !$wal_exists)) + || ($state=="done" && ($oal_exists && $progress!=0)) + } { + error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } +} + +proc check_poststep_state {rc target state} { + if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + if {$state=="move" && ($oal_exists || !$wal_exists)} { + error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } + } +} + # Run the RBU in file $rbu on target database $target until completion. # proc run_rbu {target rbu} { sqlite3rbu rbu $target $rbu while 1 { + set state [rbu state] + + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state + if {$rc!="SQLITE_OK"} break } rbu close @@ -29,10 +60,33 @@ proc run_rbu {target rbu} { proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu + set state [rbu state] + check_prestep_state $target $state set rc [rbu step] + check_poststep_state $rc $target $state rbu close if {$rc != "SQLITE_OK"} break } set rc } +proc do_rbu_vacuum_test {tn step} { + uplevel [list do_test $tn.1 { + if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } + while 1 { + if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } + set state [rbu state] + check_prestep_state test.db $state + set rc [rbu step] + check_poststep_state $rc test.db $state + if {$rc!="SQLITE_OK"} break + if {$step==1} { rbu close } + } + rbu close + } {SQLITE_DONE}] + + uplevel [list do_execsql_test $tn.2 { + PRAGMA integrity_check + } ok] +} + diff --git a/ext/rbu/rbucrash2.test b/ext/rbu/rbucrash2.test new file mode 100644 index 0000000000..5f2ba604d1 --- /dev/null +++ b/ext/rbu/rbucrash2.test @@ -0,0 +1,106 @@ +# 2017 March 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbucrash2 + +db close +forcedelete test.db-oal rbu.db +sqlite3_shutdown +sqlite3_config_uri 1 +reset_db + +# Set up a target database and an rbu update database. The target +# db is the usual "test.db", the rbu db is "test.db2". +# +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0); +} +db_save_and_close + +proc do_rbu_crash_test2 {tn script} { + + foreach {f blksz} { + test.db 512 + test.db2 512 + test.db 4096 + test.db2 4096 + } { + set bDone 0 + for {set iDelay 1} {$bDone==0} {incr iDelay} { + forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal + db_restore + + set res [ + crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ + -blocksize $blksz {} + ] + + set bDone 1 + if {$res == "1 {child process exited abnormally}"} { + set bDone 0 + } elseif {$res != "0 {}"} { + error "unexected catchsql result: $res" + } + + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz { + PRAGMA integrity_check; + } {ok} + db close + } + } +} + +for {set x 1} {$x < 10} {incr x} { + do_rbu_crash_test2 1.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu savestate + } + rbu close + } +} + +for {set x 1} {$x < 2} {incr x} { + do_rbu_crash_test2 2.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu close + sqlite3rbu rbu test.db test.db2 + } + rbu close + } +} + +finish_test + diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test index 10be9e0d53..6f6af82804 100644 --- a/ext/rbu/rbudiff.test +++ b/ext/rbu/rbudiff.test @@ -18,22 +18,17 @@ if {![info exists testdir]} { source $testdir/tester.tcl set testprefix rbudiff -if {$tcl_platform(platform)=="windows"} { - set PROG "sqldiff.exe" -} else { - set PROG "./sqldiff" -} -if {![file exe $PROG]} { - puts "rbudiff.test cannot run because $PROG is not available" - finish_test - return -} +set PROG [test_find_sqldiff] db close proc get_rbudiff_sql {db1 db2} { exec $::PROG --rbu $db1 $db2 } +proc get_vtab_rbudiff_sql {db1 db2} { + exec $::PROG --vtab --rbu $db1 $db2 +} + proc step_rbu {target rbu} { while 1 { sqlite3rbu rbu $target $rbu @@ -45,6 +40,7 @@ proc step_rbu {target rbu} { } proc apply_rbudiff {sql target} { + test_rbucount $sql forcedelete rbu.db sqlite3 rbudb rbu.db rbudb eval $sql @@ -52,15 +48,43 @@ proc apply_rbudiff {sql target} { step_rbu $target rbu.db } +proc sqlesc {id} { + set ret "'[string map {' ''} $id]'" + set ret +} + +# The only argument is the output of an [sqldiff -rbu] run. This command +# tests that the contents of the rbu_count table is correct. An exception +# is thrown if it is not. +# +proc test_rbucount {sql} { + sqlite3 tmpdb "" + tmpdb eval $sql + tmpdb eval { + SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table' + } { + set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"] + set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}] + if {$a != $b} { + tmpdb close + error "rbu_count error - tbl = $name" + } + } + tmpdb close + return "" +} + proc rbudiff_cksum {db1} { set txt "" sqlite3 dbtmp $db1 foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] { set cols [list] - dbtmp eval "PRAGMA table_info = $tbl" { lappend cols "quote( $name )" } + dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { + lappend cols "quote( $name )" + } append txt [dbtmp eval \ - "SELECT [join $cols {||'.'||}] FROM $tbl ORDER BY 1" + "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1" ] } dbtmp close @@ -116,6 +140,24 @@ foreach {tn init mod} { ); } + 4 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO x1 VALUES('u', 'v', NULL); + INSERT INTO x1 VALUES('x', 'y', 'z'); + INSERT INTO x1 VALUES('a', NULL, 'b'); + } { + INSERT INTO x1 VALUES('a', 'b', 'c'); + } + + 5 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, X''); + } { + UPDATE t1 SET b = X'' WHERE a=1; + UPDATE t1 SET b = NULL WHERE a=2; + } + } { catch { db close } @@ -146,5 +188,116 @@ foreach {tn init mod} { do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] } +#------------------------------------------------------------------------- +# Test that if the --vtab switch is present, [sqldiff] handles virtual +# table types fts[345] and rtree correctly. +# +ifcapable fts3&&fts5&&rtree { + +foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2); + INSERT INTO "x y" VALUES(1, 2, 3); + INSERT INTO "x y" VALUES(2, 4, 6); + } { + DELETE FROM "x y" WHERE rowid = 1; + INSERT INTO "x y" VALUES(3, 6, 9); + } + + 3 { + CREATE VIRTUAL TABLE 'x''y' USING fts3; + INSERT INTO 'x''y' VALUES('one two three'); + INSERT INTO 'x''y' VALUES('four five six'); + } { + DELETE FROM 'x''y' WHERE rowid = 1; + INSERT INTO 'x''y' VALUES('one two three'); + } +} { + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db close + + do_test 2.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] +} + +} + +ifcapable fts5 { + foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 3 { + creAte virTUal +tablE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + } { + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db eval { INSERT INTO t1(t1) VALUES('optimize') } + db close + + do_test 3.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + + sqlite3 db test.db + sqlite3 db2 test.db2 + do_test 3.$tn.2 { + db2 eval { SELECT * FROM t1 ORDER BY rowid } + } [db eval { SELECT * FROM t1 ORDER BY rowid }] + + do_test 3.$tn.3 { + db2 eval { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + + db close + db2 close + } +} + + finish_test diff --git a/ext/rbu/rbudor.test b/ext/rbu/rbudor.test new file mode 100644 index 0000000000..dcee7ac381 --- /dev/null +++ b/ext/rbu/rbudor.test @@ -0,0 +1,59 @@ +# 2016 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test file focuses on interactions between RBU and the feature +# enabled by SQLITE_DIRECT_OVERFLOW_READ - Direct Overflow Read. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix rbudor + +set bigA [string repeat a 5000] +set bigB [string repeat b 5000] +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + INSERT INTO t1 VALUES(1, $bigA); +} {} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, $bigB, 0); + } + rbu close +} {} + +do_test 1.2 { + sqlite3rbu rbu test.db rbu.db + while {[rbu state]!="checkpoint"} { + rbu step + } + rbu step + db eval { SELECT * FROM t1 } +} [list 1 $bigA 2 $bigB] + +do_test 1.3 { + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.4 { + SELECT * FROM t1 +} [list 1 $bigA 2 $bigB] + +finish_test + diff --git a/ext/rbu/rbufault3.test b/ext/rbu/rbufault3.test new file mode 100644 index 0000000000..e72eb14f7d --- /dev/null +++ b/ext/rbu/rbufault3.test @@ -0,0 +1,98 @@ +# 2016 April 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests for RBU vacuum operations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +source $testdir/malloc_common.tcl +set ::testprefix rbufault3 + +foreach {fault errlist} { + oom-* { + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - out of memory}} + } + + ioerr-* { + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_FSYNC} + {1 SQLITE_IOERR_READ} + {1 {SQLITE_IOERR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - SQL logic error or missing database}} + } + + cantopen* { + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database file}} + {1 SQLITE_CANTOPEN} + } + +} { + + reset_db + do_execsql_test 0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + INSERT INTO target VALUES(7, 8, 9); + CREATE INDEX i1 ON target(z); + } + faultsim_save_and_close + + do_faultsim_test 1 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist] + } + + do_faultsim_test 2 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + faultsim_save_and_close + + do_faultsim_test 3 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + +} + +finish_test + diff --git a/ext/rbu/rbufault4.test b/ext/rbu/rbufault4.test new file mode 100644 index 0000000000..fe6cc90dc9 --- /dev/null +++ b/ext/rbu/rbufault4.test @@ -0,0 +1,66 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix rbufault4 + +for {set tn 1} {1} {incr tn} { + reset_db + do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + for {set i 0} {$i < $tn} {incr i} { rbu step } + set rc [rbu close] + if {$rc!="SQLITE_OK"} { + if {$rc!="SQLITE_DONE"} {error $rc} + break + } + faultsim_save + + do_faultsim_test $tn -faults oom-t* -prep { + faultsim_restore + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while 1 { + set rc [rbu step] + if {$rc=="SQLITE_DONE"} break + if {$rc!="SQLITE_OK"} { error $rc } + } + } -test { + catch {rbu close} + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} + + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + set trc [rbu close] + if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" } + + set rc [db one {PRAGMA integrity_check}] + if {$rc!="ok"} { error "Got $rc instead of ok!" } + } +} + + + +finish_test + diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test new file mode 100644 index 0000000000..078b3b0d30 --- /dev/null +++ b/ext/rbu/rbuprogress.test @@ -0,0 +1,419 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuprogress + + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + create_db_file $filename { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + return $filename +} + + +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu bp_progress +} {0 0} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- +# +proc do_sp_test {tn bReopen target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + if {$bReopen==0} { sqlite3rbu rbu $target $rbu } + set res [list] + while 1 { + if {$bReopen} { sqlite3rbu rbu $target $rbu } + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" } + lappend res [lindex [rbu bp_progress] 0] + if {[lindex [set res] end]==10000} break + if {$bReopen} { rbu close } + } + if {[set res] != [list $reslist]} { + rbu close + error "1. reslist incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. After each of these steps, the progress remains + # at "10000 0". + # + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } + } + + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "3. reslist incorrect (expect=10000 0 got=[set res])" + } + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] {} + if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { + rbu close + error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" + } + } + + set res [rbu bp_progress] + if {[set res] != [list 10000 10000]} { + rbu close + error "5. reslist is incorrect (expect=10000 10000 got=[set res])" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach {bReopen} { 0 1 } { + reset_db + do_test 2.$bReopen.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } + } {} + do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000} + + reset_db + do_test 2.$bReopen.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db + } {rbu.db} + do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + } {} + do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000} + + reset_db + do_test 2.$bReopen.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} +} + +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + PRAGMA page_size = 4096; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) } + } { + + foreach {tn2 rbusql r1 r3} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {1666 3333 5000 6666 8333 10000} + + 2 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 1); + } + {3333 6666 10000} + {2000 4000 6000 8000 10000} + + 3 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {1666 2500 3750 5000 6250 7500 8750 10000} + + } { + + reset_db ; execsql $tbl + do_test 4.$tn.$bReopen.$tn2.0 { + execsql { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(ipk) $r1 + set R(wr) $r1 + set R(pk) $r3 + do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + nopk { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c); + } + vtab { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } + } { + + if {$tn=="vtab"} { ifcapable !fts5 break } + + foreach {tn2 rbusql r1 r2} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {5000 10000} + + 2 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + + 3 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1); + INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x'); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + } { + + reset_db ; execsql $tbl + do_test 5.$tn.$bReopen.$tn2.0 { + execsql { + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(nopk) $r1 + set R(vtab) $r2 + do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + + +finish_test + diff --git a/ext/rbu/rburesume.test b/ext/rbu/rburesume.test new file mode 100644 index 0000000000..d03894e135 --- /dev/null +++ b/ext/rbu/rburesume.test @@ -0,0 +1,254 @@ +# 2017 January 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for resumption of RBU operations in the +# case where the previous RBU process crashed. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rburesume + +forcedelete test.db-shm test.db-oal +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s; +} +db_save_and_close + +do_test 1.1 { + list [file exists test.db] \ + [file exists test.db-wal] \ + [file exists test.db-shm] \ + [file exists test.db-oal] +} {1 0 0 0} + +# Each iteration of the following loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Closes the RBU vacuum handled opened in (2). +# 4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + rbu close + if {$rc == "SQLITE_DONE"} break + + do_test 1.2.$n.1 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.2.$n.2 { + sqlite3 db2 test.db + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.3.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.3.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it 10 times. Then closes it. +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (3) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<10} {incr i} { + rbu step + } + rbu close + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.4.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.4.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +forcedelete rbu.db +do_test 2.0 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO data_t1 + SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s; + } + db2 close +} {} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU handle to apply the RBU update created in test case 2.0. +# 3. Steps the RBU handle $n times. +# 4. Takes a copy of all database files and the state db. +# 5. Opens a second RBU handle on the copy, resumes and completes the +# RBU op. Checks it worked as expected. +# +# The loop runs until $n is large enough that step (3) applies the entire +# update. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu rbu test.db rbu.db state.db + + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 2.$n.1 { + sqlite3rbu rbu test.db2 rbu.db state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 2.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {60 ok} + db2 close +} + +finish_test + diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test new file mode 100644 index 0000000000..86f4aa770e --- /dev/null +++ b/ext/rbu/rbuvacuum.test @@ -0,0 +1,392 @@ +# 2016 April 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuvacuum + +foreach step {0 1} { + + set ::testprefix rbuvacuum-step=$step + reset_db + + # Simplest possible vacuum. + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + PRAGMA integrity_check; + } {ok} + do_rbu_vacuum_test 1.1 $step + + # A vacuum that actually reclaims space. + do_execsql_test 1.2.1 { + INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {12} + do_execsql_test 1.2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; + PRAGMA page_count; + } {12} + do_rbu_vacuum_test 1.2.3 $step + do_execsql_test 1.2.4 { + PRAGMA page_count; + } {3} + + # Add an index to the table. + do_execsql_test 1.3.1 { + CREATE INDEX t1b ON t1(b); + INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {18} + do_execsql_test 1.3.2 { + DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; + PRAGMA page_count; + } {18} + do_rbu_vacuum_test 1.3.3 $step + do_execsql_test 1.3.4 { + PRAGMA page_count; + } {5} + + # WITHOUT ROWID table. + do_execsql_test 1.4.1 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 2 AND 7; + PRAGMA page_count; + } {20} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {10} + + # WITHOUT ROWID table with an index. + do_execsql_test 1.4.1 { + CREATE INDEX t2c ON t2(c); + + INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 8 AND 12; + PRAGMA page_count; + } {35} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {15} + do_execsql_test 1.4.4 { + VACUUM; + PRAGMA page_count; + } {15} + + do_execsql_test 1.5.1 { + CREATE TABLE t3(a, b, c); + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + INSERT INTO t3 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.5.2 $step + do_execsql_test 1.5.3 { + SELECT * FROM t3 + } {a b c d e f g h i} + do_execsql_test 1.5.4 { + CREATE INDEX t3a ON t3(a); + CREATE INDEX t3b ON t3(b); + CREATE INDEX t3c ON t3(c); + INSERT INTO t3 VALUES('j', 'k', 'l'); + DELETE FROM t3 WHERE a = 'g'; + } + do_rbu_vacuum_test 1.5.5 $step + do_execsql_test 1.5.6 { + SELECT rowid, * FROM t3 ORDER BY b + } {1 a b c 2 d e f 4 j k l} + + do_execsql_test 1.6.1 { + CREATE TABLE t4(a PRIMARY KEY, b, c); + INSERT INTO t4 VALUES('a', 'b', 'c'); + INSERT INTO t4 VALUES('d', 'e', 'f'); + INSERT INTO t4 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.6.2 $step + do_execsql_test 1.6.3 { + SELECT * FROM t4 + } {a b c d e f g h i} + do_execsql_test 1.6.4 { + CREATE INDEX t4a ON t4(a); + CREATE INDEX t4b ON t4(b); + CREATE INDEX t4c ON t4(c); + + INSERT INTO t4 VALUES('j', 'k', 'l'); + DELETE FROM t4 WHERE a='g'; + } + do_rbu_vacuum_test 1.6.5 $step + do_execsql_test 1.6.6 { + SELECT * FROM t4 ORDER BY b + } {a b c d e f j k l} + + reset_db + do_execsql_test 1.7.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + DELETE FROM t1 WHERE a=2; + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + DELETE FROM t1 WHERE a=6; + SELECT * FROM t1; + } {1 one 3 three 5 five} + do_rbu_vacuum_test 1.7.1 $step + do_execsql_test 1.7.2 { + INSERT INTO t1 VALUES(NULL, 'seven'); + SELECT * FROM t1; + } {1 one 3 three 5 five 7 seven} + + reset_db + do_execsql_test 1.8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + ANALYZE; + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + do_rbu_vacuum_test 1.8.1 $step + do_execsql_test 1.7.2 { + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + + reset_db + do_execsql_test 1.9.0 { + PRAGMA page_size = 8192; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 412; + PRAGMA application_id = 413; + + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + do_rbu_vacuum_test 1.9.1 $step + do_execsql_test 1.9.2 { + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + # Vacuum a database with a large sqlite_master table. + # + reset_db + do_test 1.10.1 { + for {set i 1} {$i < 50} {incr i} { + execsql "PRAGMA page_size = 1024" + execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));" + execsql " + INSERT INTO t$i VALUES(1, 2, 3); + INSERT INTO t$i VALUES(4, 5, 6); + " + } + } {} + do_rbu_vacuum_test 1.10.2 $step + + # Database with empty tables. + # + reset_db + do_execsql_test 1.11.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 VALUES(1, 2); + } + do_rbu_vacuum_test 1.11.2 $step + do_execsql_test 1.11.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} + reset_db + do_execsql_test 1.12.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + } + do_rbu_vacuum_test 1.12.2 $step + do_execsql_test 1.12.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} +} +set ::testprefix rbuvacuum + +#------------------------------------------------------------------------- +# Test some error cases: +# +# 2.1.* the db being vacuumed being in wal mode already. +# 2.2.* database modified mid vacuum. +# +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(9, 10); +} wal +do_test 2.1.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}} + +reset_db +do_execsql_test 2.2.0 { + CREATE TABLE tx(a PRIMARY KEY, b BLOB); + INSERT INTO tx VALUES(1, randomblob(900)); + INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; +} +db_save_and_close +for {set i 1} 1 {incr i} { + db_restore_and_reopen + + sqlite3rbu_vacuum rbu test.db state.db + for {set step 0} {$step<$i} {incr step} { rbu step } + rbu close + if {[file exists test.db-wal]} break + + execsql { INSERT INTO tx VALUES(20, 20) } + + do_test 2.2.$i.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step + } {SQLITE_BUSY} + do_test 2.2.$i.2 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu vacuum}} +} + +#------------------------------------------------------------------------- +# Test that a database that uses custom collation sequences can be RBU +# vacuumed. +# +reset_db +forcedelete state.db +proc noop {args} {} +proc length_cmp {x y} { + set n1 [string length $x] + set n2 [string length $y] + return [expr $n1 - $n2] +} +sqlite3_create_collation_v2 db length length_cmp noop + +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'i'); + INSERT INTO t1 VALUES(2, 'iiii'); + INSERT INTO t1 VALUES(3, 'ii'); + INSERT INTO t1 VALUES(4, 'iii'); + SELECT a FROM t1 ORDER BY b COLLATE length; +} {1 3 4 2} +do_execsql_test 3.1 { + CREATE INDEX i1 ON t1(b COLLATE length); +} + +do_test 3.2 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.3 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.4 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 1] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.5 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + set db2 [rbu db 1] + + sqlite3_create_collation_v2 $db1 length length_cmp noop + sqlite3_create_collation_v2 $db2 length length_cmp noop + + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {0 SQLITE_DONE} + +catch { db close } +finish_test + diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test new file mode 100644 index 0000000000..0a1fe3da94 --- /dev/null +++ b/ext/rbu/rbuvacuum2.test @@ -0,0 +1,235 @@ +# 2016 June 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] + +foreach step {0 1} { + set ::testprefix rbuvacuum2-$step + + #------------------------------------------------------------------------- + # Test that a database that contains fts3 tables can be vacuumed. + # + ifcapable fts3 { + reset_db + do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts3(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 1.2 $step + + do_execsql_test 1.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 1.4 { + SELECT rowid FROM t1 WHERE t1 MATCH 'fix'; + } {1} + + do_execsql_test 1.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE docid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 1.6 $step + do_execsql_test 1.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains fts5 tables can be vacuumed. + # + ifcapable fts5 { + reset_db + do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 2.2 $step + + do_execsql_test 2.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 2.4 { + SELECT rowid FROM t1 ('fix'); + } {1} + + do_execsql_test 2.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE rowid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 2.6 $step + do_execsql_test 2.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains an rtree table can be vacuumed. + # + ifcapable rtree { + reset_db + do_execsql_test 3.1 { + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO rt VALUES(1, 45, 55); + INSERT INTO rt VALUES(2, 50, 60); + INSERT INTO rt VALUES(3, 55, 65); + } + + do_rbu_vacuum_test 3.2 $step + + do_execsql_test 3.3 { + SELECT * FROM rt; + } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0} + + do_execsql_test 3.4.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.4.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + + do_rbu_vacuum_test 3.5 $step + + do_execsql_test 3.6.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.6.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + } + + ifcapable trigger { + reset_db + do_execsql_test 4.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END; + } + + do_execsql_test 4.2 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + + do_rbu_vacuum_test 4.3 $step + do_execsql_test 4.4 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + } +} + +#------------------------------------------------------------------------- +# Test that passing a NULL value as the second argument to +# sqlite3rbu_vacuum() causes it to: +# +# * Use -vacuum as the state db, and +# * Set the state db permissions to the same as those on the db file. +# +db close +if {$::tcl_platform(platform)=="unix"} { + forcedelete test.db + + sqlite3 db test.db + do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + db close + + foreach {tn perm} { + 1 00755 + 2 00666 + 3 00644 + 4 00444 + } { + forcedelete test.db-vacuum + + do_test 5.$tn.1 { + file attributes test.db -permissions $perm + sqlite3rbu_vacuum rbu test.db + rbu step + } {SQLITE_OK} + + do_test 5.$tn.2 { file exists test.db-vacuum } 1 + do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} $perm + rbu close + } +} + +#------------------------------------------------------------------------- +# Test the outcome of some other connection running a checkpoint while +# the incremental checkpoint is suspended. +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} +forcedelete test.db2 + +do_test 6.1 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu state]!="checkpoint"} { rbu step } + rbu close +} {SQLITE_OK} + +do_execsql_test 6.2 { + SELECT 1 FROM sqlite_master LIMIT 1; + PRAGMA wal_checkpoint; +} {1 0 4 4} + +do_test 6.3 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]!="SQLITE_DONE"} { rbu step } + rbu close + execsql { PRAGMA integrity_check } +} {ok} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 474e39fe8d..7dd232b4b5 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -147,14 +147,15 @@ ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -175,6 +176,7 @@ typedef struct RbuUpdateStmt RbuUpdateStmt; #if !defined(SQLITE_AMALGAMATION) typedef unsigned int u32; +typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; #endif @@ -188,6 +190,8 @@ typedef sqlite3_int64 i64; #define WAL_LOCK_CKPT 1 #define WAL_LOCK_READ0 3 +#define SQLITE_FCNTL_RBUCNT 5149216 + /* ** A structure to store values read from the rbu_state table in memory. */ @@ -200,6 +204,7 @@ struct RbuState { i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { @@ -244,6 +249,7 @@ struct RbuObjIter { int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ @@ -297,6 +303,43 @@ struct RbuFrame { /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ @@ -313,7 +356,9 @@ struct sqlite3rbu { RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -326,6 +371,10 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ }; /* @@ -351,6 +400,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -361,6 +411,11 @@ struct rbu_file { rbu_file *pMainNext; /* Next MAIN_DB file */ }; +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + /************************************************************************* ** The following three functions, found below: @@ -809,8 +864,11 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ /* ** The implementation of the rbu_target_name() SQL function. This function -** accepts one argument - the name of a table in the RBU database. If the -** table name matches the pattern: +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: ** ** data[0-9]_ ** @@ -821,21 +879,33 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). */ static void rbuTargetNameFunc( - sqlite3_context *context, + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ + sqlite3rbu *p = sqlite3_user_data(pCtx); const char *zIn; - assert( argc==1 ); + assert( argc==1 || argc==2 ); zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } } } } @@ -852,11 +922,14 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ int rc; memset(pIter, 0, sizeof(RbuObjIter)); - rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " "ORDER BY name" - ); + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, @@ -1144,6 +1217,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; @@ -1157,6 +1231,12 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; } rbuFinalize(p, pList); @@ -1222,6 +1302,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ pStmt = 0; if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; @@ -1270,6 +1351,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ rbuFinalize(p, pStmt); rbuObjIterCacheIndexedCols(p, pIter); assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); } return p->rc; @@ -1360,6 +1442,8 @@ static char *rbuObjIterGetIndexCols( for(i=0; pIter->abTblPk[i]==0; i++); assert( inTblCol ); zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; }else{ zCol = "rbu_rowid"; } @@ -1823,6 +1907,14 @@ static void rbuTmpInsertFunc( int rc = SQLITE_OK; int i; + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); } @@ -1892,7 +1984,7 @@ static int rbuObjIterPrepareAll( } /* And to delete index entries */ - if( p->rc==SQLITE_OK ){ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError( p->dbMain, &pIter->pDelete, &p->zErrmsg, sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) @@ -1902,6 +1994,15 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", @@ -1928,7 +2029,9 @@ static int rbuObjIterPrepareAll( sqlite3_free(zWhere); sqlite3_free(zBind); }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ const char *zWrite; /* Imposter table name */ @@ -1955,8 +2058,10 @@ static int rbuObjIterPrepareAll( ); } - /* Create the DELETE statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere @@ -1964,7 +2069,7 @@ static int rbuObjIterPrepareAll( ); } - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ const char *zRbuRowid = ""; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; @@ -2014,10 +2119,16 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM '%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, pIter->zDataTbl, zLimit ) ); @@ -2112,11 +2223,15 @@ static int rbuGetUpdateStmt( return p->rc; } -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ sqlite3 *db = 0; if( p->rc==SQLITE_OK ){ const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); sqlite3_close(db); @@ -2126,17 +2241,113 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ return db; } +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. */ -static void rbuOpenDatabase(sqlite3rbu *p){ - assert( p->rc==SQLITE_OK ); - assert( p->dbMain==0 && p->dbRbu==0 ); +static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); - p->eStage = 0; - p->dbMain = rbuOpenDbhandle(p, p->zTarget); - p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + } + } /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ @@ -2147,6 +2358,105 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>=RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 @@ -2161,7 +2471,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -2210,9 +2520,9 @@ static void rbuFileSuffix3(const char *zBase, char *z){ #endif { int i, sz; - sz = sqlite3Strlen30(z); + sz = (int)strlen(z)&0xffffff; for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); } #endif } @@ -2300,16 +2610,35 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && p->nFrame>0 ){ p->eStage = RBU_STAGE_CKPT; p->nStep = (pState ? pState->nRow : 0); p->aBuf = rbuMalloc(p, p->pgsz); p->iWalCksum = rbuShmChecksum(p); } - if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } } } @@ -2420,9 +2749,15 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); + const char *zMove = zBase; + char *zOal; + char *zWal; - char *zWal = sqlite3_mprintf("%s-wal", zBase); - char *zOal = sqlite3_mprintf("%s-oal", zBase); + if( rbuIsVacuum(p) ){ + zMove = sqlite3_db_filename(p->dbRbu, "main"); + } + zOal = sqlite3_mprintf("%s-oal", zMove); + zWal = sqlite3_mprintf("%s-wal", zMove); assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2443,8 +2778,8 @@ static void rbuMoveOalFile(sqlite3rbu *p){ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); p->dbMain = 0; p->dbRbu = 0; @@ -2476,7 +2811,7 @@ static void rbuMoveOalFile(sqlite3rbu *p){ #endif if( p->rc==SQLITE_OK ){ - rbuOpenDatabase(p); + rbuOpenDatabase(p, 0); rbuSetupCheckpoint(p, 0); } } @@ -2567,6 +2902,17 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ assert( p->rc==SQLITE_OK ); assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ pWriter = pIter->pDelete; @@ -2595,19 +2941,24 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ p->rc = sqlite3_bind_value(pWriter, i+1, pVal); if( p->rc ) return; } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT , rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + if( pIter->zIdx==0 ){ + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT , rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, + rbuIsVacuum(p) ? "rowid" : "rbu_rowid" + ); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } } if( p->rc==SQLITE_OK ){ sqlite3_step(pWriter); @@ -2638,11 +2989,14 @@ static int rbuStep(sqlite3rbu *p){ ); assert( eType!=RBU_UPDATE || pIter->zIdx==0 ); - if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){ + if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){ rbuBadControlError(p); } else if( eType==RBU_REPLACE ){ - if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE); + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); + } if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); } else if( eType!=RBU_UPDATE ){ @@ -2652,6 +3006,7 @@ static int rbuStep(sqlite3rbu *p){ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ int i; @@ -2682,13 +3037,18 @@ static int rbuStep(sqlite3rbu *p){ /* ** Increment the schema cookie of the main database opened by p->dbMain. +** +** Or, if this is an RBU vacuum, set the schema cookie of the main db +** opened by p->dbMain to one more than the schema cookie of the main +** db opened by p->dbRbu. */ static void rbuIncrSchemaCookie(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ + sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -2716,6 +3076,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); @@ -2729,6 +3090,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, @@ -2737,8 +3099,9 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, - RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_COOKIE, (i64)pFd->iCookie, + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -2752,6 +3115,92 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + +/* +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. +*/ +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + + assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + /* ** Step the RBU object. */ @@ -2760,13 +3209,22 @@ int sqlite3rbu_step(sqlite3rbu *p){ switch( p->eStage ){ case RBU_STAGE_OAL: { RbuObjIter *pIter = &p->objiter; + + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( rbuIsVacuum(p) && p->nProgress==0 && p->rc==SQLITE_OK ){ + rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); + } + while( p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); @@ -2835,9 +3293,26 @@ int sqlite3rbu_step(sqlite3rbu *p){ p->rc = SQLITE_DONE; } }else{ - RbuFrame *pFrame = &p->aFrame[p->nStep]; - rbuCheckpointFrame(p, pFrame); - p->nStep++; + /* At one point the following block copied a single frame from the + ** wal file to the database file. So that one call to sqlite3rbu_step() + ** checkpointed a single frame. + ** + ** However, if the sector-size is larger than the page-size, and the + ** application calls sqlite3rbu_savestate() or close() immediately + ** after this step, then rbu_step() again, then a power failure occurs, + ** then the database page written here may be damaged. Work around + ** this by checkpointing frames until the next page in the aFrame[] + ** lies on a different disk sector to the current one. */ + u32 iSector; + do{ + RbuFrame *pFrame = &p->aFrame[p->nStep]; + iSector = (pFrame->iDbPage-1) / p->nPagePerSector; + rbuCheckpointFrame(p, pFrame); + p->nStep++; + }while( p->nStepnFrame + && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector) + && p->rc==SQLITE_OK + ); } p->nProgress++; } @@ -2853,90 +3328,6 @@ int sqlite3rbu_step(sqlite3rbu *p){ } } -/* -** Free an RbuState object allocated by rbuLoadState(). -*/ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} - -/* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). -** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. -*/ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; -} - /* ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero ** otherwise. Either or both argument may be NULL. Two NULL values are @@ -3033,18 +3424,109 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } /* -** Open and return a new RBU handle. +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. */ -sqlite3rbu *sqlite3rbu_open( +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} + + +static sqlite3rbu *openRbuHandle( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; - size_t nTarget = strlen(zTarget); + size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); - size_t nState = zState ? strlen(zState) : 0; - size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1; p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ @@ -3054,21 +3536,34 @@ sqlite3rbu *sqlite3rbu_open( memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); - /* Open the target database */ + /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zRbu = &p->zTarget[nTarget+1]; - memcpy(p->zRbu, zRbu, nRbu+1); - if( zState ){ - p->zState = &p->zRbu[nRbu+1]; - memcpy(p->zState, zState, nState+1); + char *pCsr = (char*)&p[1]; + int bRetry = 0; + if( zTarget ){ + p->zTarget = pCsr; + memcpy(p->zTarget, zTarget, nTarget+1); + pCsr += nTarget+1; + } + p->zRbu = pCsr; + memcpy(p->zRbu, zRbu, nRbu+1); + pCsr += nRbu+1; + if( zState ){ + p->zState = rbuMPrintf(p, "%s", zState); } - rbuOpenDatabase(p); - } - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + /* If the first attempt to open the database file fails and the bRetry + ** flag it set, this means that the db was not opened because it seemed + ** to be a wal-mode db. But, this may have happened due to an earlier + ** RBU vacuum operation leaving an old wal file in the directory. + ** If this is the case, it will have been checkpointed and deleted + ** when the handle was closed and a second attempt to open the + ** database may succeed. */ + rbuOpenDatabase(p, &bRetry); + if( bRetry ){ + rbuOpenDatabase(p, 0); + } + } if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); @@ -3077,9 +3572,11 @@ sqlite3rbu *sqlite3rbu_open( if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; + p->nPhaseOneStep = pState->nPhaseOneStep; } p->nProgress = pState->nProgress; p->iOalSz = pState->iOalSz; @@ -3097,38 +3594,27 @@ sqlite3rbu *sqlite3rbu_open( } } - if( p->rc==SQLITE_OK + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) - && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie - ){ - /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on - ** page 1 of the database file. */ - p->rc = SQLITE_BUSY; - p->zErrmsg = sqlite3_mprintf("database modified during rbu update"); + && pState->eStage!=0 + ){ + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); + if( pFd->iCookie!=pState->iCookie ){ + /* At this point (pTargetFd->iCookie) contains the value of the + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on + ** page 1 of the database file. */ + p->rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", + (rbuIsVacuum(p) ? "vacuum" : "update") + ); + } } if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; - - /* Open transactions both databases. The *-oal file is opened or - ** created at this point. */ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - } - - /* Check if the main database is a zipvfs db. If it is, set the upper - ** level pager to use "journal_mode=off". This prevents it from - ** generating a large journal using a temp file. */ - if( p->rc==SQLITE_OK ){ - int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); - if( frc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); - } - } + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -3139,12 +3625,34 @@ sqlite3rbu *sqlite3rbu_open( ** update finished. */ if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){ p->rc = SQLITE_DONE; - } + p->eStage = RBU_STAGE_DONE; + }else{ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } - if( p->rc==SQLITE_OK ){ - rbuSetupOal(p, pState); - } + /* Open transactions both databases. The *-oal file is opened or + ** created at this point. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec( + db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + + if( p->rc==SQLITE_OK ){ + rbuSetupOal(p, pState); + } + } }else if( p->eStage==RBU_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==RBU_STAGE_CKPT ){ @@ -3162,6 +3670,44 @@ sqlite3rbu *sqlite3rbu_open( return p; } +/* +** Allocate and return an RBU handle with all fields zeroed except for the +** error code, which is set to SQLITE_MISUSE. +*/ +static sqlite3rbu *rbuMisuseError(void){ + sqlite3rbu *pRet; + pRet = sqlite3_malloc64(sizeof(sqlite3rbu)); + if( pRet ){ + memset(pRet, 0, sizeof(sqlite3rbu)); + pRet->rc = SQLITE_MISUSE; + } + return pRet; +} + +/* +** Open and return a new RBU handle. +*/ +sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +){ + if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); } + /* TODO: Check that zTarget and zRbu are non-NULL */ + return openRbuHandle(zTarget, zRbu, zState); +} + +/* +** Open a handle to begin or resume an RBU VACUUM operation. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +){ + if( zTarget==0 ){ return rbuMisuseError(); } + /* TODO: Check that both arguments are non-NULL */ + return openRbuHandle(0, zTarget, zState); +} /* ** Return the database handle used by pRbu. @@ -3182,7 +3728,7 @@ sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ */ static void rbuEditErrmsg(sqlite3rbu *p){ if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ - int i; + unsigned int i; size_t nErrmsg = strlen(p->zErrmsg); for(i=0; i<(nErrmsg-8); i++){ if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ @@ -3207,6 +3753,12 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } + /* Sync the db file if currently doing an incremental checkpoint */ + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ @@ -3216,9 +3768,19 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close any open statement handles. */ rbuObjIterFinalize(&p->objiter); + /* If this is an RBU vacuum handle and the vacuum has either finished + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() + ** specifying the current target and state databases to start a new + ** vacuum from scratch. */ + if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ + int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0); + if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2; + } + /* Close the open database handle and VFS object. */ - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3226,6 +3788,7 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ rbuEditErrmsg(p); rc = p->rc; *pzErrmsg = p->zErrmsg; + sqlite3_free(p->zState); sqlite3_free(p); }else{ rc = SQLITE_NOMEM; @@ -3243,9 +3806,75 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} + +/* +** Return the current state of the RBU vacuum or update operation. +*/ +int sqlite3rbu_state(sqlite3rbu *p){ + int aRes[] = { + 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE, + 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE + }; + + assert( RBU_STAGE_OAL==1 ); + assert( RBU_STAGE_MOVE==2 ); + assert( RBU_STAGE_CKPT==4 ); + assert( RBU_STAGE_DONE==5 ); + assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL ); + assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE ); + assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT ); + assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE ); + + if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){ + return SQLITE_RBU_STATE_ERROR; + }else{ + assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE ); + assert( p->eStage==RBU_STAGE_OAL + || p->eStage==RBU_STAGE_MOVE + || p->eStage==RBU_STAGE_CKPT + || p->eStage==RBU_STAGE_DONE + ); + return aRes[p->eStage]; + } +} + int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; - if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); @@ -3254,6 +3883,12 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } + /* Sync the db file */ + if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; @@ -3384,6 +4019,22 @@ static u32 rbuGetU32(u8 *aBuf){ + ((u32)aBuf[3]); } +/* +** Write an unsigned 32-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU32(u8 *aBuf, u32 iVal){ + aBuf[0] = (iVal >> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3409,6 +4060,35 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( pRbu && rbuIsVacuum(pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + && pRbu->rc==SQLITE_OK + ){ + sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0; + rbuPutU32(&aBuf[52], iRoot); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ + + if( iAmt>100 ){ + memset(&aBuf[100], 0, iAmt-100); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + aBuf[100] = 0x0D; + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3483,7 +4163,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3495,7 +4188,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3558,6 +4253,12 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + sqlite3rbu *pRbu = (sqlite3rbu*)pArg; + pRbu->nRbu++; + pRbu->pRbuFd = p; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ @@ -3721,6 +4422,33 @@ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ return pDb; } +/* +** A main database named zName has just been opened. The following +** function returns a pointer to a buffer owned by SQLite that contains +** the name of the *-wal file this db connection will use. SQLite +** happens to pass a pointer to this buffer when using xAccess() +** or xOpen() to operate on the *-wal file. +*/ +static const char *rbuMainToWal(const char *zName, int flags){ + int n = (int)strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + return z; +} + /* ** Open an rbu file handle. */ @@ -3756,6 +4484,7 @@ static int rbuVfsOpen( rbu_file *pFd = (rbu_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; + int oflags = flags; memset(pFd, 0, sizeof(rbu_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; @@ -3768,23 +4497,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - pFd->zWal = z; + pFd->zWal = rbuMainToWal(zName, flags); } else if( flags & SQLITE_OPEN_WAL ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); @@ -3794,10 +4507,17 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - size_t nCopy = strlen(zName); - char *zCopy = sqlite3_malloc64(nCopy+2); + const char *zBase = zName; + size_t nCopy; + char *zCopy; + if( rbuIsVacuum(pDb->pRbu) ){ + zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); + } + nCopy = strlen(zBase); + zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ - memcpy(zCopy, zName, nCopy); + memcpy(zCopy, zBase, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; @@ -3812,8 +4532,17 @@ static int rbuVfsOpen( } } + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) + ){ + assert( oflags & SQLITE_OPEN_MAIN_DB ); + oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + zOpen = 0; + } + if( rc==SQLITE_OK ){ - rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f1a0f3cd84..12112c14e1 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -104,7 +104,7 @@ ** may also be named data_, where is any sequence ** of zero or more numeric characters (0-9). This can be significant because ** tables within the RBU database are always processed in order sorted by -** name. By judicious selection of the the portion of the names +** name. By judicious selection of the portion of the names ** of the RBU tables the user can therefore control the order in which they ** are processed. This can be useful, for example, to ensure that "external ** content" FTS4 tables are updated before their underlying content tables. @@ -314,6 +314,44 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ); +/* +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This @@ -400,6 +438,86 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +int sqlite3rbu_state(sqlite3rbu *pRbu); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 3fa85b7569..b1a2252741 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -17,11 +17,19 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) #include "sqlite3rbu.h" -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif #include -/* From main.c (apparently...) */ +/* From main.c */ extern const char *sqlite3ErrName(int); +extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ Tcl_Interp *interp = (Tcl_Interp*)sqlite3_user_data(pCtx); @@ -48,7 +56,7 @@ void test_rbu_delta(sqlite3_context *pCtx, int nArg, sqlite3_value **apVal){ } -static int test_sqlite3rbu_cmd( +static int SQLITE_TCLAPI test_sqlite3rbu_cmd( ClientData clientData, Tcl_Interp *interp, int objc, @@ -66,6 +74,10 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ + {"db", 3, "RBU"}, /* 6 */ + {"state", 2, ""}, /* 7 */ + {"progress", 2, ""}, /* 8 */ {0,0,0} }; int iCmd; @@ -136,6 +148,46 @@ static int test_sqlite3rbu_cmd( break; } + case 5: /* bp_progress */ { + int one, two; + Tcl_Obj *pObj; + sqlite3rbu_bp_progress(pRbu, &one, &two); + + pObj = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two)); + Tcl_SetObjResult(interp, pObj); + break; + } + + case 6: /* db */ { + int bArg; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bArg) ){ + ret = TCL_ERROR; + }else{ + char zBuf[50]; + sqlite3 *db = sqlite3rbu_db(pRbu, bArg); + if( sqlite3TestMakePointerStr(interp, zBuf, (void*)db) ){ + ret = TCL_ERROR; + }else{ + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + } + } + break; + } + case 7: /* state */ { + const char *aRes[] = { 0, "oal", "move", "checkpoint", "done", "error" }; + int eState = sqlite3rbu_state(pRbu); + assert( eState>0 && eState<=5 ); + Tcl_SetResult(interp, (char*)aRes[eState], TCL_STATIC); + break; + } + case 8: /* progress */ { + sqlite3_int64 nStep = sqlite3rbu_progress(pRbu); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nStep)); + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; @@ -147,7 +199,7 @@ static int test_sqlite3rbu_cmd( /* ** Tclcmd: sqlite3rbu CMD ?? */ -static int test_sqlite3rbu( +static int SQLITE_TCLAPI test_sqlite3rbu( ClientData clientData, Tcl_Interp *interp, int objc, @@ -174,10 +226,38 @@ static int test_sqlite3rbu( return TCL_OK; } +/* +** Tclcmd: sqlite3rbu_vacuum CMD +*/ +static int SQLITE_TCLAPI test_sqlite3rbu_vacuum( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3rbu *pRbu = 0; + const char *zCmd; + const char *zTarget; + const char *zStateDb = 0; + + if( objc!=3 && objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB ?STATE-DB?"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + if( objc==4 ) zStateDb = Tcl_GetString(objv[3]); + + pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + /* ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT */ -static int test_sqlite3rbu_create_vfs( +static int SQLITE_TCLAPI test_sqlite3rbu_create_vfs( ClientData clientData, Tcl_Interp *interp, int objc, @@ -212,7 +292,7 @@ static int test_sqlite3rbu_create_vfs( /* ** Tclcmd: sqlite3rbu_destroy_vfs NAME */ -static int test_sqlite3rbu_destroy_vfs( +static int SQLITE_TCLAPI test_sqlite3rbu_destroy_vfs( ClientData clientData, Tcl_Interp *interp, int objc, @@ -233,7 +313,7 @@ static int test_sqlite3rbu_destroy_vfs( /* ** Tclcmd: sqlite3rbu_internal_test */ -static int test_sqlite3rbu_internal_test( +static int SQLITE_TCLAPI test_sqlite3rbu_internal_test( ClientData clientData, Tcl_Interp *interp, int objc, @@ -261,6 +341,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3rbu", test_sqlite3rbu }, + { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum }, { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs }, { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs }, { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test }, @@ -273,7 +354,11 @@ int SqliteRbu_Init(Tcl_Interp *interp){ } #else -#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif int SqliteRbu_Init(Tcl_Interp *interp){ return TCL_OK; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) */ #endif /* defined(SQLITE_TEST) */ diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 012c48d6d1..42f08a9628 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -68,6 +68,7 @@ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; @@ -116,13 +117,16 @@ struct Rtree { sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - int nBusy; /* Current number of users of this structure */ + u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - @@ -132,8 +136,10 @@ struct Rtree { RtreeNode *pDeleted; int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; + /* Statements to read/write/delete a record from xxx_node */ - sqlite3_stmt *pReadNode; sqlite3_stmt *pWriteNode; sqlite3_stmt *pDeleteNode; @@ -362,6 +368,58 @@ struct RtreeMatchArg { # define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGAMATION +# define testcase(X) +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +#elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +#else +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +#endif +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + /* ** Functions to deserialize a 16 bit integer, 32 bit real number and ** 64 bit integer. The deserialized value is returned. @@ -370,24 +428,47 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; +#else pCoord->u = ( (((u32)p[0]) << 24) + (((u32)p[1]) << 16) + (((u32)p[2]) << 8) + (((u32)p[3]) << 0) ); +#endif } static i64 readInt64(u8 *p){ - return ( - (((i64)p[0]) << 56) + - (((i64)p[1]) << 48) + - (((i64)p[2]) << 40) + - (((i64)p[3]) << 32) + - (((i64)p[4]) << 24) + - (((i64)p[5]) << 16) + - (((i64)p[6]) << 8) + - (((i64)p[7]) << 0) +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) ); +#endif } /* @@ -395,23 +476,43 @@ static i64 readInt64(u8 *p){ ** 64 bit integer. The value returned is the number of bytes written ** to the argument buffer (always 2, 4 and 8 respectively). */ -static int writeInt16(u8 *p, int i){ +static void writeInt16(u8 *p, int i){ p[0] = (i>> 8)&0xFF; p[1] = (i>> 0)&0xFF; - return 2; } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else i = pCoord->u; p[0] = (i>>24)&0xFF; p[1] = (i>>16)&0xFF; p[2] = (i>> 8)&0xFF; p[3] = (i>> 0)&0xFF; +#endif return 4; } static int writeInt64(u8 *p, i64 i){ +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else p[0] = (i>>56)&0xFF; p[1] = (i>>48)&0xFF; p[2] = (i>>40)&0xFF; @@ -420,6 +521,7 @@ static int writeInt64(u8 *p, i64 i){ p[5] = (i>>16)&0xFF; p[6] = (i>> 8)&0xFF; p[7] = (i>> 0)&0xFF; +#endif return 8; } @@ -502,6 +604,17 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ return pNode; } +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); + } +} + /* ** Obtain a reference to an r-tree node. */ @@ -511,9 +624,8 @@ static int nodeAcquire( RtreeNode *pParent, /* Either the parent node or NULL */ RtreeNode **ppNode /* OUT: Acquired node */ ){ - int rc; - int rc2 = SQLITE_OK; - RtreeNode *pNode; + int rc = SQLITE_OK; + RtreeNode *pNode = 0; /* Check if the requested node is already in the hash table. If so, ** increase its reference count and return it. @@ -529,28 +641,45 @@ static int nodeAcquire( return SQLITE_OK; } - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); - rc = sqlite3_step(pRtree->pReadNode); - if( rc==SQLITE_ROW ){ - const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); - if( !pNode ){ - rc2 = SQLITE_NOMEM; - }else{ - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); - } + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + } + } + if( pRtree->pNodeBlob==0 ){ + char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); + if( zTab==0 ) return SQLITE_NOMEM; + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + &pRtree->pNodeBlob); + sqlite3_free(zTab); + } + if( rc ){ + nodeBlobReset(pRtree); + *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB; + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); + nodeReference(pParent); } } - rc = sqlite3_reset(pRtree->pReadNode); - if( rc==SQLITE_OK ) rc = rc2; /* If the root node was just loaded, set pRtree->iDepth to the height ** of the r-tree structure. A height of zero means all data is stored on @@ -602,7 +731,7 @@ static void nodeOverwriteCell( int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; p += writeInt64(p, pCell->iRowid); - for(ii=0; ii<(pRtree->nDim*2); ii++){ + for(ii=0; iinDim2; ii++){ p += writeCoord(p, &pCell->aCoord[ii]); } pNode->isDirty = 1; @@ -736,13 +865,16 @@ static void nodeGetCell( ){ u8 *pData; RtreeCoord *pCoord; - int ii; + int ii = 0; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); pCoord = pCell->aCoord; - for(ii=0; iinDim*2; ii++){ - readCoord(&pData[ii*4], &pCoord[ii]); - } + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim2 ); } @@ -793,7 +925,9 @@ static void rtreeReference(Rtree *pRtree){ static void rtreeRelease(Rtree *pRtree){ pRtree->nBusy--; if( pRtree->nBusy==0 ){ - sqlite3_finalize(pRtree->pReadNode); + pRtree->inWrTrans = 0; + pRtree->nCursor = 0; + nodeBlobReset(pRtree); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -831,6 +965,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ if( !zCreate ){ rc = SQLITE_NOMEM; }else{ + nodeBlobReset(pRtree); rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); } @@ -846,6 +981,7 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ */ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; RtreeCursor *pCsr; pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); @@ -853,6 +989,7 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ memset(pCsr, 0, sizeof(RtreeCursor)); pCsr->base.pVtab = pVTab; rc = SQLITE_OK; + pRtree->nCursor++; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -885,10 +1022,13 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; + assert( pRtree->nCursor>0 ); freeCursorConstraints(pCsr); sqlite3_free(pCsr->aPoint); for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); + pRtree->nCursor--; + nodeBlobReset(pRtree); return SQLITE_OK; } @@ -911,15 +1051,22 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ** false. a[] is the four bytes of the on-disk record to be decoded. ** Store the results in "r". ** -** There are three versions of this macro, one each for little-endian and -** big-endian processors and a third generic implementation. The endian- -** specific implementations are much faster and are preferred if the -** processor endianness is known at compile-time. The SQLITE_BYTEORDER -** macro is part of sqliteInt.h and hence the endian-specific -** implementation will only be used if this module is compiled as part -** of the amalgamation. +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. */ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -927,7 +1074,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#elif SQLITE_BYTEORDER==4321 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -954,10 +1101,10 @@ static int rtreeCallbackConstraint( sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ int *peWithin /* OUT: visibility of the cell */ ){ - int i; /* Loop counter */ sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ int nCoord = pInfo->nCoord; /* No. of coordinates */ int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); @@ -967,13 +1114,41 @@ static int rtreeCallbackConstraint( pInfo->iRowid = readInt64(pCellData); } pCellData += 8; - for(i=0; iop==RTREE_MATCH ){ + int eWithin = 0; rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, - nCoord, aCoord, &i); - if( i==0 ) *peWithin = NOT_WITHIN; + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; *prScore = RTREE_ZERO; }else{ pInfo->aCoord = aCoord; @@ -1009,6 +1184,7 @@ static void rtreeNonleafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ case RTREE_LE: case RTREE_LT: @@ -1049,6 +1225,7 @@ static void rtreeLeafConstraint( assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE || p->op==RTREE_GT || p->op==RTREE_EQ ); pCellData += 8 + p->iCoord*4; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ case RTREE_LE: if( xN <= p->u.rValue ) return; break; @@ -1117,7 +1294,7 @@ static int rtreeSearchPointCompare( } /* -** Interchange to search points in a cursor. +** Interchange two search points in a cursor. */ static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ RtreeSearchPoint t = p->aPoint[i]; @@ -1365,7 +1542,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ if( rScoreeWithin = eWithin; + p->eWithin = (u8)eWithin; p->id = x.id; p->iCell = x.iCell; RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); @@ -1424,7 +1601,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - if( rc ) return rc; nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -1542,7 +1718,7 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ - RtreeSearchPoint *p; /* Search point for the the leaf */ + RtreeSearchPoint *p; /* Search point for the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); i64 iNode = 0; rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); @@ -1553,7 +1729,7 @@ static int rtreeFilter( p->id = iNode; p->eWithin = PARTLY_WITHIN; rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); - p->iCell = iCell; + p->iCell = (u8)iCell; RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); }else{ pCsr->atEOF = 1; @@ -1586,7 +1762,7 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } - p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ @@ -1601,7 +1777,7 @@ static int rtreeFilter( } if( rc==SQLITE_OK ){ RtreeSearchPoint *pNew; - pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); if( pNew==0 ) return SQLITE_NOMEM; pNew->id = 1; pNew->iCell = 0; @@ -1619,19 +1795,6 @@ static int rtreeFilter( return rc; } -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} - /* ** Rtree virtual table module xBestIndex method. There are three ** table scan strategies to choose from (in order from most to @@ -1711,7 +1874,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** a single row. */ pIdxInfo->estimatedCost = 30.0; - setEstimatedRows(pIdxInfo, 1); + pIdxInfo->estimatedRows = 1; return SQLITE_OK; } @@ -1729,7 +1892,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + '0'; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -1743,7 +1906,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; - setEstimatedRows(pIdxInfo, nRow); + pIdxInfo->estimatedRows = nRow; return rc; } @@ -1753,9 +1916,26 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ RtreeDValue area = (RtreeDValue)1; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].i - p->aCoord[8].i; + case 4: area *= p->aCoord[7].i - p->aCoord[6].i; + case 3: area *= p->aCoord[5].i - p->aCoord[4].i; + case 2: area *= p->aCoord[3].i - p->aCoord[2].i; + default: area *= p->aCoord[1].i - p->aCoord[0].i; + } } return area; } @@ -1765,11 +1945,12 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ ** of the objects size in each dimension. */ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin = (RtreeDValue)0; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); - } + ii -= 2; + }while( ii>=0 ); return margin; } @@ -1777,17 +1958,19 @@ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ ** Store the union of cells p1 and p2 in p1. */ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; + int ii = 0; if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); - } + ii += 2; + }while( iinDim2 ); }else{ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); - } + ii += 2; + }while( iinDim2 ); } } @@ -1798,7 +1981,7 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + for(ii=0; iinDim2; ii+=2){ RtreeCoord *a1 = &p1->aCoord[ii]; RtreeCoord *a2 = &p2->aCoord[ii]; if( (!isInt && (a2[0].fa1[1].f)) @@ -1833,7 +2016,7 @@ static RtreeDValue cellOverlap( for(ii=0; iinDim*2); jj+=2){ + for(jj=0; jjnDim2; jj+=2){ RtreeDValue x1, x2; x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); @@ -2800,6 +2983,53 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){ } #endif /* !defined(SQLITE_RTREE_INT_ONLY) */ +/* +** A constraint has failed while inserting a row into an rtree table. +** Assuming no OOM error occurs, this function sets the error message +** (at pRtree->base.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + /* ** The xUpdate method for rtree module virtual tables. @@ -2842,7 +3072,7 @@ static int rtreeUpdate( ** This problem was discovered after years of use, so we silently ignore ** these kinds of misdeclared tables to avoid breaking any legacy. */ - assert( nData<=(pRtree->nDim*2 + 3) ); + assert( nData<=(pRtree->nDim2 + 3) ); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -2850,7 +3080,7 @@ static int rtreeUpdate( cell.aCoord[ii].f = rtreeValueDown(azData[ii+3]); cell.aCoord[ii+1].f = rtreeValueUp(azData[ii+4]); if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } @@ -2861,7 +3091,7 @@ static int rtreeUpdate( cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]); cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]); if( cell.aCoord[ii].i>cell.aCoord[ii+1].i ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } @@ -2882,7 +3112,7 @@ static int rtreeUpdate( if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ rc = rtreeDeleteRowid(pRtree, cell.iRowid); }else{ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, 0); goto constraint; } } @@ -2932,6 +3162,27 @@ constraint: return rc; } +/* +** Called when a transaction starts. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans++; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + /* ** The xRename method for rtree module virtual tables. */ @@ -2953,6 +3204,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ return rc; } + /* ** This function populates the pRtree->nRowEst variable with an estimate ** of the number of rows in the virtual table. If possible, this is based @@ -2965,6 +3217,13 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ int rc; i64 nRow = 0; + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; @@ -3005,15 +3264,15 @@ static sqlite3_module rtreeModule = { rtreeColumn, /* xColumn - read data */ rtreeRowid, /* xRowid - read data */ rtreeUpdate, /* xUpdate - write data */ - 0, /* xBegin - begin transaction */ - 0, /* xSync - sync transaction */ - 0, /* xCommit - commit transaction */ - 0, /* xRollback - rollback transaction */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ 0, /* xFindFunction - function overloading */ rtreeRename, /* xRename - rename the table */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ }; static int rtreeSqlInit( @@ -3025,10 +3284,9 @@ static int rtreeSqlInit( ){ int rc = SQLITE_OK; - #define N_STATEMENT 9 + #define N_STATEMENT 8 static const char *azSql[N_STATEMENT] = { - /* Read and write the xxx_node table */ - "SELECT data FROM '%q'.'%q_node' WHERE nodeno = :1", + /* Write the xxx_node table */ "INSERT OR REPLACE INTO '%q'.'%q_node' VALUES(:1, :2)", "DELETE FROM '%q'.'%q_node' WHERE nodeno = :1", @@ -3066,15 +3324,14 @@ static int rtreeSqlInit( } } - appStmt[0] = &pRtree->pReadNode; - appStmt[1] = &pRtree->pWriteNode; - appStmt[2] = &pRtree->pDeleteNode; - appStmt[3] = &pRtree->pReadRowid; - appStmt[4] = &pRtree->pWriteRowid; - appStmt[5] = &pRtree->pDeleteRowid; - appStmt[6] = &pRtree->pReadParent; - appStmt[7] = &pRtree->pWriteParent; - appStmt[8] = &pRtree->pDeleteParent; + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; rc = rtreeQueryStat1(db, pRtree); for(i=0; ibase.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; - pRtree->nDim = (argc-4)/2; - pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; - pRtree->eCoordType = eCoordType; + pRtree->nDim = (u8)((argc-4)/2); + pRtree->nDim2 = pRtree->nDim*2; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); @@ -3287,7 +3545,8 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); - tree.nDim = sqlite3_value_int(apArg[0]); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); + tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); @@ -3300,7 +3559,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nodeGetCell(&tree, &node, ii, &cell); sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid); nCell = (int)strlen(zCell); - for(jj=0; jj8 AND x1<16 AND y0>2 AND y1<25; +} {1} +do_test rtreeG-1.2log { + set ::log +} {} + +db close +sqlite3 db test.db +do_execsql_test rtreeG-1.3 { + SELECT id from t1 WHERE x0>8 AND x1<16 AND y0>2 AND y1<25; +} {1} +do_test rtreeG-1.3log { + set ::log +} {} + +do_execsql_test rtreeG-1.4 { + DROP TABLE t1; +} {} +do_test rtreeG-1.4log { + set ::log +} {} + +db close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +sqlite3 db test.db + +finish_test diff --git a/ext/session/changeset.c b/ext/session/changeset.c new file mode 100644 index 0000000000..edb43ff8e5 --- /dev/null +++ b/ext/session/changeset.c @@ -0,0 +1,416 @@ +/* +** 2014-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement the "changeset" command line +** utility for displaying and transforming changesets generated by +** the Sessions extension. +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include + + +/* +** Show a usage message on stderr then quit. +*/ +static void usage(const char *argv0){ + fprintf(stderr, "Usage: %s FILENAME COMMAND ...\n", argv0); + fprintf(stderr, + "COMMANDs:\n" + " apply DB Apply the changeset to database file DB\n" + " concat FILE2 OUT Concatenate FILENAME and FILE2 into OUT\n" + " dump Show the complete content of the changeset\n" + " invert OUT Write an inverted changeset into file OUT\n" + " sql Give a pseudo-SQL rendering of the changeset\n" + ); + exit(1); +} + +/* +** Read the content of a disk file into an in-memory buffer +*/ +static void readFile(const char *zFilename, int *pSz, void **ppBuf){ + FILE *f; + int sz; + void *pBuf; + f = fopen(zFilename, "rb"); + if( f==0 ){ + fprintf(stderr, "cannot open \"%s\" for reading\n", zFilename); + exit(1); + } + fseek(f, 0, SEEK_END); + sz = (int)ftell(f); + rewind(f); + pBuf = sqlite3_malloc( sz ? sz : 1 ); + if( pBuf==0 ){ + fprintf(stderr, "cannot allocate %d to hold content of \"%s\"\n", + sz, zFilename); + exit(1); + } + if( sz>0 ){ + if( fread(pBuf, sz, 1, f)!=1 ){ + fprintf(stderr, "cannot read all %d bytes of \"%s\"\n", sz, zFilename); + exit(1); + } + fclose(f); + } + *pSz = sz; + *ppBuf = pBuf; +} + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' +}; + +/* +** Render an sqlite3_value as an SQL string. +*/ +static void renderValue(sqlite3_value *pVal){ + switch( sqlite3_value_type(pVal) ){ + case SQLITE_FLOAT: { + double r1; + char zBuf[50]; + r1 = sqlite3_value_double(pVal); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + printf("%s", zBuf); + break; + } + case SQLITE_INTEGER: { + printf("%lld", sqlite3_value_int64(pVal)); + break; + } + case SQLITE_BLOB: { + char const *zBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + int i; + printf("x'"); + for(i=0; i>4)&0x0F]); + putchar(hexdigits[(zBlob[i])&0x0F]); + } + putchar('\''); + break; + } + case SQLITE_TEXT: { + const unsigned char *zArg = sqlite3_value_text(pVal); + putchar('\''); + while( zArg[0] ){ + putchar(zArg[0]); + if( zArg[0]=='\'' ) putchar(zArg[0]); + zArg++; + } + putchar('\''); + break; + } + default: { + assert( sqlite3_value_type(pVal)==SQLITE_NULL ); + printf("NULL"); + break; + } + } +} + +/* +** Number of conflicts seen +*/ +static int nConflict = 0; + +/* +** The conflict callback +*/ +static int conflictCallback( + void *pCtx, + int eConflict, + sqlite3_changeset_iter *pIter +){ + int op, bIndirect, nCol, i; + const char *zTab; + unsigned char *abPK; + const char *zType = ""; + const char *zOp = ""; + const char *zSep = " "; + + nConflict++; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + sqlite3changeset_pk(pIter, &abPK, 0); + switch( eConflict ){ + case SQLITE_CHANGESET_DATA: zType = "DATA"; break; + case SQLITE_CHANGESET_NOTFOUND: zType = "NOTFOUND"; break; + case SQLITE_CHANGESET_CONFLICT: zType = "PRIMARY KEY"; break; + case SQLITE_CHANGESET_FOREIGN_KEY: zType = "FOREIGN KEY"; break; + case SQLITE_CHANGESET_CONSTRAINT: zType = "CONSTRAINT"; break; + } + switch( op ){ + case SQLITE_UPDATE: zOp = "UPDATE of"; break; + case SQLITE_INSERT: zOp = "INSERT into"; break; + case SQLITE_DELETE: zOp = "DELETE from"; break; + } + printf("%s conflict on %s table %s with primary key", zType, zOp, zTab); + for(i=0; i0 && fwrite(pOutBuf, szOut, 1, out)!=1 ){ + fprintf(stderr, "unable to write all %d bytes of output to \"%s\"\n", + szOut, zOut); + } + fclose(out); + sqlite3_free(pOutBuf); + sqlite3_free(pB); + }else + + /* changeset FILENAME dump + ** Show the complete content of the changeset in FILENAME + */ + if( strcmp(argv[2],"dump")==0 ){ + int cnt = 0; + int i; + sqlite3_changeset_iter *pIter; + rc = sqlite3changeset_start(&pIter, sz, pBuf); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3changeset_start() returns %d\n", rc); + exit(1); + } + while( sqlite3changeset_next(pIter)==SQLITE_ROW ){ + int op, bIndirect, nCol; + const char *zTab; + unsigned char *abPK; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + cnt++; + printf("%d: %s table=[%s] indirect=%d nColumn=%d\n", + cnt, op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : "DELETE", + zTab, bIndirect, nCol); + sqlite3changeset_pk(pIter, &abPK, 0); + for(i=0; i0 && fwrite(pOutBuf, szOut, 1, out)!=1 ){ + fprintf(stderr, "unable to write all %d bytes of output to \"%s\"\n", + szOut, zOut); + } + fclose(out); + sqlite3_free(pOutBuf); + }else + + /* changeset FILE sql + ** Show the content of the changeset as pseudo-SQL + */ + if( strcmp(argv[2],"sql")==0 ){ + int cnt = 0; + char *zPrevTab = 0; + char *zSQLTabName = 0; + sqlite3_changeset_iter *pIter = 0; + rc = sqlite3changeset_start(&pIter, sz, pBuf); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite3changeset_start() returns %d\n", rc); + exit(1); + } + printf("BEGIN;\n"); + while( sqlite3changeset_next(pIter)==SQLITE_ROW ){ + int op, bIndirect, nCol; + const char *zTab; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + cnt++; + if( zPrevTab==0 || strcmp(zPrevTab,zTab)!=0 ){ + sqlite3_free(zPrevTab); + sqlite3_free(zSQLTabName); + zPrevTab = sqlite3_mprintf("%s", zTab); + if( !isalnum(zTab[0]) || sqlite3_strglob("*[^a-zA-Z0-9]*",zTab)==0 ){ + zSQLTabName = sqlite3_mprintf("\"%w\"", zTab); + }else{ + zSQLTabName = sqlite3_mprintf("%s", zTab); + } + printf("/****** Changes for table %s ***************/\n", zSQLTabName); + } + switch( op ){ + case SQLITE_DELETE: { + unsigned char *abPK; + int i; + const char *zSep = " "; + sqlite3changeset_pk(pIter, &abPK, 0); + printf("/* %d */ DELETE FROM %s WHERE", cnt, zSQLTabName); + for(i=0; i2; +} { + {INSERT t2 0 X. {} {i 3 t y}} + {INSERT t2 0 X. {} {i 4 t y}} +} + +do_iterator_test 6.1.7 * { + SELECT indirect(1); + DELETE FROM t2 WHERE x = 4; + SELECT indirect(0); + INSERT INTO t2 VALUES(4, 'new'); +} { + {UPDATE t2 0 X. {i 4 t y} {{} {} t new}} +} + +do_iterator_test 6.1.8 * { + CREATE TABLE t3(a, b PRIMARY KEY); + CREATE TABLE t4(a, b PRIMARY KEY); + CREATE TRIGGER t4t AFTER UPDATE ON t4 BEGIN + UPDATE t3 SET a = new.a WHERE b = new.b; + END; + + SELECT indirect(1); + INSERT INTO t3 VALUES('one', 1); + INSERT INTO t4 VALUES('one', 1); + SELECT indirect(0); + UPDATE t4 SET a = 'two' WHERE b = 1; +} { + {INSERT t3 1 .X {} {t two i 1}} + {INSERT t4 0 .X {} {t two i 1}} +} + +sqlite3session S db main +do_execsql_test 6.2.1 { + SELECT indirect(0); + SELECT indirect(-1); + SELECT indirect(45); + SELECT indirect(-100); +} {0 0 1 1} +S delete + +#------------------------------------------------------------------------- +# Test that if a conflict-handler that has been passed either NOTFOUND or +# CONSTRAINT returns REPLACE - the sqlite3changeset_apply() call returns +# MISUSE and rolls back any changes made so far. +# +# 7.1.*: NOTFOUND conflict-callback. +# 7.2.*: CONSTRAINT conflict-callback. +# +proc xConflict {args} {return REPLACE} +test_reset + +do_execsql_test 7.1.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); +} +do_test 7.1.2 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b NOT NULL); + INSERT INTO t1 VALUES(1, 'one'); + } db2 +} {} +do_test 7.1.3 { + set changeset [changeset_from_sql { + UPDATE t1 SET b = 'five' WHERE a = 1; + UPDATE t1 SET b = 'six' WHERE a = 2; + }] + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x +} [list \ + {UPDATE t1 0 X. {i 1 t one} {{} {} t five}} \ + {UPDATE t1 0 X. {i 2 t two} {{} {} t six}} \ +] +do_test 7.1.4 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg +} {1 SQLITE_MISUSE} +do_test 7.1.5 { execsql { SELECT * FROM t1 } db2 } {1 one} + +do_test 7.2.1 { + set changeset [changeset_from_sql { UPDATE t1 SET b = NULL WHERE a = 1 }] + + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x +} [list \ + {UPDATE t1 0 X. {i 1 t five} {{} {} n {}}} \ +] +do_test 7.2.2 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg +} {1 SQLITE_MISUSE} +do_test 7.2.3 { execsql { SELECT * FROM t1 } db2 } {1 one} + +#------------------------------------------------------------------------- +# Test that if a conflict-handler returns ABORT, application of the +# changeset is rolled back and the sqlite3changeset_apply() method returns +# SQLITE_ABORT. +# +# Also test that the same thing happens if a conflict handler returns an +# unrecognized integer value. Except, in this case SQLITE_MISUSE is returned +# instead of SQLITE_ABORT. +# +foreach {tn conflict_return apply_return} { + 1 ABORT SQLITE_ABORT + 2 567 SQLITE_MISUSE +} { + test_reset + proc xConflict {args} [list return $conflict_return] + + do_test 8.$tn.0 { + do_common_sql { + CREATE TABLE t1(x, y, PRIMARY KEY(x, y)); + INSERT INTO t1 VALUES('x', 'y'); + } + execsql { INSERT INTO t1 VALUES('w', 'w') } + + set changeset [changeset_from_sql { DELETE FROM t1 WHERE 1 }] + + set x [list] + sqlite3session_foreach c $changeset { lappend x $c } + set x + } [list \ + {DELETE t1 0 XX {t w t w} {}} \ + {DELETE t1 0 XX {t x t y} {}} \ + ] + + do_test 8.$tn.1 { + list [catch {sqlite3changeset_apply db2 $changeset xConflict} msg] $msg + } [list 1 $apply_return] + + do_test 8.$tn.2 { + execsql {SELECT * FROM t1} db2 + } {x y} +} + + +#------------------------------------------------------------------------- +# Try to cause an infinite loop as follows: +# +# 1. Have a changeset insert a row that causes a CONFLICT callback, +# 2. Have the conflict handler return REPLACE, +# 3. After the session module deletes the conflicting row, have a trigger +# re-insert it. +# 4. Goto step 1... +# +# This doesn't work, as the second invocation of the conflict handler is a +# CONSTRAINT, not a CONFLICT. There is at most one CONFLICT callback for +# each change in the changeset. +# +test_reset +proc xConflict {type args} { + if {$type == "CONFLICT"} { return REPLACE } + return OMIT +} +do_test 9.1 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + } + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('x', 2); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + INSERT INTO t1 VALUES(old.a, old.b); + END; + } db2 +} {} +do_test 9.2 { + set changeset [changeset_from_sql { INSERT INTO t1 VALUES('x', 1) }] + sqlite3changeset_apply db2 $changeset xConflict +} {} +do_test 9.3 { + execsql { SELECT * FROM t1 } db2 +} {x 2} + +#------------------------------------------------------------------------- +# +test_reset +db function enable [list S enable] + +do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('x', 'X'); +} + +do_iterator_test 10.1 t1 { + INSERT INTO t1 VALUES('y', 'Y'); + SELECT enable(0); + INSERT INTO t1 VALUES('z', 'Z'); + SELECT enable(1); +} { + {INSERT t1 0 X. {} {t y t Y}} +} + +sqlite3session S db main +do_execsql_test 10.2 { + SELECT enable(0); + SELECT enable(-1); + SELECT enable(1); + SELECT enable(-1); +} {0 0 1 1} +S delete + +finish_test diff --git a/ext/session/session3.test b/ext/session/session3.test new file mode 100644 index 0000000000..ba316348ef --- /dev/null +++ b/ext/session/session3.test @@ -0,0 +1,214 @@ +# 2011 March 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. More +# specifically, it focuses on testing the session modules response to +# database schema modifications and mismatches. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session3 + +#------------------------------------------------------------------------- +# These tests - session3-1.* - verify that the session module behaves +# correctly when confronted with a schema mismatch when applying a +# changeset (in function sqlite3changeset_apply()). +# +# session3-1.1.*: Table does not exist in target db. +# session3-1.2.*: Table has wrong number of columns in target db. +# session3-1.3.*: Table has wrong PK columns in target db. +# +db close +sqlite3_shutdown +test_sqlite3_log log +sqlite3 db test.db + +proc log {code msg} { lappend ::log $code $msg } + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +do_test 1.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + set ::log +} {SQLITE_SCHEMA {sqlite3changeset_apply(): no such table: t1}} + +do_test 1.2.0 { + execsql { CREATE TABLE t1(a PRIMARY KEY, b, c) } db2 +} {} +do_test 1.2.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + set ::log +} {} +do_test 1.2.2 { + db2 eval { SELECT * FROM t1 } +} {5 6 {} 7 8 {}} + +do_test 1.3.0 { + execsql { + DROP TABLE t1; + CREATE TABLE t1(a, b PRIMARY KEY); + } db2 +} {} +do_test 1.3.1 { + set ::log {} + do_then_apply_sql { + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); + } + set ::log +} {SQLITE_SCHEMA {sqlite3changeset_apply(): primary key mismatch for table t1}} + +#------------------------------------------------------------------------- +# These tests - session3-2.* - verify that the session module behaves +# correctly when the schema of an attached table is modified during the +# session. +# +# session3-2.1.*: Table is dropped midway through the session. +# session3-2.2.*: Table is dropped and recreated with a different # cols. +# session3-2.3.*: Table is dropped and recreated with a different PK. +# +# In all of these scenarios, the call to sqlite3session_changeset() will +# return SQLITE_SCHEMA. Also: +# +# session3-2.4.*: Table is dropped and recreated with an identical schema. +# In this case sqlite3session_changeset() returns SQLITE_OK. +# + +do_test 2.1 { + execsql { CREATE TABLE t2(a, b PRIMARY KEY) } + sqlite3session S db main + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.2.1 { + S delete + sqlite3session S db main + execsql { CREATE TABLE t2(a, b PRIMARY KEY, c) } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.2 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c, d); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.3 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + INSERT INTO t2 VALUES(4, 5); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} +do_test 2.2.4 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2, 3); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY, c, d); + INSERT INTO t2 VALUES(4, 5, 6, 7); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.3 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + CREATE TABLE t2(a PRIMARY KEY, b); + } + list [catch { S changeset } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 2.4 { + S delete + sqlite3session S db main + execsql { + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + S attach t2 + execsql { + INSERT INTO t2 VALUES(1, 2); + DROP TABLE t2; + CREATE TABLE t2(a, b PRIMARY KEY); + } + list [catch { S changeset } msg] $msg +} {0 {}} + +S delete + + +catch { db close } +catch { db2 close } +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize + +finish_test diff --git a/ext/session/session4.test b/ext/session/session4.test new file mode 100644 index 0000000000..120a230b57 --- /dev/null +++ b/ext/session/session4.test @@ -0,0 +1,67 @@ +# 2011 March 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session4 + +do_test 1.0 { + execsql { + CREATE TABLE x(a, b, c, d, e, PRIMARY KEY(c, e)); + INSERT INTO x VALUES(65.21, X'28B0', 16.35, NULL, 'doers'); + INSERT INTO x VALUES(NULL, 78.49, 2, X'60', -66); + INSERT INTO x VALUES('cathedral', NULL, 35, NULL, X'B220937E80A2D8'); + INSERT INTO x VALUES(NULL, 'masking', -91.37, NULL, X'596D'); + INSERT INTO x VALUES(19, 'domains', 'espouse', -94, 'throw'); + } + + set changeset [changeset_from_sql { + DELETE FROM x WHERE e = -66; + UPDATE x SET a = 'parameterizable', b = 31.8 WHERE c = 35; + INSERT INTO x VALUES(-75.61, -17, 16.85, NULL, X'D73DB02678'); + }] + set {} {} +} {} + + +# This currently causes crashes. sqlite3changeset_invert() does not handle +# corrupt changesets well. +if 0 { + do_test 1.1 { + for {set i 0} {$i < [string length $changeset]} {incr i} { + set before [string range $changeset 0 [expr $i-1]] + set after [string range $changeset [expr $i+1] end] + for {set j 10} {$j < 260} {incr j} { + set x [binary format "a*ca*" $before $j $after] + catch { sqlite3changeset_invert $x } + } + } + } {} +} + +do_test 1.2 { + set x [binary format "ca*" 0 [string range $changeset 1 end]] + list [catch { sqlite3changeset_invert $x } msg] $msg +} {1 SQLITE_CORRUPT} + +do_test 1.3 { + set x [binary format "ca*" 0 [string range $changeset 1 end]] + list [catch { sqlite3changeset_apply db $x xConflict } msg] $msg +} {1 SQLITE_CORRUPT} + +finish_test diff --git a/ext/session/session5.test b/ext/session/session5.test new file mode 100644 index 0000000000..9b8f9ffe12 --- /dev/null +++ b/ext/session/session5.test @@ -0,0 +1,408 @@ +# 2011 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for the session module. +# Specifically, for the sqlite3changeset_concat() command. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session5 + +# Organization of tests: +# +# session5-1.*: Simple tests to check the concat() function produces +# correct results. +# +# session5-2.*: More complicated tests. +# +# session5-3.*: Schema mismatch errors. +# +# session5-4.*: Test the concat cases that indicate that the database +# was modified in between recording of the two changesets +# being concatenated (i.e. two changesets that INSERT rows +# with the same PK values). +# + +proc do_concat_test {tn args} { + + set subtest 0 + foreach sql $args { + incr subtest + sqlite3session S db main ; S attach * + execsql $sql + + set c [S changeset] + if {[info commands s_prev] != ""} { + set c_concat [sqlite3changeset_concat $c_prev $c] + set c_two [s_prev changeset] + s_prev delete + + set h_concat [changeset_to_list $c_concat] + set h_two [changeset_to_list $c_two] + + do_test $tn.$subtest [list set {} $h_concat] $h_two + } + set c_prev $c + rename S s_prev + } + + catch { s_prev delete } +} + +#------------------------------------------------------------------------- +# Test cases session5-1.* - simple tests. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); +} + +do_concat_test 1.1.1 { + INSERT INTO t1 VALUES(1, 'one'); +} { + INSERT INTO t1 VALUES(2, 'two'); +} + +do_concat_test 1.1.2 { + UPDATE t1 SET b = 'five' WHERE a = 1; +} { + UPDATE t1 SET b = 'six' WHERE a = 2; +} + +do_concat_test 1.1.3 { + DELETE FROM t1 WHERE a = 1; +} { + DELETE FROM t1 WHERE a = 2; +} + + +# 1.2.1: INSERT + DELETE -> (none) +# 1.2.2: INSERT + UPDATE -> INSERT +# +# 1.2.3: DELETE + INSERT (matching data) -> (none) +# 1.2.4: DELETE + INSERT (non-matching data) -> UPDATE +# +# 1.2.5: UPDATE + UPDATE (matching data) -> (none) +# 1.2.6: UPDATE + UPDATE (non-matching data) -> UPDATE +# 1.2.7: UPDATE + DELETE -> DELETE +# +do_concat_test 1.2.1 { + INSERT INTO t1 VALUES('x', 'y'); +} { + DELETE FROM t1 WHERE a = 'x'; +} +do_concat_test 1.2.2 { + INSERT INTO t1 VALUES(5.0, 'five'); +} { + UPDATE t1 SET b = 'six' WHERE a = 5.0; +} + +do_execsql_test 1.2.3.1 "INSERT INTO t1 VALUES('I', 'one')" +do_concat_test 1.2.3.2 { + DELETE FROM t1 WHERE a = 'I'; +} { + INSERT INTO t1 VALUES('I', 'one'); +} +do_concat_test 1.2.4 { + DELETE FROM t1 WHERE a = 'I'; +} { + INSERT INTO t1 VALUES('I', 'two'); +} +do_concat_test 1.2.5 { + UPDATE t1 SET b = 'five' WHERE a = 'I'; +} { + UPDATE t1 SET b = 'two' WHERE a = 'I'; +} +do_concat_test 1.2.6 { + UPDATE t1 SET b = 'six' WHERE a = 'I'; +} { + UPDATE t1 SET b = 'seven' WHERE a = 'I'; +} +do_concat_test 1.2.7 { + UPDATE t1 SET b = 'eight' WHERE a = 'I'; +} { + DELETE FROM t1 WHERE a = 'I'; +} + + +#------------------------------------------------------------------------- +# Test cases session5-2.* - more complex tests. +# +db function indirect indirect +proc indirect {{x -1}} { + S indirect $x + s_prev indirect $x +} +do_concat_test 2.1 { + CREATE TABLE abc(a, b, c PRIMARY KEY); + INSERT INTO abc VALUES(NULL, NULL, 1); + INSERT INTO abc VALUES('abcdefghijkl', NULL, 2); +} { + DELETE FROM abc WHERE c = 1; + UPDATE abc SET c = 1 WHERE c = 2; +} { + INSERT INTO abc VALUES('abcdefghijkl', NULL, 2); + INSERT INTO abc VALUES(1.0, 2.0, 3); +} { + UPDATE abc SET a = a-1; +} { + CREATE TABLE def(d, e, f, PRIMARY KEY(e, f)); + INSERT INTO def VALUES('x', randomblob(11000), 67); + INSERT INTO def SELECT d, e, f+1 FROM def; + INSERT INTO def SELECT d, e, f+2 FROM def; + INSERT INTO def SELECT d, e, f+4 FROM def; +} { + DELETE FROM def WHERE rowid>4; +} { + INSERT INTO def SELECT d, e, f+4 FROM def; +} { + INSERT INTO abc VALUES(22, 44, -1); +} { + UPDATE abc SET c=-2 WHERE c=-1; + UPDATE abc SET c=-3 WHERE c=-2; +} { + UPDATE abc SET c=-4 WHERE c=-3; +} { + UPDATE abc SET a=a+1 WHERE c=-3; + UPDATE abc SET a=a+1 WHERE c=-3; +} { + UPDATE abc SET a=a+1 WHERE c=-3; + UPDATE abc SET a=a+1 WHERE c=-3; +} { + INSERT INTO abc VALUES('one', 'two', 'three'); +} { + SELECT indirect(1); + UPDATE abc SET a='one point five' WHERE c = 'three'; +} { + SELECT indirect(0); + UPDATE abc SET a='one point six' WHERE c = 'three'; +} { + CREATE TABLE x1(a, b, PRIMARY KEY(a)); + SELECT indirect(1); + INSERT INTO x1 VALUES(1, 2); +} { + SELECT indirect(1); + UPDATE x1 SET b = 3 WHERE a = 1; +} + +catch {db close} +forcedelete test.db +sqlite3 db test.db +do_concat_test 2.2 { + CREATE TABLE t1(a, b, PRIMARY KEY(b)); + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('string', 1); + INSERT INTO t1 VALUES(4, 2); + INSERT INTO t1 VALUES(X'FFAAFFAAFFAA', 3); +} { + INSERT INTO t2 VALUES('one', 'two'); + INSERT INTO t2 VALUES(1, NULL); + UPDATE t1 SET a = 5 WHERE a = 2; +} { + DELETE FROM t2 WHERE a = 1; + UPDATE t1 SET a = 4 WHERE a = 2; + INSERT INTO t2 VALUES('x', 'y'); +} + +do_test 2.3.0 { + catch {db close} + forcedelete test.db + sqlite3 db test.db + + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 "INSERT INTO x1 VALUES($i*4, $i);" + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE a = $i*4;" + } + set {} {} +} {} +do_concat_test 2.3 { + CREATE TABLE x1(a PRIMARY KEY, b) +} $sql1 $sql2 $sql1 $sql2 + +do_concat_test 2.4 { + CREATE TABLE x2(a PRIMARY KEY, b); + CREATE TABLE x3(a PRIMARY KEY, b); + + INSERT INTO x2 VALUES('a', 'b'); + INSERT INTO x2 VALUES('x', 'y'); + INSERT INTO x3 VALUES('a', 'b'); +} { + INSERT INTO x2 VALUES('c', 'd'); + INSERT INTO x3 VALUES('e', 'f'); + INSERT INTO x3 VALUES('x', 'y'); +} + +do_concat_test 2.5 { + UPDATE x3 SET b = 'Y' WHERE a = 'x' +} { + DELETE FROM x3 WHERE a = 'x' +} { + DELETE FROM x2 WHERE a = 'a' +} { + INSERT INTO x2 VALUES('a', 'B'); +} + +for {set k 1} {$k <=10} {incr k} { + do_test 2.6.$k.1 { + drop_all_tables + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 "INSERT INTO x1 VALUES(randomblob(20+(random()%10)), $i);" + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE rowid = $i;" + } + set {} {} + } {} + do_concat_test 2.6.$k { + CREATE TABLE x1(a PRIMARY KEY, b) + } $sql1 $sql2 $sql1 $sql2 +} + +for {set k 1} {$k <=10} {incr k} { + do_test 2.7.$k.1 { + drop_all_tables + set sql1 "" + set sql2 "" + for {set i 1} {$i < 120} {incr i} { + append sql1 { + INSERT INTO x1 VALUES( + CASE WHEN random()%2 THEN random() ELSE randomblob(20+random()%10) END, + CASE WHEN random()%2 THEN random() ELSE randomblob(20+random()%10) END + ); + } + } + for {set i 1} {$i < 120} {incr i} { + append sql2 "DELETE FROM x1 WHERE rowid = $i;" + } + set {} {} + } {} + do_concat_test 2.7.$k { + CREATE TABLE x1(a PRIMARY KEY, b) + } $sql1 $sql2 $sql1 $sql2 +} + + +#------------------------------------------------------------------------- +# Test that schema incompatibilities are detected correctly. +# +# session5-3.1: Incompatible number of columns. +# session5-3.2: Incompatible PK definition. +# + +do_test 3.1 { + db close + forcedelete test.db + sqlite3 db test.db + + execsql { CREATE TABLE t1(a PRIMARY KEY, b) } + set c1 [changeset_from_sql { INSERT INTO t1 VALUES(1, 2) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(a PRIMARY KEY, b, c); + } + set c2 [changeset_from_sql { INSERT INTO t1 VALUES(2, 3, 4) }] + + list [catch { sqlite3changeset_concat $c1 $c2 } msg] $msg +} {1 SQLITE_SCHEMA} + +do_test 3.2 { + db close + forcedelete test.db + sqlite3 db test.db + + execsql { CREATE TABLE t1(a PRIMARY KEY, b) } + set c1 [changeset_from_sql { INSERT INTO t1 VALUES(1, 2) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(a, b PRIMARY KEY); + } + set c2 [changeset_from_sql { INSERT INTO t1 VALUES(2, 3) }] + + list [catch { sqlite3changeset_concat $c1 $c2 } msg] $msg +} {1 SQLITE_SCHEMA} + +#------------------------------------------------------------------------- +# Test that concat() handles these properly: +# +# session5-4.1: INSERT + INSERT +# session5-4.2: UPDATE + INSERT +# session5-4.3: DELETE + UPDATE +# session5-4.4: DELETE + DELETE +# + +proc do_concat_test2 {tn sql1 sqlX sql2 expected} { + sqlite3session S db main ; S attach * + execsql $sql1 + set ::c1 [S changeset] + S delete + + execsql $sqlX + + sqlite3session S db main ; S attach * + execsql $sql2 + set ::c2 [S changeset] + S delete + + uplevel do_test $tn [list { + changeset_to_list [sqlite3changeset_concat $::c1 $::c2] + }] [list [normalize_list $expected]] +} + +drop_all_tables db +do_concat_test2 4.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('key', 'value'); +} { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'xxx'); +} { + {INSERT t1 0 X. {} {t key t value}} +} +do_concat_test2 4.2 { + UPDATE t1 SET b = 'yyy'; +} { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'value'); +} { + {UPDATE t1 0 X. {t key t xxx} {{} {} t yyy}} +} +do_concat_test2 4.3 { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'www'); +} { + UPDATE t1 SET b = 'valueX' WHERE a = 'key'; +} { + {DELETE t1 0 X. {t key t value} {}} +} +do_concat_test2 4.4 { + DELETE FROM t1 WHERE a = 'key'; +} { + INSERT INTO t1 VALUES('key', 'ttt'); +} { + DELETE FROM t1 WHERE a = 'key'; +} { + {DELETE t1 0 X. {t key t valueX} {}} +} + +finish_test diff --git a/ext/session/session6.test b/ext/session/session6.test new file mode 100644 index 0000000000..8a1f172cde --- /dev/null +++ b/ext/session/session6.test @@ -0,0 +1,90 @@ +# 2011 July 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite sessions extension. +# Specifically, it tests that sessions work when the database is modified +# using incremental blob handles. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session6 + +proc do_then_apply_tcl {tcl {dbname main}} { + proc xConflict args { return "OMIT" } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + eval $tcl + sqlite3changeset_apply db2 [S changeset] xConflict + } msg] + + catch { S delete } + if {$rc} {error $msg} +} + +test_sqlite3_log x +proc x {args} {puts $args} + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(c PRIMARY KEY, d); +} + +# Test a blob update. +# +do_test 1.1 { + do_then_apply_tcl { + db eval { INSERT INTO t1 VALUES(1, 'helloworld') } + db eval { INSERT INTO t2 VALUES(2, 'onetwothree') } + } + compare_db db db2 +} {} +do_test 1.2 { + do_then_apply_tcl { + set fd [db incrblob t1 b 1] + puts -nonewline $fd 1234567890 + close $fd + } + compare_db db db2 +} {} + +# Test an attached database. +# +do_test 2.1 { + forcedelete test.db3 + file copy test.db2 test.db3 + execsql { ATTACH 'test.db3' AS aux; } + + do_then_apply_tcl { + set fd [db incrblob aux t2 d 1] + puts -nonewline $fd fourfivesix + close $fd + } aux + + sqlite3 db3 test.db3 + compare_db db2 db3 +} {} + + +db3 close +db2 close + +finish_test diff --git a/ext/session/session8.test b/ext/session/session8.test new file mode 100644 index 0000000000..9f70fe2829 --- /dev/null +++ b/ext/session/session8.test @@ -0,0 +1,91 @@ +# 2011 July 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix session8 + +proc noop {args} {} + +# Like [dbcksum] in tester.tcl. Except this version is not sensitive +# to changes in the value of implicit IPK columns. +# +proc udbcksum {db dbname} { + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master"]\n + foreach tab $alltab { + append txt [lsort [$db eval "SELECT * FROM $dbname.$tab"]]\n + } + return [md5 $txt] +} + +proc do_then_undo {tn sql} { + set ck1 [udbcksum db main] + + sqlite3session S db main + S attach * + db eval $sql + + set ck2 [udbcksum db main] + + set invert [sqlite3changeset_invert [S changeset]] + S delete + sqlite3changeset_apply db $invert noop + + set ck3 [udbcksum db main] + + set a [expr {$ck1==$ck2}] + set b [expr {$ck1==$ck3}] + uplevel [list do_test $tn.1 "set {} $a" 0] + uplevel [list do_test $tn.2 "set {} $b" 1] +} + +do_execsql_test 1.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES("abc", "xyz"); +} +do_then_undo 1.2 { INSERT INTO t1 VALUES(3, 4); } +do_then_undo 1.3 { DELETE FROM t1 WHERE b=2; } +do_then_undo 1.4 { UPDATE t1 SET b = 3 WHERE a = 1; } + +do_execsql_test 2.1 { + CREATE TABLE t2(a, b PRIMARY KEY); + INSERT INTO t2 VALUES(1, 2); + INSERT INTO t2 VALUES('abc', 'xyz'); +} +do_then_undo 1.2 { INSERT INTO t2 VALUES(3, 4); } +do_then_undo 1.3 { DELETE FROM t2 WHERE b=2; } +do_then_undo 1.4 { UPDATE t1 SET a = '123' WHERE b = 'xyz'; } + +do_execsql_test 3.1 { + CREATE TABLE t3(a, b, c, d, e, PRIMARY KEY(c, e)); + INSERT INTO t3 VALUES('x', 45, 0.0, 'abcdef', 12); + INSERT INTO t3 VALUES(45, 0.0, 'abcdef', 12, 'x'); + INSERT INTO t3 VALUES(0.0, 'abcdef', 12, 'x', 45); +} + +do_then_undo 3.2 { UPDATE t3 SET b=b||b WHERE e!='x' } +do_then_undo 3.3 { UPDATE t3 SET a = 46 } + +finish_test diff --git a/ext/session/session9.test b/ext/session/session9.test new file mode 100644 index 0000000000..ebb88ffade --- /dev/null +++ b/ext/session/session9.test @@ -0,0 +1,287 @@ +# 2013 July 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that the sessions module handles foreign key constraint +# violations when applying changesets as required. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix session9 + + +#-------------------------------------------------------------------- +# Basic tests. +# +proc populate_db {} { + drop_all_tables + execsql { + PRAGMA foreign_keys = 1; + CREATE TABLE p1(a PRIMARY KEY, b); + CREATE TABLE c1(a PRIMARY KEY, b REFERENCES p1); + CREATE TABLE c2(a PRIMARY KEY, + b REFERENCES p1 DEFERRABLE INITIALLY DEFERRED + ); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + } +} + +proc capture_changeset {sql} { + sqlite3session S db main + + foreach t [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $t + } + execsql $sql + set ret [S changeset] + S delete + + return $ret +} + +do_test 1.1 { + populate_db + set cc [capture_changeset { + INSERT INTO c1 VALUES('ii', 2); + INSERT INTO c2 VALUES('iii', 3); + }] + set {} {} +} {} + +proc xConflict {args} { + lappend ::xConflict {*}$args + return $::conflictret +} + +foreach {tn delrow trans conflictargs conflictret} { + 1 2 0 {FOREIGN_KEY 1} OMIT + 2 3 0 {FOREIGN_KEY 1} OMIT + 3 2 1 {FOREIGN_KEY 1} OMIT + 4 3 1 {FOREIGN_KEY 1} OMIT + 5 2 0 {FOREIGN_KEY 1} ABORT + 6 3 0 {FOREIGN_KEY 1} ABORT + 7 2 1 {FOREIGN_KEY 1} ABORT + 8 3 1 {FOREIGN_KEY 1} ABORT +} { + + set A(OMIT) {0 {}} + set A(ABORT) {1 SQLITE_CONSTRAINT} + do_test 1.2.$tn.1 { + populate_db + execsql { DELETE FROM p1 WHERE a=($delrow+0) } + if {$trans} { execsql BEGIN } + + set ::xConflict [list] + list [catch {sqlite3changeset_apply db $::cc xConflict} msg] $msg + } $A($conflictret) + + do_test 1.2.$tn.2 { set ::xConflict } $conflictargs + + set A(OMIT) {1 1} + set A(ABORT) {0 0} + do_test 1.2.$tn.3 { + execsql { SELECT count(*) FROM c1 UNION ALL SELECT count(*) FROM c2 } + } $A($conflictret) + + do_test 1.2.$tn.4 { expr ![sqlite3_get_autocommit db] } $trans + do_test 1.2.$tn.5 { + if { $trans } { execsql COMMIT } + } {} +} + +#-------------------------------------------------------------------- +# Test that closing a transaction clears the defer_foreign_keys flag. +# +foreach {tn open noclose close} { + 1 BEGIN {} COMMIT + 2 BEGIN {} ROLLBACK + + 3 {SAVEPOINT one} {} {RELEASE one} + 4 {SAVEPOINT one} {ROLLBACK TO one} {RELEASE one} +} { + execsql $open + do_execsql_test 2.$tn.1 { PRAGMA defer_foreign_keys } {0} + + do_execsql_test 2.$tn.2 { + PRAGMA defer_foreign_keys = 1; + PRAGMA defer_foreign_keys; + } {1} + + execsql $noclose + do_execsql_test 2.$tn.3 { PRAGMA defer_foreign_keys } {1} + + execsql $close + do_execsql_test 2.$tn.4 { PRAGMA defer_foreign_keys } {0} +} + +#-------------------------------------------------------------------- +# Test that a cyclic relationship can be inserted and deleted. +# +# This situation does not come up in practice, but testing it serves to +# show that it does not matter which order parent and child keys +# are processed in internally when applying a changeset. +# +drop_all_tables + +do_execsql_test 3.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(x PRIMARY KEY, y); +} + +# Create changesets as follows: +# +# $cc1 - Insert a row into t1. +# $cc2 - Insert a row into t2. +# $cc - Combination of $cc1 and $cc2. +# +# $ccdel1 - Delete the row from t1. +# $ccdel2 - Delete the row from t2. +# $ccdel - Combination of $cc1 and $cc2. +# +do_test 3.2 { + set cc1 [capture_changeset { + INSERT INTO t1 VALUES('one', 'value one'); + }] + set ccdel1 [capture_changeset { DELETE FROM t1; }] + set cc2 [capture_changeset { + INSERT INTO t2 VALUES('value one', 'one'); + }] + set ccdel2 [capture_changeset { DELETE FROM t2; }] + set cc [capture_changeset { + INSERT INTO t1 VALUES('one', 'value one'); + INSERT INTO t2 VALUES('value one', 'one'); + }] + set ccdel [capture_changeset { + DELETE FROM t1; + DELETE FROM t2; + }] + set {} {} +} {} + +# Now modify the database schema to create a cyclic foreign key dependency +# between tables t1 and t2. This means that although changesets $cc and +# $ccdel can be applied, none of the others may without violating the +# foreign key constraints. +# +do_test 3.3 { + + drop_all_tables + execsql { + CREATE TABLE t1(a PRIMARY KEY, b REFERENCES t2); + CREATE TABLE t2(x PRIMARY KEY, y REFERENCES t1); + } + + + proc conflict_handler {args} { return "ABORT" } + sqlite3changeset_apply db $cc conflict_handler + + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {one {value one} {value one} one} + +do_test 3.3.1 { + list [catch {sqlite3changeset_apply db $::ccdel1 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +do_test 3.3.2 { + list [catch {sqlite3changeset_apply db $::ccdel2 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +do_test 3.3.4.1 { + list [catch {sqlite3changeset_apply db $::ccdel conflict_handler} msg] $msg +} {0 {}} +do_execsql_test 3.3.4.2 { + SELECT * FROM t1; + SELECT * FROM t2; +} {} + +do_test 3.5.1 { + list [catch {sqlite3changeset_apply db $::cc1 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} +do_test 3.5.2 { + list [catch {sqlite3changeset_apply db $::cc2 conflict_handler} msg] $msg +} {1 SQLITE_CONSTRAINT} + +#-------------------------------------------------------------------- +# Test that if a change that affects FK processing is not applied +# due to a separate constraint, SQLite does not get confused and +# increment FK counters anyway. +# +drop_all_tables +do_execsql_test 4.1 { + CREATE TABLE p1(x PRIMARY KEY, y); + CREATE TABLE c1(a PRIMARY KEY, b REFERENCES p1); + INSERT INTO p1 VALUES(1,1); +} + +do_execsql_test 4.2.1 { + BEGIN; + PRAGMA defer_foreign_keys = 1; + INSERT INTO c1 VALUES('x', 'x'); +} +do_catchsql_test 4.2.2 { COMMIT } {1 {FOREIGN KEY constraint failed}} +do_catchsql_test 4.2.3 { ROLLBACK } {0 {}} + +do_execsql_test 4.3.1 { + BEGIN; + PRAGMA defer_foreign_keys = 1; + INSERT INTO c1 VALUES(1, 1); +} +do_catchsql_test 4.3.2 { + INSERT INTO c1 VALUES(1, 'x') +} {1 {UNIQUE constraint failed: c1.a}} + +do_catchsql_test 4.3.3 { COMMIT } {0 {}} +do_catchsql_test 4.3.4 { BEGIN ; COMMIT } {0 {}} + +#-------------------------------------------------------------------- +# Test that if a DELETE change cannot be applied due to an +# SQLITE_CONSTRAINT error thrown by a trigger program, things do not +# go awry. + +drop_all_tables +reset_db +do_execsql_test 5.1 { + CREATE TABLE x1(x PRIMARY KEY, y); + CREATE TABLE x2(x PRIMARY KEY, y); + INSERT INTO x2 VALUES(1, 1); + INSERT INTO x1 VALUES(1, 1); +} + +set ::cc [changeset_from_sql { DELETE FROM x1; }] + +do_execsql_test 5.2 { + INSERT INTO x1 VALUES(1, 1); + CREATE TRIGGER tr1 AFTER DELETE ON x1 BEGIN + INSERT INTO x2 VALUES(old.x, old.y); + END; +} {} + +proc conflict_handler {args} { return "ABORT" } +do_test 5.3 { + list [catch {sqlite3changeset_apply db $::cc conflict_handler} msg] $msg +} {1 SQLITE_ABORT} + +do_execsql_test 5.4 { + SELECT * FROM X1; +} {1 1} + +finish_test diff --git a/ext/session/sessionA.test b/ext/session/sessionA.test new file mode 100644 index 0000000000..0e0a14e38a --- /dev/null +++ b/ext/session/sessionA.test @@ -0,0 +1,106 @@ +# 2013 July 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests that filter callbacks work as required. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionA + + +forcedelete test.db2 +sqlite3 db2 test.db2 +foreach {tn db} {1 db 2 db2} { + do_test 1.$tn.1 { + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t3(a PRIMARY KEY, b); + } $db + } {} +} + +proc tbl_filter {zTbl} { + return $::table_filter($zTbl) +} + +do_test 2.1 { + set ::table_filter(t1) 1 + set ::table_filter(t2) 0 + set ::table_filter(t3) 1 + + sqlite3session S db main + S table_filter tbl_filter + + execsql { + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t3 VALUES('e', 'f'); + } + + set changeset [S changeset] + S delete + sqlite3changeset_apply db2 $changeset xConflict + + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } db2 +} {a b e f} + +#------------------------------------------------------------------------- +# Test that filter callbacks passed to sqlite3changeset_apply() are +# invoked correctly. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(x PRIMARY KEY, y); +} + +do_test 3.2 { + execsql BEGIN + set ::cs [changeset_from_sql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES('x', 'y'); + }] + execsql ROLLBACK + set {} {} +} {} + +proc filter {x y} { + return [string equal $x $y] +} + +do_test 3.3 { + sqlite3changeset_apply db $::cs {} [list filter t1] + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2} + +do_test 3.4 { + execsql { DELETE FROM t1 } + sqlite3changeset_apply db $::cs {} [list filter t2] + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {x y} + +finish_test diff --git a/ext/session/sessionB.test b/ext/session/sessionB.test new file mode 100644 index 0000000000..eae41f76ad --- /dev/null +++ b/ext/session/sessionB.test @@ -0,0 +1,508 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for sessions SQLite extension. +# Specifically, this file contains tests for "patchset" changes. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionB + +# +# 1.*: Test that the blobs returned by the session_patchset() API are +# as expected. Also the sqlite3_changeset_iter functions. +# +# 2.*: Test that patchset blobs are handled by sqlite3changeset_apply(). +# +# 3.*: Test that sqlite3changeset_invert() works with patchset blobs. +# Correct behaviour is to return SQLITE_CORRUPT. + +proc do_sql2patchset_test {tn sql res} { + sqlite3session S db main + S attach * + execsql $sql + uplevel [list do_patchset_test $tn S $res] + S delete +} + +#------------------------------------------------------------------------- +# Run simple tests of the _patchset() API. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d, PRIMARY KEY(d, a)); + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(5, 6, 7, 8); + INSERT INTO t1 VALUES(9, 10, 11, 12); +} + +do_test 1.1 { + sqlite3session S db main + S attach t1 + execsql { + INSERT INTO t1 VALUES('w', 'x', 'y', 'z'); + DELETE FROM t1 WHERE d=4; + UPDATE t1 SET c = 14 WHERE a=5; + } +} {} + +do_patchset_test 1.2 S { + {UPDATE t1 0 X..X {i 5 {} {} {} {} i 8} {{} {} {} {} i 14 {} {}}} + {INSERT t1 0 X..X {} {t w t x t y t z}} + {DELETE t1 0 X..X {i 1 {} {} {} {} i 4} {}} +} + +do_test 1.3 { + S delete +} {} + +do_sql2patchset_test 1.4 { + DELETE FROM t1; +} { + {DELETE t1 0 X..X {i 5 {} {} {} {} i 8} {}} + {DELETE t1 0 X..X {t w {} {} {} {} t z} {}} + {DELETE t1 0 X..X {i 9 {} {} {} {} i 12} {}} +} + +do_sql2patchset_test 1.5 { + INSERT INTO t1 VALUES(X'61626364', NULL, NULL, 4.2); + INSERT INTO t1 VALUES(4.2, NULL, NULL, X'61626364'); +} { + {INSERT t1 0 X..X {} {f 4.2 n {} n {} b abcd}} + {INSERT t1 0 X..X {} {b abcd n {} n {} f 4.2}} +} + +do_sql2patchset_test 1.6 { + UPDATE t1 SET b=45 WHERE typeof(a)=='blob'; + UPDATE t1 SET c='zzzz' WHERE typeof(a)!='blob'; +} { + {UPDATE t1 0 X..X {f 4.2 {} {} {} {} b abcd} {{} {} {} {} t zzzz {} {}}} + {UPDATE t1 0 X..X {b abcd {} {} {} {} f 4.2} {{} {} i 45 {} {} {} {}}} +} + +do_sql2patchset_test 1.7 { + UPDATE t1 SET b='xyz' WHERE typeof(a)=='blob'; + UPDATE t1 SET c='xyz' WHERE typeof(a)!='blob'; + UPDATE t1 SET b=45 WHERE typeof(a)=='blob'; + UPDATE t1 SET c='zzzz' WHERE typeof(a)!='blob'; +} { +} + +do_sql2patchset_test 1.8 { + DELETE FROM t1; +} { + {DELETE t1 0 X..X {f 4.2 {} {} {} {} b abcd} {}} + {DELETE t1 0 X..X {b abcd {} {} {} {} f 4.2} {}} +} + +#------------------------------------------------------------------------- +# Run simple tests of _apply() with patchset objects. +# +reset_db + +proc noop {args} { error $args } +proc exec_rollback_replay {sql} { + sqlite3session S db main + S attach * + execsql BEGIN + execsql $sql + set patchset [S patchset] + S delete + execsql ROLLBACK + sqlite3changeset_apply db $patchset noop +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(b,c)); + CREATE TABLE t3(w, x, y, z, PRIMARY KEY(w)); +} + +do_test 2.1 { + exec_rollback_replay { + INSERT INTO t2 VALUES(1, 2, 3, 4); + INSERT INTO t2 VALUES('w', 'x', 'y', 'z'); + } + execsql { SELECT * FROM t2 } +} {1 2 3 4 w x y z} + +do_test 2.2 { + exec_rollback_replay { + DELETE FROM t2 WHERE a=1; + UPDATE t2 SET d = 'a'; + } + execsql { SELECT * FROM t2 } +} {w x y a} + +#------------------------------------------------------------------------- +# sqlite3changeset_invert() +# +reset_db + +do_execsql_test 3.1 { CREATE TABLE t1(x PRIMARY KEY, y) } +do_test 3.2 { + sqlite3session S db main + S attach * + execsql { INSERT INTO t1 VALUES(1, 2) } + set patchset [S patchset] + S delete + list [catch { sqlite3changeset_invert $patchset } msg] [set msg] +} {1 SQLITE_CORRUPT} + + +#------------------------------------------------------------------------- +# sqlite3changeset_concat() +# +reset_db + +proc do_patchconcat_test {tn args} { + set bRevert 0 + if {[lindex $args 0] == "-revert"} { + set bRevert 1 + set args [lrange $args 1 end] + } + set nSql [expr [llength $args]-1] + set res [lindex $args $nSql] + set patchlist [list] + + execsql BEGIN + if {$bRevert} { execsql { SAVEPOINT x } } + foreach sql [lrange $args 0 end-1] { + sqlite3session S db main + S attach * + execsql $sql + lappend patchlist [S patchset] + S delete + if {$bRevert} { execsql { ROLLBACK TO x } } + } + execsql ROLLBACK + + set patch [lindex $patchlist 0] + foreach p [lrange $patchlist 1 end] { + set patch [sqlite3changeset_concat $patch $p] + } + + set x [list] + sqlite3session_foreach c $patch { lappend x $c } + + uplevel [list do_test $tn [list set {} $x] [list {*}$res]] +} + +do_execsql_test 4.1.1 { + CREATE TABLE t1(x PRIMARY KEY, y, z); +} +do_patchconcat_test 4.1.2 { + INSERT INTO t1 VALUES(1, 2, 3); +} { + INSERT INTO t1 VALUES(4, 5, 6); +} { + {INSERT t1 0 X.. {} {i 1 i 2 i 3}} + {INSERT t1 0 X.. {} {i 4 i 5 i 6}} +} + +do_execsql_test 4.2.1 { + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_patchconcat_test 4.2.2 { + UPDATE t1 SET z = 'abc' WHERE x=1 +} { + UPDATE t1 SET z = 'def' WHERE x=4 +} { + {UPDATE t1 0 X.. {i 1 {} {} {} {}} {{} {} {} {} t abc}} + {UPDATE t1 0 X.. {i 4 {} {} {} {}} {{} {} {} {} t def}} +} + +do_patchconcat_test 4.2.3 { + DELETE FROM t1 WHERE x=1; +} { + DELETE FROM t1 WHERE x=4; +} { + {DELETE t1 0 X.. {i 1 {} {} {} {}} {}} + {DELETE t1 0 X.. {i 4 {} {} {} {}} {}} +} + + +do_execsql_test 4.3.1 { + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(c, b)); + INSERT INTO t2 VALUES('.', 1, 1, '.'); + INSERT INTO t2 VALUES('.', 1, 2, '.'); + INSERT INTO t2 VALUES('.', 2, 1, '.'); + INSERT INTO t2 VALUES('.', 2, 2, '.'); +} + +# INSERT + INSERT +do_patchconcat_test 4.3.2 -revert { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + INSERT INTO t2 VALUES('b', 'a', 'a', 'b'); +} { + {INSERT t2 0 .XX. {} {t a t a t a t a}} +} + +# INSERT + DELETE +do_patchconcat_test 4.3.3 { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + DELETE FROM t2 WHERE c = 'a'; +} { +} + +# INSERT + UPDATE +do_patchconcat_test 4.3.4 { + INSERT INTO t2 VALUES('a', 'a', 'a', 'a'); +} { + UPDATE t2 SET d = 'b' WHERE c='a'; +} { + {INSERT t2 0 .XX. {} {t a t a t a t b}} +} + +# UPDATE + UPDATE +do_patchconcat_test 4.3.5 { + UPDATE t2 SET a = 'a' WHERE c=1 AND b=2; +} { + UPDATE t2 SET d = 'd' WHERE c=1 AND b=2; +} { + {UPDATE t2 0 .XX. {{} {} i 2 i 1 {} {}} {t a {} {} {} {} t d}} +} + +# UPDATE + DELETE +do_patchconcat_test 4.3.6 { + UPDATE t2 SET a = 'a' WHERE c=1 AND b=2; +} { + DELETE FROM t2 WHERE c=1 AND b=2; +} { + {DELETE t2 0 .XX. {{} {} i 2 i 1 {} {}} {}} +} + +# DELETE + INSERT +do_patchconcat_test 4.3.7 { + DELETE FROM t2 WHERE b=1; +} { + INSERT INTO t2 VALUES('x', 1, 2, '.'); +} { + {DELETE t2 0 .XX. {{} {} i 1 i 1 {} {}} {}} + {UPDATE t2 0 .XX. {{} {} i 1 i 2 {} {}} {t x {} {} {} {} t .}} +} + +# DELETE + UPDATE +do_patchconcat_test 4.3.8 -revert { + DELETE FROM t2 WHERE b=1 AND c=2; +} { + UPDATE t2 SET a=5 WHERE b=1 AND c=2; +} { + {DELETE t2 0 .XX. {{} {} i 1 i 2 {} {}} {}} +} + +# DELETE + UPDATE +do_patchconcat_test 4.3.9 -revert { + DELETE FROM t2 WHERE b=1 AND c=2; +} { + DELETE FROM t2 WHERE b=1; +} { + {DELETE t2 0 .XX. {{} {} i 1 i 1 {} {}} {}} + {DELETE t2 0 .XX. {{} {} i 1 i 2 {} {}} {}} +} + +#------------------------------------------------------------------------- +# More rigorous testing of the _patchset(), _apply and _concat() APIs. +# +# The inputs to each test are a populate database and a list of DML +# statements. This test determines that the final database is the same +# if: +# +# 1) the statements are executed directly on the database. +# +# 2) a single patchset is collected while executing the statements and +# then applied to a copy of the original database file. +# +# 3) individual patchsets are collected for statement while executing +# them and concatenated together before being applied to a copy of +# the original database. The concatenation is done in a couple of +# different ways - linear, pairwise etc. +# +# All tests, as it happens, are run with both changesets and patchsets. +# But the focus is on patchset capabilities. +# + +# Return a checksum of the contents of the database file. Implicit IPK +# columns are not included in the checksum - just modifying rowids does +# not change the database checksum. +# +proc databasecksum {db} { + set alltab [$db eval {SELECT name FROM sqlite_master WHERE type='table'}] + foreach tab $alltab { + $db eval "SELECT * FROM $tab LIMIT 1" res { } + set slist [list] + foreach col [lsort $res(*)] { + lappend slist "quote($col)" + } + set sql "SELECT [join $slist ,] FROM $tab" + append txt "[lsort [$db eval $sql]]\n" + } + return [md5 $txt] +} + +proc do_patchset_test {tn tstcmd lSql} { + if {$tstcmd != "patchset" && $tstcmd != "changeset"} { + error "have $tstcmd: must be patchset or changeset" + } + + foreach fname {test.db2 test.db3 test.db4 test.db5} { + forcedelete $fname + forcecopy test.db $fname + } + + # Execute the SQL statements on [db]. Collect a patchset for each + # individual statement, as well as a single patchset for the entire + # operation. + sqlite3session S db main + S attach * + foreach sql $lSql { + sqlite3session T db main + T attach * + db eval $sql + lappend lPatch [T $tstcmd] + T delete + } + set patchset [S $tstcmd] + S delete + + # Calculate a checksum for the final database. + set cksum [databasecksum db] + + # 1. Apply the single large patchset to test.db2 + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $patchset noop + uplevel [list do_test $tn.1 { databasecksum db2 } $cksum ] + db2 close + + # 2. Apply each of the single-statement patchsets to test.db3 + sqlite3 db2 test.db3 + foreach p $lPatch { + sqlite3changeset_apply db2 $p noop + } + uplevel [list do_test $tn.2 { databasecksum db2 } $cksum ] + db2 close + + # 3. Concatenate all single-statement patchsets into a single large + # patchset, then apply it to test.db4. + # + sqlite3 db2 test.db4 + set big "" + foreach p $lPatch { + set big [sqlite3changeset_concat $big $p] + } + sqlite3changeset_apply db2 $big noop + uplevel [list do_test $tn.3 { databasecksum db2 } $cksum ] + db2 close + + # 4. Concatenate all single-statement patchsets pairwise into a single + # large patchset, then apply it to test.db5. Pairwise concatenation: + # + # a b c d e f g h i j k + # -> {a b} {c d} {e f} {g h} {i j} k + # -> {a b c d} {e f g h} {i j k} + # -> {a b c d e f g h} {i j k} + # -> {a b c d e f g h i j k} + # -> APPLY! + # + sqlite3 db2 test.db5 + set L $lPatch + while {[llength $L] > 1} { + set O [list] + for {set i 0} {$i < [llength $L]} {incr i 2} { + if {$i==[llength $L]-1} { + lappend O [lindex $L $i] + } else { + set i1 [expr $i+1] + lappend O [sqlite3changeset_concat [lindex $L $i] [lindex $L $i1]] + } + } + set L $O + } + sqlite3changeset_apply db2 [lindex $L 0] noop + uplevel [list do_test $tn.4 { databasecksum db2 } $cksum ] + db2 close +} + +proc do_patchset_changeset_test {tn initsql args} { + foreach tstcmd {patchset changeset} { + reset_db + execsql $initsql + set x 0 + foreach sql $args { + incr x + set lSql [split $sql ";"] + uplevel [list do_patchset_test $tn.$tstcmd.$x $tstcmd $lSql] + } + } +} + +do_patchset_changeset_test 5.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} { + INSERT INTO t1 VALUES(4, 5, 6); + DELETE FROM t1 WHERE a=1; +} { + INSERT INTO t1 VALUES(7, 8, 9); + UPDATE t1 SET c = 5; + INSERT INTO t1 VALUES(10, 11, 12); + UPDATE t1 SET c = 6; + INSERT INTO t1 VALUES(13, 14, 15); +} { + UPDATE t1 SET c=c+1; + DELETE FROM t1 WHERE (a%2); +} + +do_patchset_changeset_test 5.2 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(a, b, c, d, PRIMARY KEY(c, b)); +} { + INSERT INTO t1 VALUES(x'00', 0, 'zero'); + INSERT INTO t1 VALUES(x'01', 1, 'one'); + INSERT INTO t1 VALUES(x'02', 4, 'four'); + INSERT INTO t1 VALUES(x'03', 9, 'nine'); + INSERT INTO t1 VALUES(x'04', 16, 'sixteen'); + INSERT INTO t1 VALUES(x'05', 25, 'twenty-five'); +} { + UPDATE t1 SET a = b WHERE b<=4; + INSERT INTO t2 SELECT NULL, * FROM t1; + DELETE FROM t1 WHERE b=25; +} { + DELETE FROM t2; + INSERT INTO t2 SELECT NULL, * FROM t1; + DELETE FROM t1; + INSERT INTO t1 SELECT b, c, d FROM t2; + UPDATE t1 SET b = b+1; + UPDATE t1 SET b = b+1; + UPDATE t1 SET b = b+1; +} + +set initsql { CREATE TABLE t1(a, b, c, PRIMARY KEY(c, b)); } +for {set i 0} {$i < 1000} {incr i} { + append insert "INSERT INTO t1 VALUES($i, $i, $i);" + append delete "DELETE FROM t1 WHERE b=$i;" +} +do_patchset_changeset_test 5.3 \ + $initsql $insert $delete \ + $insert $delete \ + "$insert $delete" \ + $delete + + +finish_test diff --git a/ext/session/sessionC.test b/ext/session/sessionC.test new file mode 100644 index 0000000000..390a31deb6 --- /dev/null +++ b/ext/session/sessionC.test @@ -0,0 +1,198 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionC + +#------------------------------------------------------------------------- +# Test the outcome of a DELETE operation made as part of applying a +# changeset failing with SQLITE_CONSTRAINT. This may happen if an +# ON DELETE RESTRICT foreign key action is triggered, or if a trigger +# program raises a constraint somehow. +# +# UPDATE: The above is no longer true, as "PRAGMA defer_foreign_keys" +# now disables "RESTRICT" processing. The test below has been rewritten +# to use a trigger instead of a foreign key to test this case. +# +do_execsql_test 1.0 { + PRAGMA foreign_keys = 1; + + CREATE TABLE p(a PRIMARY KEY, b, c); + CREATE TABLE c(d PRIMARY KEY, e /* REFERENCES p ON DELETE RESTRICT */); + + CREATE TRIGGER restrict_trig BEFORE DELETE ON p BEGIN + SELECT raise(ABORT, 'error!') FROM c WHERE e=old.a; + END; + + INSERT INTO p VALUES('one', 1, 1); + INSERT INTO p VALUES('two', 2, 2); + INSERT INTO p VALUES('three', 3, 3); + + INSERT INTO c VALUES(1, 'one'); + INSERT INTO c VALUES(3, 'three'); +} + +do_test 1.1 { + execsql BEGIN + set C [changeset_from_sql { + INSERT INTO c VALUES(4, 'one'); + DELETE FROM p WHERE a='two'; + }] + execsql ROLLBACK + execsql { + INSERT INTO c VALUES(2, 'two'); + } +} {} + +do_test 1.2.1 { + proc xConflict {args} { return "ABORT" } + catch { sqlite3changeset_apply db $C xConflict } msg + set msg +} {SQLITE_ABORT} +do_execsql_test 1.2.2 { SELECT * FROM c } {1 one 3 three 2 two} + +do_test 1.3.1 { + proc xConflict {args} { return "OMIT" } + catch { sqlite3changeset_apply db $C xConflict } msg + set msg +} {} +do_execsql_test 1.3.2 { SELECT * FROM c } {1 one 3 three 2 two 4 one} +do_execsql_test 1.3.3 { + SELECT * FROM p; +} {one 1 1 two 2 2 three 3 3} + + +#------------------------------------------------------------------------- +# Test that concatenating a changeset with a patchset does not work. +# Any attempt to do so returns SQLITE_ERROR. +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(t, v PRIMARY KEY); + INSERT INTO x1 VALUES(12, 55); + INSERT INTO x1 VALUES(55, 14); +} + +do_test 2.1 { + execsql BEGIN + + sqlite3session S1 db main + S1 attach * + execsql { + UPDATE x1 SET t=13 WHERE v=55; + INSERT INTO x1 VALUES(99, 123); + } + set patchset [S1 patchset] + S1 delete + + sqlite3session S1 db main + S1 attach * + execsql { + UPDATE x1 SET t=56 WHERE v=14; + INSERT INTO x1 VALUES(22, 998); + } + set changeset [S1 changeset] + S1 delete + + execsql ROLLBACK +} {} + +do_test 2.2 { + set rc [catch { sqlite3changeset_concat $patchset $changeset } msg] + list $rc $msg +} {1 SQLITE_ERROR} + +do_test 2.3 { + set rc [catch { sqlite3changeset_concat $changeset $patchset } msg] + list $rc $msg +} {1 SQLITE_ERROR} + +do_test 2.4 { + set rc [catch { sqlite3changeset_concat {} $patchset } msg] + list $rc $msg +} [list 0 $patchset] + +do_test 2.5 { + set rc [catch { sqlite3changeset_concat $patchset {} } msg] + list $rc $msg +} [list 0 $patchset] + +do_test 2.6 { + set rc [catch { sqlite3changeset_concat {} $changeset } msg] + list $rc $msg +} [list 0 $changeset] + +do_test 2.7 { + set rc [catch { sqlite3changeset_concat $changeset {} } msg] + list $rc $msg +} [list 0 $changeset] + +do_test 2.8 { + set rc [catch { sqlite3changeset_concat {} {} } msg] + list $rc $msg +} [list 0 {}] + + +#------------------------------------------------------------------------- +# Test that the xFilter argument to sqlite3changeset_apply() works. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a PRIMARY KEY, b); + CREATE TABLE t3(a PRIMARY KEY, b); +} +do_test 3.1 { + execsql BEGIN + set changeset [changeset_from_sql { + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t3 VALUES(3, 3); + }] + execsql ROLLBACK +} {} +do_test 3.2 { + proc xFilter {zName} { + if {$zName == "t1"} { return 1 } + return 0 + } + sqlite3changeset_apply db $changeset noop xFilter + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 1} +do_test 3.3 { + proc xFilter {zName} { + if {$zName == "t3"} { return 1 } + return 0 + } + sqlite3changeset_apply db $changeset noop xFilter + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 1 3 3} + + + +finish_test + diff --git a/ext/session/sessionD.test b/ext/session/sessionD.test new file mode 100644 index 0000000000..b8572782e2 --- /dev/null +++ b/ext/session/sessionD.test @@ -0,0 +1,225 @@ +# 2014 August 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file focuses on the sqlite3session_diff() function. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionD + +proc scksum {db dbname} { + + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master ORDER BY type,name,sql"] + foreach tab $alltab { + set cols [list] + db eval "PRAGMA $dbname.table_info = $tab" x { + lappend cols "quote($x(name))" + } + set cols [join $cols ,] + append txt [db eval "SELECT $cols FROM $tab ORDER BY $cols"] + } + return [md5 $txt] +} + +proc do_diff_test {tn setup} { + reset_db + forcedelete test.db2 + execsql { ATTACH 'test.db2' AS aux } + execsql $setup + + sqlite3session S db main + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set C [S changeset] + S delete + + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok + db2 close + + set cksum [scksum db main] + uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] +} + +# Ensure that the diff produced by comparing the current contents of [db] +# with itself is empty. +proc do_empty_diff_test {tn} { + forcedelete test.db2 + forcecopy test.db test.db2 + + execsql { ATTACH 'test.db2' AS aux } + sqlite3session S db main + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set ::C [S changeset] + S delete + + uplevel [list do_test $tn {string length $::C} 0] +} + + +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t2(a PRIMARY KEY, b); + INSERT INTO t2 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t2(a PRIMARY KEY, b); +} + +do_test 1.1 { + sqlite3session S db main + S attach t2 + S diff aux t2 + set C [S changeset] + S delete +} {} + +do_test 1.2 { + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + db2 close + db eval { SELECT * FROM aux.t2 } +} {1 one 2 two} + +do_diff_test 2.1 { + CREATE TABLE aux.t1(x, y, PRIMARY KEY(y)); + CREATE TABLE t1(x, y, PRIMARY KEY(y)); + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(NULL, 'xyz'); + INSERT INTO t1 VALUES(4.5, 5.5); +} + +do_diff_test 2.2 { + CREATE TABLE aux.t1(x, y, PRIMARY KEY(y)); + CREATE TABLE t1(x, y, PRIMARY KEY(y)); + + INSERT INTO aux.t1 VALUES(1, 2); + INSERT INTO aux.t1 VALUES(NULL, 'xyz'); + INSERT INTO aux.t1 VALUES(4.5, 5.5); +} + +do_diff_test 2.3 { + CREATE TABLE aux.t1(a PRIMARY KEY, b TEXT); + CREATE TABLE t1(a PRIMARY KEY, b TEXT); + + INSERT INTO aux.t1 VALUES(1, 'one'); + INSERT INTO aux.t1 VALUES(2, 'two'); + INSERT INTO aux.t1 VALUES(3, 'three'); + + INSERT INTO t1 VALUES(1, 'I'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'III'); +} + +do_diff_test 2.4 { + CREATE TABLE aux.t1(a, b, c, d, PRIMARY KEY(c, b, a)); + CREATE TABLE t1(a, b, c, d, PRIMARY KEY(c, b, a)); + + INSERT INTO t1 VALUES('hvkzyipambwdqlvwv','',-458331.50,X'DA51ED5E84'); + INSERT INTO t1 VALUES(X'C5C6B5DD','jjxrath',40917,830244); + INSERT INTO t1 VALUES(-204877.54,X'1704C253D5F3AFA8',155120.88,NULL); + INSERT INTO t1 + VALUES('ckmqmzoeuvxisxqy',X'EB5A5D3A1DD22FD1','tidhjcbvbppdt',-642987.37); + INSERT INTO t1 VALUES(-851726,-161992,-469943,-159541); + INSERT INTO t1 VALUES(X'4A6A667F858938',185083,X'7A',NULL); + + INSERT INTO aux.t1 VALUES(415075.74,'auawczkb',X'',X'57B4FAAF2595'); + INSERT INTO aux.t1 VALUES(727637,711560,-181340,'hphuo'); + INSERT INTO aux.t1 + VALUES(-921322.81,662959,'lvlgwdgxaurr','ajjrzrbhqflsutnymgc'); + INSERT INTO aux.t1 VALUES(-146061,-377892,X'4E','gepvpvvuhszpxabbb'); + INSERT INTO aux.t1 VALUES(-851726,-161992,-469943,-159541); + INSERT INTO aux.t1 VALUES(X'4A6A667F858938',185083,X'7A',NULL); + INSERT INTO aux.t1 VALUES(-204877.54,X'1704C253D5F3AFA8',155120.88, 4); + INSERT INTO aux.t1 + VALUES('ckmqmzoeuvxisxqy',X'EB5A5D3A1DD22FD1','tidgtsplhjcbvbppdt',-642987.3); +} + +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(4, 5, 6); + INSERT INTO t2 VALUES(7, 8, 9); + + CREATE TABLE t3(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO t3 VALUES(1, 2, 3); + INSERT INTO t3 VALUES(4, 5, 6); + INSERT INTO t3 VALUES(7, 8, 9); +} +do_empty_diff_test 3.1 + + +#------------------------------------------------------------------------- +# Test some error cases: +# +# 1) schema mismatches between the two dbs, and +# 2) tables with no primary keys. This is not actually an error, but +# should not add any changes to the session object. +# +reset_db +forcedelete test.db2 +do_execsql_test 4.0 { + ATTACH 'test.db2' AS ixua; + CREATE TABLE ixua.t1(a, b, c); + CREATE TABLE main.t1(a, b, c); + INSERT INTO main.t1 VALUES(1, 2, 3); + + CREATE TABLE ixua.t2(a PRIMARY KEY, b, c); + CREATE TABLE main.t2(a PRIMARY KEY, b, x); +} + +do_test 4.1.1 { + sqlite3session S db main + S attach t1 + list [catch { S diff ixua t1 } msg] $msg +} {0 {}} +do_test 4.1.2 { + string length [S changeset] +} {0} +S delete + +do_test 4.2.2 { + sqlite3session S db main + S attach t2 + list [catch { S diff ixua t2 } msg] $msg +} {1 {SQLITE_SCHEMA - table schemas do not match}} +S delete + +finish_test + diff --git a/ext/session/sessionE.test b/ext/session/sessionE.test new file mode 100644 index 0000000000..27e530ec7f --- /dev/null +++ b/ext/session/sessionE.test @@ -0,0 +1,117 @@ +# 2015 June 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that operations on tables without primary keys +# are ignored. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionE + +# +# Test plan: +# +# 1.*: Test that non-PK tables are not auto-attached. +# 2.*: Test that explicitly attaching a non-PK table is a no-op. +# 3.*: Test that sqlite3session_diff() on a non-PK table is a no-op. +# + + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); +} +do_test 1.1 { + sqlite3session S db main + S attach * + breakpoint + execsql { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(1, 2); + } +} {} +do_changeset_test 1.2 S { + {INSERT t2 0 X. {} {i 1 i 2}} +} +S delete + +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); +} +do_test 2.1 { + sqlite3session S db main + S attach t1 + S attach t2 + breakpoint + execsql { + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES(5, 6); + } +} {} +do_changeset_test 2.2 S { + {INSERT t2 0 X. {} {i 3 i 4}} + {INSERT t2 0 X. {} {i 5 i 6}} +} +S delete + +reset_db +forcedelete test.db2 +do_execsql_test 3.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(a, b); + CREATE TABLE aux.t2(a PRIMARY KEY, b); + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a PRIMARY KEY, b); + + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t2 VALUES(3, 4); +} +do_test 3.1 { + sqlite3session S db main + S attach t1 + S diff aux t1 + + S attach t2 + S diff aux t2 +} {} +do_changeset_test 3.2 S { + {INSERT t2 0 X. {} {i 3 i 4}} +} +do_execsql_test 3.3 { + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES(7, 8); +} +do_changeset_test 3.4 S { + {INSERT t2 0 X. {} {i 3 i 4}} + {INSERT t2 0 X. {} {i 7 i 8}} +} + + +S delete + +finish_test + + diff --git a/ext/session/sessionF.test b/ext/session/sessionF.test new file mode 100644 index 0000000000..ba57bf85ec --- /dev/null +++ b/ext/session/sessionF.test @@ -0,0 +1,295 @@ +# 2015 June 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that tables appear in the correct order +# within changesets and patchsets. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionF + +# +# Test plan: +# +# 1.*: Test that sqlite3session_changeset() and sqlite3session_patchset() +# output tables in the right order. +# +# 2.*: Test that sqlite3session_invert() does not modify the order of +# tables within a changeset. +# +# 3.*: Test that sqlite3session_concat outputs tables in the right order. +# + +# Create a db schema to use. +# +do_execsql_test 1.0 { + CREATE TABLE t3(e PRIMARY KEY, f); + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(c PRIMARY KEY, d); +} + +#----------------------------------------------------------------------- +# 1.* - changeset() and patchset(). +# + +foreach {tn setup result} { + 1 { + S attach * + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t3 0 X. {} {i 3 t three}} + } + + 2 { + S attach t1 + S attach * + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t3 0 X. {} {i 3 t three}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {INSERT t3 0 X. {} {i 3 t three}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t1 0 X. {} {i 1 t one}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t3 VALUES(3, 'three'); + } + + do_changeset_test 1.1.$tn.2 S $result + do_patchset_test 1.1.$tn.3 S $result + + S delete +} + +foreach {tn setup result} { + 1 { + S attach * + } { + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t3 0 X. {} {i 6 t six}} + } + + 2 { + S attach t1 + S attach * + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t3 0 X. {} {i 6 t six}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {INSERT t3 0 X. {} {i 6 t six}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t2 0 X. {} {i 5 t five}} + {INSERT t1 0 X. {} {i 1 t one}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + DELETE FROM t2; + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t2 VALUES(5, 'five'); + INSERT INTO t3 VALUES(6, 'six'); + } + + do_changeset_test 1.2.$tn.2 S $result + do_patchset_test 1.2.$tn.2 S $result + + S delete +} + +#------------------------------------------------------------------------- +# 2.* - invert() +# + +foreach {tn setup result} { + 1 { + S attach * + } { + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t1 0 X. {i 1 t one} {}} + {DELETE t3 0 X. {i 6 t six} {}} + } + + 2 { + S attach t1 + S attach * + } { + {DELETE t1 0 X. {i 1 t one} {}} + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t3 0 X. {i 6 t six} {}} + } + + 3 { + S attach t3 + S attach t2 + S attach t1 + } { + {DELETE t3 0 X. {i 6 t six} {}} + {DELETE t2 0 X. {i 4 t four} {}} + {DELETE t2 0 X. {i 5 t five} {}} + {DELETE t1 0 X. {i 1 t one} {}} + } +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup + + do_execsql_test 1.$tn.1 { + INSERT INTO t2 VALUES(2, 'two'); + INSERT INTO t1 VALUES(1, 'one'); + DELETE FROM t2; + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t2 VALUES(5, 'five'); + INSERT INTO t3 VALUES(6, 'six'); + } + + do_changeset_invert_test 2.$tn.2 S $result + + S delete +} + +#------------------------------------------------------------------------- +# 3.* - concat() +# +foreach {tn setup1 sql1 setup2 sql2 result} { + 1 { + S attach * + } { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach t2 + S attach t1 + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + } { + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t1 0 X. {} {i 3 t three}} + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + } + + 1 { + S attach t2 + S attach t1 + } { + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach * + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t1 0 X. {} {i 1 t one}} + {INSERT t1 0 X. {} {i 3 t three}} + } + + 1 { + S attach * + } { + INSERT INTO t2 VALUES(2, 'two'); + } { + S attach * + } { + INSERT INTO t1 VALUES(3, 'three'); + INSERT INTO t2 VALUES(4, 'four'); + INSERT INTO t3 VALUES(5, 'five'); + } { + {INSERT t2 0 X. {} {i 2 t two}} + {INSERT t2 0 X. {} {i 4 t four}} + {INSERT t1 0 X. {} {i 3 t three}} + {INSERT t3 0 X. {} {i 5 t five}} + } + +} { + execsql { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + } + sqlite3session S db main + eval $setup1 + execsql $sql1 + set c1 [S changeset] + S delete + + sqlite3session S db main + eval $setup2 + execsql $sql2 + set c2 [S changeset] + S delete + + set res [list] + sqlite3session_foreach x [sqlite3changeset_concat $c1 $c2] { + lappend res $x + } + + do_test 3.$tn { set res } [list {*}$result] +} + + +finish_test + diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test new file mode 100644 index 0000000000..5c057350e4 --- /dev/null +++ b/ext/session/sessionG.test @@ -0,0 +1,177 @@ +# 2016 March 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for the sessions module. +# Specifically, it tests that UNIQUE constraints are dealt with correctly. +# + + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix sessionG + + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 1.0 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + } + do_then_apply_sql { + DELETE FROM t1 WHERE a=1; + INSERT INTO t1 VALUES(4, 'one'); + } + compare_db db db2 +} {} + +do_test 1.1 { + do_then_apply_sql { + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(1, 'one'); + } + compare_db db db2 +} {} + +do_test 1.2 { + execsql { INSERT INTO t1 VALUES(5, 'five') } db2 + do_then_apply_sql { + INSERT INTO t1 VALUES(11, 'eleven'); + INSERT INTO t1 VALUES(12, 'five'); + } + execsql { SELECT * FROM t1 } db2 +} {2 two 3 three 1 one 5 five 11 eleven} + +do_test 1.3 { + execsql { SELECT * FROM t1 } +} {2 two 3 three 1 one 11 eleven 12 five} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 2.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE, c UNIQUE); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } +} {} + +do_test 2.2.1 { + # It is not possible to apply the changeset generated by the following + # SQL, as none of the three updated rows may be updated as part of the + # first pass. + do_then_apply_sql { + UPDATE t1 SET b=0 WHERE a=1; + UPDATE t1 SET b=1 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=1; + } + db2 eval { SELECT a, b FROM t1 } +} {1 1 2 2 3 3} +do_test 2.2.2 { db eval { SELECT a, b FROM t1 } } {1 3 2 1 3 2} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 3.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE, c UNIQUE); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } +} {} + +do_test 3.3 { + do_then_apply_sql { + UPDATE t1 SET b=4 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=1; + } + compare_db db db2 +} {} + +do_test 3.4 { + do_then_apply_sql { + UPDATE t1 SET b=1 WHERE a=1; + UPDATE t1 SET b=2 WHERE a=2; + UPDATE t1 SET b=3 WHERE a=3; + } + compare_db db db2 +} {} + +#------------------------------------------------------------------------- +# +reset_db +db2 close +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_test 4.1 { + do_common_sql { + CREATE TABLE t1(a PRIMARY KEY, b UNIQUE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + + CREATE TABLE t2(a PRIMARY KEY, b UNIQUE); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); + } +} {} + +do_test 4.2 { + do_then_apply_sql { + UPDATE t1 SET b=4 WHERE a=3; + UPDATE t1 SET b=3 WHERE a=2; + UPDATE t1 SET b=2 WHERE a=1; + + UPDATE t2 SET b=0 WHERE a=1; + UPDATE t2 SET b=1 WHERE a=2; + UPDATE t2 SET b=2 WHERE a=3; + } + compare_db db db2 +} {} + +do_test 4.3 { + do_then_apply_sql { + UPDATE t1 SET b=1 WHERE a=1; + UPDATE t1 SET b=2 WHERE a=2; + UPDATE t1 SET b=3 WHERE a=3; + + UPDATE t2 SET b=3 WHERE a=3; + UPDATE t2 SET b=2 WHERE a=2; + UPDATE t2 SET b=1 WHERE a=1; + } + compare_db db db2 +} {} + +finish_test + diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl new file mode 100644 index 0000000000..d4804d924f --- /dev/null +++ b/ext/session/session_common.tcl @@ -0,0 +1,171 @@ + +proc do_changeset_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + sqlite3session_foreach c [$session changeset] { lappend x [set c] } + set x + }]] [list $r] +} + +proc do_patchset_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + sqlite3session_foreach c [$session patchset] { lappend x [set c] } + set x + }]] [list $r] +} + + +proc do_changeset_invert_test {tn session res} { + set r [list] + foreach x $res {lappend r $x} + uplevel do_test $tn [list [subst -nocommands { + set x [list] + set changeset [sqlite3changeset_invert [$session changeset]] + sqlite3session_foreach c [set changeset] { lappend x [set c] } + set x + }]] [list $r] +} + + +proc do_conflict_test {tn args} { + + set O(-tables) [list] + set O(-sql) [list] + set O(-conflicts) [list] + set O(-policy) "OMIT" + + array set V $args + foreach key [array names V] { + if {![info exists O($key)]} {error "no such option: $key"} + } + array set O $args + + proc xConflict {args} [subst -nocommands { + lappend ::xConflict [set args] + return $O(-policy) + }] + proc bgerror {args} { set ::background_error $args } + + sqlite3session S db main + foreach t $O(-tables) { S attach $t } + execsql $O(-sql) + + set ::xConflict [list] + sqlite3changeset_apply db2 [S changeset] xConflict + + set conflicts [list] + foreach c $O(-conflicts) { + lappend conflicts $c + } + + after 1 {set go 1} + vwait go + + uplevel do_test $tn [list { set ::xConflict }] [list $conflicts] + S delete +} + +proc do_common_sql {sql} { + execsql $sql db + execsql $sql db2 +} + +proc changeset_from_sql {sql {dbname main}} { + if {$dbname == "main"} { + return [sql_exec_changeset db $sql] + } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + db eval $sql + S changeset + } changeset] + catch { S delete } + + if {$rc} { + error $changeset + } + return $changeset +} + +proc do_then_apply_sql {sql {dbname main}} { + proc xConflict args { return "OMIT" } + set rc [catch { + sqlite3session S db $dbname + db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { + S attach $name + } + db eval $sql + sqlite3changeset_apply db2 [S changeset] xConflict + } msg] + + catch { S delete } + + if {$rc} {error $msg} +} + +proc do_iterator_test {tn tbl_list sql res} { + sqlite3session S db main + if {[llength $tbl_list]==0} { S attach * } + foreach t $tbl_list {S attach $t} + + execsql $sql + + set r [list] + foreach v $res { lappend r $v } + + set x [list] + sqlite3session_foreach c [S changeset] { lappend x $c } + uplevel do_test $tn [list [list set {} $x]] [list $r] + + S delete +} + +# Compare the contents of all tables in [db1] and [db2]. Throw an error if +# they are not identical, or return an empty string if they are. +# +proc compare_db {db1 db2} { + + set sql {SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name} + set lot1 [$db1 eval $sql] + set lot2 [$db2 eval $sql] + + if {$lot1 != $lot2} { + puts $lot1 + puts $lot2 + error "databases contain different tables" + } + + foreach tbl $lot1 { + set col1 [list] + set col2 [list] + + $db1 eval "PRAGMA table_info = $tbl" { lappend col1 $name } + $db2 eval "PRAGMA table_info = $tbl" { lappend col2 $name } + if {$col1 != $col2} { error "table $tbl schema mismatch" } + + set sql "SELECT * FROM $tbl ORDER BY [join $col1 ,]" + set data1 [$db1 eval $sql] + set data2 [$db2 eval $sql] + if {$data1 != $data2} { + puts "$data1" + puts "$data2" + error "table $tbl data mismatch" + } + } + + return "" +} + +proc changeset_to_list {c} { + set list [list] + sqlite3session_foreach elem $c { lappend list $elem } + lsort $list +} diff --git a/ext/session/session_speed_test.c b/ext/session/session_speed_test.c new file mode 100644 index 0000000000..14b97ea07b --- /dev/null +++ b/ext/session/session_speed_test.c @@ -0,0 +1,360 @@ +/* +** 2017 January 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the source code for a standalone program used to +** test the performance of the sessions module. Compile and run: +** +** ./session_speed_test -help +** +** for details. +*/ + +#include "sqlite3.h" +#include +#include +#include +#include +#include + +/************************************************************************* +** Start of generic command line parser. +*/ +#define CMDLINE_BARE 0 +#define CMDLINE_INTEGER 1 +#define CMDLINE_STRING 2 +#define CMDLINE_BOOLEAN 3 + +typedef struct CmdLineOption CmdLineOption; +struct CmdLineOption { + const char *zText; /* Name of command line option */ + const char *zHelp; /* Help text for option */ + int eType; /* One of the CMDLINE_* values */ + int iOff; /* Offset of output variable */ +}; + +#define CMDLINE_INT32(x,y,z) {x, y, CMDLINE_INTEGER, z} +#define CMDLINE_BOOL(x,y,z) {x, y, CMDLINE_BOOLEAN, z} +#define CMDLINE_TEXT(x,y,z) {x, y, CMDLINE_STRING, z} +#define CMDLINE_NONE(x,y,z) {x, y, CMDLINE_BARE, z} + +static void option_requires_argument_error(CmdLineOption *pOpt){ + fprintf(stderr, "Option requires a%s argument: %s\n", + pOpt->eType==CMDLINE_INTEGER ? "n integer" : + pOpt->eType==CMDLINE_STRING ? " string" : " boolean", + pOpt->zText + ); + exit(1); +} + +static void ambiguous_option_error(const char *zArg){ + fprintf(stderr, "Option is ambiguous: %s\n", zArg); + exit(1); +} + +static void unknown_option_error( + const char *zArg, + CmdLineOption *aOpt, + const char *zHelp +){ + int i; + fprintf(stderr, "Unknown option: %s\n", zArg); + fprintf(stderr, "\nOptions are:\n"); + fprintf(stderr, " % -30sEcho command line options\n", "-cmdline:verbose"); + for(i=0; aOpt[i].zText; i++){ + int eType = aOpt[i].eType; + char *zOpt = sqlite3_mprintf("%s %s", aOpt[i].zText, + eType==CMDLINE_BARE ? "" : + eType==CMDLINE_INTEGER ? "N" : + eType==CMDLINE_BOOLEAN ? "BOOLEAN" : "TEXT" + ); + fprintf(stderr, " % -30s%s\n", zOpt, aOpt[i].zHelp); + sqlite3_free(zOpt); + } + if( zHelp ){ + fprintf(stderr, "\n%s\n", zHelp); + } + exit(1); +} + +static int get_integer_option(CmdLineOption *pOpt, const char *zArg){ + int i = 0; + int iRet = 0; + int bSign = 1; + if( zArg[0]=='-' ){ + bSign = -1; + i = 1; + } + while( zArg[i] ){ + if( zArg[i]<'0' || zArg[i]>'9' ) option_requires_argument_error(pOpt); + iRet = iRet*10 + (zArg[i] - '0'); + i++; + } + return (iRet*bSign); +} + +static int get_boolean_option(CmdLineOption *pOpt, const char *zArg){ + if( 0==sqlite3_stricmp(zArg, "true") ) return 1; + if( 0==sqlite3_stricmp(zArg, "1") ) return 1; + if( 0==sqlite3_stricmp(zArg, "0") ) return 0; + if( 0==sqlite3_stricmp(zArg, "false") ) return 0; + option_requires_argument_error(pOpt); + return 0; +} + +static void parse_command_line( + int argc, + char **argv, + int iStart, + CmdLineOption *aOpt, + void *pStruct, + const char *zHelp +){ + char *pOut = (char*)pStruct; + int bVerbose = 0; + int iArg; + + for(iArg=iStart; iArgzText, zArg, nArg) ){ + if( nMatch ){ + ambiguous_option_error(zArg); + } + nMatch++; + if( pOpt->eType==CMDLINE_BARE ){ + *(int*)(&pOut[pOpt->iOff]) = 1; + }else{ + iArg++; + if( iArg==argc ){ + option_requires_argument_error(pOpt); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + *(int*)(&pOut[pOpt->iOff]) = get_integer_option(pOpt, argv[iArg]); + break; + case CMDLINE_STRING: + *(const char**)(&pOut[pOpt->iOff]) = argv[iArg]; + break; + case CMDLINE_BOOLEAN: + *(int*)(&pOut[pOpt->iOff]) = get_boolean_option(pOpt, argv[iArg]); + break; + } + } + } + } + + if( nMatch==0 && 0==sqlite3_strnicmp("-cmdline:verbose", zArg, nArg) ){ + bVerbose = 1; + nMatch = 1; + } + + if( nMatch==0 ){ + unknown_option_error(zArg, aOpt, zHelp); + } + } + + if( bVerbose ){ + int iOpt; + fprintf(stdout, "Options are: "); + for(iOpt=0; aOpt[iOpt].zText; iOpt++){ + CmdLineOption *pOpt = &aOpt[iOpt]; + if( pOpt->eType!=CMDLINE_BARE || *(int*)(&pOut[pOpt->iOff]) ){ + fprintf(stdout, "%s ", pOpt->zText); + } + switch( pOpt->eType ){ + case CMDLINE_INTEGER: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_BOOLEAN: + fprintf(stdout, "%d ", *(int*)(&pOut[pOpt->iOff])); + break; + case CMDLINE_STRING: + fprintf(stdout, "%s ", *(const char**)(&pOut[pOpt->iOff])); + break; + } + } + fprintf(stdout, "\n"); + } +} +/* +** End of generic command line parser. +*************************************************************************/ + +static void abort_due_to_error(int rc){ + fprintf(stderr, "Error: %d\n"); + exit(-1); +} + +static void execsql(sqlite3 *db, const char *zSql){ + int rc = sqlite3_exec(db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); +} + +static int xConflict(void *pCtx, int eConflict, sqlite3_changeset_iter *p){ + return SQLITE_CHANGESET_ABORT; +} + +static void run_test( + sqlite3 *db, + sqlite3 *db2, + int nRow, + const char *zSql +){ + sqlite3_session *pSession = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int i; + int nChangeset; + void *pChangeset; + + /* Attach a session object to database db */ + rc = sqlite3session_create(db, "main", &pSession); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Configure the session to capture changes on all tables */ + rc = sqlite3session_attach(pSession, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Prepare the SQL statement */ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) abort_due_to_error(rc); + + /* Open a transaction */ + execsql(db, "BEGIN"); + + /* Execute the SQL statement nRow times */ + for(i=0; i +#include + +#ifndef SQLITE_AMALGAMATION +# include "sqliteInt.h" +# include "vdbeInt.h" +#endif + +typedef struct SessionTable SessionTable; +typedef struct SessionChange SessionChange; +typedef struct SessionBuffer SessionBuffer; +typedef struct SessionInput SessionInput; + +/* +** Minimum chunk size used by streaming versions of functions. +*/ +#ifndef SESSIONS_STRM_CHUNK_SIZE +# ifdef SQLITE_TEST +# define SESSIONS_STRM_CHUNK_SIZE 64 +# else +# define SESSIONS_STRM_CHUNK_SIZE 1024 +# endif +#endif + +typedef struct SessionHook SessionHook; +struct SessionHook { + void *pCtx; + int (*xOld)(void*,int,sqlite3_value**); + int (*xNew)(void*,int,sqlite3_value**); + int (*xCount)(void*); + int (*xDepth)(void*); +}; + +/* +** Session handle structure. +*/ +struct sqlite3_session { + sqlite3 *db; /* Database handle session is attached to */ + char *zDb; /* Name of database session is attached to */ + int bEnable; /* True if currently recording */ + int bIndirect; /* True if all changes are indirect */ + int bAutoAttach; /* True to auto-attach tables */ + int rc; /* Non-zero if an error has occurred */ + void *pFilterCtx; /* First argument to pass to xTableFilter */ + int (*xTableFilter)(void *pCtx, const char *zTab); + sqlite3_session *pNext; /* Next session object on same db. */ + SessionTable *pTable; /* List of attached tables */ + SessionHook hook; /* APIs to grab new and old data with */ +}; + +/* +** Instances of this structure are used to build strings or binary records. +*/ +struct SessionBuffer { + u8 *aBuf; /* Pointer to changeset buffer */ + int nBuf; /* Size of buffer aBuf */ + int nAlloc; /* Size of allocation containing aBuf */ +}; + +/* +** An object of this type is used internally as an abstraction for +** input data. Input data may be supplied either as a single large buffer +** (e.g. sqlite3changeset_start()) or using a stream function (e.g. +** sqlite3changeset_start_strm()). +*/ +struct SessionInput { + int bNoDiscard; /* If true, discard no data */ + int iCurrent; /* Offset in aData[] of current change */ + int iNext; /* Offset in aData[] of next change */ + u8 *aData; /* Pointer to buffer containing changeset */ + int nData; /* Number of bytes in aData */ + + SessionBuffer buf; /* Current read buffer */ + int (*xInput)(void*, void*, int*); /* Input stream call (or NULL) */ + void *pIn; /* First argument to xInput */ + int bEof; /* Set to true after xInput finished */ +}; + +/* +** Structure for changeset iterators. +*/ +struct sqlite3_changeset_iter { + SessionInput in; /* Input buffer or stream */ + SessionBuffer tblhdr; /* Buffer to hold apValue/zTab/abPK/ */ + int bPatchset; /* True if this is a patchset */ + int rc; /* Iterator error code */ + sqlite3_stmt *pConflict; /* Points to conflicting row, if any */ + char *zTab; /* Current table */ + int nCol; /* Number of columns in zTab */ + int op; /* Current operation */ + int bIndirect; /* True if current change was indirect */ + u8 *abPK; /* Primary key array */ + sqlite3_value **apValue; /* old.* and new.* values */ +}; + +/* +** Each session object maintains a set of the following structures, one +** for each table the session object is monitoring. The structures are +** stored in a linked list starting at sqlite3_session.pTable. +** +** The keys of the SessionTable.aChange[] hash table are all rows that have +** been modified in any way since the session object was attached to the +** table. +** +** The data associated with each hash-table entry is a structure containing +** a subset of the initial values that the modified row contained at the +** start of the session. Or no initial values if the row was inserted. +*/ +struct SessionTable { + SessionTable *pNext; + char *zName; /* Local name of table */ + int nCol; /* Number of columns in table zName */ + const char **azCol; /* Column names */ + u8 *abPK; /* Array of primary key flags */ + int nEntry; /* Total number of entries in hash table */ + int nChange; /* Size of apChange[] array */ + SessionChange **apChange; /* Hash table buckets */ +}; + +/* +** RECORD FORMAT: +** +** The following record format is similar to (but not compatible with) that +** used in SQLite database files. This format is used as part of the +** change-set binary format, and so must be architecture independent. +** +** Unlike the SQLite database record format, each field is self-contained - +** there is no separation of header and data. Each field begins with a +** single byte describing its type, as follows: +** +** 0x00: Undefined value. +** 0x01: Integer value. +** 0x02: Real value. +** 0x03: Text value. +** 0x04: Blob value. +** 0x05: SQL NULL value. +** +** Note that the above match the definitions of SQLITE_INTEGER, SQLITE_TEXT +** and so on in sqlite3.h. For undefined and NULL values, the field consists +** only of the single type byte. For other types of values, the type byte +** is followed by: +** +** Text values: +** A varint containing the number of bytes in the value (encoded using +** UTF-8). Followed by a buffer containing the UTF-8 representation +** of the text value. There is no nul terminator. +** +** Blob values: +** A varint containing the number of bytes in the value, followed by +** a buffer containing the value itself. +** +** Integer values: +** An 8-byte big-endian integer value. +** +** Real values: +** An 8-byte big-endian IEEE 754-2008 real value. +** +** Varint values are encoded in the same way as varints in the SQLite +** record format. +** +** CHANGESET FORMAT: +** +** A changeset is a collection of DELETE, UPDATE and INSERT operations on +** one or more tables. Operations on a single table are grouped together, +** but may occur in any order (i.e. deletes, updates and inserts are all +** mixed together). +** +** Each group of changes begins with a table header: +** +** 1 byte: Constant 0x54 (capital 'T') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more changes to the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), UPDATE (0x17) or DELETE (0x09). +** 1 byte: The "indirect-change" flag. +** old.* record: (delete and update only) +** new.* record: (insert and update only) +** +** The "old.*" and "new.*" records, if present, are N field records in the +** format described above under "RECORD FORMAT", where N is the number of +** columns in the table. The i'th field of each record is associated with +** the i'th column of the table, counting from left to right in the order +** in which columns were declared in the CREATE TABLE statement. +** +** The new.* record that is part of each INSERT change contains the values +** that make up the new row. Similarly, the old.* record that is part of each +** DELETE change contains the values that made up the row that was deleted +** from the database. In the changeset format, the records that are part +** of INSERT or DELETE changes never contain any undefined (type byte 0x00) +** fields. +** +** Within the old.* record associated with an UPDATE change, all fields +** associated with table columns that are not PRIMARY KEY columns and are +** not modified by the UPDATE change are set to "undefined". Other fields +** are set to the values that made up the row before the UPDATE that the +** change records took place. Within the new.* record, fields associated +** with table columns modified by the UPDATE change contain the new +** values. Fields associated with table columns that are not modified +** are set to "undefined". +** +** PATCHSET FORMAT: +** +** A patchset is also a collection of changes. It is similar to a changeset, +** but leaves undefined those fields that are not useful if no conflict +** resolution is required when applying the changeset. +** +** Each group of changes begins with a table header: +** +** 1 byte: Constant 0x50 (capital 'P') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more changes to the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), UPDATE (0x17) or DELETE (0x09). +** 1 byte: The "indirect-change" flag. +** single record: (PK fields for DELETE, PK and modified fields for UPDATE, +** full record for INSERT). +** +** As in the changeset format, each field of the single record that is part +** of a patchset change is associated with the correspondingly positioned +** table column, counting from left to right within the CREATE TABLE +** statement. +** +** For a DELETE change, all fields within the record except those associated +** with PRIMARY KEY columns are set to "undefined". The PRIMARY KEY fields +** contain the values identifying the row to delete. +** +** For an UPDATE change, all fields except those associated with PRIMARY KEY +** columns and columns that are modified by the UPDATE are set to "undefined". +** PRIMARY KEY fields contain the values identifying the table row to update, +** and fields associated with modified columns contain the new column values. +** +** The records associated with INSERT changes are in the same format as for +** changesets. It is not possible for a record associated with an INSERT +** change to contain a field set to "undefined". +*/ + +/* +** For each row modified during a session, there exists a single instance of +** this structure stored in a SessionTable.aChange[] hash table. +*/ +struct SessionChange { + int op; /* One of UPDATE, DELETE, INSERT */ + int bIndirect; /* True if this change is "indirect" */ + int nRecord; /* Number of bytes in buffer aRecord[] */ + u8 *aRecord; /* Buffer containing old.* record */ + SessionChange *pNext; /* For hash-table collisions */ +}; + +/* +** Write a varint with value iVal into the buffer at aBuf. Return the +** number of bytes written. +*/ +static int sessionVarintPut(u8 *aBuf, int iVal){ + return putVarint32(aBuf, iVal); +} + +/* +** Return the number of bytes required to store value iVal as a varint. +*/ +static int sessionVarintLen(int iVal){ + return sqlite3VarintLen(iVal); +} + +/* +** Read a varint value from aBuf[] into *piVal. Return the number of +** bytes read. +*/ +static int sessionVarintGet(u8 *aBuf, int *piVal){ + return getVarint32(aBuf, *piVal); +} + +/* Load an unaligned and unsigned 32-bit integer */ +#define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + +/* +** Read a 64-bit big-endian integer value from buffer aRec[]. Return +** the value read. +*/ +static sqlite3_int64 sessionGetI64(u8 *aRec){ + u64 x = SESSION_UINT32(aRec); + u32 y = SESSION_UINT32(aRec+4); + x = (x<<32) + y; + return (sqlite3_int64)x; +} + +/* +** Write a 64-bit big-endian integer value to the buffer aBuf[]. +*/ +static void sessionPutI64(u8 *aBuf, sqlite3_int64 i){ + aBuf[0] = (i>>56) & 0xFF; + aBuf[1] = (i>>48) & 0xFF; + aBuf[2] = (i>>40) & 0xFF; + aBuf[3] = (i>>32) & 0xFF; + aBuf[4] = (i>>24) & 0xFF; + aBuf[5] = (i>>16) & 0xFF; + aBuf[6] = (i>> 8) & 0xFF; + aBuf[7] = (i>> 0) & 0xFF; +} + +/* +** This function is used to serialize the contents of value pValue (see +** comment titled "RECORD FORMAT" above). +** +** If it is non-NULL, the serialized form of the value is written to +** buffer aBuf. *pnWrite is set to the number of bytes written before +** returning. Or, if aBuf is NULL, the only thing this function does is +** set *pnWrite. +** +** If no error occurs, SQLITE_OK is returned. Or, if an OOM error occurs +** within a call to sqlite3_value_text() (may fail if the db is utf-16)) +** SQLITE_NOMEM is returned. +*/ +static int sessionSerializeValue( + u8 *aBuf, /* If non-NULL, write serialized value here */ + sqlite3_value *pValue, /* Value to serialize */ + int *pnWrite /* IN/OUT: Increment by bytes written */ +){ + int nByte; /* Size of serialized value in bytes */ + + if( pValue ){ + int eType; /* Value type (SQLITE_NULL, TEXT etc.) */ + + eType = sqlite3_value_type(pValue); + if( aBuf ) aBuf[0] = eType; + + switch( eType ){ + case SQLITE_NULL: + nByte = 1; + break; + + case SQLITE_INTEGER: + case SQLITE_FLOAT: + if( aBuf ){ + /* TODO: SQLite does something special to deal with mixed-endian + ** floating point values (e.g. ARM7). This code probably should + ** too. */ + u64 i; + if( eType==SQLITE_INTEGER ){ + i = (u64)sqlite3_value_int64(pValue); + }else{ + double r; + assert( sizeof(double)==8 && sizeof(u64)==8 ); + r = sqlite3_value_double(pValue); + memcpy(&i, &r, 8); + } + sessionPutI64(&aBuf[1], i); + } + nByte = 9; + break; + + default: { + u8 *z; + int n; + int nVarint; + + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + if( eType==SQLITE_TEXT ){ + z = (u8 *)sqlite3_value_text(pValue); + }else{ + z = (u8 *)sqlite3_value_blob(pValue); + } + n = sqlite3_value_bytes(pValue); + if( z==0 && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + nVarint = sessionVarintLen(n); + + if( aBuf ){ + sessionVarintPut(&aBuf[1], n); + if( n ) memcpy(&aBuf[nVarint + 1], z, n); + } + + nByte = 1 + nVarint + n; + break; + } + } + }else{ + nByte = 1; + if( aBuf ) aBuf[0] = '\0'; + } + + if( pnWrite ) *pnWrite += nByte; + return SQLITE_OK; +} + + +/* +** This macro is used to calculate hash key values for data structures. In +** order to use this macro, the entire data structure must be represented +** as a series of unsigned integers. In order to calculate a hash-key value +** for a data structure represented as three such integers, the macro may +** then be used as follows: +** +** int hash_key_value; +** hash_key_value = HASH_APPEND(0, ); +** hash_key_value = HASH_APPEND(hash_key_value, ); +** hash_key_value = HASH_APPEND(hash_key_value, ); +** +** In practice, the data structures this macro is used for are the primary +** key values of modified rows. +*/ +#define HASH_APPEND(hash, add) ((hash) << 3) ^ (hash) ^ (unsigned int)(add) + +/* +** Append the hash of the 64-bit integer passed as the second argument to the +** hash-key value passed as the first. Return the new hash-key value. +*/ +static unsigned int sessionHashAppendI64(unsigned int h, i64 i){ + h = HASH_APPEND(h, i & 0xFFFFFFFF); + return HASH_APPEND(h, (i>>32)&0xFFFFFFFF); +} + +/* +** Append the hash of the blob passed via the second and third arguments to +** the hash-key value passed as the first. Return the new hash-key value. +*/ +static unsigned int sessionHashAppendBlob(unsigned int h, int n, const u8 *z){ + int i; + for(i=0; inCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; + + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + }else{ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + } + if( rc!=SQLITE_OK ) return rc; + + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); + }else{ + assert( eType==SQLITE_NULL ); + *pbNullPK = 1; + } + } + } + + *piHash = (h % pTab->nChange); + return SQLITE_OK; +} + +/* +** The buffer that the argument points to contains a serialized SQL value. +** Return the number of bytes of space occupied by the value (including +** the type byte). +*/ +static int sessionSerialLen(u8 *a){ + int e = *a; + int n; + if( e==0 ) return 1; + if( e==SQLITE_NULL ) return 1; + if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; + return sessionVarintGet(&a[1], &n) + 1 + n; +} + +/* +** Based on the primary key values stored in change aRecord, calculate a +** hash key. Assume the has table has nBucket buckets. The hash keys +** calculated by this function are compatible with those calculated by +** sessionPreupdateHash(). +** +** The bPkOnly argument is non-zero if the record at aRecord[] is from +** a patchset DELETE. In this case the non-PK fields are omitted entirely. +*/ +static unsigned int sessionChangeHash( + SessionTable *pTab, /* Table handle */ + int bPkOnly, /* Record consists of PK fields only */ + u8 *aRecord, /* Change record */ + int nBucket /* Assume this many buckets in hash table */ +){ + unsigned int h = 0; /* Value to return */ + int i; /* Used to iterate through columns */ + u8 *a = aRecord; /* Used to iterate through change record */ + + for(i=0; inCol; i++){ + int eType = *a; + int isPK = pTab->abPK[i]; + if( bPkOnly && isPK==0 ) continue; + + /* It is not possible for eType to be SQLITE_NULL here. The session + ** module does not record changes for rows with NULL values stored in + ** primary key columns. */ + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_TEXT || eType==SQLITE_BLOB + || eType==SQLITE_NULL || eType==0 + ); + assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); + + if( isPK ){ + a++; + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + h = sessionHashAppendI64(h, sessionGetI64(a)); + a += 8; + }else{ + int n; + a += sessionVarintGet(a, &n); + h = sessionHashAppendBlob(h, n, a); + a += n; + } + }else{ + a += sessionSerialLen(a); + } + } + return (h % nBucket); +} + +/* +** Arguments aLeft and aRight are pointers to change records for table pTab. +** This function returns true if the two records apply to the same row (i.e. +** have the same values stored in the primary key columns), or false +** otherwise. +*/ +static int sessionChangeEqual( + SessionTable *pTab, /* Table used for PK definition */ + int bLeftPkOnly, /* True if aLeft[] contains PK fields only */ + u8 *aLeft, /* Change record */ + int bRightPkOnly, /* True if aRight[] contains PK fields only */ + u8 *aRight /* Change record */ +){ + u8 *a1 = aLeft; /* Cursor to iterate through aLeft */ + u8 *a2 = aRight; /* Cursor to iterate through aRight */ + int iCol; /* Used to iterate through table columns */ + + for(iCol=0; iColnCol; iCol++){ + if( pTab->abPK[iCol] ){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); + + if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ + return 0; + } + a1 += n1; + a2 += n2; + }else{ + if( bLeftPkOnly==0 ) a1 += sessionSerialLen(a1); + if( bRightPkOnly==0 ) a2 += sessionSerialLen(a2); + } + } + + return 1; +} + +/* +** Arguments aLeft and aRight both point to buffers containing change +** records with nCol columns. This function "merges" the two records into +** a single records which is written to the buffer at *paOut. *paOut is +** then set to point to one byte after the last byte written before +** returning. +** +** The merging of records is done as follows: For each column, if the +** aRight record contains a value for the column, copy the value from +** their. Otherwise, if aLeft contains a value, copy it. If neither +** record contains a value for a given column, then neither does the +** output record. +*/ +static void sessionMergeRecord( + u8 **paOut, + int nCol, + u8 *aLeft, + u8 *aRight +){ + u8 *a1 = aLeft; /* Cursor used to iterate through aLeft */ + u8 *a2 = aRight; /* Cursor used to iterate through aRight */ + u8 *aOut = *paOut; /* Output cursor */ + int iCol; /* Used to iterate from 0 to nCol */ + + for(iCol=0; iColnCol; i++){ + int nOld; + u8 *aOld; + int nNew; + u8 *aNew; + + aOld = sessionMergeValue(&aOld1, &aOld2, &nOld); + aNew = sessionMergeValue(&aNew1, &aNew2, &nNew); + if( pTab->abPK[i] || nOld!=nNew || memcmp(aOld, aNew, nNew) ){ + if( pTab->abPK[i]==0 ) bRequired = 1; + memcpy(aOut, aOld, nOld); + aOut += nOld; + }else{ + *(aOut++) = '\0'; + } + } + + if( !bRequired ) return 0; + } + + /* Write the new.* vector */ + aOld1 = aOldRecord1; + aOld2 = aOldRecord2; + aNew1 = aNewRecord1; + aNew2 = aNewRecord2; + for(i=0; inCol; i++){ + int nOld; + u8 *aOld; + int nNew; + u8 *aNew; + + aOld = sessionMergeValue(&aOld1, &aOld2, &nOld); + aNew = sessionMergeValue(&aNew1, &aNew2, &nNew); + if( bPatchset==0 + && (pTab->abPK[i] || (nOld==nNew && 0==memcmp(aOld, aNew, nNew))) + ){ + *(aOut++) = '\0'; + }else{ + memcpy(aOut, aNew, nNew); + aOut += nNew; + } + } + + *paOut = aOut; + return 1; +} + +/* +** This function is only called from within a pre-update-hook callback. +** It determines if the current pre-update-hook change affects the same row +** as the change stored in argument pChange. If so, it returns true. Otherwise +** if the pre-update-hook does not affect the same row as pChange, it returns +** false. +*/ +static int sessionPreupdateEqual( + sqlite3_session *pSession, /* Session object that owns SessionTable */ + SessionTable *pTab, /* Table associated with change */ + SessionChange *pChange, /* Change to compare to */ + int op /* Current pre-update operation */ +){ + int iCol; /* Used to iterate through columns */ + u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); + for(iCol=0; iColnCol; iCol++){ + if( !pTab->abPK[iCol] ){ + a += sessionSerialLen(a); + }else{ + sqlite3_value *pVal; /* Value returned by preupdate_new/old */ + int rc; /* Error code from preupdate_new/old */ + int eType = *a++; /* Type of value from change record */ + + /* The following calls to preupdate_new() and preupdate_old() can not + ** fail. This is because they cache their return values, and by the + ** time control flows to here they have already been called once from + ** within sessionPreupdateHash(). The first two asserts below verify + ** this (that the method has already been called). */ + if( op==SQLITE_INSERT ){ + /* assert( db->pPreUpdate->pNewUnpacked || db->pPreUpdate->aNew ); */ + rc = pSession->hook.xNew(pSession->hook.pCtx, iCol, &pVal); + }else{ + /* assert( db->pPreUpdate->pUnpacked ); */ + rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); + } + assert( rc==SQLITE_OK ); + if( sqlite3_value_type(pVal)!=eType ) return 0; + + /* A SessionChange object never has a NULL value in a PK column */ + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_BLOB || eType==SQLITE_TEXT + ); + + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal = sessionGetI64(a); + a += 8; + if( eType==SQLITE_INTEGER ){ + if( sqlite3_value_int64(pVal)!=iVal ) return 0; + }else{ + double rVal; + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&rVal, &iVal, 8); + if( sqlite3_value_double(pVal)!=rVal ) return 0; + } + }else{ + int n; + const u8 *z; + a += sessionVarintGet(a, &n); + if( sqlite3_value_bytes(pVal)!=n ) return 0; + if( eType==SQLITE_TEXT ){ + z = sqlite3_value_text(pVal); + }else{ + z = sqlite3_value_blob(pVal); + } + if( memcmp(a, z, n) ) return 0; + a += n; + break; + } + } + } + + return 1; +} + +/* +** If required, grow the hash table used to store changes on table pTab +** (part of the session pSession). If a fatal OOM error occurs, set the +** session object to failed and return SQLITE_ERROR. Otherwise, return +** SQLITE_OK. +** +** It is possible that a non-fatal OOM error occurs in this function. In +** that case the hash-table does not grow, but SQLITE_OK is returned anyway. +** Growing the hash table in this case is a performance optimization only, +** it is not required for correct operation. +*/ +static int sessionGrowHash(int bPatchset, SessionTable *pTab){ + if( pTab->nChange==0 || pTab->nEntry>=(pTab->nChange/2) ){ + int i; + SessionChange **apNew; + int nNew = (pTab->nChange ? pTab->nChange : 128) * 2; + + apNew = (SessionChange **)sqlite3_malloc(sizeof(SessionChange *) * nNew); + if( apNew==0 ){ + if( pTab->nChange==0 ){ + return SQLITE_ERROR; + } + return SQLITE_OK; + } + memset(apNew, 0, sizeof(SessionChange *) * nNew); + + for(i=0; inChange; i++){ + SessionChange *p; + SessionChange *pNext; + for(p=pTab->apChange[i]; p; p=pNext){ + int bPkOnly = (p->op==SQLITE_DELETE && bPatchset); + int iHash = sessionChangeHash(pTab, bPkOnly, p->aRecord, nNew); + pNext = p->pNext; + p->pNext = apNew[iHash]; + apNew[iHash] = p; + } + } + + sqlite3_free(pTab->apChange); + pTab->nChange = nNew; + pTab->apChange = apNew; + } + + return SQLITE_OK; +} + +/* +** This function queries the database for the names of the columns of table +** zThis, in schema zDb. It is expected that the table has nCol columns. If +** not, SQLITE_SCHEMA is returned and none of the output variables are +** populated. +** +** Otherwise, if they are not NULL, variable *pnCol is set to the number +** of columns in the database table and variable *pzTab is set to point to a +** nul-terminated copy of the table name. *pazCol (if not NULL) is set to +** point to an array of pointers to column names. And *pabPK (again, if not +** NULL) is set to point to an array of booleans - true if the corresponding +** column is part of the primary key. +** +** For example, if the table is declared as: +** +** CREATE TABLE tbl1(w, x, y, z, PRIMARY KEY(w, z)); +** +** Then the four output variables are populated as follows: +** +** *pnCol = 4 +** *pzTab = "tbl1" +** *pazCol = {"w", "x", "y", "z"} +** *pabPK = {1, 0, 0, 1} +** +** All returned buffers are part of the same single allocation, which must +** be freed using sqlite3_free() by the caller. If pazCol was not NULL, then +** pointer *pazCol should be freed to release all memory. Otherwise, pointer +** *pabPK. It is illegal for both pazCol and pabPK to be NULL. +*/ +static int sessionTableInfo( + sqlite3 *db, /* Database connection */ + const char *zDb, /* Name of attached database (e.g. "main") */ + const char *zThis, /* Table name */ + int *pnCol, /* OUT: number of columns */ + const char **pzTab, /* OUT: Copy of zThis */ + const char ***pazCol, /* OUT: Array of column names for table */ + u8 **pabPK /* OUT: Array of booleans - true for PK col */ +){ + char *zPragma; + sqlite3_stmt *pStmt; + int rc; + int nByte; + int nDbCol = 0; + int nThis; + int i; + u8 *pAlloc = 0; + char **azCol = 0; + u8 *abPK = 0; + + assert( pazCol && pabPK ); + + nThis = sqlite3Strlen30(zThis); + zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); + if( !zPragma ) return SQLITE_NOMEM; + + rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0); + sqlite3_free(zPragma); + if( rc!=SQLITE_OK ) return rc; + + nByte = nThis + 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nByte += sqlite3_column_bytes(pStmt, 1); + nDbCol++; + } + rc = sqlite3_reset(pStmt); + + if( rc==SQLITE_OK ){ + nByte += nDbCol * (sizeof(const char *) + sizeof(u8) + 1); + pAlloc = sqlite3_malloc(nByte); + if( pAlloc==0 ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + azCol = (char **)pAlloc; + pAlloc = (u8 *)&azCol[nDbCol]; + abPK = (u8 *)pAlloc; + pAlloc = &abPK[nDbCol]; + if( pzTab ){ + memcpy(pAlloc, zThis, nThis+1); + *pzTab = (char *)pAlloc; + pAlloc += nThis+1; + } + + i = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nName = sqlite3_column_bytes(pStmt, 1); + const unsigned char *zName = sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; + memcpy(pAlloc, zName, nName+1); + azCol[i] = (char *)pAlloc; + pAlloc += nName+1; + abPK[i] = sqlite3_column_int(pStmt, 5); + i++; + } + rc = sqlite3_reset(pStmt); + + } + + /* If successful, populate the output variables. Otherwise, zero them and + ** free any allocation made. An error code will be returned in this case. + */ + if( rc==SQLITE_OK ){ + *pazCol = (const char **)azCol; + *pabPK = abPK; + *pnCol = nDbCol; + }else{ + *pazCol = 0; + *pabPK = 0; + *pnCol = 0; + if( pzTab ) *pzTab = 0; + sqlite3_free(azCol); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** This function is only called from within a pre-update handler for a +** write to table pTab, part of session pSession. If this is the first +** write to this table, initalize the SessionTable.nCol, azCol[] and +** abPK[] arrays accordingly. +** +** If an error occurs, an error code is stored in sqlite3_session.rc and +** non-zero returned. Or, if no error occurs but the table has no primary +** key, sqlite3_session.rc is left set to SQLITE_OK and non-zero returned to +** indicate that updates on this table should be ignored. SessionTable.abPK +** is set to NULL in this case. +*/ +static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ + if( pTab->nCol==0 ){ + u8 *abPK; + assert( pTab->azCol==0 || pTab->abPK==0 ); + pSession->rc = sessionTableInfo(pSession->db, pSession->zDb, + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + ); + if( pSession->rc==SQLITE_OK ){ + int i; + for(i=0; inCol; i++){ + if( abPK[i] ){ + pTab->abPK = abPK; + break; + } + } + } + } + return (pSession->rc || pTab->abPK==0); +} + +/* +** This function is only called from with a pre-update-hook reporting a +** change on table pTab (attached to session pSession). The type of change +** (UPDATE, INSERT, DELETE) is specified by the first argument. +** +** Unless one is already present or an error occurs, an entry is added +** to the changed-rows hash table associated with table pTab. +*/ +static void sessionPreupdateOneChange( + int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + sqlite3_session *pSession, /* Session object pTab is attached to */ + SessionTable *pTab /* Table that change applies to */ +){ + int iHash; + int bNull = 0; + int rc = SQLITE_OK; + + if( pSession->rc ) return; + + /* Load table details if required */ + if( sessionInitTable(pSession, pTab) ) return; + + /* Check the number of columns in this xPreUpdate call matches the + ** number of columns in the table. */ + if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + pSession->rc = SQLITE_SCHEMA; + return; + } + + /* Grow the hash table if required */ + if( sessionGrowHash(0, pTab) ){ + pSession->rc = SQLITE_NOMEM; + return; + } + + /* Calculate the hash-key for this change. If the primary key of the row + ** includes a NULL value, exit early. Such changes are ignored by the + ** session module. */ + rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + if( rc!=SQLITE_OK ) goto error_out; + + if( bNull==0 ){ + /* Search the hash table for an existing record for this row. */ + SessionChange *pC; + for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ + if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + } + + if( pC==0 ){ + /* Create a new change object containing all the old values (if + ** this is an SQLITE_UPDATE or SQLITE_DELETE), or just the PK + ** values (if this is an INSERT). */ + SessionChange *pChange; /* New change object */ + int nByte; /* Number of bytes to allocate */ + int i; /* Used to iterate through columns */ + + assert( rc==SQLITE_OK ); + pTab->nEntry++; + + /* Figure out how large an allocation is required */ + nByte = sizeof(SessionChange); + for(i=0; inCol; i++){ + sqlite3_value *p = 0; + if( op!=SQLITE_INSERT ){ + TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); + assert( trc==SQLITE_OK ); + }else if( pTab->abPK[i] ){ + TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p); + assert( trc==SQLITE_OK ); + } + + /* This may fail if SQLite value p contains a utf-16 string that must + ** be converted to utf-8 and an OOM error occurs while doing so. */ + rc = sessionSerializeValue(0, p, &nByte); + if( rc!=SQLITE_OK ) goto error_out; + } + + /* Allocate the change object */ + pChange = (SessionChange *)sqlite3_malloc(nByte); + if( !pChange ){ + rc = SQLITE_NOMEM; + goto error_out; + }else{ + memset(pChange, 0, sizeof(SessionChange)); + pChange->aRecord = (u8 *)&pChange[1]; + } + + /* Populate the change object. None of the preupdate_old(), + ** preupdate_new() or SerializeValue() calls below may fail as all + ** required values and encodings have already been cached in memory. + ** It is not possible for an OOM to occur in this block. */ + nByte = 0; + for(i=0; inCol; i++){ + sqlite3_value *p = 0; + if( op!=SQLITE_INSERT ){ + pSession->hook.xOld(pSession->hook.pCtx, i, &p); + }else if( pTab->abPK[i] ){ + pSession->hook.xNew(pSession->hook.pCtx, i, &p); + } + sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte); + } + + /* Add the change to the hash-table */ + if( pSession->bIndirect || pSession->hook.xDepth(pSession->hook.pCtx) ){ + pChange->bIndirect = 1; + } + pChange->nRecord = nByte; + pChange->op = op; + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + + }else if( pC->bIndirect ){ + /* If the existing change is considered "indirect", but this current + ** change is "direct", mark the change object as direct. */ + if( pSession->hook.xDepth(pSession->hook.pCtx)==0 + && pSession->bIndirect==0 + ){ + pC->bIndirect = 0; + } + } + } + + /* If an error has occurred, mark the session object as failed. */ + error_out: + if( rc!=SQLITE_OK ){ + pSession->rc = rc; + } +} + +static int sessionFindTable( + sqlite3_session *pSession, + const char *zName, + SessionTable **ppTab +){ + int rc = SQLITE_OK; + int nName = sqlite3Strlen30(zName); + SessionTable *pRet; + + /* Search for an existing table */ + for(pRet=pSession->pTable; pRet; pRet=pRet->pNext){ + if( 0==sqlite3_strnicmp(pRet->zName, zName, nName+1) ) break; + } + + if( pRet==0 && pSession->bAutoAttach ){ + /* If there is a table-filter configured, invoke it. If it returns 0, + ** do not automatically add the new table. */ + if( pSession->xTableFilter==0 + || pSession->xTableFilter(pSession->pFilterCtx, zName) + ){ + rc = sqlite3session_attach(pSession, zName); + if( rc==SQLITE_OK ){ + for(pRet=pSession->pTable; pRet->pNext; pRet=pRet->pNext); + assert( 0==sqlite3_strnicmp(pRet->zName, zName, nName+1) ); + } + } + } + + assert( rc==SQLITE_OK || pRet==0 ); + *ppTab = pRet; + return rc; +} + +/* +** The 'pre-update' hook registered by this module with SQLite databases. +*/ +static void xPreUpdate( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ +){ + sqlite3_session *pSession; + int nDb = sqlite3Strlen30(zDb); + + assert( sqlite3_mutex_held(db->mutex) ); + + for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ + SessionTable *pTab; + + /* If this session is attached to a different database ("main", "temp" + ** etc.), or if it is not currently enabled, there is nothing to do. Skip + ** to the next session object attached to this database. */ + if( pSession->bEnable==0 ) continue; + if( pSession->rc ) continue; + if( sqlite3_strnicmp(zDb, pSession->zDb, nDb+1) ) continue; + + pSession->rc = sessionFindTable(pSession, zName, &pTab); + if( pTab ){ + assert( pSession->rc==SQLITE_OK ); + sessionPreupdateOneChange(op, pSession, pTab); + if( op==SQLITE_UPDATE ){ + sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + } + } + } +} + +/* +** The pre-update hook implementations. +*/ +static int sessionPreupdateOld(void *pCtx, int iVal, sqlite3_value **ppVal){ + return sqlite3_preupdate_old((sqlite3*)pCtx, iVal, ppVal); +} +static int sessionPreupdateNew(void *pCtx, int iVal, sqlite3_value **ppVal){ + return sqlite3_preupdate_new((sqlite3*)pCtx, iVal, ppVal); +} +static int sessionPreupdateCount(void *pCtx){ + return sqlite3_preupdate_count((sqlite3*)pCtx); +} +static int sessionPreupdateDepth(void *pCtx){ + return sqlite3_preupdate_depth((sqlite3*)pCtx); +} + +/* +** Install the pre-update hooks on the session object passed as the only +** argument. +*/ +static void sessionPreupdateHooks( + sqlite3_session *pSession +){ + pSession->hook.pCtx = (void*)pSession->db; + pSession->hook.xOld = sessionPreupdateOld; + pSession->hook.xNew = sessionPreupdateNew; + pSession->hook.xCount = sessionPreupdateCount; + pSession->hook.xDepth = sessionPreupdateDepth; +} + +typedef struct SessionDiffCtx SessionDiffCtx; +struct SessionDiffCtx { + sqlite3_stmt *pStmt; + int nOldOff; +}; + +/* +** The diff hook implementations. +*/ +static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + return SQLITE_OK; +} +static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + *ppVal = sqlite3_column_value(p->pStmt, iVal); + return SQLITE_OK; +} +static int sessionDiffCount(void *pCtx){ + SessionDiffCtx *p = (SessionDiffCtx*)pCtx; + return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); +} +static int sessionDiffDepth(void *pCtx){ + return 0; +} + +/* +** Install the diff hooks on the session object passed as the only +** argument. +*/ +static void sessionDiffHooks( + sqlite3_session *pSession, + SessionDiffCtx *pDiffCtx +){ + pSession->hook.pCtx = (void*)pDiffCtx; + pSession->hook.xOld = sessionDiffOld; + pSession->hook.xNew = sessionDiffNew; + pSession->hook.xCount = sessionDiffCount; + pSession->hook.xDepth = sessionDiffDepth; +} + +static char *sessionExprComparePK( + int nCol, + const char *zDb1, const char *zDb2, + const char *zTab, + const char **azCol, u8 *abPK +){ + int i; + const char *zSep = ""; + char *zRet = 0; + + for(i=0; inCol, zDb1, zDb2, pTab->zName,zExpr); + + if( zStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare(pSession->db, zStmt, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; + pDiffCtx->pStmt = pStmt; + pDiffCtx->nOldOff = 0; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + sessionPreupdateOneChange(op, pSession, pTab); + } + rc = sqlite3_finalize(pStmt); + } + sqlite3_free(zStmt); + } + + return rc; +} + +static int sessionDiffFindModified( + sqlite3_session *pSession, + SessionTable *pTab, + const char *zFrom, + const char *zExpr +){ + int rc = SQLITE_OK; + + char *zExpr2 = sessionExprCompareOther(pTab->nCol, + pSession->zDb, zFrom, pTab->zName, pTab->azCol, pTab->abPK + ); + if( zExpr2==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zStmt = sqlite3_mprintf( + "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + ); + if( zStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare(pSession->db, zStmt, -1, &pStmt, 0); + + if( rc==SQLITE_OK ){ + SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; + pDiffCtx->pStmt = pStmt; + pDiffCtx->nOldOff = pTab->nCol; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + } + rc = sqlite3_finalize(pStmt); + } + sqlite3_free(zStmt); + } + } + + return rc; +} + +int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFrom, + const char *zTbl, + char **pzErrMsg +){ + const char *zDb = pSession->zDb; + int rc = pSession->rc; + SessionDiffCtx d; + + memset(&d, 0, sizeof(d)); + sessionDiffHooks(pSession, &d); + + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( pzErrMsg ) *pzErrMsg = 0; + if( rc==SQLITE_OK ){ + char *zExpr = 0; + sqlite3 *db = pSession->db; + SessionTable *pTo; /* Table zTbl */ + + /* Locate and if necessary initialize the target table object */ + rc = sessionFindTable(pSession, zTbl, &pTo); + if( pTo==0 ) goto diff_out; + if( sessionInitTable(pSession, pTo) ){ + rc = pSession->rc; + goto diff_out; + } + + /* Check the table schemas match */ + if( rc==SQLITE_OK ){ + int bHasPk = 0; + int bMismatch = 0; + int nCol; /* Columns in zFrom.zTbl */ + u8 *abPK; + const char **azCol = 0; + rc = sessionTableInfo(db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + if( rc==SQLITE_OK ){ + if( pTo->nCol!=nCol ){ + bMismatch = 1; + }else{ + int i; + for(i=0; iabPK[i]!=abPK[i] ) bMismatch = 1; + if( sqlite3_stricmp(azCol[i], pTo->azCol[i]) ) bMismatch = 1; + if( abPK[i] ) bHasPk = 1; + } + } + + } + sqlite3_free((char*)azCol); + if( bMismatch ){ + *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + rc = SQLITE_SCHEMA; + } + if( bHasPk==0 ){ + /* Ignore tables with no primary keys */ + goto diff_out; + } + } + + if( rc==SQLITE_OK ){ + zExpr = sessionExprComparePK(pTo->nCol, + zDb, zFrom, pTo->zName, pTo->azCol, pTo->abPK + ); + } + + /* Find new rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindNew(SQLITE_INSERT, pSession, pTo, zDb, zFrom, zExpr); + } + + /* Find old rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindNew(SQLITE_DELETE, pSession, pTo, zFrom, zDb, zExpr); + } + + /* Find modified rows */ + if( rc==SQLITE_OK ){ + rc = sessionDiffFindModified(pSession, pTo, zFrom, zExpr); + } + + sqlite3_free(zExpr); + } + + diff_out: + sessionPreupdateHooks(pSession); + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return rc; +} + +/* +** Create a session object. This session object will record changes to +** database zDb attached to connection db. +*/ +int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +){ + sqlite3_session *pNew; /* Newly allocated session object */ + sqlite3_session *pOld; /* Session object already attached to db */ + int nDb = sqlite3Strlen30(zDb); /* Length of zDb in bytes */ + + /* Zero the output value in case an error occurs. */ + *ppSession = 0; + + /* Allocate and populate the new session object. */ + pNew = (sqlite3_session *)sqlite3_malloc(sizeof(sqlite3_session) + nDb + 1); + if( !pNew ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(sqlite3_session)); + pNew->db = db; + pNew->zDb = (char *)&pNew[1]; + pNew->bEnable = 1; + memcpy(pNew->zDb, zDb, nDb+1); + sessionPreupdateHooks(pNew); + + /* Add the new session object to the linked list of session objects + ** attached to database handle $db. Do this under the cover of the db + ** handle mutex. */ + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + pOld = (sqlite3_session*)sqlite3_preupdate_hook(db, xPreUpdate, (void*)pNew); + pNew->pNext = pOld; + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + + *ppSession = pNew; + return SQLITE_OK; +} + +/* +** Free the list of table objects passed as the first argument. The contents +** of the changed-rows hash tables are also deleted. +*/ +static void sessionDeleteTable(SessionTable *pList){ + SessionTable *pNext; + SessionTable *pTab; + + for(pTab=pList; pTab; pTab=pNext){ + int i; + pNext = pTab->pNext; + for(i=0; inChange; i++){ + SessionChange *p; + SessionChange *pNextChange; + for(p=pTab->apChange[i]; p; p=pNextChange){ + pNextChange = p->pNext; + sqlite3_free(p); + } + } + sqlite3_free((char*)pTab->azCol); /* cast works around VC++ bug */ + sqlite3_free(pTab->apChange); + sqlite3_free(pTab); + } +} + +/* +** Delete a session object previously allocated using sqlite3session_create(). +*/ +void sqlite3session_delete(sqlite3_session *pSession){ + sqlite3 *db = pSession->db; + sqlite3_session *pHead; + sqlite3_session **pp; + + /* Unlink the session from the linked list of sessions attached to the + ** database handle. Hold the db mutex while doing so. */ + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + pHead = (sqlite3_session*)sqlite3_preupdate_hook(db, 0, 0); + for(pp=&pHead; ALWAYS((*pp)!=0); pp=&((*pp)->pNext)){ + if( (*pp)==pSession ){ + *pp = (*pp)->pNext; + if( pHead ) sqlite3_preupdate_hook(db, xPreUpdate, (void*)pHead); + break; + } + } + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + + /* Delete all attached table objects. And the contents of their + ** associated hash-tables. */ + sessionDeleteTable(pSession->pTable); + + /* Free the session object itself. */ + sqlite3_free(pSession); +} + +/* +** Set a table filter on a Session Object. +*/ +void sqlite3session_table_filter( + sqlite3_session *pSession, + int(*xFilter)(void*, const char*), + void *pCtx /* First argument passed to xFilter */ +){ + pSession->bAutoAttach = 1; + pSession->pFilterCtx = pCtx; + pSession->xTableFilter = xFilter; +} + +/* +** Attach a table to a session. All subsequent changes made to the table +** while the session object is enabled will be recorded. +** +** Only tables that have a PRIMARY KEY defined may be attached. It does +** not matter if the PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) +** or not. +*/ +int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zName /* Table name */ +){ + int rc = SQLITE_OK; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + + if( !zName ){ + pSession->bAutoAttach = 1; + }else{ + SessionTable *pTab; /* New table object (if required) */ + int nName; /* Number of bytes in string zName */ + + /* First search for an existing entry. If one is found, this call is + ** a no-op. Return early. */ + nName = sqlite3Strlen30(zName); + for(pTab=pSession->pTable; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zName, nName+1) ) break; + } + + if( !pTab ){ + /* Allocate new SessionTable object. */ + pTab = (SessionTable *)sqlite3_malloc(sizeof(SessionTable) + nName + 1); + if( !pTab ){ + rc = SQLITE_NOMEM; + }else{ + /* Populate the new SessionTable object and link it into the list. + ** The new object must be linked onto the end of the list, not + ** simply added to the start of it in order to ensure that tables + ** appear in the correct order when a changeset or patchset is + ** eventually generated. */ + SessionTable **ppTab; + memset(pTab, 0, sizeof(SessionTable)); + pTab->zName = (char *)&pTab[1]; + memcpy(pTab->zName, zName, nName+1); + for(ppTab=&pSession->pTable; *ppTab; ppTab=&(*ppTab)->pNext); + *ppTab = pTab; + } + } + } + + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return rc; +} + +/* +** Ensure that there is room in the buffer to append nByte bytes of data. +** If not, use sqlite3_realloc() to grow the buffer so that there is. +** +** If successful, return zero. Otherwise, if an OOM condition is encountered, +** set *pRc to SQLITE_NOMEM and return non-zero. +*/ +static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ + if( *pRc==SQLITE_OK && p->nAlloc-p->nBufnAlloc ? p->nAlloc : 128; + do { + nNew = nNew*2; + }while( nNew<(p->nBuf+nByte) ); + + aNew = (u8 *)sqlite3_realloc(p->aBuf, nNew); + if( 0==aNew ){ + *pRc = SQLITE_NOMEM; + }else{ + p->aBuf = aNew; + p->nAlloc = nNew; + } + } + return (*pRc!=SQLITE_OK); +} + +/* +** Append the value passed as the second argument to the buffer passed +** as the first. +** +** This function is a no-op if *pRc is non-zero when it is called. +** Otherwise, if an error occurs, *pRc is set to an SQLite error code +** before returning. +*/ +static void sessionAppendValue(SessionBuffer *p, sqlite3_value *pVal, int *pRc){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + int nByte = 0; + rc = sessionSerializeValue(0, pVal, &nByte); + sessionBufferGrow(p, nByte, &rc); + if( rc==SQLITE_OK ){ + rc = sessionSerializeValue(&p->aBuf[p->nBuf], pVal, 0); + p->nBuf += nByte; + }else{ + *pRc = rc; + } + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single byte to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendByte(SessionBuffer *p, u8 v, int *pRc){ + if( 0==sessionBufferGrow(p, 1, pRc) ){ + p->aBuf[p->nBuf++] = v; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single varint to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendVarint(SessionBuffer *p, int v, int *pRc){ + if( 0==sessionBufferGrow(p, 9, pRc) ){ + p->nBuf += sessionVarintPut(&p->aBuf[p->nBuf], v); + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a blob of data to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendBlob( + SessionBuffer *p, + const u8 *aBlob, + int nBlob, + int *pRc +){ + if( nBlob>0 && 0==sessionBufferGrow(p, nBlob, pRc) ){ + memcpy(&p->aBuf[p->nBuf], aBlob, nBlob); + p->nBuf += nBlob; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a string to the buffer. All bytes in the string +** up to (but not including) the nul-terminator are written to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendStr( + SessionBuffer *p, + const char *zStr, + int *pRc +){ + int nStr = sqlite3Strlen30(zStr); + if( 0==sessionBufferGrow(p, nStr, pRc) ){ + memcpy(&p->aBuf[p->nBuf], zStr, nStr); + p->nBuf += nStr; + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append the string representation of integer iVal +** to the buffer. No nul-terminator is written. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendInteger( + SessionBuffer *p, /* Buffer to append to */ + int iVal, /* Value to write the string rep. of */ + int *pRc /* IN/OUT: Error code */ +){ + char aBuf[24]; + sqlite3_snprintf(sizeof(aBuf)-1, aBuf, "%d", iVal); + sessionAppendStr(p, aBuf, pRc); +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append the string zStr enclosed in quotes (") and +** with any embedded quote characters escaped to the buffer. No +** nul-terminator byte is written. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendIdent( + SessionBuffer *p, /* Buffer to a append to */ + const char *zStr, /* String to quote, escape and append */ + int *pRc /* IN/OUT: Error code */ +){ + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + if( 0==sessionBufferGrow(p, nStr, pRc) ){ + char *zOut = (char *)&p->aBuf[p->nBuf]; + const char *zIn = zStr; + *zOut++ = '"'; + while( *zIn ){ + if( *zIn=='"' ) *zOut++ = '"'; + *zOut++ = *(zIn++); + } + *zOut++ = '"'; + p->nBuf = (int)((u8 *)zOut - p->aBuf); + } +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwse, it appends the serialized version of the value stored +** in column iCol of the row that SQL statement pStmt currently points +** to to the buffer. +*/ +static void sessionAppendCol( + SessionBuffer *p, /* Buffer to append to */ + sqlite3_stmt *pStmt, /* Handle pointing to row containing value */ + int iCol, /* Column to read value from */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int eType = sqlite3_column_type(pStmt, iCol); + sessionAppendByte(p, (u8)eType, pRc); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + sqlite3_int64 i; + u8 aBuf[8]; + if( eType==SQLITE_INTEGER ){ + i = sqlite3_column_int64(pStmt, iCol); + }else{ + double r = sqlite3_column_double(pStmt, iCol); + memcpy(&i, &r, 8); + } + sessionPutI64(aBuf, i); + sessionAppendBlob(p, aBuf, 8, pRc); + } + if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){ + u8 *z; + int nByte; + if( eType==SQLITE_BLOB ){ + z = (u8 *)sqlite3_column_blob(pStmt, iCol); + }else{ + z = (u8 *)sqlite3_column_text(pStmt, iCol); + } + nByte = sqlite3_column_bytes(pStmt, iCol); + if( z || (eType==SQLITE_BLOB && nByte==0) ){ + sessionAppendVarint(p, nByte, pRc); + sessionAppendBlob(p, z, nByte, pRc); + }else{ + *pRc = SQLITE_NOMEM; + } + } + } +} + +/* +** +** This function appends an update change to the buffer (see the comments +** under "CHANGESET FORMAT" at the top of the file). An update change +** consists of: +** +** 1 byte: SQLITE_UPDATE (0x17) +** n bytes: old.* record (see RECORD FORMAT) +** m bytes: new.* record (see RECORD FORMAT) +** +** The SessionChange object passed as the third argument contains the +** values that were stored in the row when the session began (the old.* +** values). The statement handle passed as the second argument points +** at the current version of the row (the new.* values). +** +** If all of the old.* values are equal to their corresponding new.* value +** (i.e. nothing has changed), then no data at all is appended to the buffer. +** +** Otherwise, the old.* record contains all primary key values and the +** original values of any fields that have been modified. The new.* record +** contains the new values of only those fields that have been modified. +*/ +static int sessionAppendUpdate( + SessionBuffer *pBuf, /* Buffer to append to */ + int bPatchset, /* True for "patchset", 0 for "changeset" */ + sqlite3_stmt *pStmt, /* Statement handle pointing at new row */ + SessionChange *p, /* Object containing old values */ + u8 *abPK /* Boolean array - true for PK columns */ +){ + int rc = SQLITE_OK; + SessionBuffer buf2 = {0,0,0}; /* Buffer to accumulate new.* record in */ + int bNoop = 1; /* Set to zero if any values are modified */ + int nRewind = pBuf->nBuf; /* Set to zero if any values are modified */ + int i; /* Used to iterate through columns */ + u8 *pCsr = p->aRecord; /* Used to iterate through old.* values */ + + sessionAppendByte(pBuf, SQLITE_UPDATE, &rc); + sessionAppendByte(pBuf, p->bIndirect, &rc); + for(i=0; inBuf = nRewind; + }else{ + sessionAppendBlob(pBuf, buf2.aBuf, buf2.nBuf, &rc); + } + sqlite3_free(buf2.aBuf); + + return rc; +} + +/* +** Append a DELETE change to the buffer passed as the first argument. Use +** the changeset format if argument bPatchset is zero, or the patchset +** format otherwise. +*/ +static int sessionAppendDelete( + SessionBuffer *pBuf, /* Buffer to append to */ + int bPatchset, /* True for "patchset", 0 for "changeset" */ + SessionChange *p, /* Object containing old values */ + int nCol, /* Number of columns in table */ + u8 *abPK /* Boolean array - true for PK columns */ +){ + int rc = SQLITE_OK; + + sessionAppendByte(pBuf, SQLITE_DELETE, &rc); + sessionAppendByte(pBuf, p->bIndirect, &rc); + + if( bPatchset==0 ){ + sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc); + }else{ + int i; + u8 *a = p->aRecord; + for(i=0; iaRecord)==p->nRecord ); + } + + return rc; +} + +/* +** Formulate and prepare a SELECT statement to retrieve a row from table +** zTab in database zDb based on its primary key. i.e. +** +** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +*/ +static int sessionSelectStmt( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name */ + const char *zTab, /* Table name */ + int nCol, /* Number of columns in table */ + const char **azCol, /* Names of table columns */ + u8 *abPK, /* PRIMARY KEY array */ + sqlite3_stmt **ppStmt /* OUT: Prepared SELECT statement */ +){ + int rc = SQLITE_OK; + int i; + const char *zSep = ""; + SessionBuffer buf = {0, 0, 0}; + + sessionAppendStr(&buf, "SELECT * FROM ", &rc); + sessionAppendIdent(&buf, zDb, &rc); + sessionAppendStr(&buf, ".", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " WHERE ", &rc); + for(i=0; iaRecord; + + for(i=0; inCol, pRc); + sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc); + sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc); +} + +/* +** Generate either a changeset (if argument bPatchset is zero) or a patchset +** (if it is non-zero) based on the current contents of the session object +** passed as the first argument. +** +** If no error occurs, SQLITE_OK is returned and the new changeset/patchset +** stored in output variables *pnChangeset and *ppChangeset. Or, if an error +** occurs, an SQLite error code is returned and both output variables set +** to 0. +*/ +static int sessionGenerateChangeset( + sqlite3_session *pSession, /* Session object */ + int bPatchset, /* True for patchset, false for changeset */ + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, /* First argument for xOutput */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + sqlite3 *db = pSession->db; /* Source database handle */ + SessionTable *pTab; /* Used to iterate through attached tables */ + SessionBuffer buf = {0,0,0}; /* Buffer in which to accumlate changeset */ + int rc; /* Return code */ + + assert( xOutput==0 || (pnChangeset==0 && ppChangeset==0 ) ); + + /* Zero the output variables in case an error occurs. If this session + ** object is already in the error state (sqlite3_session.rc != SQLITE_OK), + ** this call will be a no-op. */ + if( xOutput==0 ){ + *pnChangeset = 0; + *ppChangeset = 0; + } + + if( pSession->rc ) return pSession->rc; + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + + for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + if( pTab->nEntry ){ + const char *zName = pTab->zName; + int nCol; /* Number of columns in table */ + u8 *abPK; /* Primary key array */ + const char **azCol = 0; /* Table columns */ + int i; /* Used to iterate through hash buckets */ + sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ + int nRewind = buf.nBuf; /* Initial size of write buffer */ + int nNoop; /* Size of buffer after writing tbl header */ + + /* Check the table schema is still Ok. */ + rc = sessionTableInfo(db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK); + if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = SQLITE_SCHEMA; + } + + /* Write a table header */ + sessionAppendTableHdr(&buf, bPatchset, pTab, &rc); + + /* Build and compile a statement to execute: */ + if( rc==SQLITE_OK ){ + rc = sessionSelectStmt( + db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + } + + nNoop = buf.nBuf; + for(i=0; inChange && rc==SQLITE_OK; i++){ + SessionChange *p; /* Used to iterate through changes */ + + for(p=pTab->apChange[i]; rc==SQLITE_OK && p; p=p->pNext){ + rc = sessionSelectBind(pSel, nCol, abPK, p); + if( rc!=SQLITE_OK ) continue; + if( sqlite3_step(pSel)==SQLITE_ROW ){ + if( p->op==SQLITE_INSERT ){ + int iCol; + sessionAppendByte(&buf, SQLITE_INSERT, &rc); + sessionAppendByte(&buf, p->bIndirect, &rc); + for(iCol=0; iColop!=SQLITE_INSERT ){ + rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pSel); + } + + /* If the buffer is now larger than SESSIONS_STRM_CHUNK_SIZE, pass + ** its contents to the xOutput() callback. */ + if( xOutput + && rc==SQLITE_OK + && buf.nBuf>nNoop + && buf.nBuf>SESSIONS_STRM_CHUNK_SIZE + ){ + rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); + nNoop = -1; + buf.nBuf = 0; + } + + } + } + + sqlite3_finalize(pSel); + if( buf.nBuf==nNoop ){ + buf.nBuf = nRewind; + } + sqlite3_free((char*)azCol); /* cast works around VC++ bug */ + } + } + + if( rc==SQLITE_OK ){ + if( xOutput==0 ){ + *pnChangeset = buf.nBuf; + *ppChangeset = buf.aBuf; + buf.aBuf = 0; + }else if( buf.nBuf>0 ){ + rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); + } + } + + sqlite3_free(buf.aBuf); + sqlite3_exec(db, "RELEASE changeset", 0, 0, 0); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; +} + +/* +** Obtain a changeset object containing all changes recorded by the +** session object passed as the first argument. +** +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +*/ +int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); +} + +/* +** Streaming version of sqlite3session_changeset(). +*/ +int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0); +} + +/* +** Streaming version of sqlite3session_patchset(). +*/ +int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0); +} + +/* +** Obtain a patchset object containing all changes recorded by the +** session object passed as the first argument. +** +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +*/ +int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ + void **ppPatchset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset); +} + +/* +** Enable or disable the session object passed as the first argument. +*/ +int sqlite3session_enable(sqlite3_session *pSession, int bEnable){ + int ret; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( bEnable>=0 ){ + pSession->bEnable = bEnable; + } + ret = pSession->bEnable; + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return ret; +} + +/* +** Enable or disable the session object passed as the first argument. +*/ +int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect){ + int ret; + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + if( bIndirect>=0 ){ + pSession->bIndirect = bIndirect; + } + ret = pSession->bIndirect; + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + return ret; +} + +/* +** Return true if there have been no changes to monitored tables recorded +** by the session object passed as the only argument. +*/ +int sqlite3session_isempty(sqlite3_session *pSession){ + int ret = 0; + SessionTable *pTab; + + sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); + for(pTab=pSession->pTable; pTab && ret==0; pTab=pTab->pNext){ + ret = (pTab->nEntry>0); + } + sqlite3_mutex_leave(sqlite3_db_mutex(pSession->db)); + + return (ret==0); +} + +/* +** Do the work for either sqlite3changeset_start() or start_strm(). +*/ +static int sessionChangesetStart( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int nChangeset, /* Size of buffer pChangeset in bytes */ + void *pChangeset /* Pointer to buffer containing changeset */ +){ + sqlite3_changeset_iter *pRet; /* Iterator to return */ + int nByte; /* Number of bytes to allocate for iterator */ + + assert( xInput==0 || (pChangeset==0 && nChangeset==0) ); + + /* Zero the output variable in case an error occurs. */ + *pp = 0; + + /* Allocate and initialize the iterator structure. */ + nByte = sizeof(sqlite3_changeset_iter); + pRet = (sqlite3_changeset_iter *)sqlite3_malloc(nByte); + if( !pRet ) return SQLITE_NOMEM; + memset(pRet, 0, sizeof(sqlite3_changeset_iter)); + pRet->in.aData = (u8 *)pChangeset; + pRet->in.nData = nChangeset; + pRet->in.xInput = xInput; + pRet->in.pIn = pIn; + pRet->in.bEof = (xInput ? 0 : 1); + + /* Populate the output variable and return success. */ + *pp = pRet; + return SQLITE_OK; +} + +/* +** Create an iterator used to iterate through the contents of a changeset. +*/ +int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int nChangeset, /* Size of buffer pChangeset in bytes */ + void *pChangeset /* Pointer to buffer containing changeset */ +){ + return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset); +} + +/* +** Streaming version of sqlite3changeset_start(). +*/ +int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +){ + return sessionChangesetStart(pp, xInput, pIn, 0, 0); +} + +/* +** If the SessionInput object passed as the only argument is a streaming +** object and the buffer is full, discard some data to free up space. +*/ +static void sessionDiscardData(SessionInput *pIn){ + if( pIn->bEof && pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ + int nMove = pIn->buf.nBuf - pIn->iNext; + assert( nMove>=0 ); + if( nMove>0 ){ + memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove); + } + pIn->buf.nBuf -= pIn->iNext; + pIn->iNext = 0; + pIn->nData = pIn->buf.nBuf; + } +} + +/* +** Ensure that there are at least nByte bytes available in the buffer. Or, +** if there are not nByte bytes remaining in the input, that all available +** data is in the buffer. +** +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +static int sessionInputBuffer(SessionInput *pIn, int nByte){ + int rc = SQLITE_OK; + if( pIn->xInput ){ + while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){ + int nNew = SESSIONS_STRM_CHUNK_SIZE; + + if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn); + if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){ + rc = pIn->xInput(pIn->pIn, &pIn->buf.aBuf[pIn->buf.nBuf], &nNew); + if( nNew==0 ){ + pIn->bEof = 1; + }else{ + pIn->buf.nBuf += nNew; + } + } + + pIn->aData = pIn->buf.aBuf; + pIn->nData = pIn->buf.nBuf; + } + } + return rc; +} + +/* +** When this function is called, *ppRec points to the start of a record +** that contains nCol values. This function advances the pointer *ppRec +** until it points to the byte immediately following that record. +*/ +static void sessionSkipRecord( + u8 **ppRec, /* IN/OUT: Record pointer */ + int nCol /* Number of values in record */ +){ + u8 *aRec = *ppRec; + int i; + for(i=0; iaData[pIn->iNext++]; + } + + assert( apOut[i]==0 ); + if( eType ){ + apOut[i] = sqlite3ValueNew(0); + if( !apOut[i] ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + u8 *aVal = &pIn->aData[pIn->iNext]; + if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + int nByte; + pIn->iNext += sessionVarintGet(aVal, &nByte); + rc = sessionInputBuffer(pIn, nByte); + if( rc==SQLITE_OK ){ + u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0); + rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc); + } + pIn->iNext += nByte; + } + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + sqlite3_int64 v = sessionGetI64(aVal); + if( eType==SQLITE_INTEGER ){ + sqlite3VdbeMemSetInt64(apOut[i], v); + }else{ + double d; + memcpy(&d, &v, 8); + sqlite3VdbeMemSetDouble(apOut[i], d); + } + pIn->iNext += 8; + } + } + } + + return rc; +} + +/* +** The input pointer currently points to the second byte of a table-header. +** Specifically, to the following: +** +** + number of columns in table (varint) +** + array of PK flags (1 byte per column), +** + table name (nul terminated). +** +** This function ensures that all of the above is present in the input +** buffer (i.e. that it can be accessed without any calls to xInput()). +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +** The input pointer is not moved. +*/ +static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ + int rc = SQLITE_OK; + int nCol = 0; + int nRead = 0; + + rc = sessionInputBuffer(pIn, 9); + if( rc==SQLITE_OK ){ + nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); + rc = sessionInputBuffer(pIn, nRead+nCol+100); + nRead += nCol; + } + + while( rc==SQLITE_OK ){ + while( (pIn->iNext + nRead)nData && pIn->aData[pIn->iNext + nRead] ){ + nRead++; + } + if( (pIn->iNext + nRead)nData ) break; + rc = sessionInputBuffer(pIn, nRead + 100); + } + *pnByte = nRead+1; + return rc; +} + +/* +** The input pointer currently points to the first byte of the first field +** of a record consisting of nCol columns. This function ensures the entire +** record is buffered. It does not move the input pointer. +** +** If successful, SQLITE_OK is returned and *pnByte is set to the size of +** the record in bytes. Otherwise, an SQLite error code is returned. The +** final value of *pnByte is undefined in this case. +*/ +static int sessionChangesetBufferRecord( + SessionInput *pIn, /* Input data */ + int nCol, /* Number of columns in record */ + int *pnByte /* OUT: Size of record in bytes */ +){ + int rc = SQLITE_OK; + int nByte = 0; + int i; + for(i=0; rc==SQLITE_OK && iaData[pIn->iNext + nByte++]; + if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + int n; + nByte += sessionVarintGet(&pIn->aData[pIn->iNext+nByte], &n); + nByte += n; + rc = sessionInputBuffer(pIn, nByte); + }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + nByte += 8; + } + } + } + *pnByte = nByte; + return rc; +} + +/* +** The input pointer currently points to the second byte of a table-header. +** Specifically, to the following: +** +** + number of columns in table (varint) +** + array of PK flags (1 byte per column), +** + table name (nul terminated). +** +** This function decodes the table-header and populates the p->nCol, +** p->zTab and p->abPK[] variables accordingly. The p->apValue[] array is +** also allocated or resized according to the new value of p->nCol. The +** input pointer is left pointing to the byte following the table header. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code +** is returned and the final values of the various fields enumerated above +** are undefined. +*/ +static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ + int rc; + int nCopy; + assert( p->rc==SQLITE_OK ); + + rc = sessionChangesetBufferTblhdr(&p->in, &nCopy); + if( rc==SQLITE_OK ){ + int nByte; + int nVarint; + nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol); + nCopy -= nVarint; + p->in.iNext += nVarint; + nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; + p->tblhdr.nBuf = 0; + sessionBufferGrow(&p->tblhdr, nByte, &rc); + } + + if( rc==SQLITE_OK ){ + int iPK = sizeof(sqlite3_value*)*p->nCol*2; + memset(p->tblhdr.aBuf, 0, iPK); + memcpy(&p->tblhdr.aBuf[iPK], &p->in.aData[p->in.iNext], nCopy); + p->in.iNext += nCopy; + } + + p->apValue = (sqlite3_value**)p->tblhdr.aBuf; + p->abPK = (u8*)&p->apValue[p->nCol*2]; + p->zTab = (char*)&p->abPK[p->nCol]; + return (p->rc = rc); +} + +/* +** Advance the changeset iterator to the next change. +** +** If both paRec and pnRec are NULL, then this function works like the public +** API sqlite3changeset_next(). If SQLITE_ROW is returned, then the +** sqlite3changeset_new() and old() APIs may be used to query for values. +** +** Otherwise, if paRec and pnRec are not NULL, then a pointer to the change +** record is written to *paRec before returning and the number of bytes in +** the record to *pnRec. +** +** Either way, this function returns SQLITE_ROW if the iterator is +** successfully advanced to the next change in the changeset, an SQLite +** error code if an error occurs, or SQLITE_DONE if there are no further +** changes in the changeset. +*/ +static int sessionChangesetNext( + sqlite3_changeset_iter *p, /* Changeset iterator */ + u8 **paRec, /* If non-NULL, store record pointer here */ + int *pnRec /* If non-NULL, store size of record here */ +){ + int i; + u8 op; + + assert( (paRec==0 && pnRec==0) || (paRec && pnRec) ); + + /* If the iterator is in the error-state, return immediately. */ + if( p->rc!=SQLITE_OK ) return p->rc; + + /* Free the current contents of p->apValue[], if any. */ + if( p->apValue ){ + for(i=0; inCol*2; i++){ + sqlite3ValueFree(p->apValue[i]); + } + memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); + } + + /* Make sure the buffer contains at least 10 bytes of input data, or all + ** remaining data if there are less than 10 bytes available. This is + ** sufficient either for the 'T' or 'P' byte and the varint that follows + ** it, or for the two single byte values otherwise. */ + p->rc = sessionInputBuffer(&p->in, 2); + if( p->rc!=SQLITE_OK ) return p->rc; + + /* If the iterator is already at the end of the changeset, return DONE. */ + if( p->in.iNext>=p->in.nData ){ + return SQLITE_DONE; + } + + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + + op = p->in.aData[p->in.iNext++]; + if( op=='T' || op=='P' ){ + p->bPatchset = (op=='P'); + if( sessionChangesetReadTblhdr(p) ) return p->rc; + if( (p->rc = sessionInputBuffer(&p->in, 2)) ) return p->rc; + p->in.iCurrent = p->in.iNext; + op = p->in.aData[p->in.iNext++]; + } + + p->op = op; + p->bIndirect = p->in.aData[p->in.iNext++]; + if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + return (p->rc = SQLITE_CORRUPT_BKPT); + } + + if( paRec ){ + int nVal; /* Number of values to buffer */ + if( p->bPatchset==0 && op==SQLITE_UPDATE ){ + nVal = p->nCol * 2; + }else if( p->bPatchset && op==SQLITE_DELETE ){ + nVal = 0; + for(i=0; inCol; i++) if( p->abPK[i] ) nVal++; + }else{ + nVal = p->nCol; + } + p->rc = sessionChangesetBufferRecord(&p->in, nVal, pnRec); + if( p->rc!=SQLITE_OK ) return p->rc; + *paRec = &p->in.aData[p->in.iNext]; + p->in.iNext += *pnRec; + }else{ + + /* If this is an UPDATE or DELETE, read the old.* record. */ + if( p->op!=SQLITE_INSERT && (p->bPatchset==0 || p->op==SQLITE_DELETE) ){ + u8 *abPK = p->bPatchset ? p->abPK : 0; + p->rc = sessionReadRecord(&p->in, p->nCol, abPK, p->apValue); + if( p->rc!=SQLITE_OK ) return p->rc; + } + + /* If this is an INSERT or UPDATE, read the new.* record. */ + if( p->op!=SQLITE_DELETE ){ + p->rc = sessionReadRecord(&p->in, p->nCol, 0, &p->apValue[p->nCol]); + if( p->rc!=SQLITE_OK ) return p->rc; + } + + if( p->bPatchset && p->op==SQLITE_UPDATE ){ + /* If this is an UPDATE that is part of a patchset, then all PK and + ** modified fields are present in the new.* record. The old.* record + ** is currently completely empty. This block shifts the PK fields from + ** new.* to old.*, to accommodate the code that reads these arrays. */ + for(i=0; inCol; i++){ + assert( p->apValue[i]==0 ); + assert( p->abPK[i]==0 || p->apValue[i+p->nCol] ); + if( p->abPK[i] ){ + p->apValue[i] = p->apValue[i+p->nCol]; + p->apValue[i+p->nCol] = 0; + } + } + } + } + + return SQLITE_ROW; +} + +/* +** Advance an iterator created by sqlite3changeset_start() to the next +** change in the changeset. This function may return SQLITE_ROW, SQLITE_DONE +** or SQLITE_CORRUPT. +** +** This function may not be called on iterators passed to a conflict handler +** callback by changeset_apply(). +*/ +int sqlite3changeset_next(sqlite3_changeset_iter *p){ + return sessionChangesetNext(p, 0, 0); +} + +/* +** The following function extracts information on the current change +** from a changeset iterator. It may only be called after changeset_next() +** has returned SQLITE_ROW. +*/ +int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator handle */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True if change is indirect */ +){ + *pOp = pIter->op; + *pnCol = pIter->nCol; + *pzTab = pIter->zTab; + if( pbIndirect ) *pbIndirect = pIter->bIndirect; + return SQLITE_OK; +} + +/* +** Return information regarding the PRIMARY KEY and number of columns in +** the database table affected by the change that pIter currently points +** to. This function may only be called after changeset_next() returns +** SQLITE_ROW. +*/ +int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +){ + *pabPK = pIter->abPK; + if( pnCol ) *pnCol = pIter->nCol; + return SQLITE_OK; +} + +/* +** This function may only be called while the iterator is pointing to an +** SQLITE_UPDATE or SQLITE_DELETE change (see sqlite3changeset_op()). +** Otherwise, SQLITE_MISUSE is returned. +** +** It sets *ppValue to point to an sqlite3_value structure containing the +** iVal'th value in the old.* record. Or, if that particular value is not +** included in the record (because the change is an UPDATE and the field +** was not modified and is not a PK column), set *ppValue to NULL. +** +** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is +** not modified. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of old.* value to retrieve */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +){ + if( pIter->op!=SQLITE_UPDATE && pIter->op!=SQLITE_DELETE ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=pIter->nCol ){ + return SQLITE_RANGE; + } + *ppValue = pIter->apValue[iVal]; + return SQLITE_OK; +} + +/* +** This function may only be called while the iterator is pointing to an +** SQLITE_UPDATE or SQLITE_INSERT change (see sqlite3changeset_op()). +** Otherwise, SQLITE_MISUSE is returned. +** +** It sets *ppValue to point to an sqlite3_value structure containing the +** iVal'th value in the new.* record. Or, if that particular value is not +** included in the record (because the change is an UPDATE and the field +** was not modified), set *ppValue to NULL. +** +** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is +** not modified. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of new.* value to retrieve */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +){ + if( pIter->op!=SQLITE_UPDATE && pIter->op!=SQLITE_INSERT ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=pIter->nCol ){ + return SQLITE_RANGE; + } + *ppValue = pIter->apValue[pIter->nCol+iVal]; + return SQLITE_OK; +} + +/* +** The following two macros are used internally. They are similar to the +** sqlite3changeset_new() and sqlite3changeset_old() functions, except that +** they omit all error checking and return a pointer to the requested value. +*/ +#define sessionChangesetNew(pIter, iVal) (pIter)->apValue[(pIter)->nCol+(iVal)] +#define sessionChangesetOld(pIter, iVal) (pIter)->apValue[(iVal)] + +/* +** This function may only be called with a changeset iterator that has been +** passed to an SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT +** conflict-handler function. Otherwise, SQLITE_MISUSE is returned. +** +** If successful, *ppValue is set to point to an sqlite3_value structure +** containing the iVal'th value of the conflicting record. +** +** If value iVal is out-of-range or some other error occurs, an SQLite error +** code is returned. Otherwise, SQLITE_OK. +*/ +int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Index of conflict record value to fetch */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +){ + if( !pIter->pConflict ){ + return SQLITE_MISUSE; + } + if( iVal<0 || iVal>=pIter->nCol ){ + return SQLITE_RANGE; + } + *ppValue = sqlite3_column_value(pIter->pConflict, iVal); + return SQLITE_OK; +} + +/* +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +){ + if( pIter->pConflict || pIter->apValue ){ + return SQLITE_MISUSE; + } + *pnOut = pIter->nCol; + return SQLITE_OK; +} + + +/* +** Finalize an iterator allocated with sqlite3changeset_start(). +** +** This function may not be called on iterators passed to a conflict handler +** callback by changeset_apply(). +*/ +int sqlite3changeset_finalize(sqlite3_changeset_iter *p){ + int rc = SQLITE_OK; + if( p ){ + int i; /* Used to iterate through p->apValue[] */ + rc = p->rc; + if( p->apValue ){ + for(i=0; inCol*2; i++) sqlite3ValueFree(p->apValue[i]); + } + sqlite3_free(p->tblhdr.aBuf); + sqlite3_free(p->in.buf.aBuf); + sqlite3_free(p); + } + return rc; +} + +static int sessionChangesetInvert( + SessionInput *pInput, /* Input changeset */ + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, + int *pnInverted, /* OUT: Number of bytes in output changeset */ + void **ppInverted /* OUT: Inverse of pChangeset */ +){ + int rc = SQLITE_OK; /* Return value */ + SessionBuffer sOut; /* Output buffer */ + int nCol = 0; /* Number of cols in current table */ + u8 *abPK = 0; /* PK array for current table */ + sqlite3_value **apVal = 0; /* Space for values for UPDATE inversion */ + SessionBuffer sPK = {0, 0, 0}; /* PK array for current table */ + + /* Initialize the output buffer */ + memset(&sOut, 0, sizeof(SessionBuffer)); + + /* Zero the output variables in case an error occurs. */ + if( ppInverted ){ + *ppInverted = 0; + *pnInverted = 0; + } + + while( 1 ){ + u8 eType; + + /* Test for EOF. */ + if( (rc = sessionInputBuffer(pInput, 2)) ) goto finished_invert; + if( pInput->iNext>=pInput->nData ) break; + eType = pInput->aData[pInput->iNext]; + + switch( eType ){ + case 'T': { + /* A 'table' record consists of: + ** + ** * A constant 'T' character, + ** * Number of columns in said table (a varint), + ** * An array of nCol bytes (sPK), + ** * A nul-terminated table name. + */ + int nByte; + int nVar; + pInput->iNext++; + if( (rc = sessionChangesetBufferTblhdr(pInput, &nByte)) ){ + goto finished_invert; + } + nVar = sessionVarintGet(&pInput->aData[pInput->iNext], &nCol); + sPK.nBuf = 0; + sessionAppendBlob(&sPK, &pInput->aData[pInput->iNext+nVar], nCol, &rc); + sessionAppendByte(&sOut, eType, &rc); + sessionAppendBlob(&sOut, &pInput->aData[pInput->iNext], nByte, &rc); + if( rc ) goto finished_invert; + + pInput->iNext += nByte; + sqlite3_free(apVal); + apVal = 0; + abPK = sPK.aBuf; + break; + } + + case SQLITE_INSERT: + case SQLITE_DELETE: { + int nByte; + int bIndirect = pInput->aData[pInput->iNext+1]; + int eType2 = (eType==SQLITE_DELETE ? SQLITE_INSERT : SQLITE_DELETE); + pInput->iNext += 2; + assert( rc==SQLITE_OK ); + rc = sessionChangesetBufferRecord(pInput, nCol, &nByte); + sessionAppendByte(&sOut, eType2, &rc); + sessionAppendByte(&sOut, bIndirect, &rc); + sessionAppendBlob(&sOut, &pInput->aData[pInput->iNext], nByte, &rc); + pInput->iNext += nByte; + if( rc ) goto finished_invert; + break; + } + + case SQLITE_UPDATE: { + int iCol; + + if( 0==apVal ){ + apVal = (sqlite3_value **)sqlite3_malloc(sizeof(apVal[0])*nCol*2); + if( 0==apVal ){ + rc = SQLITE_NOMEM; + goto finished_invert; + } + memset(apVal, 0, sizeof(apVal[0])*nCol*2); + } + + /* Write the header for the new UPDATE change. Same as the original. */ + sessionAppendByte(&sOut, eType, &rc); + sessionAppendByte(&sOut, pInput->aData[pInput->iNext+1], &rc); + + /* Read the old.* and new.* records for the update change. */ + pInput->iNext += 2; + rc = sessionReadRecord(pInput, nCol, 0, &apVal[0]); + if( rc==SQLITE_OK ){ + rc = sessionReadRecord(pInput, nCol, 0, &apVal[nCol]); + } + + /* Write the new old.* record. Consists of the PK columns from the + ** original old.* record, and the other values from the original + ** new.* record. */ + for(iCol=0; iCol=SESSIONS_STRM_CHUNK_SIZE ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + sOut.nBuf = 0; + if( rc!=SQLITE_OK ) goto finished_invert; + } + } + + assert( rc==SQLITE_OK ); + if( pnInverted ){ + *pnInverted = sOut.nBuf; + *ppInverted = sOut.aBuf; + sOut.aBuf = 0; + }else if( sOut.nBuf>0 ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + } + + finished_invert: + sqlite3_free(sOut.aBuf); + sqlite3_free(apVal); + sqlite3_free(sPK.aBuf); + return rc; +} + + +/* +** Invert a changeset object. +*/ +int sqlite3changeset_invert( + int nChangeset, /* Number of bytes in input */ + const void *pChangeset, /* Input changeset */ + int *pnInverted, /* OUT: Number of bytes in output changeset */ + void **ppInverted /* OUT: Inverse of pChangeset */ +){ + SessionInput sInput; + + /* Set up the input stream */ + memset(&sInput, 0, sizeof(SessionInput)); + sInput.nData = nChangeset; + sInput.aData = (u8*)pChangeset; + + return sessionChangesetInvert(&sInput, 0, 0, pnInverted, ppInverted); +} + +/* +** Streaming version of sqlite3changeset_invert(). +*/ +int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + SessionInput sInput; + int rc; + + /* Set up the input stream */ + memset(&sInput, 0, sizeof(SessionInput)); + sInput.xInput = xInput; + sInput.pIn = pIn; + + rc = sessionChangesetInvert(&sInput, xOutput, pOut, 0, 0); + sqlite3_free(sInput.buf.aBuf); + return rc; +} + +typedef struct SessionApplyCtx SessionApplyCtx; +struct SessionApplyCtx { + sqlite3 *db; + sqlite3_stmt *pDelete; /* DELETE statement */ + sqlite3_stmt *pUpdate; /* UPDATE statement */ + sqlite3_stmt *pInsert; /* INSERT statement */ + sqlite3_stmt *pSelect; /* SELECT statement */ + int nCol; /* Size of azCol[] and abPK[] arrays */ + const char **azCol; /* Array of column names */ + u8 *abPK; /* Boolean array - true if column is in PK */ + + int bDeferConstraints; /* True to defer constraints */ + SessionBuffer constraints; /* Deferred constraints are stored here */ +}; + +/* +** Formulate a statement to DELETE a row from database db. Assuming a table +** structure like this: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The DELETE statement looks like this: +** +** DELETE FROM x WHERE a = :1 AND c = :3 AND (:5 OR b IS :2 AND d IS :4) +** +** Variable :5 (nCol+1) is a boolean. It should be set to 0 if we require +** matching b and d values, or 1 otherwise. The second case comes up if the +** conflict handler is invoked with NOTFOUND and returns CHANGESET_REPLACE. +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pDelete is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionDeleteRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int i; + const char *zSep = ""; + int rc = SQLITE_OK; + SessionBuffer buf = {0, 0, 0}; + int nPk = 0; + + sessionAppendStr(&buf, "DELETE FROM ", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " WHERE ", &rc); + + for(i=0; inCol; i++){ + if( p->abPK[i] ){ + nPk++; + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = ?", &rc); + sessionAppendInteger(&buf, i+1, &rc); + zSep = " AND "; + } + } + + if( nPknCol ){ + sessionAppendStr(&buf, " AND (?", &rc); + sessionAppendInteger(&buf, p->nCol+1, &rc); + sessionAppendStr(&buf, " OR ", &rc); + + zSep = ""; + for(i=0; inCol; i++){ + if( !p->abPK[i] ){ + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " IS ?", &rc); + sessionAppendInteger(&buf, i+1, &rc); + zSep = "AND "; + } + } + sessionAppendStr(&buf, ")", &rc); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pDelete, 0); + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Formulate and prepare a statement to UPDATE a row from database db. +** Assuming a table structure like this: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The UPDATE statement looks like this: +** +** UPDATE x SET +** a = CASE WHEN ?2 THEN ?3 ELSE a END, +** b = CASE WHEN ?5 THEN ?6 ELSE b END, +** c = CASE WHEN ?8 THEN ?9 ELSE c END, +** d = CASE WHEN ?11 THEN ?12 ELSE d END +** WHERE a = ?1 AND c = ?7 AND (?13 OR +** (?5==0 OR b IS ?4) AND (?11==0 OR d IS ?10) AND +** ) +** +** For each column in the table, there are three variables to bind: +** +** ?(i*3+1) The old.* value of the column, if any. +** ?(i*3+2) A boolean flag indicating that the value is being modified. +** ?(i*3+3) The new.* value of the column, if any. +** +** Also, a boolean flag that, if set to true, causes the statement to update +** a row even if the non-PK values do not match. This is required if the +** conflict-handler is invoked with CHANGESET_DATA and returns +** CHANGESET_REPLACE. This is variable "?(nCol*3+1)". +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pUpdate is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionUpdateRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int rc = SQLITE_OK; + int i; + const char *zSep = ""; + SessionBuffer buf = {0, 0, 0}; + + /* Append "UPDATE tbl SET " */ + sessionAppendStr(&buf, "UPDATE ", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, " SET ", &rc); + + /* Append the assignments */ + for(i=0; inCol; i++){ + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = CASE WHEN ?", &rc); + sessionAppendInteger(&buf, i*3+2, &rc); + sessionAppendStr(&buf, " THEN ?", &rc); + sessionAppendInteger(&buf, i*3+3, &rc); + sessionAppendStr(&buf, " ELSE ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " END", &rc); + zSep = ", "; + } + + /* Append the PK part of the WHERE clause */ + sessionAppendStr(&buf, " WHERE ", &rc); + for(i=0; inCol; i++){ + if( p->abPK[i] ){ + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " = ?", &rc); + sessionAppendInteger(&buf, i*3+1, &rc); + sessionAppendStr(&buf, " AND ", &rc); + } + } + + /* Append the non-PK part of the WHERE clause */ + sessionAppendStr(&buf, " (?", &rc); + sessionAppendInteger(&buf, p->nCol*3+1, &rc); + sessionAppendStr(&buf, " OR 1", &rc); + for(i=0; inCol; i++){ + if( !p->abPK[i] ){ + sessionAppendStr(&buf, " AND (?", &rc); + sessionAppendInteger(&buf, i*3+2, &rc); + sessionAppendStr(&buf, "=0 OR ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + sessionAppendStr(&buf, " IS ?", &rc); + sessionAppendInteger(&buf, i*3+1, &rc); + sessionAppendStr(&buf, ")", &rc); + } + } + sessionAppendStr(&buf, ")", &rc); + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pUpdate, 0); + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Formulate and prepare an SQL statement to query table zTab by primary +** key. Assuming the following table structure: +** +** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); +** +** The SELECT statement looks like this: +** +** SELECT * FROM x WHERE a = ?1 AND c = ?3 +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pSelect is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionSelectRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + return sessionSelectStmt( + db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); +} + +/* +** Formulate and prepare an INSERT statement to add a record to table zTab. +** For example: +** +** INSERT INTO main."zTab" VALUES(?1, ?2, ?3 ...); +** +** If successful, SQLITE_OK is returned and SessionApplyCtx.pInsert is left +** pointing to the prepared version of the SQL statement. +*/ +static int sessionInsertRow( + sqlite3 *db, /* Database handle */ + const char *zTab, /* Table name */ + SessionApplyCtx *p /* Session changeset-apply context */ +){ + int rc = SQLITE_OK; + int i; + SessionBuffer buf = {0, 0, 0}; + + sessionAppendStr(&buf, "INSERT INTO main.", &rc); + sessionAppendIdent(&buf, zTab, &rc); + sessionAppendStr(&buf, "(", &rc); + for(i=0; inCol; i++){ + if( i!=0 ) sessionAppendStr(&buf, ", ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + } + + sessionAppendStr(&buf, ") VALUES(?", &rc); + for(i=1; inCol; i++){ + sessionAppendStr(&buf, ", ?", &rc); + } + sessionAppendStr(&buf, ")", &rc); + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pInsert, 0); + } + sqlite3_free(buf.aBuf); + return rc; +} + +/* +** A wrapper around sqlite3_bind_value() that detects an extra problem. +** See comments in the body of this function for details. +*/ +static int sessionBindValue( + sqlite3_stmt *pStmt, /* Statement to bind value to */ + int i, /* Parameter number to bind to */ + sqlite3_value *pVal /* Value to bind */ +){ + int eType = sqlite3_value_type(pVal); + /* COVERAGE: The (pVal->z==0) branch is never true using current versions + ** of SQLite. If a malloc fails in an sqlite3_value_xxx() function, either + ** the (pVal->z) variable remains as it was or the type of the value is + ** set to SQLITE_NULL. */ + if( (eType==SQLITE_TEXT || eType==SQLITE_BLOB) && pVal->z==0 ){ + /* This condition occurs when an earlier OOM in a call to + ** sqlite3_value_text() or sqlite3_value_blob() (perhaps from within + ** a conflict-handler) has zeroed the pVal->z pointer. Return NOMEM. */ + return SQLITE_NOMEM; + } + return sqlite3_bind_value(pStmt, i, pVal); +} + +/* +** Iterator pIter must point to an SQLITE_INSERT entry. This function +** transfers new.* values from the current iterator entry to statement +** pStmt. The table being inserted into has nCol columns. +** +** New.* value $i from the iterator is bound to variable ($i+1) of +** statement pStmt. If parameter abPK is NULL, all values from 0 to (nCol-1) +** are transfered to the statement. Otherwise, if abPK is not NULL, it points +** to an array nCol elements in size. In this case only those values for +** which abPK[$i] is true are read from the iterator and bound to the +** statement. +** +** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +*/ +static int sessionBindRow( + sqlite3_changeset_iter *pIter, /* Iterator to read values from */ + int(*xValue)(sqlite3_changeset_iter *, int, sqlite3_value **), + int nCol, /* Number of columns */ + u8 *abPK, /* If not NULL, bind only if true */ + sqlite3_stmt *pStmt /* Bind values to this statement */ +){ + int i; + int rc = SQLITE_OK; + + /* Neither sqlite3changeset_old or sqlite3changeset_new can fail if the + ** argument iterator points to a suitable entry. Make sure that xValue + ** is one of these to guarantee that it is safe to ignore the return + ** in the code below. */ + assert( xValue==sqlite3changeset_old || xValue==sqlite3changeset_new ); + + for(i=0; rc==SQLITE_OK && idb, pIter, p->abPK, p->pSelect); + }else{ + rc = SQLITE_OK; + } + + if( rc==SQLITE_ROW ){ + /* There exists another row with the new.* primary key. */ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + rc = sqlite3_reset(p->pSelect); + }else if( rc==SQLITE_OK ){ + if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ + /* Instead of invoking the conflict handler, append the change blob + ** to the SessionApplyCtx.constraints buffer. */ + u8 *aBlob = &pIter->in.aData[pIter->in.iCurrent]; + int nBlob = pIter->in.iNext - pIter->in.iCurrent; + sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); + res = SQLITE_CHANGESET_OMIT; + }else{ + /* No other row with the new.* primary key. */ + res = xConflict(pCtx, eType+1, pIter); + if( res==SQLITE_CHANGESET_REPLACE ) rc = SQLITE_MISUSE; + } + } + + if( rc==SQLITE_OK ){ + switch( res ){ + case SQLITE_CHANGESET_REPLACE: + assert( pbReplace ); + *pbReplace = 1; + break; + + case SQLITE_CHANGESET_OMIT: + break; + + case SQLITE_CHANGESET_ABORT: + rc = SQLITE_ABORT; + break; + + default: + rc = SQLITE_MISUSE; + break; + } + } + + return rc; +} + +/* +** Attempt to apply the change that the iterator passed as the first argument +** currently points to to the database. If a conflict is encountered, invoke +** the conflict handler callback. +** +** If argument pbRetry is NULL, then ignore any CHANGESET_DATA conflict. If +** one is encountered, update or delete the row with the matching primary key +** instead. Or, if pbRetry is not NULL and a CHANGESET_DATA conflict occurs, +** invoke the conflict handler. If it returns CHANGESET_REPLACE, set *pbRetry +** to true before returning. In this case the caller will invoke this function +** again, this time with pbRetry set to NULL. +** +** If argument pbReplace is NULL and a CHANGESET_CONFLICT conflict is +** encountered invoke the conflict handler with CHANGESET_CONSTRAINT instead. +** Or, if pbReplace is not NULL, invoke it with CHANGESET_CONFLICT. If such +** an invocation returns SQLITE_CHANGESET_REPLACE, set *pbReplace to true +** before retrying. In this case the caller attempts to remove the conflicting +** row before invoking this function again, this time with pbReplace set +** to NULL. +** +** If any conflict handler returns SQLITE_CHANGESET_ABORT, this function +** returns SQLITE_ABORT. Otherwise, if no error occurs, SQLITE_OK is +** returned. +*/ +static int sessionApplyOneOp( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + SessionApplyCtx *p, /* changeset_apply() context */ + int(*xConflict)(void *, int, sqlite3_changeset_iter *), + void *pCtx, /* First argument for the conflict handler */ + int *pbReplace, /* OUT: True to remove PK row and retry */ + int *pbRetry /* OUT: True to retry. */ +){ + const char *zDummy; + int op; + int nCol; + int rc = SQLITE_OK; + + assert( p->pDelete && p->pUpdate && p->pInsert && p->pSelect ); + assert( p->azCol && p->abPK ); + assert( !pbReplace || *pbReplace==0 ); + + sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); + + if( op==SQLITE_DELETE ){ + + /* Bind values to the DELETE statement. If conflict handling is required, + ** bind values for all columns and set bound variable (nCol+1) to true. + ** Or, if conflict handling is not required, bind just the PK column + ** values and, if it exists, set (nCol+1) to false. Conflict handling + ** is not required if: + ** + ** * this is a patchset, or + ** * (pbRetry==0), or + ** * all columns of the table are PK columns (in this case there is + ** no (nCol+1) variable to bind to). + */ + u8 *abPK = (pIter->bPatchset ? p->abPK : 0); + rc = sessionBindRow(pIter, sqlite3changeset_old, nCol, abPK, p->pDelete); + if( rc==SQLITE_OK && sqlite3_bind_parameter_count(p->pDelete)>nCol ){ + rc = sqlite3_bind_int(p->pDelete, nCol+1, (pbRetry==0 || abPK)); + } + if( rc!=SQLITE_OK ) return rc; + + sqlite3_step(p->pDelete); + rc = sqlite3_reset(p->pDelete); + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry + ); + }else if( (rc&0xff)==SQLITE_CONSTRAINT ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, 0 + ); + } + + }else if( op==SQLITE_UPDATE ){ + int i; + + /* Bind values to the UPDATE statement. */ + for(i=0; rc==SQLITE_OK && ipUpdate, i*3+2, !!pNew); + if( pOld ){ + rc = sessionBindValue(p->pUpdate, i*3+1, pOld); + } + if( rc==SQLITE_OK && pNew ){ + rc = sessionBindValue(p->pUpdate, i*3+3, pNew); + } + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int(p->pUpdate, nCol*3+1, pbRetry==0 || pIter->bPatchset); + } + if( rc!=SQLITE_OK ) return rc; + + /* Attempt the UPDATE. In the case of a NOTFOUND or DATA conflict, + ** the result will be SQLITE_OK with 0 rows modified. */ + sqlite3_step(p->pUpdate); + rc = sqlite3_reset(p->pUpdate); + + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + /* A NOTFOUND or DATA error. Search the table to see if it contains + ** a row with a matching primary key. If so, this is a DATA conflict. + ** Otherwise, if there is no primary key match, it is a NOTFOUND. */ + + rc = sessionConflictHandler( + SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry + ); + + }else if( (rc&0xff)==SQLITE_CONSTRAINT ){ + /* This is always a CONSTRAINT conflict. */ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, 0 + ); + } + + }else{ + assert( op==SQLITE_INSERT ); + rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert); + if( rc!=SQLITE_OK ) return rc; + + sqlite3_step(p->pInsert); + rc = sqlite3_reset(p->pInsert); + if( (rc&0xff)==SQLITE_CONSTRAINT ){ + rc = sessionConflictHandler( + SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, pbReplace + ); + } + } + + return rc; +} + +/* +** Attempt to apply the change that the iterator passed as the first argument +** currently points to to the database. If a conflict is encountered, invoke +** the conflict handler callback. +** +** The difference between this function and sessionApplyOne() is that this +** function handles the case where the conflict-handler is invoked and +** returns SQLITE_CHANGESET_REPLACE - indicating that the change should be +** retried in some manner. +*/ +static int sessionApplyOneWithRetry( + sqlite3 *db, /* Apply change to "main" db of this handle */ + sqlite3_changeset_iter *pIter, /* Changeset iterator to read change from */ + SessionApplyCtx *pApply, /* Apply context */ + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int bReplace = 0; + int bRetry = 0; + int rc; + + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, &bRetry); + assert( rc==SQLITE_OK || (bRetry==0 && bReplace==0) ); + + /* If the bRetry flag is set, the change has not been applied due to an + ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and + ** a row with the correct PK is present in the db, but one or more other + ** fields do not contain the expected values) and the conflict handler + ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation, + ** but pass NULL as the final argument so that sessionApplyOneOp() ignores + ** the SQLITE_CHANGESET_DATA problem. */ + if( bRetry ){ + assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE ); + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + + /* If the bReplace flag is set, the change is an INSERT that has not + ** been performed because the database already contains a row with the + ** specified primary key and the conflict handler returned + ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row + ** before reattempting the INSERT. */ + else if( bReplace ){ + assert( pIter->op==SQLITE_INSERT ); + rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sessionBindRow(pIter, + sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); + sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pApply->pDelete); + rc = sqlite3_reset(pApply->pDelete); + } + if( rc==SQLITE_OK ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + } + } + + return rc; +} + +/* +** Retry the changes accumulated in the pApply->constraints buffer. +*/ +static int sessionRetryConstraints( + sqlite3 *db, + int bPatchset, + const char *zTab, + SessionApplyCtx *pApply, + int(*xConflict)(void*, int, sqlite3_changeset_iter*), + void *pCtx /* First argument passed to xConflict */ +){ + int rc = SQLITE_OK; + + while( pApply->constraints.nBuf ){ + sqlite3_changeset_iter *pIter2 = 0; + SessionBuffer cons = pApply->constraints; + memset(&pApply->constraints, 0, sizeof(SessionBuffer)); + + rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf); + if( rc==SQLITE_OK ){ + int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + int rc2; + pIter2->bPatchset = bPatchset; + pIter2->zTab = (char*)zTab; + pIter2->nCol = pApply->nCol; + pIter2->abPK = pApply->abPK; + sessionBufferGrow(&pIter2->tblhdr, nByte, &rc); + pIter2->apValue = (sqlite3_value**)pIter2->tblhdr.aBuf; + if( rc==SQLITE_OK ) memset(pIter2->apValue, 0, nByte); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter2) ){ + rc = sessionApplyOneWithRetry(db, pIter2, pApply, xConflict, pCtx); + } + + rc2 = sqlite3changeset_finalize(pIter2); + if( rc==SQLITE_OK ) rc = rc2; + } + assert( pApply->bDeferConstraints || pApply->constraints.nBuf==0 ); + + sqlite3_free(cons.aBuf); + if( rc!=SQLITE_OK ) break; + if( pApply->constraints.nBuf>=cons.nBuf ){ + /* No progress was made on the last round. */ + pApply->bDeferConstraints = 0; + } + } + + return rc; +} + +/* +** Argument pIter is a changeset iterator that has been initialized, but +** not yet passed to sqlite3changeset_next(). This function applies the +** changeset to the main database attached to handle "db". The supplied +** conflict handler callback is invoked to resolve any conflicts encountered +** while applying the change. +*/ +static int sessionChangesetApply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + sqlite3_changeset_iter *pIter, /* Changeset to apply */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of fifth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + int schemaMismatch = 0; + int rc; /* Return code */ + const char *zTab = 0; /* Name of current table */ + int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ + SessionApplyCtx sApply; /* changeset_apply() context object */ + int bPatchset; + + assert( xConflict!=0 ); + + pIter->in.bNoDiscard = 1; + memset(&sApply, 0, sizeof(sApply)); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0); + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3changeset_next(pIter) ){ + int nCol; + int op; + const char *zNew; + + sqlite3changeset_op(pIter, &zNew, &nCol, &op, 0); + + if( zTab==0 || sqlite3_strnicmp(zNew, zTab, nTab+1) ){ + u8 *abPK; + + rc = sessionRetryConstraints( + db, pIter->bPatchset, zTab, &sApply, xConflict, pCtx + ); + if( rc!=SQLITE_OK ) break; + + sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ + sqlite3_finalize(sApply.pDelete); + sqlite3_finalize(sApply.pUpdate); + sqlite3_finalize(sApply.pInsert); + sqlite3_finalize(sApply.pSelect); + memset(&sApply, 0, sizeof(sApply)); + sApply.db = db; + sApply.bDeferConstraints = 1; + + /* If an xFilter() callback was specified, invoke it now. If the + ** xFilter callback returns zero, skip this table. If it returns + ** non-zero, proceed. */ + schemaMismatch = (xFilter && (0==xFilter(pCtx, zNew))); + if( schemaMismatch ){ + zTab = sqlite3_mprintf("%s", zNew); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + break; + } + nTab = (int)strlen(zTab); + sApply.azCol = (const char **)zTab; + }else{ + int nMinCol = 0; + int i; + + sqlite3changeset_pk(pIter, &abPK, 0); + rc = sessionTableInfo( + db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + ); + if( rc!=SQLITE_OK ) break; + for(i=0; ibPatchset; + if( rc==SQLITE_OK ){ + rc = sqlite3changeset_finalize(pIter); + }else{ + sqlite3changeset_finalize(pIter); + } + + if( rc==SQLITE_OK ){ + rc = sessionRetryConstraints(db, bPatchset, zTab, &sApply, xConflict, pCtx); + } + + if( rc==SQLITE_OK ){ + int nFk, notUsed; + sqlite3_db_status(db, SQLITE_DBSTATUS_DEFERRED_FKS, &nFk, ¬Used, 0); + if( nFk!=0 ){ + int res = SQLITE_CHANGESET_ABORT; + sqlite3_changeset_iter sIter; + memset(&sIter, 0, sizeof(sIter)); + sIter.nCol = nFk; + res = xConflict(pCtx, SQLITE_CHANGESET_FOREIGN_KEY, &sIter); + if( res!=SQLITE_CHANGESET_OMIT ){ + rc = SQLITE_CONSTRAINT; + } + } + } + sqlite3_exec(db, "PRAGMA defer_foreign_keys = 0", 0, 0, 0); + + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0); + }else{ + sqlite3_exec(db, "ROLLBACK TO changeset_apply", 0, 0, 0); + sqlite3_exec(db, "RELEASE changeset_apply", 0, 0, 0); + } + + sqlite3_finalize(sApply.pInsert); + sqlite3_finalize(sApply.pDelete); + sqlite3_finalize(sApply.pUpdate); + sqlite3_finalize(sApply.pSelect); + sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ + sqlite3_free((char*)sApply.constraints.aBuf); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; +} + +/* +** Apply the changeset passed via pChangeset/nChangeset to the main database +** attached to handle "db". Invoke the supplied conflict handler callback +** to resolve any conflicts encountered while applying the change. +*/ +int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of fifth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); + } + return rc; +} + +/* +** Apply the changeset passed via xInput/pIn to the main database +** attached to handle "db". Invoke the supplied conflict handler callback +** to resolve any conflicts encountered while applying the change. +*/ +int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); + } + return rc; +} + +/* +** sqlite3_changegroup handle. +*/ +struct sqlite3_changegroup { + int rc; /* Error code */ + int bPatch; /* True to accumulate patchsets */ + SessionTable *pList; /* List of tables in current patch */ +}; + +/* +** This function is called to merge two changes to the same row together as +** part of an sqlite3changeset_concat() operation. A new change object is +** allocated and a pointer to it stored in *ppNew. +*/ +static int sessionChangeMerge( + SessionTable *pTab, /* Table structure */ + int bPatchset, /* True for patchsets */ + SessionChange *pExist, /* Existing change */ + int op2, /* Second change operation */ + int bIndirect, /* True if second change is indirect */ + u8 *aRec, /* Second change record */ + int nRec, /* Number of bytes in aRec */ + SessionChange **ppNew /* OUT: Merged change */ +){ + SessionChange *pNew = 0; + + if( !pExist ){ + pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange) + nRec); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SessionChange)); + pNew->op = op2; + pNew->bIndirect = bIndirect; + pNew->nRecord = nRec; + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, aRec, nRec); + }else{ + int op1 = pExist->op; + + /* + ** op1=INSERT, op2=INSERT -> Unsupported. Discard op2. + ** op1=INSERT, op2=UPDATE -> INSERT. + ** op1=INSERT, op2=DELETE -> (none) + ** + ** op1=UPDATE, op2=INSERT -> Unsupported. Discard op2. + ** op1=UPDATE, op2=UPDATE -> UPDATE. + ** op1=UPDATE, op2=DELETE -> DELETE. + ** + ** op1=DELETE, op2=INSERT -> UPDATE. + ** op1=DELETE, op2=UPDATE -> Unsupported. Discard op2. + ** op1=DELETE, op2=DELETE -> Unsupported. Discard op2. + */ + if( (op1==SQLITE_INSERT && op2==SQLITE_INSERT) + || (op1==SQLITE_UPDATE && op2==SQLITE_INSERT) + || (op1==SQLITE_DELETE && op2==SQLITE_UPDATE) + || (op1==SQLITE_DELETE && op2==SQLITE_DELETE) + ){ + pNew = pExist; + }else if( op1==SQLITE_INSERT && op2==SQLITE_DELETE ){ + sqlite3_free(pExist); + assert( pNew==0 ); + }else{ + u8 *aExist = pExist->aRecord; + int nByte; + u8 *aCsr; + + /* Allocate a new SessionChange object. Ensure that the aRecord[] + ** buffer of the new object is large enough to hold any record that + ** may be generated by combining the input records. */ + nByte = sizeof(SessionChange) + pExist->nRecord + nRec; + pNew = (SessionChange *)sqlite3_malloc(nByte); + if( !pNew ){ + sqlite3_free(pExist); + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SessionChange)); + pNew->bIndirect = (bIndirect && pExist->bIndirect); + aCsr = pNew->aRecord = (u8 *)&pNew[1]; + + if( op1==SQLITE_INSERT ){ /* INSERT + UPDATE */ + u8 *a1 = aRec; + assert( op2==SQLITE_UPDATE ); + pNew->op = SQLITE_INSERT; + if( bPatchset==0 ) sessionSkipRecord(&a1, pTab->nCol); + sessionMergeRecord(&aCsr, pTab->nCol, aExist, a1); + }else if( op1==SQLITE_DELETE ){ /* DELETE + INSERT */ + assert( op2==SQLITE_INSERT ); + pNew->op = SQLITE_UPDATE; + if( bPatchset ){ + memcpy(aCsr, aRec, nRec); + aCsr += nRec; + }else{ + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist, 0,aRec,0) ){ + sqlite3_free(pNew); + pNew = 0; + } + } + }else if( op2==SQLITE_UPDATE ){ /* UPDATE + UPDATE */ + u8 *a1 = aExist; + u8 *a2 = aRec; + assert( op1==SQLITE_UPDATE ); + if( bPatchset==0 ){ + sessionSkipRecord(&a1, pTab->nCol); + sessionSkipRecord(&a2, pTab->nCol); + } + pNew->op = SQLITE_UPDATE; + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aRec, aExist,a1,a2) ){ + sqlite3_free(pNew); + pNew = 0; + } + }else{ /* UPDATE + DELETE */ + assert( op1==SQLITE_UPDATE && op2==SQLITE_DELETE ); + pNew->op = SQLITE_DELETE; + if( bPatchset ){ + memcpy(aCsr, aRec, nRec); + aCsr += nRec; + }else{ + sessionMergeRecord(&aCsr, pTab->nCol, aRec, aExist); + } + } + + if( pNew ){ + pNew->nRecord = (int)(aCsr - pNew->aRecord); + } + sqlite3_free(pExist); + } + } + + *ppNew = pNew; + return SQLITE_OK; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp /* Changegroup object to add changeset to */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + SessionTable *pTab = 0; + + + while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec) ){ + const char *zNew; + int nCol; + int op; + int iHash; + int bIndirect; + SessionChange *pChange; + SessionChange *pExist = 0; + SessionChange **pp; + + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + break; + } + + sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); + if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ + /* Search the list for a matching table */ + int nNew = (int)strlen(zNew); + u8 *abPK; + + sqlite3changeset_pk(pIter, &abPK, 0); + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; + } + if( !pTab ){ + SessionTable **ppTab; + + pTab = sqlite3_malloc(sizeof(SessionTable) + nCol + nNew+1); + if( !pTab ){ + rc = SQLITE_NOMEM; + break; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zNew, nNew+1); + + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); + *ppTab = pTab; + }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ + rc = SQLITE_SCHEMA; + break; + } + } + + if( sessionGrowHash(pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + break; + } + iHash = sessionChangeHash( + pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + ); + + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. + */ + for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ + int bPkOnly1 = 0; + int bPkOnly2 = 0; + if( pIter->bPatchset ){ + bPkOnly1 = (*pp)->op==SQLITE_DELETE; + bPkOnly2 = op==SQLITE_DELETE; + } + if( sessionChangeEqual(pTab, bPkOnly1, (*pp)->aRecord, bPkOnly2, aRec) ){ + pExist = *pp; + *pp = (*pp)->pNext; + pTab->nEntry--; + break; + } + } + + rc = sessionChangeMerge(pTab, + pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + ); + if( rc ) break; + if( pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Serialize a changeset (or patchset) based on all changesets (or patchsets) +** added to the changegroup object passed as the first argument. +** +** If xOutput is not NULL, then the changeset/patchset is returned to the +** user via one or more calls to xOutput, as with the other streaming +** interfaces. +** +** Or, if xOutput is NULL, then (*ppOut) is populated with a pointer to a +** buffer containing the output changeset before this function returns. In +** this case (*pnOut) is set to the size of the output buffer in bytes. It +** is the responsibility of the caller to free the output buffer using +** sqlite3_free() when it is no longer required. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an SQLite +** error code. If an error occurs and xOutput is NULL, (*ppOut) and (*pnOut) +** are both set to 0 before returning. +*/ +static int sessionChangegroupOutput( + sqlite3_changegroup *pGrp, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, + int *pnOut, + void **ppOut +){ + int rc = SQLITE_OK; + SessionBuffer buf = {0, 0, 0}; + SessionTable *pTab; + assert( xOutput==0 || (ppOut==0 && pnOut==0) ); + + /* Create the serialized output changeset based on the contents of the + ** hash tables attached to the SessionTable objects in list p->pList. + */ + for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + int i; + if( pTab->nEntry==0 ) continue; + + sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc); + for(i=0; inChange; i++){ + SessionChange *p; + for(p=pTab->apChange[i]; p; p=p->pNext){ + sessionAppendByte(&buf, p->op, &rc); + sessionAppendByte(&buf, p->bIndirect, &rc); + sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc); + } + } + + if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ + rc = xOutput(pOut, buf.aBuf, buf.nBuf); + buf.nBuf = 0; + } + } + + if( rc==SQLITE_OK ){ + if( xOutput ){ + if( buf.nBuf>0 ) rc = xOutput(pOut, buf.aBuf, buf.nBuf); + }else{ + *ppOut = buf.aBuf; + *pnOut = buf.nBuf; + buf.aBuf = 0; + } + } + sqlite3_free(buf.aBuf); + + return rc; +} + +/* +** Allocate a new, empty, sqlite3_changegroup. +*/ +int sqlite3changegroup_new(sqlite3_changegroup **pp){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_changegroup *p; /* New object */ + p = (sqlite3_changegroup*)sqlite3_malloc(sizeof(sqlite3_changegroup)); + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(p, 0, sizeof(sqlite3_changegroup)); + } + *pp = p; + return rc; +} + +/* +** Add the changeset currently stored in buffer pData, size nData bytes, +** to changeset-group p. +*/ +int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pData/nData */ + int rc; /* Return code */ + + rc = sqlite3changeset_start(&pIter, nData, pData); + if( rc==SQLITE_OK ){ + rc = sessionChangesetToHash(pIter, pGrp); + } + sqlite3changeset_finalize(pIter); + return rc; +} + +/* +** Obtain a buffer containing a changeset representing the concatenation +** of all changesets added to the group so far. +*/ +int sqlite3changegroup_output( + sqlite3_changegroup *pGrp, + int *pnData, + void **ppData +){ + return sessionChangegroupOutput(pGrp, 0, 0, pnData, ppData); +} + +/* +** Streaming versions of changegroup_add(). +*/ +int sqlite3changegroup_add_strm( + sqlite3_changegroup *pGrp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pData/nData */ + int rc; /* Return code */ + + rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); + if( rc==SQLITE_OK ){ + rc = sessionChangesetToHash(pIter, pGrp); + } + sqlite3changeset_finalize(pIter); + return rc; +} + +/* +** Streaming versions of changegroup_output(). +*/ +int sqlite3changegroup_output_strm( + sqlite3_changegroup *pGrp, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + return sessionChangegroupOutput(pGrp, xOutput, pOut, 0, 0); +} + +/* +** Delete a changegroup object. +*/ +void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ + if( pGrp ){ + sessionDeleteTable(pGrp->pList); + sqlite3_free(pGrp); + } +} + +/* +** Combine two changesets together. +*/ +int sqlite3changeset_concat( + int nLeft, /* Number of bytes in lhs input */ + void *pLeft, /* Lhs input changeset */ + int nRight /* Number of bytes in rhs input */, + void *pRight, /* Rhs input changeset */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: changeset (left right) */ +){ + sqlite3_changegroup *pGrp; + int rc; + + rc = sqlite3changegroup_new(&pGrp); + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add(pGrp, nLeft, pLeft); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add(pGrp, nRight, pRight); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); + } + sqlite3changegroup_delete(pGrp); + + return rc; +} + +/* +** Streaming version of sqlite3changeset_concat(). +*/ +int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + sqlite3_changegroup *pGrp; + int rc; + + rc = sqlite3changegroup_new(&pGrp); + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add_strm(pGrp, xInputA, pInA); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_add_strm(pGrp, xInputB, pInB); + } + if( rc==SQLITE_OK ){ + rc = sqlite3changegroup_output_strm(pGrp, xOutput, pOut); + } + sqlite3changegroup_delete(pGrp); + + return rc; +} + +#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h new file mode 100644 index 0000000000..0882cf34cf --- /dev/null +++ b/ext/session/sqlite3session.h @@ -0,0 +1,1288 @@ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + +#include "sqlite3.h" + +/* +** CAPI3REF: Session Object Handle +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +void sqlite3session_delete(sqlite3_session *pSession); + + +/* +** CAPI3REF: Enable Or Disable A Session Object +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
      +**
    • The session object "indirect" flag is set when the change is +** made, or +**
    • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
    +** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +*/ +int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes is not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

    Changeset Generation

    +** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
      +**
    • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
    • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
    +** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
      +**
    • Has the same name, +**
    • Has the same set of columns declared in the same order, and +**
    • Has the same PRIMARY KEY definition. +**
    +** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
      +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
    • For each row (primary key) that exists in both tables, but features +** different non-PK values in each, an UPDATE record is added to the +** session. +**
    +** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** +** The differences between a patchset and a changeset are that: +** +**
      +**
    • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
    • The original values of any modified fields are omitted from +** UPDATE records. +**
    +** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ + void **ppPatchset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
      +**
    • [sqlite3changeset_next()] +**
    • [sqlite3changeset_op()] +**
    • [sqlite3changeset_new()] +**
    • [sqlite3changeset_old()] +**
    +** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +*/ +int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); + + +/* +** CAPI3REF: Advance A Changeset Iterator +** +** This function may only be used with iterators created by function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** If argument pzTab is not NULL, then *pzTab is set to point to a +** nul-terminated utf-8 encoded string containing the name of the table +** affected by the current change. The buffer remains valid until either +** sqlite3changeset_next() is called on the iterator or until the +** conflict-handler function returns. If pnCol is not NULL, then *pnCol is +** set to the number of columns in the table affected by the change. If +** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. Finally, if pOp is not NULL, then *pOp is set to one of +** [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], depending on the +** type of change that the iterator currently points to. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** +** For each modified table, a changeset includes the following: +** +**
      +**
    • The number of columns in the table, and +**
    • Which of those columns make up the tables PRIMARY KEY. +**
    +** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** sqlite3changeset_start(); +** while( SQLITE_ROW==sqlite3changeset_next() ){ +** // Do something with change. +** } +** rc = sqlite3changeset_finalize(); +** if( rc!=SQLITE_OK ){ +** // An error has occurred +** } +*/ +int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
      +**
    • Each DELETE change is changed to an INSERT, and +**
    • Each INSERT change is changed to a DELETE, and +**
    • For each UPDATE change, the old.* and new.* values are exchanged. +**
    +** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** sqlite3_changegroup *pGrp; +** rc = sqlite3_changegroup_new(&pGrp); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA); +** if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB); +** if( rc==SQLITE_OK ){ +** rc = sqlite3changegroup_output(pGrp, pnOut, ppOut); +** }else{ +** *ppOut = 0; +** *pnOut = 0; +** } +** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Changegroup Handle +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
      +**
    • It is created using a call to sqlite3changegroup_new(). +** +**
    • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
    • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
    • The object is deleted using a call to sqlite3changegroup_delete(). +**
    +** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
    Existing Change New Change Output Change +**
    INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
    INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
    UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
    UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
    DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
    DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    +** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. If the input changeset +** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is +** returned. Or, if an out-of-memory condition occurs during processing, this +** function returns SQLITE_NOMEM. In all cases, if an error occurs the +** final contents of the changegroup is undefined. +** +** If no error occurs, SQLITE_OK is returned. +*/ +int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +*/ +void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset to a database. This function attempts to update the +** "main" database attached to handle db with the changes found in the +** changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to this function is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument to this function as the first. If the "filter +** callback" returns zero, then no attempt is made to apply any changes to +** the table. Otherwise, if the return value is non-zero or the xFilter +** argument to this function is NULL, all changes related to the table are +** attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
      +**
    • The table has the same name as the name recorded in the +** changeset, and +**
    • The table has at least as many columns as recorded in the +** changeset, and +**
    • The table has primary key columns in the same position as +** recorded in the changeset. +**
    +** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
    +**
    DELETE Changes
    +** For each DELETE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
    INSERT Changes
    +** For each INSERT change, an attempt is made to insert the new row into +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
    UPDATE Changes
    +** For each UPDATE change, this function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
    +** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the applications conflict +** resolution strategy. +** +** All changes made by this function are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +*/ +int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
    +**
    SQLITE_CHANGESET_DATA
    +** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_NOTFOUND
    +** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    SQLITE_CHANGESET_CONFLICT
    +** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_FOREIGN_KEY
    +** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
    SQLITE_CHANGESET_CONSTRAINT
    +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    +*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
    +**
    SQLITE_CHANGESET_OMIT
    +** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
    SQLITE_CHANGESET_REPLACE
    +** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
    SQLITE_CHANGESET_ABORT
    +** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
    +*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
    Streaming functionNon-streaming equivalent
    sqlite3changeset_apply_str[sqlite3changeset_apply] +**
    sqlite3changeset_concat_str[sqlite3changeset_concat] +**
    sqlite3changeset_invert_str[sqlite3changeset_invert] +**
    sqlite3changeset_start_str[sqlite3changeset_start] +**
    sqlite3session_changeset_str[sqlite3session_changeset] +**
    sqlite3session_patchset_str[sqlite3session_patchset] +**
    +** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
    +**        int nChangeset,
+**        void *pChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
    +** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
    +**        int *pnChangeset,
+**        void **ppChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
    +** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ diff --git a/ext/session/test_session.c b/ext/session/test_session.c new file mode 100644 index 0000000000..411354cc02 --- /dev/null +++ b/ext/session/test_session.c @@ -0,0 +1,1046 @@ + +#if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_SESSION) \ + && defined(SQLITE_ENABLE_PREUPDATE_HOOK) + +#include "sqlite3session.h" +#include +#include +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +# ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +# endif +#endif + +typedef struct TestSession TestSession; +struct TestSession { + sqlite3_session *pSession; + Tcl_Interp *interp; + Tcl_Obj *pFilterScript; +}; + +typedef struct TestStreamInput TestStreamInput; +struct TestStreamInput { + int nStream; /* Maximum chunk size */ + unsigned char *aData; /* Pointer to buffer containing data */ + int nData; /* Size of buffer aData in bytes */ + int iData; /* Bytes of data already read by sessions */ +}; + +/* +** Extract an sqlite3* db handle from the object passed as the second +** argument. If successful, set *pDb to point to the db handle and return +** TCL_OK. Otherwise, return TCL_ERROR. +*/ +static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + +/************************************************************************* +** The following code is copied byte-for-byte from the sessions module +** documentation. It is used by some of the sessions modules tests to +** ensure that the example in the documentation does actually work. +*/ +/* +** Argument zSql points to a buffer containing an SQL script to execute +** against the database handle passed as the first argument. As well as +** executing the SQL script, this function collects a changeset recording +** all changes made to the "main" database file. Assuming no error occurs, +** output variables (*ppChangeset) and (*pnChangeset) are set to point +** to a buffer containing the changeset and the size of the changeset in +** bytes before returning SQLITE_OK. In this case it is the responsibility +** of the caller to eventually free the changeset blob by passing it to +** the sqlite3_free function. +** +** Or, if an error does occur, return an SQLite error code. The final +** value of (*pChangeset) and (*pnChangeset) are undefined in this case. +*/ +int sql_exec_changeset( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL script to execute */ + int *pnChangeset, /* OUT: Size of changeset blob in bytes */ + void **ppChangeset /* OUT: Pointer to changeset blob */ +){ + sqlite3_session *pSession = 0; + int rc; + + /* Create a new session object */ + rc = sqlite3session_create(db, "main", &pSession); + + /* Configure the session object to record changes to all tables */ + if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL); + + /* Execute the SQL script */ + if( rc==SQLITE_OK ) rc = sqlite3_exec(db, zSql, 0, 0, 0); + + /* Collect the changeset */ + if( rc==SQLITE_OK ){ + rc = sqlite3session_changeset(pSession, pnChangeset, ppChangeset); + } + + /* Delete the session object */ + sqlite3session_delete(pSession); + + return rc; +} +/************************************************************************/ + +/* +** Tclcmd: sql_exec_changeset DB SQL +*/ +static int SQLITE_TCLAPI test_sql_exec_changeset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zSql; + sqlite3 *db; + void *pChangeset; + int nChangeset; + int rc; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SQL"); + return TCL_ERROR; + } + if( dbHandleFromObj(interp, objv[1], &db) ) return TCL_ERROR; + zSql = (const char*)Tcl_GetString(objv[2]); + + rc = sql_exec_changeset(db, zSql, &nChangeset, &pChangeset); + if( rc!=SQLITE_OK ){ + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, "error in sql_exec_changeset()", 0); + return TCL_ERROR; + } + + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pChangeset, nChangeset)); + sqlite3_free(pChangeset); + return TCL_OK; +} + + + +#define SESSION_STREAM_TCL_VAR "sqlite3session_streams" + +/* +** Attempt to find the global variable zVar within interpreter interp +** and extract an integer value from it. Return this value. +** +** If the named variable cannot be found, or if it cannot be interpreted +** as a integer, return 0. +*/ +static int test_tcl_integer(Tcl_Interp *interp, const char *zVar){ + Tcl_Obj *pObj; + int iVal = 0; + pObj = Tcl_ObjGetVar2(interp, Tcl_NewStringObj(zVar, -1), 0, TCL_GLOBAL_ONLY); + if( pObj ) Tcl_GetIntFromObj(0, pObj, &iVal); + return iVal; +} + +static int test_session_error(Tcl_Interp *interp, int rc, char *zErr){ + extern const char *sqlite3ErrName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + if( zErr ){ + Tcl_AppendResult(interp, " - ", zErr, 0); + sqlite3_free(zErr); + } + return TCL_ERROR; +} + +static int test_table_filter(void *pCtx, const char *zTbl){ + TestSession *p = (TestSession*)pCtx; + Tcl_Obj *pEval; + int rc; + int bRes = 0; + + pEval = Tcl_DuplicateObj(p->pFilterScript); + Tcl_IncrRefCount(pEval); + rc = Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zTbl, -1)); + if( rc==TCL_OK ){ + rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL); + } + if( rc==TCL_OK ){ + rc = Tcl_GetBooleanFromObj(p->interp, Tcl_GetObjResult(p->interp), &bRes); + } + if( rc!=TCL_OK ){ + /* printf("error: %s\n", Tcl_GetStringResult(p->interp)); */ + Tcl_BackgroundError(p->interp); + } + Tcl_DecrRefCount(pEval); + + return bRes; +} + +struct TestSessionsBlob { + void *p; + int n; +}; +typedef struct TestSessionsBlob TestSessionsBlob; + +static int testStreamOutput( + void *pCtx, + const void *pData, + int nData +){ + TestSessionsBlob *pBlob = (TestSessionsBlob*)pCtx; + char *pNew; + + assert( nData>0 ); + pNew = (char*)sqlite3_realloc(pBlob->p, pBlob->n + nData); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pBlob->p = (void*)pNew; + memcpy(&pNew[pBlob->n], pData, nData); + pBlob->n += nData; + return SQLITE_OK; +} + +/* +** Tclcmd: $session attach TABLE +** $session changeset +** $session delete +** $session enable BOOL +** $session indirect INTEGER +** $session patchset +** $session table_filter SCRIPT +*/ +static int SQLITE_TCLAPI test_session_cmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + TestSession *p = (TestSession*)clientData; + sqlite3_session *pSession = p->pSession; + struct SessionSubcmd { + const char *zSub; + int nArg; + const char *zMsg; + int iSub; + } aSub[] = { + { "attach", 1, "TABLE", }, /* 0 */ + { "changeset", 0, "", }, /* 1 */ + { "delete", 0, "", }, /* 2 */ + { "enable", 1, "BOOL", }, /* 3 */ + { "indirect", 1, "BOOL", }, /* 4 */ + { "isempty", 0, "", }, /* 5 */ + { "table_filter", 1, "SCRIPT", }, /* 6 */ + { "patchset", 0, "", }, /* 7 */ + { "diff", 2, "FROMDB TBL", }, /* 8 */ + { 0 } + }; + int iSub; + int rc; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( objc!=2+aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: { /* attach */ + char *zArg = Tcl_GetString(objv[2]); + if( zArg[0]=='*' && zArg[1]=='\0' ) zArg = 0; + rc = sqlite3session_attach(pSession, zArg); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + break; + } + + case 7: /* patchset */ + case 1: { /* changeset */ + TestSessionsBlob o = {0, 0}; + if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + void *pCtx = (void*)&o; + if( iSub==7 ){ + rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx); + }else{ + rc = sqlite3session_changeset_strm(pSession, testStreamOutput, pCtx); + } + }else{ + if( iSub==7 ){ + rc = sqlite3session_patchset(pSession, &o.n, &o.p); + }else{ + rc = sqlite3session_changeset(pSession, &o.n, &o.p); + } + } + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(o.p, o.n)); + } + sqlite3_free(o.p); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + break; + } + + case 2: /* delete */ + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + + case 3: { /* enable */ + int val; + if( Tcl_GetIntFromObj(interp, objv[2], &val) ) return TCL_ERROR; + val = sqlite3session_enable(pSession, val); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 4: { /* indirect */ + int val; + if( Tcl_GetIntFromObj(interp, objv[2], &val) ) return TCL_ERROR; + val = sqlite3session_indirect(pSession, val); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 5: { /* isempty */ + int val; + val = sqlite3session_isempty(pSession); + Tcl_SetObjResult(interp, Tcl_NewBooleanObj(val)); + break; + } + + case 6: { /* table_filter */ + if( p->pFilterScript ) Tcl_DecrRefCount(p->pFilterScript); + p->interp = interp; + p->pFilterScript = Tcl_DuplicateObj(objv[2]); + Tcl_IncrRefCount(p->pFilterScript); + sqlite3session_table_filter(pSession, test_table_filter, clientData); + break; + } + + case 8: { /* diff */ + char *zErr = 0; + rc = sqlite3session_diff(pSession, + Tcl_GetString(objv[2]), + Tcl_GetString(objv[3]), + &zErr + ); + assert( rc!=SQLITE_OK || zErr==0 ); + if( rc ){ + return test_session_error(interp, rc, zErr); + } + break; + } + } + + return TCL_OK; +} + +static void SQLITE_TCLAPI test_session_del(void *clientData){ + TestSession *p = (TestSession*)clientData; + if( p->pFilterScript ) Tcl_DecrRefCount(p->pFilterScript); + sqlite3session_delete(p->pSession); + ckfree((char*)p); +} + +/* +** Tclcmd: sqlite3session CMD DB-HANDLE DB-NAME +*/ +static int SQLITE_TCLAPI test_sqlite3session( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + Tcl_CmdInfo info; + int rc; /* sqlite3session_create() return code */ + TestSession *p; /* New wrapper object */ + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE DB-NAME"); + return TCL_ERROR; + } + + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + + p = (TestSession*)ckalloc(sizeof(TestSession)); + memset(p, 0, sizeof(TestSession)); + rc = sqlite3session_create(db, Tcl_GetString(objv[3]), &p->pSession); + if( rc!=SQLITE_OK ){ + ckfree((char*)p); + return test_session_error(interp, rc, 0); + } + + Tcl_CreateObjCommand( + interp, Tcl_GetString(objv[1]), test_session_cmd, (ClientData)p, + test_session_del + ); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + +static void test_append_value(Tcl_Obj *pList, sqlite3_value *pVal){ + if( pVal==0 ){ + Tcl_ListObjAppendElement(0, pList, Tcl_NewObj()); + Tcl_ListObjAppendElement(0, pList, Tcl_NewObj()); + }else{ + Tcl_Obj *pObj; + switch( sqlite3_value_type(pVal) ){ + case SQLITE_NULL: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("n", 1)); + pObj = Tcl_NewObj(); + break; + case SQLITE_INTEGER: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("i", 1)); + pObj = Tcl_NewWideIntObj(sqlite3_value_int64(pVal)); + break; + case SQLITE_FLOAT: + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("f", 1)); + pObj = Tcl_NewDoubleObj(sqlite3_value_double(pVal)); + break; + case SQLITE_TEXT: { + const char *z = (char*)sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("t", 1)); + pObj = Tcl_NewStringObj(z, n); + break; + } + default: + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + Tcl_ListObjAppendElement(0, pList, Tcl_NewStringObj("b", 1)); + pObj = Tcl_NewByteArrayObj( + sqlite3_value_blob(pVal), + sqlite3_value_bytes(pVal) + ); + break; + } + Tcl_ListObjAppendElement(0, pList, pObj); + } +} + +typedef struct TestConflictHandler TestConflictHandler; +struct TestConflictHandler { + Tcl_Interp *interp; + Tcl_Obj *pConflictScript; + Tcl_Obj *pFilterScript; +}; + +static int test_obj_eq_string(Tcl_Obj *p, const char *z){ + int n; + int nObj; + char *zObj; + + n = (int)strlen(z); + zObj = Tcl_GetStringFromObj(p, &nObj); + + return (nObj==n && (n==0 || 0==memcmp(zObj, z, n))); +} + +static int test_filter_handler( + void *pCtx, /* Pointer to TestConflictHandler structure */ + const char *zTab /* Table name */ +){ + TestConflictHandler *p = (TestConflictHandler *)pCtx; + int res = 1; + Tcl_Obj *pEval; + Tcl_Interp *interp = p->interp; + + pEval = Tcl_DuplicateObj(p->pFilterScript); + Tcl_IncrRefCount(pEval); + + if( TCL_OK!=Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj(zTab, -1)) + || TCL_OK!=Tcl_EvalObjEx(interp, pEval, TCL_EVAL_GLOBAL) + || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &res) + ){ + Tcl_BackgroundError(interp); + } + + Tcl_DecrRefCount(pEval); + return res; +} + +static int test_conflict_handler( + void *pCtx, /* Pointer to TestConflictHandler structure */ + int eConf, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *pIter /* Handle describing change and conflict */ +){ + TestConflictHandler *p = (TestConflictHandler *)pCtx; + Tcl_Obj *pEval; + Tcl_Interp *interp = p->interp; + int ret = 0; /* Return value */ + + int op; /* SQLITE_UPDATE, DELETE or INSERT */ + const char *zTab; /* Name of table conflict is on */ + int nCol; /* Number of columns in table zTab */ + + pEval = Tcl_DuplicateObj(p->pConflictScript); + Tcl_IncrRefCount(pEval); + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0); + + if( eConf==SQLITE_CHANGESET_FOREIGN_KEY ){ + int nFk; + sqlite3changeset_fk_conflicts(pIter, &nFk); + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj("FOREIGN_KEY", -1)); + Tcl_ListObjAppendElement(0, pEval, Tcl_NewIntObj(nFk)); + }else{ + + /* Append the operation type. */ + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + /* Append the table name. */ + Tcl_ListObjAppendElement(0, pEval, Tcl_NewStringObj(zTab, -1)); + + /* Append the conflict type. */ + switch( eConf ){ + case SQLITE_CHANGESET_DATA: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("DATA",-1)); + break; + case SQLITE_CHANGESET_NOTFOUND: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("NOTFOUND",-1)); + break; + case SQLITE_CHANGESET_CONFLICT: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("CONFLICT",-1)); + break; + case SQLITE_CHANGESET_CONSTRAINT: + Tcl_ListObjAppendElement(interp, pEval,Tcl_NewStringObj("CONSTRAINT",-1)); + break; + } + + /* If this is not an INSERT, append the old row */ + if( op!=SQLITE_INSERT ){ + int i; + Tcl_Obj *pOld = Tcl_NewObj(); + for(i=0; inData - p->iData; /* Bytes of data available */ + int nRet = p->nStream; /* Bytes actually returned */ + + /* Allocate and free some space. There is no point to this, other than + ** that it allows the regular OOM fault-injection tests to cause an error + ** in this function. */ + void *pAlloc = sqlite3_malloc(10); + if( pAlloc==0 ) return SQLITE_NOMEM; + sqlite3_free(pAlloc); + + if( nRet>nReq ) nRet = nReq; + if( nRet>nRem ) nRet = nRem; + + assert( nRet>=0 ); + if( nRet>0 ){ + memcpy(pData, &p->aData[p->iData], nRet); + p->iData += nRet; + } + + *pnData = nRet; + return SQLITE_OK; +} + + +/* +** sqlite3changeset_apply DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT? +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_apply( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; /* Database handle */ + Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ + int rc; /* Return code from changeset_invert() */ + void *pChangeset; /* Buffer containing changeset */ + int nChangeset; /* Size of buffer aChangeset in bytes */ + TestConflictHandler ctx; + TestStreamInput sStr; + + memset(&sStr, 0, sizeof(sStr)); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, + "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?" + ); + return TCL_ERROR; + } + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + ctx.pConflictScript = objv[3]; + ctx.pFilterScript = objc==5 ? objv[4] : 0; + ctx.interp = interp; + + if( sStr.nStream==0 ){ + rc = sqlite3changeset_apply(db, nChangeset, pChangeset, + (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx + ); + }else{ + sStr.aData = (unsigned char*)pChangeset; + sStr.nData = nChangeset; + rc = sqlite3changeset_apply_strm(db, testStreamInput, (void*)&sStr, + (objc==5) ? test_filter_handler : 0, test_conflict_handler, (void *)&ctx + ); + } + + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** sqlite3changeset_apply_replace_all DB CHANGESET +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; /* Database handle */ + Tcl_CmdInfo info; /* Database Tcl command (objv[1]) info */ + int rc; /* Return code from changeset_invert() */ + void *pChangeset; /* Buffer containing changeset */ + int nChangeset; /* Size of buffer aChangeset in bytes */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB CHANGESET"); + return TCL_ERROR; + } + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + + rc = sqlite3changeset_apply(db, nChangeset, pChangeset, 0, replace_handler,0); + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + Tcl_ResetResult(interp); + return TCL_OK; +} + + +/* +** sqlite3changeset_invert CHANGESET +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_invert( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; /* Return code from changeset_invert() */ + TestStreamInput sIn; /* Input stream */ + TestSessionsBlob sOut; /* Output blob */ + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANGESET"); + return TCL_ERROR; + } + + memset(&sIn, 0, sizeof(sIn)); + memset(&sOut, 0, sizeof(sOut)); + sIn.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + sIn.aData = Tcl_GetByteArrayFromObj(objv[1], &sIn.nData); + + if( sIn.nStream ){ + rc = sqlite3changeset_invert_strm( + testStreamInput, (void*)&sIn, testStreamOutput, (void*)&sOut + ); + }else{ + rc = sqlite3changeset_invert(sIn.nData, sIn.aData, &sOut.n, &sOut.p); + } + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + }else{ + Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n)); + } + sqlite3_free(sOut.p); + return rc; +} + +/* +** sqlite3changeset_concat LEFT RIGHT +*/ +static int SQLITE_TCLAPI test_sqlite3changeset_concat( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; /* Return code from changeset_invert() */ + + TestStreamInput sLeft; /* Input stream */ + TestStreamInput sRight; /* Input stream */ + TestSessionsBlob sOut = {0,0}; /* Output blob */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "LEFT RIGHT"); + return TCL_ERROR; + } + + memset(&sLeft, 0, sizeof(sLeft)); + memset(&sRight, 0, sizeof(sRight)); + sLeft.aData = Tcl_GetByteArrayFromObj(objv[1], &sLeft.nData); + sRight.aData = Tcl_GetByteArrayFromObj(objv[2], &sRight.nData); + sLeft.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + sRight.nStream = sLeft.nStream; + + if( sLeft.nStream>0 ){ + rc = sqlite3changeset_concat_strm( + testStreamInput, (void*)&sLeft, + testStreamInput, (void*)&sRight, + testStreamOutput, (void*)&sOut + ); + }else{ + rc = sqlite3changeset_concat( + sLeft.nData, sLeft.aData, sRight.nData, sRight.aData, &sOut.n, &sOut.p + ); + } + + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + }else{ + Tcl_SetObjResult(interp,Tcl_NewByteArrayObj((unsigned char*)sOut.p,sOut.n)); + } + sqlite3_free(sOut.p); + return rc; +} + +/* +** sqlite3session_foreach VARNAME CHANGESET SCRIPT +*/ +static int SQLITE_TCLAPI test_sqlite3session_foreach( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + void *pChangeset; + int nChangeset; + sqlite3_changeset_iter *pIter; + int rc; + Tcl_Obj *pVarname; + Tcl_Obj *pCS; + Tcl_Obj *pScript; + int isCheckNext = 0; + + TestStreamInput sStr; + memset(&sStr, 0, sizeof(sStr)); + + if( objc>1 ){ + char *zOpt = Tcl_GetString(objv[1]); + isCheckNext = (strcmp(zOpt, "-next")==0); + } + if( objc!=4+isCheckNext ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-next? VARNAME CHANGESET SCRIPT"); + return TCL_ERROR; + } + + pVarname = objv[1+isCheckNext]; + pCS = objv[2+isCheckNext]; + pScript = objv[3+isCheckNext]; + + pChangeset = (void *)Tcl_GetByteArrayFromObj(pCS, &nChangeset); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + if( sStr.nStream==0 ){ + rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); + }else{ + sStr.aData = (unsigned char*)pChangeset; + sStr.nData = nChangeset; + rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); + } + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } + + while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ + int nCol; /* Number of columns in table */ + int nCol2; /* Number of columns in table */ + int op; /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zTab; /* Name of table change applies to */ + Tcl_Obj *pVar; /* Tcl value to set $VARNAME to */ + Tcl_Obj *pOld; /* Vector of old.* values */ + Tcl_Obj *pNew; /* Vector of new.* values */ + int bIndirect; + + char *zPK; + unsigned char *abPK; + int i; + + /* Test that _fk_conflicts() returns SQLITE_MISUSE if called on this + ** iterator. */ + int nDummy; + if( SQLITE_MISUSE!=sqlite3changeset_fk_conflicts(pIter, &nDummy) ){ + sqlite3changeset_finalize(pIter); + return TCL_ERROR; + } + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + pVar = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj( + op==SQLITE_INSERT ? "INSERT" : + op==SQLITE_UPDATE ? "UPDATE" : + "DELETE", -1 + )); + + Tcl_ListObjAppendElement(0, pVar, Tcl_NewStringObj(zTab, -1)); + Tcl_ListObjAppendElement(0, pVar, Tcl_NewBooleanObj(bIndirect)); + + zPK = ckalloc(nCol+1); + memset(zPK, 0, nCol+1); + sqlite3changeset_pk(pIter, &abPK, &nCol2); + assert( nCol==nCol2 ); + for(i=0; izCmd, p->xProc, 0, 0); + } + + return TCL_OK; +} + +#endif /* SQLITE_TEST && SQLITE_SESSION && SQLITE_PREUPDATE_HOOK */ diff --git a/ext/userauth/sqlite3userauth.h b/ext/userauth/sqlite3userauth.h index 619477cac9..c7d23b9399 100644 --- a/ext/userauth/sqlite3userauth.h +++ b/ext/userauth/sqlite3userauth.h @@ -21,6 +21,10 @@ */ #ifdef SQLITE_USER_AUTHENTICATION +#ifdef __cplusplus +extern "C" { +#endif + /* ** If a database contains the SQLITE_USER table, then the ** sqlite3_user_authenticate() interface must be invoked with an @@ -85,4 +89,8 @@ int sqlite3_user_delete( const char *zUsername /* Username to remove */ ); +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + #endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 6ce99053d3..a3d346c05c 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -22,7 +22,7 @@ ** directory as this file for additional information. */ #ifdef SQLITE_USER_AUTHENTICATION -#ifndef _SQLITEINT_H_ +#ifndef SQLITEINT_H # include "sqliteInt.h" #endif diff --git a/main.mk b/main.mk index 9763a69edc..b22d07d5a2 100644 --- a/main.mk +++ b/main.mk @@ -46,6 +46,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/session TCCX += -I$(TOP)/ext/fts5 THREADLIB += $(LIBS) @@ -74,6 +75,7 @@ LIBOBJ+= vdbe.o parse.o \ vdbetrace.o wal.o walker.o where.o wherecode.o whereexpr.o \ utf.o vtab.o +LIBOBJ += sqlite3session.o # All of the source code files. # @@ -222,6 +224,9 @@ SRC += \ $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c +SRC += \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/userauth/userauth.c \ $(TOP)/ext/userauth/sqlite3userauth.h @@ -288,6 +293,7 @@ TESTSRC = \ $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ + $(TOP)/src/test_delete.c \ $(TOP)/src/test_demovfs.c \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ @@ -319,7 +325,9 @@ TESTSRC = \ # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/csv.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ @@ -327,6 +335,7 @@ TESTSRC += \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/remember.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ @@ -381,7 +390,9 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/test_session.c # Header files used by all library source files. # @@ -443,7 +454,8 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ - sqldiff$(EXE) + sqldiff$(EXE) \ + dbhash$(EXE) # Databases containing fuzzer test cases # @@ -451,7 +463,8 @@ FUZZDATA = \ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db \ - $(TOP)/test/fuzzdata4.db + $(TOP)/test/fuzzdata4.db \ + $(TOP)/test/fuzzdata5.db # Standard options to testfixture # @@ -460,8 +473,14 @@ TESTOPTS = --verbose=file --output=test-out.txt # Extra compiler options for various shell tools # SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +DBFUZZ_OPT = +KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ +ST_OPT = -DSQLITE_THREADSAFE=0 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -480,6 +499,13 @@ sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) +dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h + $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ + $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) + +scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o + $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) + srcck1$(EXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c @@ -491,10 +517,20 @@ fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(FUZZERSHELL_OPT) $(TOP)/tool/fuzzershell.c sqlite3.c \ $(TLIBS) $(THREADLIB) -fuzzcheck$(EXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h +dbfuzz$(EXE): $(TOP)/test/dbfuzz.c sqlite3.c sqlite3.h + $(TCCX) -o dbfuzz$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + $(DBFUZZ_OPT) $(TOP)/test/dbfuzz.c sqlite3.c \ + $(TLIBS) $(THREADLIB) + +fuzzcheck$(EXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(TOP)/test/ossfuzz.c $(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) -DSQLITE_OSS_FUZZ \ + $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) $(THREADLIB) + +ossshell$(EXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h + $(TCCX) -o ossshell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) \ - $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) $(THREADLIB) + $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c $(TLIBS) $(THREADLIB) mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ @@ -534,6 +570,7 @@ target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . + cp $(TOP)/ext/session/sqlite3session.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c @@ -700,10 +737,12 @@ fts5.c: $(FTS5_SRC) $(FTS5_HDR) userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c +sqlite3session.o: $(TOP)/ext/session/sqlite3session.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/session/sqlite3session.c + sqlite3rbu.o: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c - # Rules for building test programs and for running tests # tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a @@ -736,6 +775,10 @@ sqlite3_schemalint$(EXE): $(TESTSRC) sqlite3_schemalint.c sqlite3_schemalint.c $(TESTSRC) \ -o sqlite3_schemalint$(EXE) $(LIBTCL) $(THREADLIB) +dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o + $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ + $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) + # Rules to build the 'testfixture' application. # TESTFIXTURE_FLAGS = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 @@ -748,9 +791,11 @@ testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) -amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c +amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/session/test_session.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ + $(TOP)/ext/session/test_session.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c @@ -780,6 +825,11 @@ fastfuzztest: fuzzcheck$(EXE) $(FUZZDATA) valgrindfuzz: fuzzcheck$(EXE) $(FUZZDATA) valgrind ./fuzzcheck$(EXE) --cell-size-check --limit-mem 10M --timeout 600 $(FUZZDATA) +# The veryquick.test TCL tests. +# +tcltest: ./testfixture$(EXE) + ./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS) + # A very quick test using only testfixture and omitting all the slower # tests. Designed to run in under 3 minutes on a workstation. # @@ -788,9 +838,7 @@ quicktest: ./testfixture$(EXE) # The default test case. Runs most of the faster standard TCL tests, # and fuzz tests, and sqlite3_analyzer and sqldiff tests. -# -test: $(TESTPROGS) sourcetest fastfuzztest - ./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS) +test: fastfuzztest sourcetest $(TESTPROGS) tcltest # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. @@ -846,6 +894,10 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) +changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ + $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) + fts3view$(EXE): $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \ $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB) @@ -861,8 +913,11 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o wordcount$(EXE) \ $(TOP)/test/wordcount.c sqlite3.c -speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o - $(TCC) -I. $(OTAFLAGS) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB) +speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.c + $(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) + +kvtest$(EXE): $(TOP)/test/kvtest.c sqlite3.c + $(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o $(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \ @@ -902,7 +957,6 @@ clean: rm -f mkkeywordhash mkkeywordhash.exe keywordhash.h rm -f $(PUBLISH) rm -f *.da *.bb *.bbg gmon.out - rm -rf quota2a quota2b quota2c rm -rf tsrc target_source rm -f testloadext.dll libtestloadext.so rm -f amalgamation-testfixture amalgamation-testfixture.exe @@ -916,6 +970,7 @@ clean: rm -f showjournal showjournal.exe rm -f showstat4 showstat4.exe rm -f showwal showwal.exe + rm -f changeset changeset.exe rm -f speedtest1 speedtest1.exe rm -f wordcount wordcount.exe rm -f rbu rbu.exe diff --git a/manifest b/manifest index b8f39b0c2a..90f504d2aa 100644 --- a/manifest +++ b/manifest @@ -1,24 +1,24 @@ -C Update\sshell6.test\sto\saccount\sfor\sthe\sfact\sthat\stests\sare\snow\srun\sin\sa\nseparate\sdirectory. -D 2017-03-31T08:00:03.133 -F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f +C Merge\slatest\strunk\sinto\sthis\sbranch. +D 2017-03-31T11:20:20.468 +F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66 -F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 -F VERSION c6b1f51809551d60ad001e6d87cf3ab2c7f54b6f +F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a +F README.md 2b15fae33852f2f53996774c21fb41e1d94181c4401a0e43ac93e11f2cc901b9 +F VERSION 0a0e02e16b44ea735b40118fc844311b2ab0d35b25fbeda5120aee62f973f663 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 42f7eb1fb2b06ddd6c6adab3e4388cc0a237ba40 -F autoconf/Makefile.msc b8ada5177e6f39c126a88295af5ad0255e08d280 +F autoconf/Makefile.am 1a47d071e3d5435f8f7ebff7eb6703848bbd65d4 +F autoconf/Makefile.msc 1fba0d762d115509d4fce7e333305cee172d521aaacec8bbf7aad5503605d3fb F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac d807b567f73a2b0986ccdd206b782cecb4d08fff +F autoconf/configure.ac 2893b823ecc86cea13739f6c8109a41392254d1db08235c5615e0af5722c8578 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac 93d43c79e936fb16556e22498177d7e8571efa04 +F autoconf/tea/configure.ac 8aa16e3f0a5ca7959d4af198f46934ec187d395f F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d @@ -30,13 +30,13 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure c01a159fdf7ea0171ad01ce2937283f2c7972bde x -F configure.ac 89e4e02a83d8a1528011f8395621b8c3186b4089 +F configure cea63449647267beb335e899fd8a79778912ffa55d6b179730267b8ef84aa1aa x +F configure.ac 605173e829ab64514ed89f9b53d0da1739d7b0a0 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad -F doc/lemon.html c30255bea0fd87a81f082d17a72c9dffbc3f6dd9 +F doc/lemon.html b5a3c07d33ecb8e019ce8f7660fe2dbbad9d7977 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 +F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd w ext/README.txt F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 0f3070cc3f5ede78f2b9361fb3b629ce200d7d74 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef @@ -62,7 +62,7 @@ F ext/fts2/fts2_hash.c 011a1d32de45bb1b519a1fd0048e857d6a843558 F ext/fts2/fts2_hash.h 1824b99dfd8d0225facbdb26a2c87289b2e7dcf8 F ext/fts2/fts2_icu.c 51c5cd3c04954badd329fa738c95fcdb717b5188 F ext/fts2/fts2_porter.c 2cd4a507bf3c3085fe66f59b0f2a325f65aaacf5 -F ext/fts2/fts2_tokenizer.c 3dbe8058e97afb55fff3ea844120ce3208b114cc +F ext/fts2/fts2_tokenizer.c b529493d55e55497213c37e1f31680a77746be26 F ext/fts2/fts2_tokenizer.h 27a1a99ca2d615cf7e142839b8d79e8751b4529e F ext/fts2/fts2_tokenizer1.c 07e223eecb483d448313b5f1553a4f299a7fb7a1 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 @@ -70,9 +70,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e028eb13432f108d2e22cded019fc980700e4e00 +F ext/fts3/fts3.c f0d5de1bc2155ba7cd7c0c1a751779a3a8857fa34d5c12f3b233a23fa2e79ea2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 89d0bd4595a0de384dac78e94b803de12586e8dd +F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 F ext/fts3/fts3_expr.c dfd571a24412779ac01f25c01d888c6ef7b2d0ef F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -81,48 +81,49 @@ F ext/fts3/fts3_icu.c deb46f7020d87ea7a14a433fb7a7f4bef42a9652 F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7 -F ext/fts3/fts3_test.c 11e36437b0f3f2266acea5b4787f615e4337a237 +F ext/fts3/fts3_test.c 79f2a7fbb3f672fa032e5a432ca274ea3ee93c34 F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860 -F ext/fts3/fts3_tokenizer.c 3cf21cd2212db17a88d4ef7da0fd8a80275979a1 +F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 -F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 -F ext/fts3/fts3_write.c c3863f23b6b4623c8b9d5cf31c12ce4469f78ca9 +F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d +F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c +F ext/fts3/fts3_write.c a51d48d646974ee2fb4b17fcd5da0416a5759a32dcacc2cce2ba00d5a767848e F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 5d78b668f4e9598af9147f8999632599fb0d9dd5 +F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 +F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8 +F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c4951a0fbb60e780cb69f F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 4060504b7979601d99e1385c2b5713036854979a -F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458 +F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 +F ext/fts5/fts5Int.h c629b24d2b92b99596f3b8e82289fddca06df6e1 +F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c be309fb227003c931107bfcc12d5be4f2fd2bb8c -F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337 -F ext/fts5/fts5_index.c d4f0c12e4f04bbc3a06b6da052039f2ce3e45438 -F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421 -F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e -F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 +F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c +F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 +F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4 +F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2 +F ext/fts5/fts5_storage.c 8f0e65cb33bde8f449e1c9b4be4600d18b4da6e9 +F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 -F ext/fts5/fts5_vocab.c dba72ca393d71c2588548b51380387f6b44c77a8 -F ext/fts5/fts5parse.y fcc5e92e570d38cab38488b2109cbf67468923b2 +F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7 +F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 -F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084 +F ext/fts5/test/fts5aa.test bd2d88182b9f7f30d300044048ad14683306b745 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20 -F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c +F ext/fts5/test/fts5af.test c92825778ed2adc80014832762c056bbe968ef88 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df @@ -131,40 +132,45 @@ F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9 F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e -F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f +F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a +F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 -F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c -F ext/fts5/test/fts5corrupt3.test b9558d5b0ca44a8b6247fbb5d4a47592a8976892 +F ext/fts5/test/fts5corrupt2.test 128eb6e2d26b09f4da339e581f424b3321e0fdaa +F ext/fts5/test/fts5corrupt3.test f77f65e386231daf62902466b40ff998b2c8ce4f F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69 -F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d +F ext/fts5/test/fts5determin.test 10648edb75ef1e196b10978fd21a9be0c31e09c3 +F ext/fts5/test/fts5dlidx.test 007e9390c94638760797dbec2990c97c3fa08dfe F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 -F ext/fts5/test/fts5eb.test c516ae0c934be6fd29ec95ea8b5f11f461311535 +F ext/fts5/test/fts5eb.test 14597b5ecc9b2b7039ac5e0b0fca0311a7d22005 F ext/fts5/test/fts5fault1.test e09040d3e17b8c0837101e8c79c8a874c4376fb7 F ext/fts5/test/fts5fault2.test d8c6c7f916ccbdfc10b2c69530e9dd3bc8313232 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 -F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277 +F ext/fts5/test/fts5fault4.test dcbe3043c5611edd350191ea03a8daa190f0de5a F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080 F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea -F ext/fts5/test/fts5faultB.test 92ae906284062bf081b6c854afa54dcb1aa9ef88 +F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca +F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 F ext/fts5/test/fts5hash.test 06f9309ccb4d5050a131594e9e47d0b21456837d F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d +F ext/fts5/test/fts5lastrowid.test 4fac1aba696dd6c956e03b0cf91f6f1f3aaec494 F ext/fts5/test/fts5matchinfo.test f7dde99697bcb310ea8faa8eb2714d9f4dfc0e1b F ext/fts5/test/fts5merge.test 9f65f090d214ff865c56bef4f864aaa1182af6e3 F ext/fts5/test/fts5merge2.test a6da3c16d694235938d1939f503cfa53f0943d75 +F ext/fts5/test/fts5multiclient.test 5bd54a6bc8ff818be233a50a8dec93da99098f2a F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 9d3ac53bb9cae58cb070d795db86bcb2f9fec105 @@ -172,63 +178,73 @@ F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f F ext/fts5/test/fts5plan.test 6a55ecbac9890765b0e16f8c421c7e0888cfe436 F ext/fts5/test/fts5porter.test 7cdc07bef301d70eebbfa75dcaf45c3680e1d0e1 F ext/fts5/test/fts5porter2.test 2e65633d58a1c525d5af0f6c01e5a59155bb3487 -F ext/fts5/test/fts5prefix.test efd42e00bb8e8a36383f25c838185508681c093f +F ext/fts5/test/fts5prefix.test 6ef7e875738412907b17687d25db39a25cbdaba4 F ext/fts5/test/fts5query.test f5ec25f5f2fbb70033424113cdffc101b1985a40 -F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab +F ext/fts5/test/fts5rank.test 2bdc0c5f22ccc1f9dbe9f4d0b82a491dce6f8a32 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 -F ext/fts5/test/fts5simple.test ac5006cc3d0d08b3538e1e76c7300de9f24fbed1 -F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46 -F ext/fts5/test/fts5simple3.test 8e71733b3d1b0e695011d02c68ebc5ca40b6124e -F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 +F ext/fts5/test/fts5simple.test 5da9b15ed534eb0be9f279d8a2bb2e24d30e4e38 +F ext/fts5/test/fts5simple2.test 00839031878f52391562594fdab0503e424ee071 +F ext/fts5/test/fts5simple3.test 25faa8cb8ad470c6f01f670bcc1317c19a89f091 +F ext/fts5/test/fts5synonym.test f964f9a98580dec06c995d4638ecb36eee60ae4a F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 -F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 +F ext/fts5/test/fts5tokenizer.test f64ab5fe48fa1317ff6315264bfe26e73cbdefde F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 -F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 +F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e -F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 +F ext/fts5/test/fts5vocab.test 026799f8d24befc452cbddda79822f5ae8d5f4eb F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 -F ext/fts5/tool/fts5txt2db.tcl 1343745b89ca2a1e975c23f836d0cee410052975 +F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 -F ext/icu/icu.c 4bdf4c5daedabcd8e6eb6e6a377657f7b978f9f7 +F ext/icu/icu.c 84900472a088a3a172c6c079f58a1d3a1952c332 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 -F ext/misc/amatch.c a1a8f66c29d40bd71b075546ddeddb477b17a2bb +F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 +F ext/misc/amatch.c 211108e201105e4bb0c076527b8cfd34330fc234 +F ext/misc/carray.c 40c27641010a4dc67e3690bdb7c9d36ca58b3c2d F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 +F ext/misc/csv.c 531a46cbad789fca0aa9db69a0e6c8ac9e68767d +F ext/misc/dbdump.c 34174d537318027b159e3341105866c05a4149376ede59988c82280bed144e6d F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f -F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 +F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c b9c88d5c3b6ecd8c731ffdd7f5b3d902857f8c96 +F ext/misc/json1.c 92b9e404bd79cc76d3ef2b3a9e37bf775700f7d645466fd41d2e8742e3fa16fb +F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 -F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 -F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc +F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e +F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 +F ext/misc/remember.c 8440f8d0b452c5cdefb62b57135ccd1267aa729d F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a +F ext/misc/scrub.c 1c5bfb8b0cd18b602fcb55755e84abf0023ac2fb F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 +F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 +F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 598bbc45516227701558becdd38f4e6fe8e97cc2 +F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 -F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e +F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 +F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 -F ext/rbu/rbu1.test 57601977588603e82700a43c279bd55282ffa482 +F ext/rbu/rbu.c b2c0b5e6ae1a89affc0edfc127ebfa5f637a0ce4 +F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16 F ext/rbu/rbu13.test 462ff799c4afedc3ef8a47ff818c0ffbf14ae4f2 F ext/rbu/rbu14.test 01f5dcba904aecadbaea69d4ccdc2ea43dd30560 F ext/rbu/rbu3.test 8bd4c6b87367c358981b6a47dc3d654fa60bff90 -F ext/rbu/rbu5.test 2e24fee3e615aecd99bbdd46967935a641e866f7 +F ext/rbu/rbu5.test cf1fa84d6107efd73e292a7be8e1fb31953ca45c F ext/rbu/rbu6.test 32e8ed60631b6facdb6366bd2b5f5f25245e7edb F ext/rbu/rbu7.test fd025d5ba440fcfe151fbb0e3835e1e7fe964fa1 F ext/rbu/rbu8.test 3bbf2c35d71a843c463efe93946f14ad10c3ede0 @@ -236,46 +252,80 @@ F ext/rbu/rbu9.test 0806d1772c9f4981774ff028de6656e4183082af F ext/rbu/rbuA.test c1a7b3e2d926b8f8448bb3b4ae787e314ee4b2b3 F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 -F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a +F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 -F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d +F ext/rbu/rbucrash2.test b2ecbdd7bb72c88bd217c65bd00dafa07f7f2d4d +F ext/rbu/rbudiff.test 3e605cf624d00d04d0fb1316a3acec4fbe3b3ac5 +F ext/rbu/rbudor.test 99b05cc0df613e962c2c8085cfb05686a09cf315 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 +F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca +F ext/rbu/rbufault4.test 34e70701cbec51571ffbd9fbf9d4e0f2ec495ca7 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda +F ext/rbu/rbuprogress.test 1849d4e0e50616edf5ce75ce7db86622e656b5cf +F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 5956f8bee63b5ab2b04e65c1801ea0f5920dac92 -F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde -F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 +F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0 +F ext/rbu/rbuvacuum2.test 2074ab14fe66e1c7e7210c62562650dcd215bbaa +F ext/rbu/sqlite3rbu.c 2a89efba9eeba8e6c89a498dc195e8efbdde2694 +F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba +F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 +F ext/rtree/rtree.c 0acd285bfacc347579a5df9fe947212fb99e2775a40c43f027c3a16936c58e7e F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test 96a80c08440c932cd72aac50660e7af2612d2cda +F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba -F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc +F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482 F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test b5eb13849545dfd271a54ff16784cb00d8792aea -F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf +F ext/rtree/rtreeA.test ac8b503931f2f397cc3c3303354e1e085dcadb86 F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e -F ext/rtree/rtreeC.test 90aaaffe2fd4f0dcd12289cad5515f6d41f45ffd -F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca +F ext/rtree/rtreeC.test c0a9c67f2efa98b6fae12acb8a28348d231a481d +F ext/rtree/rtreeD.test bdfaaf26df8b4eea7364039aca9150bc1e1f8825 F ext/rtree/rtreeE.test 45a147a64a76306172819562309681d8e90f94bb F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4 +F ext/rtree/rtreeG.test 3b185719630795f38594f64cd7d1de86a33f91f1 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 9c5777af3d2921c7b4ae4954e8e5697502289d28 F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 +F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a +F ext/session/session1.test c8a50e0e8581dc1a00e832aa59bb61f180404d44 +F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 +F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 +F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 +F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 +F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 +F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b +F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069 +F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f +F ext/session/sessionB.test 886252dcb7e692e62ef7e357456200912e367823 +F ext/session/sessionC.test 97556f5164ac29f2344b24bd7de6a3a35a95c390 +F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2 +F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26 +F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce +F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60 +F ext/session/session_common.tcl 7776eda579773113b30c7abfd4545c445228cb73 +F ext/session/session_speed_test.c edc1f96fd5e0e4b16eb03e2a73041013d59e8723 +F ext/session/sessionat.test b25d61d663ebc795506bf74079dc4ba0092fad25 +F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 +F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 +F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc +F ext/session/sqlite3session.c 13642d9c754cc18f17e141f82860d269e2adf920 +F ext/session/sqlite3session.h d4db650adfcc7a4360e9f12a09c2d117b1db6b53 +F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 +F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e +F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk e3b21467eb50c2dff9bf4003a02aea753bfe7e35 +F main.mk e6f8f6895b75e4615a284b546933ca88fd56f3b85b560f4fb0da731abaa14f80 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -287,164 +337,167 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c 1bb0709b3048e24217b80ec6bd78a3e99a47c01b -F src/analyze.c ab57b6763dd4c6170a20673d14882c033affd188 -F src/attach.c 771153bd1f4ab0b97a44a13dde2c7e5e1efeba22 -F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c f60f0aa55d25d853ffde53d0b0370a7bb7ee41ce -F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63 -F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 -F src/btree.c 6eee126fe9d1f57118de9be2be840a4c6e691828 -F src/btree.h a5008b9afe56e8e54ade6c436a910f112defcca9 -F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 -F src/build.c 213cbf84e99dd834e6ea46615633656d7ef79321 +F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121 +F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421 +F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43 +F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 +F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b +F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 +F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca +F src/btree.c 64ff65a01851a34c8145e5bc767df9e57d2f3c1acdc6aba334794b7c40c684e6 +F src/btree.h bf64dfeeddeebdb775a5eba0098bbc00d073290d +F src/btreeInt.h cd55d39d9916270837a88c12e701047cba0729b0 +F src/build.c 43f903c9082040ced2b421543cb0300c2973647d F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 -F src/date.c 0b73e681c11fca867fec554750c07fe0d4e417c1 -F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b -F src/delete.c 48802aa3ee6339f576d074336d3ae1b5f40e240f -F src/expr.c c329d581e5d631153456369684d7d4bcd94c907d -F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 5cb42d9a59e2a590776fd3fc8ff6f61d40df3c6e -F src/func.c 552d300265aed09eea21f68ac742a440550c0062 -F src/global.c 884d4c7eba9f5fc25c96a23b21520da19b7713e2 -F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 -F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 -F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 723d5d708cdb61bdd47c00b9f07c75be45aefc09 -F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c 9e2a41adcaff16ebc1ebff1f336cbf33de55396f -F src/main.c 787fc7ccd341ae80695fb3954ab95b84b7b3a146 -F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec +F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c +F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae +F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d +F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c +F src/expr.c f12a581f342a6fd85d14c31e4fb84f16b3dd107f54d7728dddb62cebc79d7ce1 +F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 +F src/fkey.c 2e9aabe1aee76273aff8a84ee92c464e095400ae +F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174 +F src/global.c 4a34512d82fc5aa13c802db06bcfff5e1d3de955 +F src/hash.c 63d0ee752a3b92d4695b2b1f5259c4621b2cfebd +F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 +F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da +F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 +F src/insert.c d4bb3a135948553d18cf992f76f7ed7b18aa0327f250607b5a6671e55d9947d5 +F src/legacy.c e88ed13c2d531decde75d42c2e35623fb9ce3cb0 +F src/loadext.c a72909474dadce771d3669bf84bf689424f6f87d471fee898589c3ef9b2acfd9 +F src/main.c 36974edf7461f708b337389c59b30ec91257420c129c1f5ee022bc2c2784d320 +F src/malloc.c 89c98e3619d362dcffa5c1c639b364b65b474751 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b +F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memjournal.c 6423a0817ffd8c7a04ef9e5fb974b6b9dd71f8b6 -F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 +F src/memjournal.c 95752936c11dc6995672d1dd783cd633eea0cc95 +F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 +F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c f89e3ca1c2e3d5015b847aec60371c474acbac82 -F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf +F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 +F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c f5bac8e74aaefc4ea520e43b4540793c3b8a9e8f -F src/os_win.c f0d7aa603eb6262143d7169a222aea07c4fca91d -F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d -F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 -F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 -F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df -F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 -F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051 -F src/pragma.c 3ca4a32189ffa5d0b2a908c9d55bd7fcd2717a62 -F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c -F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e -F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 -F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 -F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e -F src/select.c 137b31daa84d57d67847bf621bb54f3353e2077b -F src/shell.c 4aba7844546ffd02b728a9509226a6e3c0462f77 +F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 +F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 +F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a +F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a +F src/pager.c ff1232b3088a39806035ecfac4fffeb22717d80b +F src/pager.h f2a99646c5533ffe11afa43e9e0bea74054e4efa +F src/parse.y 48b03113704ee8bd78ee6996d81de7fbee22e105 +F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 +F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 +F src/pcache1.c 1195a21fe28e223e024f900b2011e80df53793f0356a24caace4188b098540dc +F src/pragma.c 157c323aef421d55905b42162ba16bc874c0dce3102c543b4092c202f881f2c6 +F src/pragma.h c9c763958fec92b04125571472c9500b351c5f7f +F src/prepare.c b1140c3d0cf59bc85ace00ce363153041b424b7a +F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 +F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 +F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 +F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac +F src/select.c 2496d0cc6368dabe7ad2e4c7f5ed3ad9aa3b4d11cd90f33fa1d1ef72493f43aa +F src/shell.c e524688c2544167f835ba43e24309f8707ca60c8ab6eb5c263a12c8618a233b8 F src/shell_indexes.c 23c4b7f7bb8e9b0a8b912c389a85922939bb0769 -F src/sqlite.h.in 9f0fee10494624d03686fabcc270ce8d12fcb65b +F src/sqlite.h.in 3db120c750634c0acc1ca291140a4cae75cb41cb0507f53c31b838c040954366 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h 8c95b3589d084ea68e947aa84abc32f185ed6155 -F src/sqliteLimit.h 7b28cf72cbd52f178bfc97ea266445e351f2cd24 -F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba -F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 -F src/tclsqlite.c 54dca4eaf3a9cb0a4b845ef54ffa99225c30a52e -F src/test1.c f14a6f9e2cff6cba4d83e2b0c52857f61886cead -F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b -F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f -F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e -F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 -F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 -F src/test7.c 9c89a4f1ed6bb13af0ed805b8d782bd83fcd57e3 -F src/test8.c fa262391d3edea6490a71bfaa8fed477ccbbac75 -F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 -F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 -F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 -F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 -F src/test_bestindex.c 29af3cc3b963ffe5760c85d142b9b3e5302c1e3d -F src/test_blob.c b2551a9b5573232db5f66f292307c37067937239 -F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 0dee90328e3dedf8ba002ee94b6a7e7ea7726fe4 -F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 -F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc -F src/test_fs.c f10f840ca4f8c72e4837908bd8347ac4bcab074b -F src/test_func.c 37453d346cfcf118774efd5bf6187f7e6a7e3254 -F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd -F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 -F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 -F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 -F src/test_journal.c 5360fbe1d1e4416ca36290562fd5a2e3f70f32aa +F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 +F src/sqliteInt.h 22a0a101c51a0fae4156088f7c613b99a73d20ba6153a1400d1ae3011cf1082c +F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b +F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 +F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 +F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79 +F src/test1.c 8a98191a1da8e100f77cdb5cc716df67d405028d +F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 +F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee +F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 +F src/test5.c 328aae2c010c57a9829d255dc099d6899311672d +F src/test6.c 004ad42f121f693b8cbe060d1a330678abc61620 +F src/test7.c 5612e9aecf934d6df7bba6ce861fdf5ba5456010 +F src/test8.c 4f4904721167b32f7a4fa8c7b32a07a673d6cc86 +F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 +F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a +F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 +F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 +F src/test_bestindex.c d23f80d334c59662af69191854c76b8d3d0c8c96 +F src/test_blob.c f65ac717da2618691cf9dad094e6da0219dcd208 +F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 +F src/test_config.c edcba290248dc18736dd814c9b95863c6762e0b35753048d8cbe5bf65f7abfbb +F src/test_delete.c af7eab5702f853fb1c62a5f7665e2234cf1ae17b +F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e +F src/test_devsym.c 4e58dec2602d8e139ca08659f62a62450587cb58 +F src/test_fs.c e16cbe68d3b107e00a907c20a9a02629870eb69b +F src/test_func.c a4fdab3363b436c1b12660e9362ce3f3782b7b5e +F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d +F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 +F src/test_intarray.c 988fc61cb0ff539f4172c0d95f15287c92516f64 +F src/test_intarray.h f3b7672f5d1056eac563c0d6ea8480a660b1475c +F src/test_journal.c 619f2aa10e0d7a5f87c0f06825bc61dfce1c6b9c7f3ad990fb13de6c3b8874a3 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c 96df9381a1ff1f6d3805ff7231b9baf1386aaabf -F src/test_multiplex.c 6a088d8d9d4aad4bec45dd8878af11b15900702d -F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 -F src/test_mutex.c dbdfaff8580071f2212a0deae3325a93a737819c -F src/test_onefile.c 38f7cbe79d5bafe95bde683cc3a53b8ca16daf10 -F src/test_osinst.c 5423dc1d355f594371f27dd292ca54bd320b8196 +F src/test_malloc.c c05f6c40bd6c8bfe5f1718212f81fd5687f91766 +F src/test_multiplex.c e054459f7633f3ff8ce1245da724f9a8be189e4e +F src/test_multiplex.h 5436d03f2d0501d04f3ed50a75819e190495b635 +F src/test_mutex.c 7f4337ba23ee6b1d2ec81c189653608cb069926a +F src/test_onefile.c 416f87a28d6d673352d33fc4b1c7d39db878e50f +F src/test_osinst.c 98ef31ff03d55497829ca0f6c74a9f4e1aa48690 F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 180813f43683be5725458fc1ff13ac455d8e722d +F src/test_quota.c 6cb9297115b551f433a9ad1741817a9831abed99 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d -F src/test_rtree.c 43fff4c5a01576d6d213f27472598801a247890c -F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 +F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b +F src/test_schema.c f575932cb6274d12147a77e13ea4b49d52408513 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe -F src/test_sqllog.c 0d138a8180a312bf996b37fa66da5c5799d4d57b -F src/test_superlock.c 06797157176eb7085027d9dd278c0d7a105e3ec9 -F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b -F src/test_tclvar.c d86412527da65468ee6fa1b8607c65d0af736bc4 -F src/test_thread.c af391ec03d23486dffbcc250b7e58e073f172af9 -F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 +F src/test_sqllog.c 11e6ce7575f489155c604ac4b439f2ac1d3d5aef +F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e +F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 +F src/test_tclvar.c df9fe1213c2634687a9ca0b0bec0d2119d359ae3 +F src/test_thread.c 911d15fb14e19c0c542bdc8aabf981c2f10a4858 +F src/test_vfs.c f0186261a24de2671d080bcd8050732f0cb64f6e F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 -F src/test_windirent.c 8f5fada630348558d5745b334702f301da1ffc61 -F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5 +F src/test_windirent.c 17f91f5f2aa1bb7328abb49414c363b5d2a9d3ff +F src/test_windirent.h 5d67483a55442e31e1bde0f4a230e6e932ad5906 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 3d338cdd00d916ce8a05c397001d64ed58e6fe1c -F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 -F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 -F src/update.c 56b3db7edff0110360a12b76af97c39ebe3ea8b8 +F src/tokenize.c de2ec4fed5aa9770791be7528a08597a77a36b2ffbbeb0c2cb6951e80357730b +F src/treeview.c 84d0ac737e1231702679f0289180021e19c5cc186ec413e8dcb704a887c76ec8 +F src/trigger.c c9f0810043b265724fdb1bdd466894f984dfc182 +F src/update.c 456d4a4656f8a03c2abc88a51b19172197400e58 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 34ef7be420f82415ec48131404995ddb6ee7502f -F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 -F src/vdbe.c 87ae3a5657fefed7875f3eb30e7ababd48013d71 -F src/vdbe.h c743791f723049db94f009e3e30958952bc2d512 -F src/vdbeInt.h 49a74fe5ece206d2d8666ba9afaf1abeda5f123b -F src/vdbeapi.c 95b1f8e527240a18a9aea41a655b013bf07a7009 -F src/vdbeaux.c 2c15cf88de4df97428318c8cfac0dea873dae451 -F src/vdbeblob.c 3b570b730109e8f653d9d2081649f6e7015113db -F src/vdbemem.c 9b0cb32cc267ef026515f15a3594d5ff91fe4dfc -F src/vdbesort.c 307460bfa4de4d1c3901fcd42089159131e34062 -F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484 -F src/vtab.c fd69fd398e23e57ea4ea377d8a44b6998fc569c7 +F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 +F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 +F src/vdbe.c f1acf5744cef62cbfd0b503d84289f840b6cdc980ac47b0d9632dfdb89cc79eb +F src/vdbe.h caa5346d52bae2a3c8c1dcfa60a7a4dc878a9e3865cb8239da55808b316c8158 +F src/vdbeInt.h 5db089ce18c4feff8820ec6e4cac2d2c82e03d4b1d96f10a6e43832147b8dffe +F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 +F src/vdbeaux.c ecd0468611925d218e1eb4b3f538907904b136f0e15e333291a232b521bfcef1 +F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 +F src/vdbemem.c 3b5a9a5b375458d3e12a50ae1aaa41eeec2175fd +F src/vdbesort.c eda25cb2d1727efca6f7862fea32b8aa33c0face +F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 +F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b -F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c -F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c b8566be7e2e195053238874d4f3bc4abf9e36b40 -F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34 -F src/wherecode.c 863aedf086131743763c1960637fde904eadc442 -F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a +F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 +F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 +F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 +F src/where.c 30c790ab98b759ac3d7fa564e1321489151fac7d1587525bc5b9698394a41ecf +F src/whereInt.h 2d50c2b74a33be44cb68fdecee30b4d93552f1f4 +F src/wherecode.c 677e95413c472c0b413023b6b69a47f40fce1b04 +F src/whereexpr.c 130cdd1a43af71b19755270fb1224874cf55158c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd +F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 -F test/alter.test 2facdddf08d0d48e75dc6cc312cd2b030f4835dd +F test/alter.test b820ab9dcf85f8e3a65bc8326accb2f0c7be64ef F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 -F test/alter3.test b3568d11c38c4599c92f24242eda34144d78dc10 -F test/alter4.test c461150723ac957f3b2214aa0b11552cd72023ec +F test/alter3.test 4d79934d812eaeacc6f22781a080f8cfe012fdc3 +F test/alter4.test b6d7b86860111864f6cddb54af313f5862dda23b F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 3eb35a4af972f98422e5dc0586501b17d103d321 @@ -460,7 +513,7 @@ F test/analyzeB.test a4c1c3048f6d9e090eb76e83eecb18bcf6d31a70 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 F test/analyzeD.test 42af58de25a6436502e43006e9e59e2d71bcb0cf F test/analyzeE.test 8684e8ac5722fb97c251887ad97e5d496a98af1d -F test/analyzeF.test 5d1fe1024ba2dfea3c18bede8c1ccef8aba1ab34 +F test/analyzeF.test f423125b557f11ad71bb29765ef9c34b6dcf4ab7 F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -468,15 +521,16 @@ F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a F test/async5.test 383ab533fdb9f7ad228cc99ee66e1acb34cc0dc0 F test/atof1.test ff0b0156fd705b67c506e1f2bfe9e26102bea9bd -F test/attach.test 437107943f14d131cf5efc2ae5305a94d7cb1d58 +F test/attach.test 9976867b1a6585018f0966857aa82ed44e0bd6dc F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d -F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27 +F test/attach3.test c59d92791070c59272e00183b7353eeb94915976 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 -F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa -F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 -F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 +F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e +F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 +F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f +F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec +F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 @@ -486,17 +540,19 @@ F test/autovacuum.test 92c24eedbdb68e49f3fb71f26f9ce6d8988cac15 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d -F test/backup.test b79299a536a4c6d919094786595b95be56d02014 +F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 -F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 +F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c -F test/bestindex1.test d5ba89a7941a941476d8d69be39b146aaed3614c +F test/bestindex1.test 0cf1bd2d7b97d3a3a8c10736125274f64765c4ee F test/bestindex2.test 4a06b8922ab2fd09434870da8d1cdf525aaf7060 +F test/bestindex3.test 578b6a52dab819e63f28e3640e04b32c85aed320 +F test/bestindex4.test 4cb5ff7dbaebadb87d366f51969271778423b455 F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -521,14 +577,14 @@ F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test bf6f0308bbbba1e770dac13aa08e5c2ac61c7324 +F test/capi3.test f0c66919e43d42e1572a69be039e4527a931b00f F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test 06f6261f9e9b4ef6f76afcd9900f3665408af1c8 +F test/capi3c.test 0b9edb0c2156a964b9271cd5ea7ae56736cc2fcb F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 -F test/cffault.test aadc1f61f8811cb600e3e069acbf8796f472a096 -F test/check.test 85a84bfc4be0e83f668747211c7cd45a6721d485 +F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef +F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/close.test 83947daf3b700631f90f4850ddaab455be4af73d F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -542,6 +598,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 +F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db @@ -549,7 +606,7 @@ F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97 -F test/corrupt2.test cb787825d761b0f869764d6990531382840de872 +F test/corrupt2.test e4964cee73dda57a90958e0087a6b388b1d9cb58 F test/corrupt3.test e676f478fe602915d721472811f6f410b75ddc7e F test/corrupt4.test 8d1d86b850fcc43e417450454f2044e52d55778a F test/corrupt5.test 8ead52af76006f3286e9396cb41898018ccea107 @@ -559,7 +616,7 @@ F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 53e56dafd180addcdadb402244b8cb9771d2ba26 F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 0c46574f8d4f27ecc799b1b5c4cbf9b1817bce9a +F test/corruptC.test 46ec43bd90a02fd7b37ad8a7a949c55aa5717f89 F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 82ccf4f8f543fdbedd4aa42c709cb077f7374c62 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 @@ -567,6 +624,7 @@ F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 075fe1d75aa1d84e2949be56b6264376c41502e4 F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 +F test/corruptK.test 814a59ec699d8546b4e29005fba3d16e933ef2fe F test/cost.test 1eedbfd868f806f3fa08ff072b04cf270dcf61c8 F test/count.test cb2e0f934c6eb33670044520748d2ecccd46259c F test/coveridxscan.test b629e896b14df2f000a99b8d170d80589c46562c @@ -582,16 +640,20 @@ F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c -F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 +F test/csv01.test e0ba3caaa57e4c667a0b45977689fb8082f14348 +F test/ctime.test ff6c38e822459d6ca743c34901caf57740b08b54 F test/cursorhint.test 7bc346788390475e77a345da2b92270d04d35856 -F test/date.test 984ac1e3e5e031386866f034006148d3972b4a65 -F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/cursorhint2.test 8457e93d97f665f23f97cdbc8477d16e3480331b +F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 +F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e +F test/dbstatus.test 73149851b3aff14fc6db478e58f9083a66422cf5 +F test/dbstatus2.test e93ab03bfae6d62d4d935f20de928c19ca0ed0ab F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d -F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f +F test/delete.test acc38fca8ee4851467705b1c2cfea64cd26667e5 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab -F test/delete4.test 3ac0b8d23689ba764c2e8b78c1b56b8f1b942fa2 +F test/delete4.test 21d2113217eeaacac2d99defe14fe6611615ae86 +F test/delete_db.test c70a43629dd4d3e1dd03fdaf7a22153af6a69d92 F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 @@ -606,23 +668,24 @@ F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test d4c6059d44dcd4b636de9aae322766062b471844 F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 -F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 03a84a6fa9bd3472112d6bd4599f5269f5f74803 -F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 +F test/e_dropview.test 21ce09c361227ddbc9819a5608ee2700c276bdd5 +F test/e_expr.test d0dd488e0527cbc560ae2dea1939cc44cb9b9924 +F test/e_fkey.test 54cc0046d2d952d6c42b0dd94414e7a8f75f79f4 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 3de217e95094d3d165992a6de1164bbc4bd92dc7 F test/e_reindex.test 2bebf7b393e519198b7c654407221cf171a439b8 F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 -F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 +F test/e_select.test 16651bb681e83a1a2875ff4a595ed2b4b4dee375 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 25385396082b67fd02ae0038b95a3b3575fe0519 -F test/e_vacuum.test 4d5b391384bb7d56bb9337d956f08035332421fc +F test/e_vacuum.test 1b8b4772d05374aa1b8958669138bbb4213ee26a F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 4c0613a0c76e7a9d5c4c211e1b4cbcc1143914df +F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 6807f7a7740a00361ca8d0ccd66bc60c8dc5f2b6 @@ -630,24 +693,24 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c -F test/exclusive.test f48243eaf40e0957215501a12f510a8644d13a02 -F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 +F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 +F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 -F test/expr.test 79c3e7502d9e571553b85f0ecc8ff2ac7d0e4931 +F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac +F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7 -F test/filectrl.test 7c13f96457435238da99aff7343ad6a3a4885787 -F test/filefmt.test cb34663f126cbc2d358af552dcaf5c72769b0146 -F test/fkey1.test 13e3d48236a2b9f5c5ebd232eef9b3ab682a8a2c -F test/fkey2.test f3d27ecba480a348c328965d154214719bb158a9 +F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3 +F test/filefmt.test e4edbdc637ca9576ccf4337a3cce627d9df7a56c +F test/fkey1.test ba64806ff9a04eecab2679caad377ae99a5e94e4 +F test/fkey2.test 155809016fad6b2a1491facf2ac53a551bc57c2c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 5a373303f201ac03c22ba1ef17a733d3f56e611a -F test/fkey6.test ce3c39cd4f910184f5fd4a48db6783c59956224f +F test/fkey6.test d078a1e323a740062bed38df32b8a736fd320dc0 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 -F test/fkey8.test 7bd1dd0174a0e29a90c62c517b9e2a410a0b345d +F test/fkey8.test e5372e32cdb4481f121ec3550703eeb7b4e0762c F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/fordelete.test eb93a2f34137bb87bdab88fcab06c0bd92719aff F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb @@ -705,12 +768,12 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2 F test/fts3atoken.test 4b4c16fdcfc972f2cdbba212375a060a86ccf5f1 F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9 -F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221 +F test/fts3aux1.test ca32c138aa777b476271df2da2c5538b7a43ada1 F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3conf.test 1c8b8adb0ab28a41b68d1514df44380bd7353402 +F test/fts3conf.test 60317efd562080e198b5bdc9fcd222ce32cf01d7 F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 @@ -720,7 +783,7 @@ F test/fts3defer2.test c540f5f5c2840f70c68fd9b597df817ec7170468 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 -F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162 +F test/fts3expr.test 9466627007804d855bf9df2a0cfb3dac23686fdc F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3 @@ -751,6 +814,7 @@ F test/fts4growth.test e5390da74619cacc389711bac9349640b32c4f9a F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d F test/fts4langid.test 65a7332c9bc257919e259a304aa8a38c41655b9d +F test/fts4lastrowid.test fa5e157955a3121615ef3e16ff5196e96c9e1e64 F test/fts4merge.test d2b39f6b1bd4a9738a13540e2d044cba11c43d47 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 @@ -765,41 +829,44 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 -F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 +F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test b47377143f0c80f91ed29d722861077ff34415d5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 -F test/fuzzcheck.c 93bb9d309888634615e21ef98d1c30d51483e942 +F test/fuzzcheck.c 2152602232c96d9c790eff3013e1369ce59de3203fa0b75bc613531448454e61 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba F test/fuzzdata4.db 1882f0055fb63214d8407ddc7aca9b0b1c59af21 +F test/fuzzdata5.db 9f0cdcc5c6e83b90cf9ae343bd07f684d2da2de7 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 +F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/hexlit.test d7b0a5f41123df1e43985b91b8b2e70f95282d21 +F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 -F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f +F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 +F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 73956798bace8982909c00476b216714a6d0559a F test/ieee754.test 806fc0ce7f305f57e3331eaceeddcfec9339e607 F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test 61a24ae38d4b64ec69f06ccdf022992f68a98176 +F test/in.test 20c5529986998949908f889c8208b2cd894b2cc9 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test acc710c12af118df5f8645eaba9479f5619eed81 +F test/in5.test 6c006e0bcd7351b69350ef566e65f244023489e9 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 -F test/incrblob4.test f26502a5697893e5acea268c910f16478c2f0fab +F test/incrblob4.test 21a52a6843a56cdcce968c6a86b72a7066d0e6ba F test/incrblob_err.test 69f9247fed50278d48ea710d1a8f9cdb09e4c0b8 -F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e +F test/incrblobfault.test 74dd8ac108304cea0b4a0df6df63a1567e558758 F test/incrcorrupt.test 6c567fbf870aa9e91866fe52ce6f200cd548939a F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 -F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac +F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 @@ -807,10 +874,11 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 81bc47890b8abfb181bc35f8d10b56c069803386 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a -F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c +F test/index6.test b4fc812290067a578b98bb2667b676db89e202a7 +F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb +F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 -F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2 +F test/indexexpr1.test 038b3befa74e5a75126b6e9dd2ae5df61c1c7cf7 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -818,10 +886,12 @@ F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test a20432f1c0fbbcff8f11d0e6ab4acb8c9db58023 F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 -F test/instr.test 737bbf80685232033f3abedc6ae92f75860b5dd2 -F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 -F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc -F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 +F test/instr.test 9a8802f28437d8ade53fedfc47b2ca599b4e48ba +F test/instrfault.test 0f870b218ea17cd477bb19ed330eecdb460dd53a +F test/intarray.test 46d95b457916638c5d8b1af21fb174804b3acf8b +F test/interrupt.test 16ea879ec728cb76414c148c5f24afd5d1f91054 +F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d +F test/intpkey.test ac71107a49a06492b69b82aafaf225400598d3c8 F test/io.test f95bca1783b01ea7761671560d023360d2dfa4cc F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 @@ -830,30 +900,33 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test f9d4a28dec81c6e9dc21b73518e024d73b5ebf57 -F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 +F test/join2.test a48f723c5692e2cbb23a9297ac2720cb77d51a70 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 8a5c0be6f0c260a5c7177c3b8f07c7856141038a +F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 -F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 +F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test ef42283f0b60d8bacbc2243448e7c84988578e52 +F test/json101.test c0897616f32d95431f37fd291cb78742181980ac F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 +F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff +F test/kvtest.c b9a9822dda05a1aa481215a52e2fc93cd8b22ee5 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test 81632c437a947bf1f7130b19537da6a1a844806a +F test/like.test 0603f4fa0dad50987f70032c05800cbfa8985302 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/like3.test 3608a2042b6f922f900fbfd5d3ce4e7eca57f7c4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e -F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b +F test/limit2.test e35f57bd3a62d7c5dcb5ac4306e675c75f974809 +F test/loadext.test d077450695ddb5c1ea3ad7d48e5f5850fe732ad9 F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test b984ab9034e7389be0d863fe4e64cbbc4d2028f5 +F test/lock.test be4fe08118fb988fed741f429b7dd5d65e1c90db F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 @@ -867,7 +940,7 @@ F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 -F test/malloc5.test 02ed7c5313f0a68d95f2dfca8c8962132bd1f04b +F test/malloc5.test a591be066ebc6ad34adaa4b38939a037a59ebd80ad681e79dedac7b4a9e7bc09 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d @@ -881,18 +954,19 @@ F test/mallocE.test db1ed69d7eded1b080952e2a7c37f364ad241b08 F test/mallocF.test 2d5c590ebc2fc7f0dcebdf5aa8498b9aed69107e F test/mallocG.test 0ff91b65c50bdaba680fb75d87fe4ad35bb7934f F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb -F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 +F test/mallocI.test 6c23a71df077fa5d387be90e7e669c5b368ca38a F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 27cb5566a6e5f2d76f9d4aa2eca45524401fd61e F test/mallocL.test fb311ff80afddf3b1a75e52289081f4754d901dc +F test/mallocM.test 78bbe9d3da84a5c679123cdb40d7b2010b18fc46e13897e4f253c6ba6fbff134 F test/malloc_common.tcl aac62499b76be719fac31e7a3e54a7fd53272e7f F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test 0311bfb4edd2615e3aa56c7a9cf44574e4229077 +F test/memsubsys1.test 6d268d0ae90f8d61a2356a1838665654d83de518 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 -F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd +F test/minmax.test 6751e87b409fe11b02e70a306d846fa544e25a41 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f @@ -900,24 +974,25 @@ F test/misc1.test 6430dabfb4b4fa480633590118964201f94d3ccc F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3 -F test/misc5.test f96428ea95b3820aafc6f1c50cf48a09e4597ee1 +F test/misc5.test fff0f75e834bc09a39f6079320dd8c37de956f4f F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test 21ac9d35a5e110279ae9e1588b8914f54de1c60b +F test/misc8.test ba03aaa08f02d62fbb8d3b2f5595c1b33aa9bbc5 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test d2cfc1635171c434dcff0ece2f1c8e0a658807ce F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 -F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 +F test/multiplex.test dc0d67b66f84b484a83cb8bbdf3f0a7f49562ccd F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test e0a44072d98189003deae4b091106f085d94bea8 +F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dacc57f80859c06a433d30839336fe227d2038b3 -F test/nolock.test 96e922d2d3db71c2dd6557c98e8027a28277b415 +F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d +F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -937,32 +1012,35 @@ F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/oserror.test b32dc34f2363ef18532e3a0a7358e3e7e321974f +F test/ossfuzz.c f5abed3177f719df3c3109901fcdd26b9fb7f581c8da50fc26f3a81ddfb2c2ae +F test/ossshell.c 296ab63067841bd1b1e97b46a0b2af48ee7f69d50d1a723008bee12dd7122622 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 841868017e9dd3cb459b8d78862091a7d9cff21d F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test ae9ee0db5a30aecda9db8290ce3dd12e5f7bbaa1 +F test/pagerfault.test 42ff797b1e6426c141cc7ee8b7417c9f27427950 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 -F test/pageropt.test f8d840f0fed84854dbd12b4c17fb3506eeb8224c +F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 +F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test af8067e4b6d684518956f8e2a744f42bde99014d -F test/pragma.test afbf028be1c35b68f57db8eb015c4a3c59d8f28e +F test/permutations.test af720e7d139e7e5417341d0f0eef2b911c0b067852138dc2f5b6a451b5725118 +F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f -F test/pragma3.test 3f1984a04657331f838df5c519b443c2088df922 +F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed +F test/pragma4.test 6e85b6eab8e61ffc9c7db59d842276674e8e3264 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 +F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test 36cd78b178c4eb0401d4f25754ef410fbd9df2a7 +F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 @@ -970,9 +1048,9 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 -F test/regexp2.test aa7ffcc21350007a78361b82bcf3b74d12227144 +F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl a246ecb14ed594bf44bf77bd21df873971d779bf +F test/releasetest.tcl cb06c4df0de4872e65b178316f8a87ccf7624d59 x F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5 @@ -980,6 +1058,16 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d +F test/rowvalue.test 44f3492f415cc9f374e8388a5eb61503eaca5230 +F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b +F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 +F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea +F test/rowvalue5.test c81c7d8cf36711ab37675ad7376084ae2a359cb6 +F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 +F test/rowvalue7.test 5d06ff19d9e6969e574a2e662a531dd0c67801a8 +F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 +F test/rowvalue9.test d8dd2c6ecac432dadaa79e41dc2434f007be1b6b +F test/rowvaluefault.test 7cd9ccc6c2fbdd881672984087aad0491bb75504 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d @@ -987,33 +1075,34 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 +F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 -F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5 +F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 F test/schema5.test 29699b4421f183c8f0e88bd28ce7d75d13ea653e F test/securedel.test 21749c32ccc30f1ea9e4b9f33295a6521ec20fa0 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/select1.test be62204d2bd9a5a8a149e9974cfddce893d8f686 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 -F test/select4.test 453631158540e5f685b81cac5b7e8bd8c6b4c5fc +F test/select4.test 5389d9895968d1196c457d59b3ee6515d771d328 F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0 -F test/select7.test 95e370c42d47c3c52377d05e9ffc01ccff7c1f61 +F test/select7.test f659f231489349e8c5734e610803d7654207318f F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7 +F test/selectA.test 101e722370ac6e84978c2958b8931c78b10a1709 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 +F test/selectC.test e25243f8ca503e06f252eb0218976d07cfeceac3 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test e8600e379589e85e9fefd2fe4d44a4cdd63f6982 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 +F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 F test/shared3.test ab693f9b6e156b8bfb2a0ad94f29fe69602a5d38 @@ -1026,68 +1115,75 @@ F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test a45b424ec23a4b946848ff9e1af5186c677162ab -F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862 -F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 -F test/shell4.test f43e250139dc5dc5f0f2ec1752c50284a1ede102 -F test/shell5.test c04e9f9f948305706b88377c464c7f08ce7479f9 +F test/shell1.test 50226a3a66bbd42a902e9f7698f768927eb33a56e9cfc55b7c157c38eb3e80ac +F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b +F test/shell3.test 9b95ba643eaa228376f06a898fb410ee9b6e57c1 +F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d +F test/shell5.test 50a732c1c2158b1cd62cf53975ce1ea7ce6b9dc9 F test/shell6.test 05c2488ff2bb11ac300b9170e8f5b6e03d58982c77035cdca5dbf036f0d3c5b7 +F test/shell7.test 07751911b294698e0c5df67bcbd29e7d2f0f2907 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test d37a75b4be4eb9dedeb69b4f38b1d0a74b5021d7 +F test/skipscan1.test 8ab5d2c7c5cd3fe7f172d366e6e74e887cb33cb4 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b -F test/snapshot.test c03eb5ba1602df33c1edc1d12393d6ca7a282a46 -F test/snapshot_fault.test 25973aeb1b86a280800e0bcf1eb5ce70e9ef57ab +F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f +F test/snapshot2.test 867652ed4a13282dce218723535fad1c7b44c3c4 +F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c -F test/sort5.test d3041ce3c475aa04142a959ae56ef6593f98a99f +F test/sort5.test 30cc17768e0c06ecb048e08efec59c11811fd186 F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded -F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 +F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 1478cb3fb64ad30f291ddca87ca9dbd72ff552aa +F test/speedtest1.c 7b1ab42b097b484c18d99e1d1c71a6a0c9c87a7a F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 -F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5 +F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a -F test/stat.test b65bad7120c52583b8f0054d99eff80718119a77 +F test/stat.test f8f1279ffffabe6df825723af18cc6e0ae70a893 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1 F test/stmt.test 64844332db69cf1a735fcb3e11548557fc95392f F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f -F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 +F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 -F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899 +F test/sync.test 2f84bdbc2b2df1fcb0220575b4b9f8cea94b7529 +F test/sync2.test 29af00fe9e468a1e20315d1edced8f8a32ac156df983ae9b53802e7d472675d5 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test f977868fa8bb7beb4b2072883190411653473906 +F test/tabfunc01.test 699251cb99651415218a891384510a685c7ab012 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test c6d9f546f79d15d0134c1e06583fb3ee0c3afad3 +F test/tclsqlite.test 1d73b9203b1ca8798d7d7310742b8d3febc0d56e F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 +F test/tempdb2.test 27e41ed540b2f9b056c2e77e9bddc1b875358507 +F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 -F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl d6dee24773e20320ab64d39bb5ce6e1a91b6029e +F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e +F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 +F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc +F test/tester.tcl 581f0185434daf7026ccede4c07e8d1479186ec5 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1100,13 +1196,14 @@ F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 F test/threadtest3.c 38a612ea62854349ed66372f330a40d73c5cf956 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 +F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 -F test/tkt-2d1a5c67d.test d371279946622698ab393ff88cad9f5f6d82960b +F test/tkt-2d1a5c67d.test be1326f3061caec85085f4c9ee4490561ca037c0 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58 -F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac +F test/tkt-313723c356.test 4b306ad45c736cedf2f5221f6155b92143244b6d F test/tkt-385a5b56b9.test c0a06ada41d7f06b1686da0e718553f853771d1e F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 @@ -1117,23 +1214,23 @@ F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 F test/tkt-4ef7e3cfca.test 3965ae11cc9cf6e334f9d7d3c1e20bf8d56254b1 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 -F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa +F test/tkt-5d863f876e.test 726e76d725f6fe0eb2fc8a522b721b79807380ee F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf -F test/tkt-78e04e52ea.test 813779f8888f3ca226df656c4eef078f9635f3c9 +F test/tkt-78e04e52ea.test 1b2e6bf4f1d9887b216b6da774e5f25915ec8118 F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 -F test/tkt-80e031a00f.test f50046f474ecf67ad5c50cd9200da04ff887d7cd +F test/tkt-80e031a00f.test 9ee36348b761bf7c14261e002b75a4c0d5a04d4c F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 -F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 +F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 @@ -1216,7 +1313,7 @@ F test/tkt3541.test 5dc257bde9bc833ab9cc6844bf170b998dbb950a F test/tkt3554.test f599967f279077bace39220cbe76085c7b423725 F test/tkt3581.test 1966b7193f1e3f14951cce8c66907ae69454e9a3 F test/tkt35xx.test f38c1b03713179d414969187c941466e44945b35 -F test/tkt3630.test 929f64852103054125200bc825c316d5f75d42f7 +F test/tkt3630.test 9a934c58c259f89a0ae6bb6bb846c56285a6fd0f F test/tkt3718.test 3b59dcb5c4e7754dacd91e7fd353a61492cc402a F test/tkt3731.test 0c5f4cbffe102d43c3b2188af91a9e36348f974b F test/tkt3757.test 10cd679a88675c880533083fc79ac04324525595 @@ -1225,7 +1322,7 @@ F test/tkt3762.test 4d439ff7abdc8d9323150269d182c37c2d514576 F test/tkt3773.test 7bca904d2a647a6a4a291bd86d7fd7c73855b789 F test/tkt3791.test a6624b9a80b216a26cf473607f42f3e51898c267 F test/tkt3793.test d90ffd75c52413908d15e1c44fc2ea9c80fcc449 -F test/tkt3810.test 90fa0635dfa7da9680c8cd3513350a49b3a8ae12 +F test/tkt3810.test 09608d81c63a6ff3aaf7bc70717909c51f5f4048 F test/tkt3824.test 150aa00bb6220672e5f0eb14dc8eaa36750425f0 F test/tkt3832.test 2300d10d57562b89875b72148338ac3e14f8847d F test/tkt3838.test 292e72489101cd1320d7278dc111c173ebf334d4 @@ -1244,6 +1341,7 @@ F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4 F test/trace.test 6f676313e3ebd2a50585036d2f212a3319dd5836 F test/trace2.test f5cb67ad3bc09e0c58e8cca78dfd0b5639259983 +F test/trace3.test 56ab944fddacf628b118cc298503fc45c2e50ab0 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 @@ -1262,30 +1360,36 @@ F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 302d8995f5ffe63bbc15053abb3ef7a39cf5a092 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 15fa63f1097db1f83dd62d121616006978063d1f +F test/triggerF.test 55b1eb13433997faac3a4948c1d8252f6c8c636b +F test/triggerG.test 175cafdc6399d85231a09e82e051b0e45a2fd1f23dd08ae715bc359716149ab6 F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff -F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 +F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 -F test/uri.test 6630ecbdea2aac10df3c89dbae2243f4c2c353e4 +F test/update2.test 81772e5c9b93d1006d66d65e9a1327281bdec4e8 +F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 +F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 +F test/vacuum5.test c87234e8ca4107f349da4edbeda3e4ea5adc93f3 +F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test f6c3a39e0c819891265e1d0754e99960d81ef6c9 -F test/vtab1.test 7c4b81abd88361ada9cbe414c459efca26be6bda +F test/view.test 765802c7a66d37fabd5ac8e2f2dbe572b43eb9ab +F test/vtab1.test ed4a576231d8a36e70fd18e2b79b621b31e7f22a F test/vtab2.test f8cd1bb9aba7143eba97812d9617880a36d247ad F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 8e73ed268f3d596bc3590f45fc948fb40f28e9c3 @@ -1299,18 +1403,18 @@ F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796 F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test d5024aa42754962f6bb0afd261681686488e7afe -F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e -F test/vtabH.test 5f5157a1501d9889ec35c1a1832f69612dd31444 +F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b +F test/vtabH.test 5f9253eb9e41ba9fe94f4aa3e9230191893d7764 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 25ae059e900dbb584e0775627e45415ba5940df1 -F test/wal3.test 5dd734147f1f8f958c5261a1f2775d346d7013ce +F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7 +F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 22cb252502c5ff851925add2ca8a7e2af012e1d6 -F test/wal6.test a9d6aa635b9d63607dabdc11406f5f96ca986635 +F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 +F test/wal6.test b602704e4b066199bc89d91ca9000f335dcf4572 F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8 @@ -1323,20 +1427,21 @@ F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af +F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20 F test/walfault.test 1f8389f7709877e9b4cc679033d71d6fe529056b F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 -F test/walprotocol.test 059cb75484a1ecf6357a2c1b3324b8156749221e +F test/walprotocol.test 0b92feb132ccebd855494d917d3f6c2d717ace20 F test/walro.test 4ab7ac01b77c2f894235c699d59e3e3c7f15a160 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test f0c325563acde44f2c4ea6ba348e9e29f7121757 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 -F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e +F test/where3.test 54cdeb02157acc979de41530b804ae7b09552bf1 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1346,7 +1451,7 @@ F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 -F test/whereD.test 51366b07cb6f546cd30cc803f7e754f063b940de +F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501 @@ -1360,83 +1465,90 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d +F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/with1.test cef099a491eac9874f2c28bd2dc86394fb3e47b3 F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87 -F test/with3.test 511bacdbe41c49cf34f9fd1bd3245fe1575bca98 +F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test 1a7b9bd51b899928d327052df9741d2fe8dbe701 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 -F test/without_rowid3.test aad4f9d383e199349b6c7e508a778f7dff5dff79 +F test/without_rowid3.test 2724c787a51a5dce09d078453a758117b4b728f1 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test 89b1c587bd92a0590e440da33e7666bf4891572a F test/without_rowid6.test 1f99644e6508447fb050f73697350c7ceca3392e -F test/wordcount.c 2a0a6c0d0e8e8bbbac1f06d72a6791828c37c0cf +F test/wordcount.c 06efb84b7c48a4973c2c24ea06c93d00bce24389 F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa -F test/zerodamage.test 2d725c214b883e25ae6bb85ef228ecdfa03c6a7b +F test/zerodamage.test e59a56443d6298ecf7435f618f0b27654f0c849e F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 -F tool/GetTclKit.bat 629d87562e0487c386db630033931d12d62e6372 +F tool/GetTclKit.bat 6afa640edc7810725aec61c3076ac617c4aaf0b7 F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/addopcodes.tcl 2b089684eb8b7d0db64cf9d8e6d2fe1b6d279e8d -F tool/build-all-msvc.bat 3e4e4043b53f1aede4308e0d2567bbd773614630 x +F tool/addopcodes.tcl 10c889c4a65ec6c5604e4a47306fa77ff57ae189 +F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 -F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x +F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 +F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 -F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa +F tool/fuzzershell.c dbf6c26eef936ec78cb0707570de3a4308b2507e F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 251f5c3f21b553240cbdd42dd187a51bb2372cd3 -F tool/lempar.c d5114c7d13aa3af1e27ff3d02e4dea6eadec7ddf +F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f +F tool/lemon.c 5ccba178a8e8a4b21e1c9232944d23973da38ad7 +F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 +F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 -F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 +F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca +F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 F tool/mkkeywordhash.c f7f3b342211ac6a14258b9726d5b97cf4f548f22 -F tool/mkmsvcmin.tcl 2f12f7fa8858bbe61cf81820a2da96c79ed1ca8d +F tool/mkmsvcmin.tcl 95b37e202cbed873aa8ffdbb493b9db45927be2b F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c -F tool/mkopcodeh.tcl 385c62d78c38b2d92146dcb5abd319dbbc33506d +F tool/mkopcodeh.tcl a01d2c1d8a6205b03fc635adf3735b4c523befd3 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl f0d5bb266d1d388cf86fce5ba01a891e95d72d41 +F tool/mkpragmatab.tcl 2ffe6d5fdc2d3381621d6c77978ba054466e757f F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835 -F tool/mksqlite3c.tcl 9382100c82a0a08e63265780536a6c8efead4b65 -F tool/mksqlite3h.tcl 1d41ab59bffb025121f75b76e183125ce41b3ec8 +F tool/mksqlite3c-noext.tcl fef88397668ae83166735c41af99d79f56afaabb +F tool/mksqlite3c.tcl 06b2e6a0f21cc0a5d70fbbd136b3e0a96470645e +F tool/mksqlite3h.tcl b9836752c3d08f9fab2dfc0017ca9fd5d90ac863 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b -F tool/mkvsix.tcl 4abcaf3267171b2faadaf9b82a0dfbaa6e98f8b7 +F tool/mkvsix.tcl b9e0777a213c23156b6542842c238479e496ebf5 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 34d7ac01fe4fd18e3637f64abe12c40eca0f6b97 +F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b -F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0 +F tool/replace.tcl 60f91e8dd06ab81f74d213ecbd9c9945f32ac048 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a +F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 F tool/schemalint.tcl 49690356702d6cac07e2bb1790eac73862e92926 -F tool/showdb.c 82dca79a999b2701c62417636345e9974151fdad +F tool/showdb.c e6bc9dba233bf1b57ca0a525a2bba762db4e223de84990739db3f09c46151b1e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 -F tool/showstat4.c bda40d6e395df7edb6e9ea630784d3d762c35b4b +F tool/showstat4.c b14159aa062f661b394ba37b6b7b94bfb8012ab9 F tool/showwal.c ec79959834f7b21f1e0a2aa52bb7c056d2203977 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe -F tool/spaceanal.tcl 93c1fdc9733c525b17a2024c7df193daa002e037 +F tool/spaceanal.tcl ab7d9bf68062907282a64b3e12ccbfad47193c5a +F tool/speed-check.sh 9630ba0468b609c52f48309243d4eb6e9c34deda F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 5a26205111e6fa856d9b1535b1637744dcdb930b -F tool/srcck1.c 4f65e1a6748e42f24c0ea629dddc934d821c729a +F tool/sqldiff.c 3fb48a6c6669d2d2c59a7f072a410dd2583579b4 +F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d -F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f +F tool/symbols.sh c5a617b8c61a0926747a56c65f5671ef8ac0e148 F tool/tostr.tcl 96022f35ada2194f6f8ccf6fd95809e90ed277c4 F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 -F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598 +F tool/warnings.sh d9ded5274b3cb947e98e70d587f71d6263f803b5 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F vsixtest/App.xaml b76d3b48860e7454775c47ea38ffea9c4abe3e85 F vsixtest/App.xaml.cpp c465147f50871165c60ca16955219f6c5812d6d8 @@ -1459,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 249cd361b840913794b7cd2f2d42777dcd547a60 -R b5c9ea0357d4d69dec4a40a72a5670a9 +P 1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab c8000e94cca59dabf83d6cb75b40441aaf793d29880582dc4baa17246449b5fe +R a9131615092ed98b863a64f66624daea U dan -Z 84e2a466cf9b475b6898bdfe1581ffe0 +Z 924aec6608b5903c59e65e7507c45eb8 diff --git a/manifest.uuid b/manifest.uuid index 8cfc3a9b80..1b590b1f1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e3622de8b51fdb219eb1934533a95fe6dc5158f44d358bc2efdcb53a4a9fdab \ No newline at end of file +cb721d0b36268a7b0ef493fa4d7f6bcbaa9ead8b1990e3c3fae015fa1d545226 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 642c1fb677..067cbb896d 100644 --- a/src/alter.c +++ b/src/alter.c @@ -413,7 +413,7 @@ void sqlite3AlterRenameTable( pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; db->flags |= SQLITE_PreferBuiltin; /* Get a NULL terminated version of the new table name. */ @@ -504,7 +504,7 @@ void sqlite3AlterRenameTable( sqlite3NestedParse(pParse, "UPDATE \"%w\".%s SET " "sql = sqlite_rename_parent(sql, %Q, %Q) " - "WHERE %s;", zDb, SCHEMA_TABLE(iDb), zTabName, zName, zWhere); + "WHERE %s;", zDb, MASTER_NAME, zTabName, zName, zWhere); sqlite3DbFree(db, zWhere); } } @@ -528,7 +528,7 @@ void sqlite3AlterRenameTable( "ELSE name END " "WHERE tbl_name=%Q COLLATE nocase AND " "(type='table' OR type='index' OR type='trigger');", - zDb, SCHEMA_TABLE(iDb), zName, zName, zName, + zDb, MASTER_NAME, zName, zName, zName, #ifndef SQLITE_OMIT_TRIGGER zName, #endif @@ -601,6 +601,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; @@ -610,7 +611,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ pCol = &pNew->aCol[pNew->nCol-1]; pDflt = pCol->pDflt; @@ -688,23 +689,25 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ "UPDATE \"%w\".%s SET " "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " "WHERE type = 'table' AND name = %Q", - zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, + zDb, MASTER_NAME, pNew->addColOffset, zCol, pNew->addColOffset+1, zTab ); sqlite3DbFree(db, zCol); db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then the file - ** format to 2. If the default value of the new column is not NULL, - ** the file format be 3. Back when this feature was first added - ** in 2006, we went to the trouble to upgrade the file format to the - ** minimum support values. But 10-years on, we can assume that all - ** extent versions of SQLite support file-format 4, so we always and - ** unconditionally upgrade to 4. + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. */ - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, - SQLITE_MAX_FILE_FORMAT); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); /* Reload the schema of the modified table. */ reloadTableSchema(pParse, pTab, pTab->zName); @@ -770,7 +773,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); if( !pNew ) goto exit_begin_add_column; pParse->pNewTable = pNew; - pNew->nRef = 1; + pNew->nTabRef = 1; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; @@ -790,7 +793,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ } pNew->pSchema = db->aDb[iDb].pSchema; pNew->addColOffset = pTab->addColOffset; - pNew->nRef = 1; + pNew->nTabRef = 1; /* Begin a transaction and increment the schema cookie. */ sqlite3BeginWriteOperation(pParse, 0, iDb); diff --git a/src/analyze.c b/src/analyze.c index cd59ae681d..495cc954ac 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -210,14 +210,14 @@ static void openStatTable( for(i=0; izName))==0 ){ + if( (pStat = sqlite3FindTable(db, zTab, pDb->zDbSName))==0 ){ if( aTable[i].zCols ){ /* The sqlite_statN table does not exist. Create it. Note that a ** side-effect of the CREATE TABLE statement is to leave the rootpage ** of the new table in register pParse->regRoot. This is important ** because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); aRoot[i] = pParse->regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; @@ -232,7 +232,7 @@ static void openStatTable( if( zWhere ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - pDb->zName, zTab, zWhereType, zWhere + pDb->zDbSName, zTab, zWhereType, zWhere ); }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ @@ -290,6 +290,7 @@ struct Stat4Accum { Stat4Sample *aBest; /* Array of nCol best samples */ int iMin; /* Index in a[] of entry with minimum score */ int nSample; /* Current number of samples */ + int nMaxEqZero; /* Max leading 0 in anEq[] for any a[] entry */ int iGet; /* Index of current sample accessed by stat_get() */ Stat4Sample *a; /* Array of mxSample Stat4Sample objects */ sqlite3 *db; /* Database connection, for malloc() */ @@ -554,6 +555,13 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ assert( IsStat4 || nEqZero==0 ); #ifdef SQLITE_ENABLE_STAT4 + /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in Stat4Accum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with Stat4Sample.anEq[m]==0 for (m>=n). */ + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } if( pNew->isPSample==0 ){ Stat4Sample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); @@ -651,12 +659,22 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } } - /* Update the anEq[] fields of any samples already collected. */ + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ for(i=p->nSample-1; i>=0; i--){ int j; - for(j=iChng; jnCol; j++){ - if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); + } + + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } } + p->nMaxEqZero = iChng; } #endif @@ -797,6 +815,12 @@ static const FuncDef statPushFuncdef = { ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a poiner to a Stat4Accum object, never a +** NULL. +** ** If neither STAT3 nor STAT4 are enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the @@ -994,7 +1018,7 @@ static void analyzeOneTable( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -1180,7 +1204,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knCol ); + assert( k>=0 && knColumn ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -1364,27 +1388,14 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -1394,10 +1405,11 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -1526,7 +1538,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -1535,6 +1551,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -1615,7 +1632,7 @@ static void initAvgEq(Index *pIdx){ } } - if( nDist100>nSum100 ){ + if( nDist100>nSum100 && sumEqp, sqlite3_column_blob(pStmt, 4), pSample->n); + if( pSample->n ){ + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + } pIdx->nSample++; } rc = sqlite3_finalize(pStmt); @@ -1826,39 +1845,47 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ analysisInfo sInfo; HashElem *i; char *zSql; - int rc; + int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - sqlite3DefaultRowEst(pIdx); + pIdx->hasStat1 = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif } - /* Check to make sure the sqlite_stat1 table exists */ - sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)==0 ){ - return SQLITE_ERROR; - } - /* Load new statistics out of the sqlite_stat1 table */ - zSql = sqlite3MPrintf(db, - "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); - if( zSql==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); - sqlite3DbFree(db, zSql); + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } } + /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); + } /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1867,7 +1894,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; diff --git a/src/attach.c b/src/attach.c index ea378a40a2..393c46ccb6 100644 --- a/src/attach.c +++ b/src/attach.c @@ -97,7 +97,7 @@ static void attachFunc( goto attach_error; } for(i=0; inDb; i++){ - char *z = db->aDb[i].zName; + char *z = db->aDb[i].zDbSName; assert( z && zName ); if( sqlite3StrICmp(z, zName)==0 ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); @@ -137,6 +137,7 @@ static void attachFunc( rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); sqlite3_free( zPath ); db->nDb++; + db->skipBtreeMutex = 0; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; zErrDyn = sqlite3MPrintf(db, "database is already attached"); @@ -162,8 +163,8 @@ static void attachFunc( sqlite3BtreeLeave(aNew->pBt); } aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - aNew->zName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zName==0 ){ + aNew->zDbSName = sqlite3DbStrDup(db, zName); + if( rc==SQLITE_OK && aNew->zDbSName==0 ){ rc = SQLITE_NOMEM_BKPT; } @@ -192,7 +193,7 @@ static void attachFunc( case SQLITE_NULL: /* No key specified. Use the key from the main database */ sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey>0 || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ + if( nKey || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); } break; @@ -275,7 +276,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; + if( sqlite3StrICmp(pDb->zDbSName, zName)==0 ) break; } if( i>=db->nDb ){ @@ -325,6 +326,7 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; @@ -433,7 +435,7 @@ void sqlite3FixInit( db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zName; + pFix->zDb = db->aDb[iDb].zDbSName; pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; @@ -530,7 +532,7 @@ int sqlite3FixExpr( return 1; } } - if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; + if( ExprHasProperty(pExpr, EP_TokenOnly|EP_Leaf) ) break; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; }else{ diff --git a/src/auth.c b/src/auth.c index 9768fc2fc0..77a95d4a8f 100644 --- a/src/auth.c +++ b/src/auth.c @@ -107,10 +107,11 @@ int sqlite3AuthReadCol( const char *zCol, /* Column name */ int iDb /* Index of containing database. */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zName; /* Name of attached database */ - int rc; /* Auth callback return code */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ + if( db->init.busy ) return SQLITE_OK; rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser diff --git a/src/backup.c b/src/backup.c index 455671a1ad..165144d965 100644 --- a/src/backup.c +++ b/src/backup.c @@ -83,22 +83,16 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int i = sqlite3FindDbName(pDb, zDb); if( i==1 ){ - Parse *pParse; + Parse sParse; int rc = 0; - pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); - if( pParse==0 ){ - sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM_BKPT; - }else{ - pParse->db = pDb; - if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); - rc = SQLITE_ERROR; - } - sqlite3DbFree(pErrorDb, pParse->zErrMsg); - sqlite3ParserReset(pParse); - sqlite3StackFree(pErrorDb, pParse); + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParserReset(&sParse); if( rc ){ return 0; } @@ -196,7 +190,6 @@ sqlite3_backup *sqlite3_backup_init( p->isAttached = 0; if( 0==p->pSrc || 0==p->pDest - || setDestPgsz(p)==SQLITE_NOMEM || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK ){ /* One (or both) of the named databases did not exist or an OOM @@ -384,14 +377,6 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = SQLITE_OK; } - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); - } - /* If there is no open read-transaction on the source database, open ** one now. If a transaction is opened here, then it will be closed ** before this function exits. @@ -401,6 +386,24 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ bCloseTrans = 1; } + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + ){ + p->bDestLocked = 1; + sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); + } + /* Do not allow backup if the destination database is in WAL mode ** and the page sizes are different between source and destination */ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); @@ -777,10 +780,10 @@ int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement ** checks this assumption - (p->rc) should be set to either SQLITE_DONE - ** or an error code. - */ + ** or an error code. */ sqlite3_backup_step(&b, 0x7FFFFFFF); assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; diff --git a/src/bitvec.c b/src/bitvec.c index 9d13ba9185..bd4a09429b 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -293,7 +293,7 @@ u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold ** up to N bits. Let I be an integer between 0 and N. 0<=Imutex) ); for(i=0; inDb; i++){ p = db->aDb[i].pBt; - if( p ) sqlite3BtreeEnter(p); + if( p && p->sharable ){ + sqlite3BtreeEnter(p); + skipOk = 0; + } } + db->skipBtreeMutex = skipOk; } -void sqlite3BtreeLeaveAll(sqlite3 *db){ +void sqlite3BtreeEnterAll(sqlite3 *db){ + if( db->skipBtreeMutex==0 ) btreeEnterAll(db); +} +static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){ int i; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); @@ -201,6 +209,9 @@ void sqlite3BtreeLeaveAll(sqlite3 *db){ if( p ) sqlite3BtreeLeave(p); } } +void sqlite3BtreeLeaveAll(sqlite3 *db){ + if( db->skipBtreeMutex==0 ) btreeLeaveAll(db); +} #ifndef NDEBUG /* diff --git a/src/btree.c b/src/btree.c index 18e6be9f38..aecf06e63f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -450,6 +450,15 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ static int cursorOwnsBtShared(BtCursor *p){ assert( cursorHoldsMutex(p) ); return (p->pBtree->db==p->pBt->db); @@ -490,6 +499,7 @@ static void invalidateAllOverflowCache(BtShared *pBt){ */ static void invalidateIncrblobCursors( Btree *pBtree, /* The database file to check */ + Pgno pgnoRoot, /* The table that might be changing */ i64 iRow, /* The rowid that might be changing */ int isClearTable /* True if all rows are being deleted */ ){ @@ -500,7 +510,7 @@ static void invalidateIncrblobCursors( for(p=pBtree->pBt->pCursor; p; p=p->pNext){ if( (p->curFlags & BTCF_Incrblob)!=0 ){ pBtree->hasIncrblobCur = 1; - if( isClearTable || p->info.nKey==iRow ){ + if( p->pgnoRoot==pgnoRoot && (isClearTable || p->info.nKey==iRow) ){ p->eState = CURSOR_INVALID; } } @@ -509,7 +519,7 @@ static void invalidateIncrblobCursors( #else /* Stub function when INCRBLOB is omitted */ - #define invalidateIncrblobCursors(x,y,z) + #define invalidateIncrblobCursors(w,x,y,z) #endif /* SQLITE_OMIT_INCRBLOB */ /* @@ -609,23 +619,21 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ ** the key. */ static int saveCursorKey(BtCursor *pCur){ - int rc; + int rc = SQLITE_OK; assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. */ - if( 0==pCur->curIntKey ){ - void *pKey = sqlite3Malloc( pCur->nKey ); + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ pCur->pKey = pKey; }else{ @@ -756,26 +764,23 @@ static int btreeMoveto( ){ int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ - char *pFree = 0; if( pKey ){ assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeAllocUnpackedRecord( - pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree - ); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); if( pIdxKey->nField==0 ){ - sqlite3DbFree(pCur->pKeyInfo->db, pFree); - return SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_BKPT; + goto moveto_done; } }else{ pIdxKey = 0; } rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); - if( pFree ){ - sqlite3DbFree(pCur->pKeyInfo->db, pFree); +moveto_done: + if( pIdxKey ){ + sqlite3DbFree(pCur->pKeyInfo->db, pIdxKey); } return rc; } @@ -1313,17 +1318,18 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ /* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. ** ** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a ** b-tree page so that there are no freeblocks or fragment bytes, all ** unused bytes are contained in the unallocated space region, and all ** cells are packed tightly at the end of the page. */ -static int defragmentPage(MemPage *pPage){ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ int i; /* Loop counter */ int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ @@ -1338,7 +1344,6 @@ static int defragmentPage(MemPage *pPage){ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); @@ -1350,9 +1355,56 @@ static int defragmentPage(MemPage *pPage){ cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); - usableSize = pPage->pBt->usableSize; - cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + + /* pageFindSlot() has already verified that free blocks are sorted + ** in order of offset within the page, and that no block extends + ** past the end of the page. Provided the two free slots do not + ** overlap, this guarantees that the memmove() calls below will not + ** overwrite the usableSize byte buffer, even if the database page + ** is corrupt. */ + assert( iFree2==0 || iFree2>iFree ); + assert( iFree+get2byte(&data[iFree+2]) <= usableSize ); + assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize ); + + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_BKPT; + sz2 = get2byte(&data[iFree2+2]); + assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + } + cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddrnFree ){ + return SQLITE_CORRUPT_BKPT; + } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; - data[hdr+7] = 0; memset(&data[iCellFirst], 0, cbrk-iCellFirst); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } return SQLITE_OK; } @@ -1533,10 +1587,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); - rc = defragmentPage(pPage); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + assert( gap+2+nByte<=top ); } @@ -1598,8 +1652,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( data[iPtr+1]==0 && data[iPtr]==0 ){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ - while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; @@ -1676,11 +1733,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior - ** table b-tree page. */ + /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an + ** interior table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf - ** table b-tree page. */ + /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a + ** leaf table b-tree page. */ assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; if( pPage->leaf ){ @@ -1694,11 +1751,11 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior - ** index b-tree page. */ + /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an + ** interior index b-tree page. */ assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf - ** index b-tree page. */ + /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a + ** leaf index b-tree page. */ assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; @@ -1733,7 +1790,7 @@ static int btreeInitPage(MemPage *pPage){ assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); if( !pPage->isInit ){ - u16 pc; /* Address of a freeblock within pPage->aData[] */ + int pc; /* Address of a freeblock within pPage->aData[] */ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ BtShared *pBt; /* The main btree structure */ @@ -1813,25 +1870,30 @@ static int btreeInitPage(MemPage *pPage){ ** freeblocks. */ pc = get2byte(&data[hdr+1]); nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ - while( pc>0 ){ - u16 next, size; - if( pciCellLast ){ + if( pc>0 ){ + u32 next, size; + if( pc0 && next<=pc+size+3) || pc+size>usableSize ){ - /* Free blocks must be in ascending order. And the last byte of - ** the free-block must lie on the database page. */ - return SQLITE_CORRUPT_BKPT; + while( 1 ){ + if( pc>iCellLast ){ + return SQLITE_CORRUPT_BKPT; /* Freeblock off the end of the page */ + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + nFree = nFree + size; + if( next<=pc+size+3 ) break; + pc = next; + } + if( next>0 ){ + return SQLITE_CORRUPT_BKPT; /* Freeblock not in ascending order */ + } + if( pc+size>(unsigned int)usableSize ){ + return SQLITE_CORRUPT_BKPT; /* Last freeblock extends past page end */ } - nFree = nFree + size; - pc = next; } /* At this point, nFree contains the sum of the offset to the start @@ -2272,7 +2334,7 @@ int sqlite3BtreeOpen( goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, - EXTRA_SIZE, flags, vfsFlags, pageReinit); + sizeof(MemPage), flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); @@ -2330,9 +2392,9 @@ int sqlite3BtreeOpen( #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ + pBt->nRef = 1; if( p->sharable ){ MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) - pBt->nRef = 1; MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); @@ -2360,12 +2422,12 @@ int sqlite3BtreeOpen( for(i=0; inDb; i++){ if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ while( pSib->pPrev ){ pSib = pSib->pPrev; } - if( p->pBtpBt ){ + if( (uptr)p->pBt<(uptr)pSib->pBt ){ p->pNext = pSib; p->pPrev = 0; pSib->pPrev = p; }else{ - while( pSib->pNext && pSib->pNext->pBtpBt ){ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ pSib = pSib->pNext; } p->pNext = pSib->pNext; @@ -2385,12 +2447,14 @@ int sqlite3BtreeOpen( btree_open_out: if( rc!=SQLITE_OK ){ if( pBt && pBt->pPager ){ - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, 0); } sqlite3_free(pBt); sqlite3_free(p); *ppBtree = 0; }else{ + sqlite3_file *pFile; + /* If the B-Tree was successfully opened, set the pager-cache size to the ** default value. Except, when opening on an existing shared pager-cache, ** do not change the pager-cache size. @@ -2398,11 +2462,17 @@ btree_open_out: if( sqlite3BtreeSchema(p, 0, 0)==0 ){ sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); } + + pFile = sqlite3PagerFile(pBt->pPager); + if( pFile->pMethods ){ + sqlite3OsFileControlHint(pFile, SQLITE_FCNTL_PDB, (void*)&pBt->db); + } } if( mutexOpen ){ assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); return rc; } @@ -2526,7 +2596,7 @@ int sqlite3BtreeClose(Btree *p){ ** Clean out and delete the BtShared object. */ assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, p->db); if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } @@ -2792,6 +2862,31 @@ int sqlite3BtreeGetAutoVacuum(Btree *p){ #endif } +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt,safety_level) +#endif /* ** Get a reference to pPage1 of the database file. This will @@ -2865,26 +2960,15 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ -#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS - sqlite3 *db; - Db *pDb; - if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ - while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 - ){ - pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; - sqlite3PagerSetFlags(pBt->pPager, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); - } - } -#endif + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); if( isOpen==0 ){ releasePage(pPage1); return SQLITE_OK; } } rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); } #endif @@ -3273,14 +3357,11 @@ static int setChildPtrmaps(MemPage *pPage){ int nCell; /* Number of cells in page pPage */ int rc; /* Return code */ BtShared *pBt = pPage->pBt; - u8 isInitOrig = pPage->isInit; Pgno pgno = pPage->pgno; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); rc = btreeInitPage(pPage); - if( rc!=SQLITE_OK ){ - goto set_child_ptrmaps_out; - } + if( rc!=SQLITE_OK ) return rc; nCell = pPage->nCell; for(i=0; iisInit = isInitOrig; return rc; } @@ -3328,7 +3407,6 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } put4byte(pPage->aData, iTo); }else{ - u8 isInitOrig = pPage->isInit; int i; int nCell; int rc; @@ -3342,12 +3420,14 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; pPage->xParseCell(pPage, pCell, &info); - if( info.nLocalaData+pPage->maskPage - && iFrom==get4byte(pCell+info.nSize-4) - ){ - put4byte(pCell+info.nSize-4, iTo); - break; + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_BKPT; + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } } }else{ if( get4byte(pCell)==iFrom ){ @@ -3364,8 +3444,6 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ } put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); } - - pPage->isInit = isInitOrig; } return SQLITE_OK; } @@ -4024,7 +4102,12 @@ int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); - rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ pBt->nPage = 0; @@ -4260,48 +4343,39 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){ return pCur && pCur->eState==CURSOR_VALID; } #endif /* NDEBUG */ - -/* -** Set *pSize to the size of the buffer needed to hold the value of -** the key for the current entry. If the cursor is not pointing -** to a valid entry, *pSize is set to 0. -** -** For a table with the INTKEY flag set, this routine returns the key -** itself, not the number of bytes in the key. -** -** The caller must position the cursor prior to invoking this routine. -** -** This routine cannot fail. It always returns SQLITE_OK. -*/ -int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - getCellInfo(pCur); - *pSize = pCur->info.nKey; - return SQLITE_OK; +int sqlite3BtreeCursorIsValidNN(BtCursor *pCur){ + assert( pCur!=0 ); + return pCur->eState==CURSOR_VALID; } /* -** Set *pSize to the number of bytes of data in the entry the -** cursor currently points to. +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. +*/ +i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); + getCellInfo(pCur); + return pCur->info.nKey; +} + +/* +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. ** ** The caller must guarantee that the cursor is pointing to a non-NULL ** valid entry. In other words, the calling procedure must guarantee ** that the cursor has Cursor.eState==CURSOR_VALID. -** -** Failure is not possible. This function always returns SQLITE_OK. -** It might just as well be a procedure (returning void) but we continue -** to return an integer result code for historical reasons. */ -int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - assert( cursorOwnsBtShared(pCur) ); +u32 sqlite3BtreePayloadSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 ); - assert( pCur->iPageapPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); - *pSize = pCur->info.nPayload; - return SQLITE_OK; + return pCur->info.nPayload; } /* @@ -4419,7 +4493,6 @@ static int copyPayload( ** ** 0: The operation is a read. Populate the overflow cache. ** 1: The operation is a write. Populate the overflow cache. -** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -4427,13 +4500,13 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the current cursor entry uses one or more overflow pages and the -** eOp argument is not 2, this function may allocate space for and lazily -** populates the overflow page-list cache array (BtCursor.aOverflow). +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** -** Once an overflow page-list cache has been allocated, it may be +** Once an overflow page-list cache has been allocated, it must be ** invalidated if some other cursor writes to the same table, or if ** the cursor is moved to a different row. Additionally, in auto-vacuum ** mode, the following events may invalidate an overflow page-list cache. @@ -4455,25 +4528,26 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - unsigned char * const pBufStart = pBuf; - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ #endif assert( pPage ); + assert( eOp==0 || eOp==1 ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; -#ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = offset+amt==pCur->info.nPayload; -#endif assert( offset+amt <= pCur->info.nPayload ); - if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ - /* Trying to read or write past the end of the data is an error */ + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ return SQLITE_CORRUPT_BKPT; } @@ -4483,7 +4557,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -4499,53 +4573,46 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. - ** Except, do not allocate aOverflow[] for eOp==2. ** ** The aOverflow[] array is sized at one entry for each overflow page ** in the overflow chain. The page number of the first overflow page is ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array ** means "not yet known" (the cache is lazily populated). */ - if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ Pgno *aNew = (Pgno*)sqlite3Realloc( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + return SQLITE_NOMEM_BKPT; }else{ pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } - if( rc==SQLITE_OK ){ - memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); - pCur->curFlags |= BTCF_ValidOvfl; + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + }else{ + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); } } - /* If the overflow page-list cache has been allocated and the - ** entry for the first required overflow page is valid, skip - ** directly to it. - */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 - && pCur->aOverflow[offset/ovflSize] - ){ - iIdx = (offset/ovflSize); - nextPage = pCur->aOverflow[iIdx]; - offset = (offset%ovflSize); - } - - for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ - + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ /* If required, populate the overflow page-list cache. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert( pCur->aOverflow[iIdx]==0 - || pCur->aOverflow[iIdx]==nextPage - || CORRUPT_DB ); - pCur->aOverflow[iIdx] = nextPage; - } + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -4553,11 +4620,7 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. - ** - ** Note that the aOverflow[] array must be allocated because eOp!=2 - ** here. If eOp==2, then offset==0 and this branch is never taken. */ - assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ @@ -4571,7 +4634,7 @@ static int accessPayload( ** range of data that is being read (eOp==0) or written (eOp!=0). */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; + sqlite3_file *fd; /* File from which to do direct overflow read */ #endif int a = amt; if( a + offset > ovflSize ){ @@ -4583,27 +4646,25 @@ static int accessPayload( ** ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) the database is file-backed, and - ** 4) there is no open write-transaction, and - ** 5) the database is not a WAL database, - ** 6) all data from the page is being read. - ** 7) at least 4 bytes have already been read into the output buffer + ** 3) there is no open write-transaction, and + ** 4) the database is file-backed, and + ** 5) the page is not in the WAL file + ** 6) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( (eOp&0x01)==0 /* (1) */ + if( eOp==0 /* (1) */ && offset==0 /* (2) */ - && (bEnd || a==ovflSize) /* (6) */ - && pBt->inTransaction==TRANS_READ /* (4) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && pBt->pPage1->aData[19]==0x01 /* (5) */ - && &pBuf[-4]>=pBufStart /* (7) */ + && pBt->inTransaction==TRANS_READ /* (3) */ + && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ + && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ + && &pBuf[-4]>=pBufStart /* (6) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; - assert( aWrite>=pBufStart ); /* hence (7) */ + assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); @@ -4614,41 +4675,49 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, - ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) + (eOp==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } } amt -= a; + if( amt==0 ) return rc; pBuf += a; } + if( rc ) break; + iIdx++; } } if( rc==SQLITE_OK && amt>0 ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_BKPT; /* Overflow chain ends prematurely */ } return rc; } /* -** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transferred into pBuf[]. The transfer +** Read part of the payload for the row at which that cursor pCur is currently +** pointing. "amt" bytes will be transferred into pBuf[]. The transfer ** begins at "offset". ** -** The caller must ensure that pCur is pointing to a valid row -** in the table. +** pCur can be pointing to either a table or an index b-tree. +** If pointing to a table btree, then the content section is read. If +** pCur is pointing to an index b-tree then the key section is read. +** +** For sqlite3BtreePayload(), the caller must ensure that pCur is pointing +** to a valid row in the table. For sqlite3BtreePayloadChecked(), the +** cursor might be invalid or might need to be restored before being read. ** ** Return SQLITE_OK on success or an error code if anything goes ** wrong. An error is returned if "offset+amt" is larger than ** the available payload. */ -int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ +int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); @@ -4657,33 +4726,34 @@ int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ } /* -** Read part of the data associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer -** begins at "offset". -** -** Return SQLITE_OK on success or an error code if anything goes -** wrong. An error is returned if "offset+amt" is larger than -** the available payload. +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. */ -int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - int rc; - #ifndef SQLITE_OMIT_INCRBLOB +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ + int rc; if ( pCur->eState==CURSOR_INVALID ){ return SQLITE_ABORT; } -#endif - assert( cursorOwnsBtShared(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - rc = accessPayload(pCur, offset, amt, pBuf, 0); - } - return rc; + rc = btreeRestoreCursorPosition(pCur); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); } +int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); + } +} +#endif /* SQLITE_OMIT_INCRBLOB */ /* ** Return a pointer to payload information from the entry that the @@ -4738,10 +4808,7 @@ static const void *fetchPayload( ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ -const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ - return fetchPayload(pCur, pAmt); -} -const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ +const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ return fetchPayload(pCur, pAmt); } @@ -4773,7 +4840,7 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ pCur, pCur->curPagerFlags); } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th @@ -4857,9 +4924,12 @@ static int moveToRoot(BtCursor *pCur){ } if( pCur->iPage>=0 ){ - while( pCur->iPage ){ - assert( pCur->apPage[pCur->iPage]!=0 ); - releasePageNotNull(pCur->apPage[pCur->iPage--]); + if( pCur->iPage ){ + do{ + assert( pCur->apPage[pCur->iPage]!=0 ); + releasePageNotNull(pCur->apPage[pCur->iPage--]); + }while( pCur->iPage); + goto skip_init; } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; @@ -4870,7 +4940,7 @@ static int moveToRoot(BtCursor *pCur){ 0, pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; - return rc; + return rc; } pCur->iPage = 0; pCur->curIntKey = pCur->apPage[0]->intKey; @@ -4893,10 +4963,12 @@ static int moveToRoot(BtCursor *pCur){ return SQLITE_CORRUPT_BKPT; } +skip_init: pCur->aiIdx[0] = 0; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); + pRoot = pCur->apPage[0]; if( pRoot->nCell>0 ){ pCur->eState = CURSOR_VALID; }else if( !pRoot->leaf ){ @@ -5074,19 +5146,37 @@ int sqlite3BtreeMovetoUnpacked( assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + assert( pCur->eState!=CURSOR_VALID || (pIdxKey==0)==(pCur->curIntKey!=0) ); /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->curIntKey + if( pIdxKey==0 + && pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 ){ if( pCur->info.nKey==intKey ){ *pRes = 0; return SQLITE_OK; } - if( (pCur->curFlags & BTCF_AtLast)!=0 && pCur->info.nKeyinfo.nKeycurFlags & BTCF_AtLast)!=0 ){ + *pRes = -1; + return SQLITE_OK; + } + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this ase, only a little more slowely */ + if( pCur->info.nKey+1==intKey && !pCur->skipNext ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, pRes); + if( rc ) return rc; + if( *pRes==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } + } + } } } @@ -5152,16 +5242,16 @@ int sqlite3BtreeMovetoUnpacked( if( lwr>upr ){ c = +1; break; } }else{ assert( nCellKey==intKey ); - pCur->curFlags |= BTCF_ValidNKey; - pCur->info.nKey = nCellKey; pCur->aiIdx[pCur->iPage] = (u16)idx; if( !pPage->leaf ){ lwr = idx; goto moveto_next_layer; }else{ + pCur->curFlags |= BTCF_ValidNKey; + pCur->info.nKey = nCellKey; + pCur->info.nSize = 0; *pRes = 0; - rc = SQLITE_OK; - goto moveto_finish; + return SQLITE_OK; } } assert( lwr+upr>=0 ); @@ -5222,7 +5312,8 @@ int sqlite3BtreeMovetoUnpacked( goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; @@ -5272,7 +5363,7 @@ moveto_next_layer: } moveto_finish: pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); return rc; } @@ -5292,6 +5383,30 @@ int sqlite3BtreeEof(BtCursor *pCur){ return (CURSOR_VALID!=pCur->eState); } +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->apPage[pCur->iPage]->leaf==0) ) return -1; + + for(n=1, i=0; i<=pCur->iPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + /* ** Advance the cursor to the next entry in the database. If ** successful then set *pRes=0. If the cursor @@ -5470,7 +5585,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ moveToParent(pCur); } assert( pCur->info.nSize==0 ); - assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 ); + assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 ); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -5986,30 +6101,28 @@ static void freePage(MemPage *pPage, int *pRC){ static int clearCell( MemPage *pPage, /* The page that contains the Cell */ unsigned char *pCell, /* First byte of the Cell */ - u16 *pnSize /* Write the size of the Cell here */ + CellInfo *pInfo /* Size information about the cell */ ){ BtShared *pBt = pPage->pBt; - CellInfo info; Pgno ovflPgno; int rc; int nOvfl; u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->xParseCell(pPage, pCell, &info); - *pnSize = info.nSize; - if( info.nLocal==info.nPayload ){ + pPage->xParseCell(pPage, pCell, pInfo); + if( pInfo->nLocal==pInfo->nPayload ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } - if( pCell+info.nSize-1 > pPage->aData+pPage->maskPage ){ + if( pCell+pInfo->nSize-1 > pPage->aData+pPage->maskPage ){ return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ } - ovflPgno = get4byte(pCell + info.nSize - 4); + ovflPgno = get4byte(pCell + pInfo->nSize - 4); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; - nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; + nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; assert( nOvfl>0 || - (CORRUPT_DB && (info.nPayload + ovflPageSize)nPayload + ovflPageSize)childPtrSize; - nPayload = nData + nZero; - if( pPage->intKeyLeaf ){ - nHeader += putVarint32(&pCell[nHeader], nPayload); - }else{ - assert( nData==0 ); - assert( nZero==0 ); - } - nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - - /* Fill in the payload size */ if( pPage->intKey ){ - pSrc = pData; - nSrc = nData; - nData = 0; - }else{ - assert( nKey<=0x7fffffff && pKey!=0 ); - nPayload = (int)nKey; - pSrc = pKey; - nSrc = (int)nKey; + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + }else{ + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); } + + /* Fill in the payload */ if( nPayload<=pPage->maxLocal ){ n = nHeader + nPayload; testcase( n==3 ); @@ -6145,12 +6251,12 @@ static int fillInCell( ** Use a call to btreeParseCellPtr() to verify that the values above ** were computed correctly. */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG { CellInfo info; pPage->xParseCell(pPage, pCell, &info); assert( nHeader==(int)(info.pPayload - pCell) ); - assert( info.nKey==nKey ); + assert( info.nKey==pX->nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); } @@ -6235,10 +6341,6 @@ static int fillInCell( pSrc += n; nSrc -= n; spaceLeft -= n; - if( nSrc==0 ){ - nSrc = nData; - pSrc = pData; - } } releasePage(pToRelease); return SQLITE_OK; @@ -6260,7 +6362,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ if( *pRC ) return; - assert( idx>=0 && idxnCell ); assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -6305,6 +6406,8 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. +** +** *pRC must be SQLITE_OK when this routine is called. */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -6320,8 +6423,7 @@ static void insertCell( u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - if( *pRC ) return; - + assert( *pRC==SQLITE_OK ); assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); @@ -6343,7 +6445,10 @@ static void insertCell( put4byte(pCell, iChild); } j = pPage->nOverflow++; - assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; @@ -6395,7 +6500,7 @@ static void insertCell( /* ** A CellArray object contains a cache of pointers and sizes for a -** consecutive sequence of cells that might be held multiple pages. +** consecutive sequence of cells that might be held on multiple pages. */ typedef struct CellArray CellArray; struct CellArray { @@ -6540,8 +6645,8 @@ static int pageInsertArray( u8 *pSlot; sz = cachedCellSize(pCArray, i); if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ + if( (pData - pBegin)apCell[i] will never overlap on a well-formed @@ -6703,7 +6808,7 @@ static int editPage( for(i=0; iapCell[i+iNew]; int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ pCell = &pTmp[pCell - aData]; } assert( 0==memcmp(pCell, &aData[iOff], @@ -6827,8 +6932,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + if( rc==SQLITE_OK ){ + insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + } /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); @@ -7081,7 +7188,7 @@ static int balance_nonroot( nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; if( (i--)==0 ) break; - if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ + if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); szNew[i] = pParent->xCellSize(pParent, apDiv[i]); @@ -7273,7 +7380,6 @@ static int balance_nonroot( for(i=0; inFree; - if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); } @@ -7348,7 +7454,7 @@ static int balance_nonroot( assert( r szLeft-(b.szCell[r]+2)) ){ + && (bBulk || szRight+b.szCell[d]+2 > szLeft-(b.szCell[r]+(i==k-1?0:2)))){ break; } szRight += b.szCell[d] + 2; @@ -7671,7 +7777,7 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0]); + rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) @@ -7920,33 +8026,39 @@ static int balance(BtCursor *pCur){ /* -** Insert a new record into the BTree. The key is given by (pKey,nKey) -** and the data is given by (pData,nData). The cursor is used only to -** define what table the record should be inserted into. The cursor -** is left pointing at a random location. +** Insert a new record into the BTree. The content of the new record +** is described by the pX object. The pCur cursor is used only to +** define what table the record should be inserted into, and is left +** pointing at a random location. ** -** For an INTKEY table, only the nKey value of the key is used. pKey is -** ignored. For a ZERODATA table, the pData and nData are both ignored. +** For a table btree (used for rowid tables), only the pX.nKey value of +** the key is used. The pX.pKey value must be NULL. The pX.nKey is the +** rowid or INTEGER PRIMARY KEY of the row. The pX.nData,pData,nZero fields +** hold the content of the row. +** +** For an index btree (used for indexes and WITHOUT ROWID tables), the +** key is an arbitrary byte sequence stored in pX.pKey,nKey. The +** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to -** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already -** been performed. seekResult is the search result returned (a negative -** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an entry that is larger than -** (pKey, nKey)). +** MovetoUnpacked() to seek cursor pCur to (pKey,nKey) has already +** been performed. In other words, if seekResult!=0 then the cursor +** is currently pointing to a cell that will be adjacent to the cell +** to be inserted. If seekResult<0 then pCur points to a cell that is +** smaller then (pKey,nKey). If seekResult>0 then pCur points to a cell +** that is larger than (pKey,nKey). ** -** If the seekResult parameter is non-zero, then the caller guarantees that -** cursor pCur is pointing at the existing copy of a row that is to be -** overwritten. If the seekResult parameter is 0, then cursor pCur may -** point to any entry or to no entry at all and so this function has to seek -** the cursor before the new key can be inserted. +** If seekResult==0, that means pCur is pointing at some unknown location. +** In that case, this routine must seek the cursor to the correct insertion +** point for (pKey,nKey) before doing the insertion. For index btrees, +** if pX->nMem is non-zero, then pX->aMem contains pointers to the unpacked +** key values and pX->aMem can be used instead of pX->pKey to avoid having +** to decode the key. */ int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ - const void *pKey, i64 nKey, /* The key of the new record */ - const void *pData, int nData, /* The data of the new record */ - int nZero, /* Number of extra 0 bytes to append to data */ - int appendBias, /* True if this is likely an append */ + const BtreePayload *pX, /* Content of the row to be inserted */ + int flags, /* True if this is likely an append */ int seekResult /* Result of prior MovetoUnpacked() call */ ){ int rc; @@ -7959,6 +8071,8 @@ int sqlite3BtreeInsert( unsigned char *oldCell; unsigned char *newCell = 0; + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND))==flags ); + if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); return pCur->skipNext; @@ -7975,7 +8089,7 @@ int sqlite3BtreeInsert( ** keys with no associated data. If the cursor was opened expecting an ** intkey table, the caller should be inserting integer keys with a ** blob of associated data. */ - assert( (pKey==0)==(pCur->pKeyInfo==0) ); + assert( (pX->pKey==0)==(pCur->pKeyInfo==0) ); /* Save the positions of any other cursors open on this table. ** @@ -7994,44 +8108,64 @@ int sqlite3BtreeInsert( } if( pCur->pKeyInfo==0 ){ - assert( pKey==0 ); + assert( pX->pKey==0 ); /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ - invalidateIncrblobCursors(p, nKey, 0); + invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. */ + assert( (flags & BTREE_SAVEPOSITION)==0 || + ((pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey) ); /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary ** btreeMoveto() call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 - && pCur->info.nKey==nKey-1 ){ - loc = -1; + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ + loc = 0; + }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 + && pCur->info.nKey==pX->nKey-1 ){ + loc = -1; }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc); + rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); if( rc ) return rc; } - }else if( loc==0 ){ - rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + }else if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ + if( pX->nMem ){ + UnpackedRecord r; + r.pKeyInfo = pCur->pKeyInfo; + r.aMem = pX->aMem; + r.nField = pX->nMem; + r.default_rc = 0; + r.errCode = 0; + r.r1 = 0; + r.r2 = 0; + r.eqSeen = 0; + rc = sqlite3BtreeMovetoUnpacked(pCur, &r, 0, flags!=0, &loc); + }else{ + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, flags!=0, &loc); + } if( rc ) return rc; } assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || nKey>=0 ); + assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, nKey, nData, pPage->pgno, + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); newCell = pBt->pTmpSpace; assert( newCell!=0 ); - rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); + rc = fillInCell(pPage, newCell, pX, &szNew); if( rc ) goto end_insert; assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(pBt) ); idx = pCur->aiIdx[pCur->iPage]; if( loc==0 ){ - u16 szOld; + CellInfo info; assert( idxnCell ); rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ @@ -8041,8 +8175,19 @@ int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - rc = clearCell(pPage, oldCell, &szOld); - dropCell(pPage, idx, szOld, &rc); + rc = clearCell(pPage, oldCell, &info); + if( info.nSize==szNew && info.nLocal==info.nPayload ){ + /* Overwrite the old cell with the new if they are the same size. + ** We could also try to do this if the old cell is smaller, then add + ** the leftover space to the free list. But experiments show that + ** doing that is no faster then skipping this optimization and just + ** calling dropCell() and insertCell(). */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ + if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; + } + dropCell(pPage, idx, info.nSize, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); @@ -8051,6 +8196,7 @@ int sqlite3BtreeInsert( assert( pPage->leaf ); } insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); /* If no error has occurred and pPage has an overflow cell, call balance() @@ -8074,7 +8220,8 @@ int sqlite3BtreeInsert( ** row without seeking the cursor. This can be a big performance boost. */ pCur->info.nSize = 0; - if( rc==SQLITE_OK && pPage->nOverflow ){ + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); @@ -8084,6 +8231,20 @@ int sqlite3BtreeInsert( ** from trying to save the current position of the cursor. */ pCur->apPage[pCur->iPage]->nOverflow = 0; pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + rc = moveToRoot(pCur); + if( pCur->pKeyInfo ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } } assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); @@ -8116,7 +8277,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ - u16 szCell; /* Size of the cell being deleted */ + CellInfo info; /* Size of the cell being deleted */ int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ @@ -8135,6 +8296,28 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ pPage = pCur->apPage[iCellDepth]; pCell = findCell(pPage, iCellIdx); + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + /* If the page containing the entry to delete is not a leaf page, move ** the cursor to the largest entry in the tree that is smaller than ** the entry being deleted. This cell will replace the cell being deleted @@ -8158,29 +8341,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); - } - - /* If the bPreserve flag is set to true, then the cursor position must - ** be preserved following this delete operation. If the current delete - ** will cause a b-tree rebalance, then this is done by saving the cursor - ** key and leaving the cursor in CURSOR_REQUIRESEEK state before - ** returning. - ** - ** Or, if the current delete will not cause a rebalance, then the cursor - ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately - ** before or after the deleted entry. In this case set bSkipnext to true. */ - if( bPreserve ){ - if( !pPage->leaf - || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) - ){ - /* A b-tree rebalance will be required after deleting this entry. - ** Save the cursor key. */ - rc = saveCursorKey(pCur); - if( rc ) return rc; - }else{ - bSkipnext = 1; - } + invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); } /* Make the page containing the entry to be deleted writable. Then free any @@ -8188,8 +8349,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell, &szCell); - dropCell(pPage, iCellIdx, szCell, &rc); + rc = clearCell(pPage, pCell, &info); + dropCell(pPage, iCellIdx, info.nSize, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -8210,7 +8371,9 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + if( rc==SQLITE_OK ){ + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } @@ -8437,7 +8600,7 @@ static int clearDatabasePage( unsigned char *pCell; int i; int hdr; - u16 szCell; + CellInfo info; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ @@ -8457,7 +8620,7 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell, &szCell); + rc = clearCell(pPage, pCell, &info); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ @@ -8505,7 +8668,7 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ /* Invalidate all incrblob cursors open on table iTable (assuming iTable ** is the root of a table b-tree - if it is not, the following call is ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); + invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); @@ -8548,27 +8711,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( sqlite3BtreeHoldsMutex(p) ); assert( p->inTrans==TRANS_WRITE ); - - /* It is illegal to drop a table if any cursors are open on the - ** database. This is because in auto-vacuum mode the backend may - ** need to move another root-page to fill a gap left by the deleted - ** root page. If an open cursor was using this page a problem would - ** occur. - ** - ** This error is caught long before control reaches this point. - */ - if( NEVER(pBt->pCursor) ){ - sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); - return SQLITE_LOCKED_SHAREDCACHE; - } - - /* - ** It is illegal to drop the sqlite_master table on page 1. But again, - ** this error is caught long before reaching this point. - */ - if( NEVER(iTable<2) ){ - return SQLITE_CORRUPT_BKPT; - } + assert( iTable>=2 ); rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; @@ -9476,7 +9619,7 @@ int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ if( pBt->inTransaction!=TRANS_NONE ){ rc = SQLITE_LOCKED; }else{ - rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt); + rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); } sqlite3BtreeLeave(p); } @@ -9699,4 +9842,14 @@ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } int sqlite3BtreeSharable(Btree *p){ return p->sharable; } + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} #endif diff --git a/src/btree.h b/src/btree.h index 9ba233ac33..0e017f5300 100644 --- a/src/btree.h +++ b/src/btree.h @@ -13,8 +13,8 @@ ** subsystem. See comments in the source code for a detailed description ** of what each interface routine does. */ -#ifndef _BTREE_H_ -#define _BTREE_H_ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. @@ -39,6 +39,7 @@ typedef struct Btree Btree; typedef struct BtCursor BtCursor; typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; int sqlite3BtreeOpen( @@ -89,7 +90,9 @@ int sqlite3BtreeIsInReadTrans(Btree*); int sqlite3BtreeIsInBackup(Btree*); void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif int sqlite3BtreeSavepoint(Btree *, int, int); const char *sqlite3BtreeGetFilename(Btree *); @@ -246,30 +249,60 @@ int sqlite3BtreeCursorHasMoved(BtCursor*); int sqlite3BtreeCursorRestore(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*, u8 flags); -/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ #define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ #define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ -int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, - const void *pData, int nData, - int nZero, int bias, int seekResult); +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to their +** key and pData,nData,nZero set to zero. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. NULL for indexes */ + struct Mem *aMem; /* First of nMem value in the unpacked pKey */ + u16 nMem; /* Number of aMem[] value. Might be zero */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int flags, int seekResult); int sqlite3BtreeFirst(BtCursor*, int *pRes); int sqlite3BtreeLast(BtCursor*, int *pRes); int sqlite3BtreeNext(BtCursor*, int *pRes); int sqlite3BtreeEof(BtCursor*); int sqlite3BtreePrevious(BtCursor*, int *pRes); -int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); -int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); -const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); -int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); -int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); +i64 sqlite3BtreeIntegerKey(BtCursor*); +int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); +const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +u32 sqlite3BtreePayloadSize(BtCursor*); char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); +i64 sqlite3BtreeRowCountEst(BtCursor*); +#ifndef SQLITE_OMIT_INCRBLOB +int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); @@ -279,6 +312,7 @@ int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); #endif +int sqlite3BtreeCursorIsValidNN(BtCursor*); #ifndef SQLITE_OMIT_BTREECOUNT int sqlite3BtreeCount(BtCursor *, i64 *); @@ -303,11 +337,13 @@ void sqlite3BtreeCursorList(Btree*); void sqlite3BtreeEnterAll(sqlite3*); int sqlite3BtreeSharable(Btree*); void sqlite3BtreeEnterCursor(BtCursor*); + int sqlite3BtreeConnectionCount(Btree*); #else # define sqlite3BtreeEnter(X) # define sqlite3BtreeEnterAll(X) # define sqlite3BtreeSharable(X) 0 # define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE @@ -332,4 +368,4 @@ void sqlite3BtreeCursorList(Btree*); #endif -#endif /* _BTREE_H_ */ +#endif /* SQLITE_BTREE_H */ diff --git a/src/btreeInt.h b/src/btreeInt.h index 6cd090257c..b01163c33f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -259,37 +259,39 @@ typedef struct CellInfo CellInfo; #define PTF_LEAF 0x08 /* -** As each page of the file is loaded into memory, an instance of the following -** structure is appended and initialized to zero. This structure stores -** information about the page that is decoded from the raw file page. +** An instance of this object stores information about each a single database +** page that has been loaded into memory. The information in this object +** is derived from the raw on-disk page content. ** -** The pParent field points back to the parent page. This allows us to -** walk up the BTree from any leaf to the root. Care must be taken to -** unref() the parent page pointer when this page is no longer referenced. -** The pageDestructor() routine handles that chore. +** As each database page is loaded into memory, the pager allocats an +** instance of this object and zeros the first 8 bytes. (This is the +** "extra" information associated with each page of the pager.) ** ** Access to all fields of this structure is controlled by the mutex ** stored in MemPage.pBt->mutex. */ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ - u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ + u8 bBusy; /* Prevent endless loops on corrupt database files */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ + Pgno pgno; /* Page number for this page */ + /* Only the first 8 bytes (above) are zeroed by pager.c when a new page + ** is allocated. All fields that follow must be initialized before use */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ - u8 bBusy; /* Prevent endless loops on corrupt database files */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ u16 nFree; /* Number of free bytes on the page */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ - u16 aiOvfl[5]; /* Insert the i-th overflow cell before the aiOvfl-th + u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th ** non-overflow cell */ - u8 *apOvfl[5]; /* Pointers to the body of overflow cells */ + u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ BtShared *pBt; /* Pointer to BtShared that this page is part of */ u8 *aData; /* Pointer to disk image of the page data */ u8 *aDataEnd; /* One byte past the end of usable data */ @@ -298,16 +300,8 @@ struct MemPage { DbPage *pDbPage; /* Pager page handle */ u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ - Pgno pgno; /* Page number for this page */ }; -/* -** The in-memory image of a disk page has the auxiliary information appended -** to the end. EXTRA_SIZE is the number of bytes of space needed to hold -** that extra information. -*/ -#define EXTRA_SIZE sizeof(MemPage) - /* ** A linked list of the following structures is stored at BtShared.pLock. ** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor @@ -698,11 +692,9 @@ struct IntegrityCk { */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) diff --git a/src/build.c b/src/build.c index e89c744461..2ce7c90747 100644 --- a/src/build.c +++ b/src/build.c @@ -30,10 +30,10 @@ ** codeTableLocks() functions. */ struct TableLock { - int iDb; /* The database containing the table to be locked */ - int iTab; /* The root page of the table to be locked */ - u8 isWriteLock; /* True for write lock. False for a read lock */ - const char *zName; /* Name of the table */ + int iDb; /* The database containing the table to be locked */ + int iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zLockName; /* Name of the table */ }; /* @@ -59,6 +59,8 @@ void sqlite3TableLock( TableLock *p; assert( iDb>=0 ); + if( iDb==1 ) return; + if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; for(i=0; inTableLock; i++){ p = &pToplevel->aTableLock[i]; if( p->iDb==iDb && p->iTab==iTab ){ @@ -75,7 +77,7 @@ void sqlite3TableLock( p->iDb = iDb; p->iTab = iTab; p->isWriteLock = isWriteLock; - p->zName = zName; + p->zLockName = zName; }else{ pToplevel->nTableLock = 0; sqlite3OomFault(pToplevel->db); @@ -97,7 +99,7 @@ static void codeTableLocks(Parse *pParse){ TableLock *p = &pParse->aTableLock[i]; int p1 = p->iDb; sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, - p->zName, P4_STATIC); + p->zLockName, P4_STATIC); } } #else @@ -146,15 +148,14 @@ void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ - while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); #if SQLITE_USER_AUTHENTICATION if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_AUTH_USER; sqlite3ErrorMsg(pParse, "user not authenticated"); + pParse->rc = SQLITE_AUTH_USER; return; } } @@ -173,14 +174,16 @@ void sqlite3FinishCoding(Parse *pParse){ assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); for(iDb=0; iDbnDb; iDb++){ + Schema *pSchema; if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pParse->cookieValue[iDb], /* P3 */ - db->aDb[iDb].pSchema->iGeneration /* P4 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); VdbeComment((v, @@ -231,16 +234,6 @@ void sqlite3FinishCoding(Parse *pParse){ }else{ pParse->rc = SQLITE_ERROR; } - - /* We are done with this Parse object. There is no need to de-initialize it */ -#if 0 - pParse->colNamesSet = 0; - pParse->nTab = 0; - pParse->nMem = 0; - pParse->nSet = 0; - pParse->nVar = 0; - DbMaskZero(pParse->cookieMask); -#endif } /* @@ -260,8 +253,7 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ char *zSql; char *zErrMsg = 0; sqlite3 *db = pParse->db; -# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) - char saveBuf[SAVE_SZ]; + char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ @@ -272,12 +264,12 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ return; /* A malloc must have failed */ } pParse->nested++; - memcpy(saveBuf, &pParse->nVar, SAVE_SZ); - memset(&pParse->nVar, 0, SAVE_SZ); + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); sqlite3RunParser(pParse, zSql, &zErrMsg); sqlite3DbFree(db, zErrMsg); sqlite3DbFree(db, zSql); - memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); pParse->nested--; } @@ -316,14 +308,22 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ return 0; } #endif - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); - if( p ) break; + while(1){ + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDatabase==0 || sqlite3StrICmp(zDatabase, db->aDb[j].zDbSName)==0 ){ + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); + if( p ) return p; + } + } + /* Not found. If the name we were looking for was temp.sqlite_master + ** then change the name to sqlite_temp_master and try again. */ + if( sqlite3StrICmp(zName, MASTER_NAME)!=0 ) break; + if( sqlite3_stricmp(zDatabase, db->aDb[1].zDbSName)!=0 ) break; + zName = TEMP_MASTER_NAME; } - return p; + return 0; } /* @@ -338,7 +338,7 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ */ Table *sqlite3LocateTable( Parse *pParse, /* context in which to report errors */ - int isView, /* True if looking for a VIEW rather than a TABLE */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ const char *zName, /* Name of the table we are looking for */ const char *zDbase /* Name of the database. Might be NULL */ ){ @@ -352,24 +352,29 @@ Table *sqlite3LocateTable( p = sqlite3FindTable(pParse->db, zName, zDbase); if( p==0 ){ - const char *zMsg = isView ? "no such view" : "no such table"; + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ /* If zName is the not the name of a table in the schema created using ** CREATE, then check to see if it is the name of an virtual table that ** can be an eponymous virtual table. */ Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ + pMod = sqlite3PragmaVtabRegister(pParse->db, zName); + } if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ return pMod->pEpoTab; } } #endif - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + if( (flags & LOCATE_NOERR)==0 ){ + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + pParse->checkSchema = 1; } - pParse->checkSchema = 1; } return p; @@ -386,18 +391,18 @@ Table *sqlite3LocateTable( */ Table *sqlite3LocateTableItem( Parse *pParse, - int isView, + u32 flags, struct SrcList_item *p ){ const char *zDb; assert( p->pSchema==0 || p->zDatabase==0 ); if( p->pSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); - zDb = pParse->db->aDb[iDb].zName; + zDb = pParse->db->aDb[iDb].zDbSName; }else{ zDb = p->zDatabase; } - return sqlite3LocateTable(pParse, isView, p->zName, zDb); + return sqlite3LocateTable(pParse, flags, p->zName, zDb); } /* @@ -421,7 +426,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zDbSName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; @@ -490,8 +495,8 @@ void sqlite3CollapseDatabaseArray(sqlite3 *db){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zName); - pDb->zName = 0; + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; continue; } if( jpnBytesFreed to measure the memory ** used by the Table object. */ -void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ +static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ - assert( !pTable || pTable->nRef>0 ); - - /* Do not delete the table until the reference count reaches zero. */ - if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; - /* Record the number of outstanding lookaside allocations in schema Tables ** prior to doing any free() operations. Since schema Tables do not use ** lookaside, this number should not change. */ @@ -611,8 +610,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema ); - if( !db || db->pnBytesFreed==0 ){ + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 @@ -641,6 +641,13 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Verify that no lookaside memory was used by schema tables */ assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); } +void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + deleteTable(db, pTable); +} + /* ** Unlink the given table from the hash tables and the delete the @@ -691,7 +698,7 @@ char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ */ void sqlite3OpenMasterTable(Parse *p, int iDb){ Vdbe *v = sqlite3GetVdbe(p); - sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); + sqlite3TableLock(p, iDb, MASTER_ROOT, 1, MASTER_NAME); sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, MASTER_ROOT, iDb, 5); if( p->nTab==0 ){ p->nTab = 1; @@ -709,7 +716,10 @@ int sqlite3FindDbName(sqlite3 *db, const char *zName){ if( zName ){ Db *pDb; for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( 0==sqlite3StrICmp(pDb->zName, zName) ) break; + if( 0==sqlite3_stricmp(pDb->zDbSName, zName) ) break; + /* "main" is always an acceptable alias for the primary database + ** even if it has been renamed using SQLITE_DBCONFIG_MAINDBNAME. */ + if( i==0 && 0==sqlite3_stricmp("main", zName) ) break; } } return i; @@ -768,7 +778,7 @@ int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy ); + assert( db->init.iDb==0 || db->init.busy || (db->flags & SQLITE_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; } @@ -879,7 +889,7 @@ void sqlite3StartTable( SQLITE_CREATE_VIEW, SQLITE_CREATE_TEMP_VIEW }; - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } @@ -898,7 +908,7 @@ void sqlite3StartTable( ** collisions. */ if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } @@ -928,7 +938,7 @@ void sqlite3StartTable( pTable->zName = zName; pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; - pTable->nRef = 1; + pTable->nTabRef = 1; pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -1056,9 +1066,6 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ memcpy(z, pName->z, pName->n); z[pName->n] = 0; sqlite3Dequote(z); - zType = z + sqlite3Strlen30(z) + 1; - memcpy(zType, pType->z, pType->n); - zType[pType->n] = 0; for(i=0; inCol; i++){ if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); @@ -1086,7 +1093,12 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ pCol->affinity = SQLITE_AFF_BLOB; pCol->szEst = 1; }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, pType->z, pType->n); + zType[pType->n] = 0; + sqlite3Dequote(zType); pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; pParse->constraintName.n = 0; @@ -1103,6 +1115,7 @@ void sqlite3AddNotNull(Parse *pParse, int onError){ p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; p->aCol[p->nCol-1].notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; } /* @@ -1282,10 +1295,10 @@ void sqlite3AddPrimaryKey( int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ ){ Table *pTab = pParse->pNewTable; - const char *zName = 0; + Column *pCol = 0; int iCol = -1, i; int nTerm; - if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab==0 ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); @@ -1294,8 +1307,8 @@ void sqlite3AddPrimaryKey( pTab->tabFlags |= TF_HasPrimaryKey; if( pList==0 ){ iCol = pTab->nCol - 1; - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zName = pTab->aCol[iCol].zName; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; nTerm = 1; }else{ nTerm = pList->nExpr; @@ -1307,8 +1320,8 @@ void sqlite3AddPrimaryKey( const char *zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ - pTab->aCol[iCol].colFlags |= COLFLAG_PRIMKEY; - zName = pTab->aCol[iCol].zName; + pCol = &pTab->aCol[iCol]; + pCol->colFlags |= COLFLAG_PRIMKEY; break; } } @@ -1316,8 +1329,8 @@ void sqlite3AddPrimaryKey( } } if( nTerm==1 - && zName - && sqlite3StrICmp(sqlite3StrNext(zName), "INTEGER")==0 + && pCol + && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 && sortOrder!=SQLITE_SO_DESC ){ pTab->iPKey = iCol; @@ -1331,12 +1344,8 @@ void sqlite3AddPrimaryKey( "INTEGER PRIMARY KEY"); #endif }else{ - Index *p; - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, - 0, sortOrder, 0); - if( p ){ - p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - } + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); pList = 0; } @@ -1455,6 +1464,9 @@ CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** set back to prior value. But schema changes are infrequent ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. */ void sqlite3ChangeCookie(Parse *pParse, int iDb){ sqlite3 *db = pParse->db; @@ -1653,21 +1665,23 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** -** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert the OP_CreateTable into an OP_CreateIndex. There is ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. -** (2) Bypass the creation of the sqlite_master table entry +** (3) Bypass the creation of the sqlite_master table entry ** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. -** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. -** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus ** columns are part of KeyInfo.nXField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. */ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; @@ -1677,6 +1691,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) + */ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + } + + /* The remaining transformations only apply to b-tree tables, not to + ** virtual tables */ + if( IN_DECLARE_VTAB ) return; + /* Convert the OP_CreateTable opcode that would normally create the ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. @@ -1698,9 +1726,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( pList==0 ) return; pList->a[0].sortOrder = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); - pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); - if( pPk==0 ) return; - pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( db->mallocFailed ) return; + pPk = sqlite3PrimaryKeyIndex(pTab); pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); @@ -1728,19 +1757,11 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } pPk->nKeyCol = j; } - pPk->isCovering = 1; assert( pPk!=0 ); + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; nPk = pPk->nKeyCol; - /* Make sure every column of the PRIMARY KEY is NOT NULL. (Except, - ** do not enforce this for imposter tables.) */ - if( !db->init.imposterTable ){ - for(i=0; iaCol[pPk->aiColumn[i]].notNull = OE_Abort; - } - pPk->uniqNotNull = 1; - } - /* The root page of the PRIMARY KEY is the table root page */ pPk->tnum = pTab->tnum; @@ -1984,7 +2005,7 @@ void sqlite3EndTable( "UPDATE %Q.%s " "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, MASTER_NAME, zType, p->zName, p->zName, @@ -1999,13 +2020,13 @@ void sqlite3EndTable( /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( p->tabFlags & TF_Autoincrement ){ + if( (p->tabFlags & TF_Autoincrement)!=0 ){ Db *pDb = &db->aDb[iDb]; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zName + pDb->zDbSName ); } } @@ -2129,7 +2150,9 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif assert( pTable ); @@ -2175,44 +2198,55 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - if( pTable->pCheck ){ + pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + if( pSel ){ + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; db->lookaside.bDisable++; - sqlite3ColumnsFromExprList(pParse, pTable->pCheck, - &pTable->nCol, &pTable->aCol); - db->lookaside.bDisable--; - }else{ - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); - if( pSel ){ - n = pParse->nTab; - sqlite3SrcListAssignCursors(pParse, pSel->pSrc); - pTable->nCol = -1; - db->lookaside.bDisable++; #ifndef SQLITE_OMIT_AUTHORIZATION - xAuth = db->xAuth; - db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); - db->xAuth = xAuth; + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif - db->lookaside.bDisable--; - pParse->nTab = n; - if( pSelTab ){ - assert( pTable->aCol==0 ); - pTable->nCol = pSelTab->nCol; - pTable->aCol = pSelTab->aCol; - pSelTab->nCol = 0; - pSelTab->aCol = 0; - sqlite3DeleteTable(db, pSelTab); - assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - }else{ - pTable->nCol = 0; - nErr++; + pParse->nTab = n; + if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); } - sqlite3SelectDelete(db, pSel); - } else { + }else if( pSelTab ){ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + }else{ + pTable->nCol = 0; nErr++; } + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDelete(db, pSel); + db->lookaside.bDisable--; + } else { + nErr++; } pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ @@ -2308,7 +2342,7 @@ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ */ sqlite3NestedParse(pParse, "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); + pParse->db->aDb[iDb].zDbSName, MASTER_NAME, iTable, r1, r1); #endif sqlite3ReleaseTempReg(pParse, r1); } @@ -2384,7 +2418,7 @@ static void sqlite3ClearStatTables( const char *zName /* Name of index or table */ ){ int i; - const char *zDbName = pParse->db->aDb[iDb].zName; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); @@ -2437,7 +2471,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName + pDb->zDbSName, pTab->zName ); } #endif @@ -2451,7 +2485,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ */ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + pDb->zDbSName, MASTER_NAME, pTab->zName); if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -2484,6 +2518,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ assert( pName->nSrc==1 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; @@ -2504,7 +2539,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ { int code; const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zArg2 = 0; if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ goto exit_drop_table; @@ -2745,7 +2780,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -2761,6 +2796,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; @@ -2784,8 +2820,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); - if( IsUniqueIndex(pIndex) && pKey!=0 ){ + if( IsUniqueIndex(pIndex) ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; sqlite3VdbeGoto(v, j2); addr2 = sqlite3VdbeCurrentAddr(v); @@ -2797,7 +2832,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); @@ -2854,12 +2889,8 @@ Index *sqlite3AllocateIndexObject( ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added ** to the table currently under construction. -** -** If the index is created successfully, return a pointer to the new Index -** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) */ -Index *sqlite3CreateIndex( +void sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ Token *pName1, /* First part of index name. May be NULL */ Token *pName2, /* Second part of index name. May be NULL */ @@ -2869,9 +2900,9 @@ Index *sqlite3CreateIndex( Token *pStart, /* The CREATE token that begins this statement */ Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist /* Omit error if index already exists */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ ){ - Index *pRet = 0; /* Pointer to return */ Table *pTab = 0; /* Table to be indexed */ Index *pIndex = 0; /* The index to be created */ char *zName = 0; /* Name of the index */ @@ -2889,7 +2920,10 @@ Index *sqlite3CreateIndex( char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){ + if( db->mallocFailed || pParse->nErr>0 ){ + goto exit_create_index; + } + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ @@ -2998,7 +3032,7 @@ Index *sqlite3CreateIndex( goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ if( !ifNotExist ){ sqlite3ErrorMsg(pParse, "index %s already exists", zName); }else{ @@ -3015,13 +3049,20 @@ Index *sqlite3CreateIndex( if( zName==0 ){ goto exit_create_index; } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_DECLARE_VTAB ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION { - const char *zDb = pDb->zName; + const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; } @@ -3078,7 +3119,7 @@ Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; + pIndex->idxType = idxType; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -3258,7 +3299,7 @@ Index *sqlite3CreateIndex( pIdx->onError = pIndex->onError; } } - pRet = pIdx; + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; goto exit_create_index; } } @@ -3270,6 +3311,7 @@ Index *sqlite3CreateIndex( assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; + assert( !IN_DECLARE_VTAB ); assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, pIndex->zName, pIndex); @@ -3335,7 +3377,7 @@ Index *sqlite3CreateIndex( */ sqlite3NestedParse(pParse, "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, MASTER_NAME, pIndex->zName, pTab->zName, iMem, @@ -3351,7 +3393,7 @@ Index *sqlite3CreateIndex( sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp1(v, OP_Expire, 0); + sqlite3VdbeAddOp0(v, OP_Expire); } sqlite3VdbeJumpHere(v, pIndex->tnum); @@ -3376,7 +3418,6 @@ Index *sqlite3CreateIndex( pIndex->pNext = pOther->pNext; pOther->pNext = pIndex; } - pRet = pIndex; pIndex = 0; } @@ -3387,7 +3428,6 @@ exit_create_index: sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); - return pRet; } /* @@ -3415,11 +3455,15 @@ void sqlite3DefaultRowEst(Index *pIdx){ int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + /* Set the first entry (number of rows in the index) to the estimated - ** number of rows in the table. Or 10, if the estimated number of rows - ** in the table is less than that. */ + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. But do not let the estimate drop below 10. */ a[0] = pIdx->pTable->nRowLogEst; - if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + if( pIdx->pPartIdxWhere!=0 ) a[0] -= 10; assert( 10==sqlite3LogEst(2) ); + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is ** 6 and each subsequent value (if any) is 5. */ @@ -3470,7 +3514,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ { int code = SQLITE_DROP_INDEX; Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ goto exit_drop_index; @@ -3488,7 +3532,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName + db->aDb[iDb].zDbSName, MASTER_NAME, pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); @@ -3631,7 +3675,7 @@ SrcList *sqlite3SrcListEnlarge( /* Allocate additional space if needed */ if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; - int nAlloc = pSrc->nSrc+nExtra; + int nAlloc = pSrc->nSrc*2+nExtra; int nGot; pNew = sqlite3DbRealloc(db, pSrc, sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); @@ -3709,9 +3753,12 @@ SrcList *sqlite3SrcListAppend( pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; - pList->nSrc = 0; + pList->nSrc = 1; + memset(&pList->a[0], 0, sizeof(pList->a[0])); + pList->a[0].iCursor = -1; + }else{ + pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); } - pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); if( db->mallocFailed ){ sqlite3SrcListDelete(db, pList); return 0; @@ -4009,15 +4056,13 @@ int sqlite3OpenTempDatabase(Parse *pParse){ */ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3 *db = pToplevel->db; - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDb>=0 && iDbdb->nDb ); + assert( pParse->db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbdb, iDb, 0) ); if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ DbMaskSet(pToplevel->cookieMask, iDb); - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); } @@ -4033,7 +4078,7 @@ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ sqlite3CodeVerifySchema(pParse, i); } } @@ -4280,7 +4325,7 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -4301,10 +4346,6 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ /* ** Return a KeyInfo structure that is appropriate for the given Index. ** -** The KeyInfo structure for an index is cached in the Index object. -** So there might be multiple references to the returned pointer. The -** caller should not try to modify the KeyInfo object. -** ** The caller should invoke sqlite3KeyInfoUnref() on the returned object ** when it has finished using it. */ diff --git a/src/ctime.c b/src/ctime.c index f1bb69c16a..20e94edba5 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -45,10 +45,19 @@ static const char * const azCompileOpt[] = { #if SQLITE_CHECK_PAGES "CHECK_PAGES", #endif +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif #if SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG "DEBUG", #endif #if SQLITE_DEFAULT_LOCKING_MODE @@ -57,6 +66,15 @@ static const char * const azCompileOpt[] = { #if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif +#if SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#if SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif +#if SQLITE_DIRECT_OVERFLOW_READ + "DIRECT_OVERFLOW_READ", +#endif #if SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif @@ -64,7 +82,7 @@ static const char * const azCompileOpt[] = { "DISABLE_LFS", #endif #if SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES", + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), #endif #if SQLITE_ENABLE_API_ARMOR "ENABLE_API_ARMOR", @@ -143,6 +161,9 @@ static const char * const azCompileOpt[] = { #if SQLITE_ENABLE_UPDATE_DELETE_LIMIT "ENABLE_UPDATE_DELETE_LIMIT", #endif +#if defined(SQLITE_ENABLE_URI_00_ERROR) + "ENABLE_URI_00_ERROR", +#endif #if SQLITE_HAS_CODEC "HAS_CODEC", #endif @@ -218,9 +239,6 @@ static const char * const azCompileOpt[] = { #if SQLITE_OMIT_BTREECOUNT "OMIT_BTREECOUNT", #endif -#if SQLITE_OMIT_BUILTIN_TEST - "OMIT_BUILTIN_TEST", -#endif #if SQLITE_OMIT_CAST "OMIT_CAST", #endif @@ -383,6 +401,9 @@ static const char * const azCompileOpt[] = { #if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif +#if SQLITE_UNTESTABLE + "UNTESTABLE" +#endif #if SQLITE_USE_ALLOCA "USE_ALLOCA", #endif diff --git a/src/date.c b/src/date.c index ef05d44086..b49d506bc4 100644 --- a/src/date.c +++ b/src/date.c @@ -50,22 +50,33 @@ #ifndef SQLITE_OMIT_DATETIME_FUNCS +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So declare a substitute. The substitute function itself is +** defined in "os_win.c". +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +struct tm *__cdecl localtime(const time_t *); +#endif /* ** A structure for holding a single date and time. */ typedef struct DateTime DateTime; struct DateTime { - sqlite3_int64 iJD; /* The julian day number times 86400000 */ - int Y, M, D; /* Year, month, and day */ - int h, m; /* Hour and minutes */ - int tz; /* Timezone offset in minutes */ - double s; /* Seconds */ - char validYMD; /* True (1) if Y,M,D are valid */ - char validHMS; /* True (1) if h,m,s are valid */ - char validJD; /* True (1) if iJD is valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validJD; /* True (1) if iJD is valid */ + char rawS; /* Raw numeric value stored in s */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char validTZ; /* True (1) if tz is valid */ + char tzSet; /* Timezone was set explicitly */ + char isError; /* An overflow has occurred */ }; @@ -213,6 +224,7 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ s = 0; } p->validJD = 0; + p->rawS = 0; p->validHMS = 1; p->h = h; p->m = m; @@ -222,6 +234,14 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ return 0; } +/* +** Put the DateTime object into its error state. +*/ +static void datetimeError(DateTime *p){ + memset(p, 0, sizeof(*p)); + p->isError = 1; +} + /* ** Convert from YYYY-MM-DD HH:MM:SS to julian day. We always assume ** that the YYYY-MM-DD is according to the Gregorian calendar. @@ -241,6 +261,10 @@ static void computeJD(DateTime *p){ M = 1; D = 1; } + if( Y<-4713 || Y>9999 || p->rawS ){ + datetimeError(p); + return; + } if( M<=2 ){ Y--; M += 12; @@ -321,6 +345,21 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ } } +/* +** Input "r" is a numeric quantity which might be a julian day number, +** or the number of seconds since 1970. If the value if r is within +** range of a julian day number, install it as such and set validJD. +** If the value is a valid unix timestamp, put it in p->s and set p->rawS. +*/ +static void setRawDateNumber(DateTime *p, double r){ + p->s = r; + p->rawS = 1; + if( r>=0.0 && r<5373484.5 ){ + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; + } +} + /* ** Attempt to parse the given string into a julian day number. Return ** the number of errors. @@ -350,13 +389,30 @@ static int parseDateOrTime( }else if( sqlite3StrICmp(zDate,"now")==0){ return setDateTimeToCurrent(context, p); }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ - p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); - p->validJD = 1; + setRawDateNumber(p, r); return 0; } return 1; } +/* The julian day number for 9999-12-31 23:59:59.999 is 5373484.4999999. +** Multiplying this by 86400000 gives 464269060799999 as the maximum value +** for DateTime.iJD. +** +** But some older compilers (ex: gcc 4.2.1 on older Macs) cannot deal with +** such a large integer literal, so we have to encode it. +*/ +#define INT_464269060799999 ((((i64)0x1a640)<<32)|0x1072fdff) + +/* +** Return TRUE if the given julian day number is within range. +** +** The input is the JulianDay times 86400000. +*/ +static int validJulianDay(sqlite3_int64 iJD){ + return iJD>=0 && iJD<=INT_464269060799999; +} + /* ** Compute the Year, Month, and Day from the julian day number. */ @@ -368,6 +424,7 @@ static void computeYMD(DateTime *p){ p->M = 1; p->D = 1; }else{ + assert( validJulianDay(p->iJD) ); Z = (int)((p->iJD + 43200000)/86400000); A = (int)((Z - 1867216.25)/36524.25); A = Z + 1 + A - (A/4); @@ -398,6 +455,7 @@ static void computeHMS(DateTime *p){ s -= p->h*3600; p->m = s/60; p->s += s - p->m*60; + p->rawS = 0; p->validHMS = 1; } @@ -418,6 +476,7 @@ static void clearYMD_HMS_TZ(DateTime *p){ p->validTZ = 0; } +#ifndef SQLITE_OMIT_LOCALTIME /* ** On recent Windows platforms, the localtime_s() function is available ** as part of the "Secure CRT". It is essentially equivalent to @@ -436,7 +495,6 @@ static void clearYMD_HMS_TZ(DateTime *p){ #define HAVE_LOCALTIME_S 1 #endif -#ifndef SQLITE_OMIT_LOCALTIME /* ** The following routine implements the rough equivalent of localtime_r() ** using whatever operating-system specific localtime facility that @@ -459,14 +517,14 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #endif sqlite3_mutex_enter(mutex); pX = localtime(t); -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; #endif if( pX ) *pTm = *pX; sqlite3_mutex_leave(mutex); rc = pX==0; #else -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; #endif #if HAVE_LOCALTIME_R @@ -537,13 +595,38 @@ static sqlite3_int64 localtimeOffset( y.validYMD = 1; y.validHMS = 1; y.validJD = 0; + y.rawS = 0; y.validTZ = 0; + y.isError = 0; computeJD(&y); *pRc = SQLITE_OK; return y.iJD - x.iJD; } #endif /* SQLITE_OMIT_LOCALTIME */ +/* +** The following table defines various date transformations of the form +** +** 'NNN days' +** +** Where NNN is an arbitrary floating-point number and "days" can be one +** of several units of time. +*/ +static const struct { + u8 eType; /* Transformation type code */ + u8 nName; /* Length of th name */ + char *zName; /* Name of the transformation */ + double rLimit; /* Maximum NNN value for this transform */ + double rXform; /* Constant used for this transform */ +} aXformType[] = { + { 0, 6, "second", 464269060800.0, 86400000.0/(24.0*60.0*60.0) }, + { 0, 6, "minute", 7737817680.0, 86400000.0/(24.0*60.0) }, + { 0, 4, "hour", 128963628.0, 86400000.0/24.0 }, + { 0, 3, "day", 5373485.0, 86400000.0 }, + { 1, 5, "month", 176546.0, 30.0*86400000.0 }, + { 2, 4, "year", 14713.0, 365.0*86400000.0 }, +}; + /* ** Process a modifier to a date-time stamp. The modifiers are ** as follows: @@ -568,17 +651,15 @@ static sqlite3_int64 localtimeOffset( ** to context pCtx. If the error is an unrecognized modifier, no error is ** written to pCtx. */ -static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ +static int parseModifier( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* The text of the modifier */ + int n, /* Length of zMod in bytes */ + DateTime *p /* The date/time value to be modified */ +){ int rc = 1; - int n; double r; - char *z, zBuf[30]; - z = zBuf; - for(n=0; niJD += localtimeOffset(p, pCtx, &rc); clearYMD_HMS_TZ(p); @@ -598,16 +679,21 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ /* ** unixepoch ** - ** Treat the current value of p->iJD as the number of + ** Treat the current value of p->s as the number of ** seconds since 1970. Convert to a real julian day number. */ - if( strcmp(z, "unixepoch")==0 && p->validJD ){ - p->iJD = (p->iJD + 43200)/86400 + 21086676*(i64)10000000; - clearYMD_HMS_TZ(p); - rc = 0; + if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){ + r = p->s*1000.0 + 210866760000000.0; + if( r>=0.0 && r<464269060800000.0 ){ + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)r; + p->validJD = 1; + p->rawS = 0; + rc = 0; + } } #ifndef SQLITE_OMIT_LOCALTIME - else if( strcmp(z, "utc")==0 ){ + else if( sqlite3_stricmp(z, "utc")==0 ){ if( p->tzSet==0 ){ sqlite3_int64 c1; computeJD(p); @@ -633,7 +719,7 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** weekday N where 0==Sunday, 1==Monday, and so forth. If the ** date is already on the appropriate weekday, this is a no-op. */ - if( strncmp(z, "weekday ", 8)==0 + if( sqlite3_strnicmp(z, "weekday ", 8)==0 && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; @@ -656,23 +742,24 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** Move the date backwards to the beginning of the current day, ** or month or year. */ - if( strncmp(z, "start of ", 9)!=0 ) break; + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); p->validHMS = 1; p->h = p->m = 0; p->s = 0.0; + p->rawS = 0; p->validTZ = 0; p->validJD = 0; - if( strcmp(z,"month")==0 ){ + if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; rc = 0; - }else if( strcmp(z,"year")==0 ){ - computeYMD(p); + }else if( sqlite3_stricmp(z,"year")==0 ){ p->M = 1; p->D = 1; rc = 0; - }else if( strcmp(z,"day")==0 ){ + }else if( sqlite3_stricmp(z,"day")==0 ){ rc = 0; } break; @@ -690,6 +777,7 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ case '8': case '9': { double rRounder; + int i; for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ rc = 1; @@ -718,46 +806,48 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ rc = 0; break; } + + /* If control reaches this point, it means the transformation is + ** one of the forms like "+NNN days". */ z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); if( n>10 || n<3 ) break; - if( z[n-1]=='s' ){ z[n-1] = 0; n--; } + if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); - rc = 0; + rc = 1; rRounder = r<0 ? -0.5 : +0.5; - if( n==3 && strcmp(z,"day")==0 ){ - p->iJD += (sqlite3_int64)(r*86400000.0 + rRounder); - }else if( n==4 && strcmp(z,"hour")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/24.0) + rRounder); - }else if( n==6 && strcmp(z,"minute")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0)) + rRounder); - }else if( n==6 && strcmp(z,"second")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0*60.0)) + rRounder); - }else if( n==5 && strcmp(z,"month")==0 ){ - int x, y; - computeYMD_HMS(p); - p->M += (int)r; - x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; - p->Y += x; - p->M -= x*12; - p->validJD = 0; - computeJD(p); - y = (int)r; - if( y!=r ){ - p->iJD += (sqlite3_int64)((r - y)*30.0*86400000.0 + rRounder); + for(i=0; i-aXformType[i].rLimit && rM += (int)r; + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + p->validJD = 0; + r -= (int)r; + break; + } + case 2: { /* Special processing to add years */ + int y = (int)r; + computeYMD_HMS(p); + p->Y += y; + p->validJD = 0; + r -= (int)r; + break; + } + } + computeJD(p); + p->iJD += (sqlite3_int64)(r*aXformType[i].rXform + rRounder); + rc = 0; + break; } - }else if( n==4 && strcmp(z,"year")==0 ){ - int y = (int)r; - computeYMD_HMS(p); - p->Y += y; - p->validJD = 0; - computeJD(p); - if( y!=r ){ - p->iJD += (sqlite3_int64)((r - y)*365.0*86400000.0 + rRounder); - } - }else{ - rc = 1; } clearYMD_HMS_TZ(p); break; @@ -784,7 +874,7 @@ static int isDate( sqlite3_value **argv, DateTime *p ){ - int i; + int i, n; const unsigned char *z; int eType; memset(p, 0, sizeof(*p)); @@ -793,8 +883,7 @@ static int isDate( } if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT || eType==SQLITE_INTEGER ){ - p->iJD = (sqlite3_int64)(sqlite3_value_double(argv[0])*86400000.0 + 0.5); - p->validJD = 1; + setRawDateNumber(p, sqlite3_value_double(argv[0])); }else{ z = sqlite3_value_text(argv[0]); if( !z || parseDateOrTime(context, (char*)z, p) ){ @@ -803,8 +892,11 @@ static int isDate( } for(i=1; iisError || !validJulianDay(p->iJD) ) return 1; return 0; } @@ -1103,7 +1195,6 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); - sqlite3 *db; sqlite3_int64 iT; struct tm *pTm; struct tm sNow; diff --git a/src/dbstat.c b/src/dbstat.c index b8e79b0864..13974bb979 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -58,10 +58,10 @@ */ #define VTAB_SCHEMA \ "CREATE TABLE xx( " \ - " name STRING, /* Name of table or index */" \ - " path INTEGER, /* Path to page from root */" \ + " name TEXT, /* Name of table or index */" \ + " path TEXT, /* Path to page from root */" \ " pageno INTEGER, /* Page number */" \ - " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" \ + " pagetype TEXT, /* 'internal', 'leaf' or 'overflow' */" \ " ncell INTEGER, /* Cells on page (0 for overflow) */" \ " payload INTEGER, /* Bytes of payload on this page */" \ " unused INTEGER, /* Bytes of unused space on this page */" \ @@ -602,7 +602,7 @@ static int statFilter( " UNION ALL " "SELECT name, rootpage, type" " FROM \"%w\".%s WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zDbSName, zMaster); if( zSql==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -656,7 +656,7 @@ static int statColumn( default: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); int iDb = pCsr->iDb; - sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); break; } } diff --git a/src/delete.c b/src/delete.c index 0fe064bc45..0683f9b9dd 100644 --- a/src/delete.c +++ b/src/delete.c @@ -36,7 +36,7 @@ Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ - pTab->nRef++; + pTab->nTabRef++; } if( sqlite3IndexedByLookup(pParse, pItem) ){ pTab = 0; @@ -102,7 +102,7 @@ void sqlite3MaterializeView( if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); assert( pFrom->a[0].pOn==0 ); assert( pFrom->a[0].pUsing==0 ); } @@ -143,7 +143,7 @@ Expr *sqlite3LimitWhere( */ if( pOrderBy && (pLimit == 0) ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* We only need to generate a select expression if there @@ -164,17 +164,17 @@ Expr *sqlite3LimitWhere( ** ); */ - pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup_2; + pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0); + if( pSelectRowid == 0 ) goto limit_where_cleanup; pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup_2; + if( pEList == 0 ) goto limit_where_cleanup; /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); if( pSelectSrc == 0 ) { sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup_2; + goto limit_where_cleanup; } /* generate the SELECT expression tree. */ @@ -183,22 +183,12 @@ Expr *sqlite3LimitWhere( if( pSelect == 0 ) return 0; /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ - pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pWhereRowid == 0 ) goto limit_where_cleanup_1; - pInClause = sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0); - if( pInClause == 0 ) goto limit_where_cleanup_1; - - pInClause->x.pSelect = pSelect; - pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeightAndFlags(pParse, pInClause); + pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0); + pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0) : 0; + sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; - /* something went wrong. clean up anything allocated. */ -limit_where_cleanup_1: - sqlite3SelectDelete(pParse->db, pSelect); - return 0; - -limit_where_cleanup_2: +limit_where_cleanup: sqlite3ExprDelete(pParse->db, pWhere); sqlite3ExprListDelete(pParse->db, pOrderBy); sqlite3ExprDelete(pParse->db, pLimit); @@ -222,7 +212,6 @@ void sqlite3DeleteFrom( ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ - const char *zDb; /* Name of database holding pTab */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ @@ -249,11 +238,12 @@ void sqlite3DeleteFrom( int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + int bComplex; /* True if there are triggers or FKs or + ** subqueries in the WHERE clause */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ - int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -281,7 +271,6 @@ void sqlite3DeleteFrom( #else # define pTrigger 0 # define isView 0 -# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -299,8 +288,8 @@ void sqlite3DeleteFrom( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - zDb = db->aDb[iDb].zName; - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; @@ -366,6 +355,9 @@ void sqlite3DeleteFrom( && pWhere==0 && !bComplex && !IsVirtual(pTab) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + && db->xPreUpdateCallback==0 +#endif ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); @@ -380,7 +372,8 @@ void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { - u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE; + if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ @@ -456,7 +449,7 @@ void sqlite3DeleteFrom( nKey = 0; /* Zero tells OP_Found to use a composite key */ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, sqlite3IndexAffinityStr(pParse->db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk); }else{ /* Add the rowid of the row to be deleted to the RowSet */ nKey = 1; /* OP_Seek always uses a single rowid */ @@ -480,7 +473,7 @@ void sqlite3DeleteFrom( if( !isView ){ int iAddrOnce = 0; if( eOnePass==ONEPASS_MULTI ){ - iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, @@ -502,7 +495,7 @@ void sqlite3DeleteFrom( } }else if( pPk ){ addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey); + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); assert( nKey==0 ); /* OP_Found will use a composite key */ }else{ addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); @@ -526,12 +519,8 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = -1; - if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ - iIdxNoSeek = aiCurOnePass[1]; - } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); } /* End of the loop over all rowids/primary-keys. */ @@ -545,14 +534,6 @@ void sqlite3DeleteFrom( sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); } - - /* Close the cursors open on the table and its indexes. */ - if( !isView && !IsVirtual(pTab) ){ - if( !pPk ) sqlite3VdbeAddOp1(v, OP_Close, iDataCur); - for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur + i); - } - } } /* End non-truncate path */ /* Update the sqlite_sequence table by storing the content of the @@ -619,15 +600,17 @@ delete_from_cleanup: ** ** If eMode is ONEPASS_MULTI, then this call is being made as part ** of a ONEPASS delete that affects multiple rows. In this case, if -** iIdxNoSeek is a valid cursor number (>=0), then its position should -** be preserved following the delete operation. Or, if iIdxNoSeek is not -** a valid cursor number, the position of iDataCur should be preserved -** instead. +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. ** ** iIdxNoSeek: -** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an -** index cursor (from within array of cursors starting at iIdxCur) that -** already points to the index entry to be deleted. +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -698,13 +681,18 @@ void sqlite3GenerateRowDelete( /* If any BEFORE triggers were coded, then seek the cursor to the ** row to be deleted again. It may be that the BEFORE triggers moved - ** the cursor or of already deleted the row that the cursor was + ** the cursor or already deleted the row that the cursor was ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. */ if( addrStart=0 ); + iIdxNoSeek = -1; } /* Do FK processing. This call checks that any FK constraints that @@ -715,18 +703,25 @@ void sqlite3GenerateRowDelete( /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). */ + ** fire the INSTEAD OF triggers). + ** + ** If variable 'count' is non-zero, then this OP_Delete instruction should + ** invoke the update-hook. The pre-update-hook, on the other hand should + ** be invoked unless table pTab is a system table. The difference is that + ** the update-hook is not invoked for rows removed by REPLACE, but the + ** pre-update-hook is. + */ if( pTab->pSelect==0 ){ u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - if( count ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + if( pParse->nested==0 ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); } if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } - if( iIdxNoSeek>=0 ){ + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); } if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; @@ -880,6 +875,10 @@ int sqlite3GenerateIndexKey( } if( regOut ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + if( pIdx->pTable->pSelect ){ + const char *zAff = sqlite3IndexAffinityStr(pParse->db, pIdx); + sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); + } } sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; diff --git a/src/expr.c b/src/expr.c index 3672d02df1..ce948be69e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -14,6 +14,18 @@ */ #include "sqliteInt.h" +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +char sqlite3TableColumnAffinity(Table *pTab, int iCol){ + assert( iColnCol ); + return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; +} + /* ** Return the 'affinity' of the expression pExpr if any. ** @@ -39,21 +51,21 @@ char sqlite3ExprAffinity(Expr *pExpr){ assert( pExpr->flags&EP_xIsSelect ); return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); } + if( op==TK_REGISTER ) op = pExpr->op2; #ifndef SQLITE_OMIT_CAST if( op==TK_CAST ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) - && pExpr->pTab!=0 - ){ - /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = pExpr->iColumn; - if( j<0 ) return SQLITE_AFF_INTEGER; - assert( pExpr->pTab && jpTab->nCol ); - return pExpr->pTab->aCol[j].affinity; + if( op==TK_AGG_COLUMN || op==TK_COLUMN ){ + return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); + } + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft->flags&EP_xIsSelect ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); } return pExpr->affinity; } @@ -219,7 +231,7 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( !aff ){ + }else if( aff==0 ){ aff = SQLITE_AFF_BLOB; } return aff; @@ -309,6 +321,274 @@ static int codeCompare( return addr; } +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +int sqlite3ExprIsVector(Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +int sqlite3ExprVectorSize(Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField /* Which column of the vector to return */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( pVector->flags & EP_xIsSelect ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT. Not deleted. + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** iTable: 0 or the number of columns on the LHS of an assignment + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); + if( pRet ){ + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + assert( pRet==0 || pRet->iTable==0 ); + }else{ + if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; + pRet = sqlite3ExprDup(pParse->db, pVector, 0); + } + return pRet; +} +#endif /* !define(SQLITE_OMIT_SUBQUERY) */ + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrDone = sqlite3VdbeMakeLabel(v); + + if( nLeft!=sqlite3ExprVectorSize(pRight) ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return; + } + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + p5 |= SQLITE_STOREP2; + if( opx==TK_LE ) opx = TK_LT; + if( opx==TK_GE ) opx = TK_GT; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL, *pR; + int r1, r2; + assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); + r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); + r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); + codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); + testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + if( i>0 ) sqlite3ExprCachePop(pParse); + if( i==nLeft-1 ){ + break; + } + if( opx==TK_EQ ){ + sqlite3VdbeAddOp2(v, OP_IfNot, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else if( opx==TK_NE ){ + sqlite3VdbeAddOp2(v, OP_If, dest, addrDone); VdbeCoverage(v); + p5 |= SQLITE_KEEPNULL; + }else{ + assert( op==TK_LT || op==TK_GT || op==TK_LE || op==TK_GE ); + sqlite3VdbeAddOp2(v, OP_ElseNotEq, 0, addrDone); + VdbeCoverageIf(v, op==TK_LT); + VdbeCoverageIf(v, op==TK_GT); + VdbeCoverageIf(v, op==TK_LE); + VdbeCoverageIf(v, op==TK_GE); + if( i==nLeft-2 ) opx = op; + } + } + sqlite3VdbeResolveLabel(v, addrDone); +} + #if SQLITE_MAX_EXPR_DEPTH>0 /* ** Check that argument nHeight is less than or equal to the maximum @@ -444,7 +724,7 @@ void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** is allocated to hold the integer text and the dequote flag is ignored. */ Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ int op, /* Expression opcode */ const Token *pToken, /* Token argument. Might be NULL */ int dequote /* True to dequote */ @@ -471,15 +751,13 @@ Expr *sqlite3ExprAlloc( pNew->flags |= EP_IntValue; pNew->u.iValue = iValue; }else{ - int c; pNew->u.zToken = (char*)&pNew[1]; assert( pToken->z!=0 || pToken->n==0 ); if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; - if( dequote && nExtra>=3 - && ((c = pToken->z[0])=='\'' || c=='"' || c=='[' || c=='`') ){ + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; sqlite3Dequote(pNew->u.zToken); - if( c=='"' ) pNew->flags |= EP_DblQuoted; } } } @@ -545,15 +823,19 @@ Expr *sqlite3PExpr( Parse *pParse, /* Parsing context */ int op, /* Expression opcode */ Expr *pLeft, /* Left operand */ - Expr *pRight, /* Right operand */ - const Token *pToken /* Argument token */ + Expr *pRight /* Right operand */ ){ Expr *p; if( op==TK_AND && pParse->nErr==0 ){ /* Take advantage of short-circuit false optimization for AND */ p = sqlite3ExprAnd(pParse->db, pLeft, pRight); }else{ - p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & TKFLG_MASK; + p->iAgg = -1; + } sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); } if( p ) { @@ -562,6 +844,22 @@ Expr *sqlite3PExpr( return p; } +/* +** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due +** do a memory allocation failure) then delete the pSelect object. +*/ +void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ + if( pExpr ){ + pExpr->x.pSelect = pSelect; + ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, pExpr); + }else{ + assert( pParse->db->mallocFailed ); + sqlite3SelectDelete(pParse->db, pSelect); + } +} + + /* ** If the expression is always either TRUE or FALSE (respectively), ** then return 1. If one cannot determine the truth value of the @@ -640,7 +938,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ ** variable number. ** ** Wildcards of the form "?nnn" are assigned the number "nnn". We make -** sure "nnn" is not too be to avoid a denial of service attack when +** sure "nnn" is not too big to avoid a denial of service attack when ** the SQL statement comes from an external source. ** ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number @@ -648,28 +946,34 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ ** instance of the wildcard, the next sequential variable number is ** assigned. */ -void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ +void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ sqlite3 *db = pParse->db; const char *z; + ynVar x; if( pExpr==0 ) return; assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); + assert( n==sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); - pExpr->iColumn = (ynVar)(++pParse->nVar); + x = (ynVar)(++pParse->nVar); }else{ - ynVar x = 0; - u32 n = sqlite3Strlen30(z); + int doAdd = 0; if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - pExpr->iColumn = x = (ynVar)i; + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -677,44 +981,32 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - x = 0; + return; } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; + x = (ynVar)i; + if( x>pParse->nVar ){ + pParse->nVar = (int)x; + doAdd = 1; + }else if( sqlite3VListNumToName(pParse->pVList, x)==0 ){ + doAdd = 1; } }else{ /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable ** number as the prior appearance of the same name, or if the name ** has never appeared before, reuse the same variable number */ - ynVar i; - for(i=0; inzVar; i++){ - if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ - pExpr->iColumn = x = (ynVar)i+1; - break; - } - } - if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); - } - if( x>0 ){ - if( x>pParse->nzVar ){ - char **a; - a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ){ - assert( db->mallocFailed ); /* Error reported through mallocFailed */ - return; - } - pParse->azVar = a; - memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); - pParse->nzVar = x; - } - if( z[0]!='?' || pParse->azVar[x-1]==0 ){ - sqlite3DbFree(db, pParse->azVar[x-1]); - pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + x = (ynVar)sqlite3VListNameToNum(pParse->pVList, z, n); + if( x==0 ){ + x = (ynVar)(++pParse->nVar); + doAdd = 1; } } - } - if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( doAdd ){ + pParse->pVList = sqlite3VListAdd(db, pParse->pVList, z, n, x); + } + } + pExpr->iColumn = x; + if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); } } @@ -722,26 +1014,36 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ /* ** Recursively delete an expression tree. */ -void sqlite3ExprDelete(sqlite3 *db, Expr *p){ - if( p==0 ) return; +static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ + assert( p!=0 ); /* Sanity check: Assert that the IntValue is non-negative if it exists */ assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); - if( !ExprHasProperty(p, EP_TokenOnly) ){ +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( p->x.pSelect==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ assert( p->x.pList==0 || p->pRight==0 ); - sqlite3ExprDelete(db, p->pLeft); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); sqlite3ExprDelete(db, p->pRight); - if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( ExprHasProperty(p, EP_xIsSelect) ){ sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); } } + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( !ExprHasProperty(p, EP_Static) ){ sqlite3DbFree(db, p); } } +void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p ) sqlite3ExprDeleteNN(db, p); +} /* ** Return the number of bytes allocated for the expression structure @@ -793,7 +1095,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==(flags&EXPRDUP_REDUCE) ){ + if( 0==flags || p->op==TK_SELECT_COLUMN ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -855,88 +1157,94 @@ static int dupedExprSize(Expr *p, int flags){ ** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ - Expr *pNew = 0; /* Value to return */ - assert( flags==0 || flags==EXPRDUP_REDUCE ); +static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ + Expr *pNew; /* Value to return */ + u8 *zAlloc; /* Memory space from which to build Expr object */ + u32 staticFlag; /* EP_Static if space not obtained from malloc */ + assert( db!=0 ); - if( p ){ - const int isReduced = (flags&EXPRDUP_REDUCE); - u8 *zAlloc; - u32 staticFlag = 0; + assert( p ); + assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || isReduced ); + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + }else{ + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + staticFlag = 0; + } + pNew = (Expr *)zAlloc; - /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; - staticFlag = EP_Static; + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, dupFlags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, flags)); + nToken = 0; + } + if( dupFlags ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + u32 nSize = (u32)exprStructSize(p); + memcpy(zAlloc, p, nSize); + if( nSizeu.zToken string (if any). - */ - const unsigned nStructSize = dupedExprStructSize(p, flags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; + /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; + + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } + + if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - nToken = 0; + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); } - if( isReduced ){ - assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); - }else{ - u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); - if( nSizepLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + zAlloc += dupedExprNodeSize(p, dupFlags); + if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; } - - /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */ - pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); - pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); - pNew->flags |= staticFlag; - - /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; - memcpy(zToken, p->u.zToken, nToken); + if( pzBuffer ){ + *pzBuffer = zAlloc; } - - if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ - /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ - pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, isReduced); + }else{ + if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + if( pNew->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->iColumn==0 || p->pRight==0 ); + assert( p->pRight==0 || p->pRight==p->pLeft ); }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, isReduced); - } - } - - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, flags); - if( ExprHasProperty(pNew, EP_Reduced) ){ - pNew->pLeft = exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc); - pNew->pRight = exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc); - } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly) ){ pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); - pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } - } } return pNew; @@ -988,12 +1296,13 @@ static With *withDup(sqlite3 *db, With *p){ */ Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ assert( flags==0 || flags==EXPRDUP_REDUCE ); - return exprDup(db, p, flags, 0); + return p ? exprDup(db, p, flags, 0) : 0; } ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; struct ExprList_item *pItem, *pOldItem; int i; + Expr *pPriorSelectCol = 0; assert( db!=0 ); if( p==0 ) return 0; pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); @@ -1008,7 +1317,24 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ pOldItem = p->a; for(i=0; inExpr; i++, pItem++, pOldItem++){ Expr *pOldExpr = pOldItem->pExpr; + Expr *pNewExpr; pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); + if( pOldExpr + && pOldExpr->op==TK_SELECT_COLUMN + && (pNewExpr = pItem->pExpr)!=0 + ){ + assert( pNewExpr->iColumn==0 || i>0 ); + if( pNewExpr->iColumn==0 ){ + assert( pOldExpr->pLeft==pOldExpr->pRight ); + pPriorSelectCol = pNewExpr->pLeft = pNewExpr->pRight; + }else{ + assert( i>0 ); + assert( pItem[-1].pExpr!=0 ); + assert( pNewExpr->iColumn==pItem[-1].pExpr->iColumn+1 ); + assert( pPriorSelectCol==pItem[-1].pExpr->pLeft ); + pNewExpr->pLeft = pPriorSelectCol; + } + } pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); pItem->sortOrder = pOldItem->sortOrder; @@ -1059,7 +1385,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ } pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ - pTab->nRef++; + pTab->nTabRef++; } pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); @@ -1092,33 +1418,41 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } return pNew; } -Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew, *pPrior; +Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + Select *p; + assert( db!=0 ); - if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); - pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); - pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); - pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); - pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); - pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); - pNew->op = p->op; - pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); - if( pPrior ) pPrior->pNext = pNew; - pNew->pNext = 0; - pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); - pNew->iLimit = 0; - pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->nSelectRow = p->nSelectRow; - pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); - return pNew; + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = withDup(db, p->pWith); + sqlite3SelectSetName(pNew, p->zSelName); + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + + return pRet; } #else Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ @@ -1175,6 +1509,75 @@ no_mem: return 0; } +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the RHS, append +** TK_SELECT_COLUMN expressions. +*/ +ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + + /* If the RHS is a vector, then we can immediately check to see that + ** the size of the RHS and LHS match. But if the RHS is a SELECT, + ** wildcards ("*") in the result set of the SELECT must be expanded before + ** we can do the size check, so defer the size check until code generation. + */ + if( pExpr->op!=TK_SELECT && pColumns->nId!=(n=sqlite3ExprVectorSize(pExpr)) ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + + for(i=0; inId; i++){ + Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i); + pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); + if( pList ){ + assert( pList->nExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + + if( pExpr->op==TK_SELECT ){ + if( pList && pList->a[iFirst].pExpr ){ + Expr *pFirst = pList->a[iFirst].pExpr; + assert( pFirst->op==TK_SELECT_COLUMN ); + + /* Store the SELECT statement in pRight so it will be deleted when + ** sqlite3ExprListDelete() is called */ + pFirst->pRight = pExpr; + pExpr = 0; + + /* Remember the size of the LHS in iTable so that we can check that + ** the RHS and LHS sizes match during code generation. */ + pFirst->iTable = pColumns->nId; + } + } + +vector_append_error: + sqlite3ExprDelete(db, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; +} + /* ** Set the sort order for the last element on the given ExprList. */ @@ -1210,7 +1613,7 @@ void sqlite3ExprListSetName( pItem = &pList->a[pList->nExpr-1]; assert( pItem->zName==0 ); pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote && pItem->zName ) sqlite3Dequote(pItem->zName); + if( dequote ) sqlite3Dequote(pItem->zName); } } @@ -1259,10 +1662,9 @@ void sqlite3ExprListCheckLength( /* ** Delete an entire expression list. */ -void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ +static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ int i; struct ExprList_item *pItem; - if( pList==0 ) return; assert( pList->a!=0 || pList->nExpr==0 ); for(pItem=pList->a, i=0; inExpr; i++, pItem++){ sqlite3ExprDelete(db, pItem->pExpr); @@ -1272,6 +1674,9 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ sqlite3DbFree(db, pList->a); sqlite3DbFree(db, pList); } +void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + if( pList ) exprListDeleteNN(db, pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -1283,7 +1688,8 @@ u32 sqlite3ExprListFlags(const ExprList *pList){ if( pList ){ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( ALWAYS(pExpr) ) m |= pExpr->flags; + assert( pExpr!=0 ); + m |= pExpr->flags; } } return m; @@ -1454,6 +1860,7 @@ int sqlite3ExprContainsSubquery(Expr *p){ */ int sqlite3ExprIsInteger(Expr *p, int *pValue){ int rc = 0; + if( p==0 ) return 0; /* Can only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ @@ -1579,8 +1986,8 @@ static Select *isCandidateForInOpt(Expr *pX){ Select *p; SrcList *pSrc; ExprList *pEList; - Expr *pRes; Table *pTab; + int i; if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ p = pX->x.pSelect; @@ -1603,23 +2010,18 @@ static Select *isCandidateForInOpt(Expr *pX){ assert( pTab->pSelect==0 ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - pRes = pEList->a[0].pExpr; - if( pRes->op!=TK_COLUMN ) return 0; /* Result is a column */ - assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } return p; } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Code an OP_Once instruction and allocate space for its flag. Return the -** address of the new instruction. -*/ -int sqlite3CodeOnce(Parse *pParse){ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); -} - +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that checks the left-most column of index table iCur to see if ** it contains any NULL entries. Cause the register at regHasNull to be set @@ -1635,6 +2037,7 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ VdbeComment((v, "first_entry_in(%d)", iCur)); sqlite3VdbeJumpHere(v, addr1); } +#endif #ifndef SQLITE_OMIT_SUBQUERY @@ -1679,7 +2082,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: ** -** SELECT FROM +** SELECT , ... FROM
    ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then @@ -1695,14 +2098,14 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected is guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or it -** has a UNIQUE constraint or UNIQUE index. +** An epheremal table must be used unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. ** ** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used ** for fast set membership tests) then an epheremal table must -** be used unless is an INTEGER PRIMARY KEY or an index can -** be found with as its left-most column. +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and ** if the RHS of the IN operator is a list (not a subquery) then this @@ -1723,9 +2126,26 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** the value in that register will be NULL if the b-tree contains one or more ** NULL values, and it will be some non-NULL value if the b-tree contains no ** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. */ #ifndef SQLITE_OMIT_SUBQUERY -int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ +int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The right-hand side (RHS) of the IN operator */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap /* Mapping from Index fields to RHS fields */ +){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ @@ -1735,36 +2155,46 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } + /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new - ** ephemeral table. - */ + ** ephemeral table. */ if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table
    . */ - Expr *pExpr; /* Expression */ - i16 iCol; /* Index of column */ i16 iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ pTab = p->pSrc->a[0].pTab; - pExpr = p->pEList->a[0].pExpr; - iCol = (i16)pExpr->iColumn; - + /* Code an OP_Transaction and OP_TableLock for
    . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - /* This function is only called from two places. In both cases the vdbe - ** has already been allocated. So assume sqlite3GetVdbe() is always - ** successful here. - */ - assert(v); - if( iCol<0 ){ - int iAddr = sqlite3CodeOnce(pParse); + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); @@ -1773,44 +2203,114 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; - /* The collation sequence used by the comparison. If an index is to - ** be used in place of a temp-table, it must be ordered according - ** to this collation sequence. */ - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); - - /* Check that the affinity that will be used to perform the - ** comparison is the same as the affinity of the column. If - ** it is not, it is not possible to use any index. - */ - int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); - - for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ - if( (pIdx->aiColumn[0]==iCol) - && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) - ){ - int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); - eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - - if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ - *prRhsHasNull = ++pParse->nMem; - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); - } - sqlite3VdbeJumpHere(v, iAddr); + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); } } - } - } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnnColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); +#ifndef SQLITE_OMIT_EXPLAIN + sqlite3VdbeAddOp4(v, OP_Explain, 0, 0, 0, + sqlite3MPrintf(db, "USING INDEX %s FOR IN-OPERATOR",pIdx->zName), + P4_DYNAMIC); +#endif + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ /* If no preexisting index is available for the IN clause ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery - ** and the RHS is not contant or has two or fewer terms, + ** and the RHS is not constant or has two or fewer terms, ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ @@ -1821,7 +2321,6 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ ){ eType = IN_INDEX_NOOP; } - if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. @@ -1843,10 +2342,85 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ }else{ pX->iTable = iTab; } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); +} +#endif + +/* +** Expression pExpr is a vector that has been used in a context where +** it is not permitted. If pExpr is a sub-select vector, this routine +** loads the Parse object with a message of the form: +** +** "sub-select returns N columns - expected 1" +** +** Or, if it is a regular scalar vector: +** +** "row value misused" +*/ +void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->flags & EP_xIsSelect ){ + sqlite3SubselectError(pParse, pExpr->x.pSelect->pEList->nExpr, 1); + }else +#endif + { + sqlite3ErrorMsg(pParse, "row value misused"); + } +} + /* ** Generate code for scalar subqueries used as a subquery expression, EXISTS, ** or IN operators. Examples: @@ -1872,7 +2446,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ ** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the -** result. For IN operators or if an error occurs, the return value is 0. +** result. For a multi-column SELECT, the result is stored in a contiguous +** array of registers and the return value is the register of the left-most +** result column. Return 0 for IN operators or if an error occurs. */ #ifndef SQLITE_OMIT_SUBQUERY int sqlite3CodeSubselect( @@ -1887,8 +2463,8 @@ int sqlite3CodeSubselect( if( NEVER(v==0) ) return 0; sqlite3ExprCachePush(pParse); - /* This code must be run in its entirety every time it is encountered - ** if any of the following is true: + /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables @@ -1898,7 +2474,7 @@ int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); + jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN @@ -1914,17 +2490,18 @@ int sqlite3CodeSubselect( switch( pExpr->op ){ case TK_IN: { - char affinity; /* Affinity of the LHS of the IN */ int addr; /* Address of OP_OpenEphemeral instruction */ Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - - affinity = sqlite3ExprAffinity(pLeft); + int nVal; /* Size of vector pLeft */ + + nVal = sqlite3ExprVectorSize(pLeft); + assert( !isRowid || nVal==1 ); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' ** expression it is handled the same way. An ephemeral table is - ** filled with single-field index keys representing the results - ** from the SELECT or the . + ** filled with index keys representing the results from the + ** SELECT or the . ** ** If the 'x' expression is a column value, or the SELECT... ** statement returns a column value, then the affinity of that @@ -1935,8 +2512,9 @@ int sqlite3CodeSubselect( ** is used. */ pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, + pExpr->iTable, (isRowid?0:nVal)); + pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -1945,27 +2523,36 @@ int sqlite3CodeSubselect( ** table allocated and opened above. */ Select *pSelect = pExpr->x.pSelect; - SelectDest dest; - ExprList *pEList; + ExprList *pEList = pSelect->pEList; assert( !isRowid ); - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affSdst = (u8)affinity; - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3KeyInfoUnref(pKeyInfo); - return 0; + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + SelectDest dest; + int i; + sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3DbFree(pParse->db, dest.zAffSdst); + sqlite3KeyInfoUnref(pKeyInfo); + return 0; + } + sqlite3DbFree(pParse->db, dest.zAffSdst); + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } } - pEList = pSelect->pEList; - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** @@ -1974,11 +2561,13 @@ int sqlite3CodeSubselect( ** that columns affinity when building index keys. If is not ** a column, use numeric affinity. */ + char affinity; /* Affinity of the LHS of the IN */ int i; ExprList *pList = pExpr->x.pList; struct ExprList_item *pItem; int r1, r2, r3; + affinity = sqlite3ExprAffinity(pLeft); if( !affinity ){ affinity = SQLITE_AFF_BLOB; } @@ -2018,7 +2607,7 @@ int sqlite3CodeSubselect( }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); sqlite3ExprCacheAffinityChange(pParse, r3, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pExpr->iTable, r2, r3, 1); } } } @@ -2034,26 +2623,37 @@ int sqlite3CodeSubselect( case TK_EXISTS: case TK_SELECT: default: { - /* If this has to be a scalar SELECT. Generate code to put the - ** value of this select in a memory cell and record the number - ** of the memory cell in iColumn. If this is an EXISTS, write - ** an integer 0 (not exists) or 1 (exists) into a memory cell - ** and record that memory cell in iColumn. + /* Case 3: (SELECT ... FROM ...) + ** or: EXISTS(SELECT ... FROM ...) + ** + ** For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. */ Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECt result */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; - sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; dest.iSdst = dest.iSDParm; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; @@ -2061,8 +2661,8 @@ int sqlite3CodeSubselect( VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, - &sqlite3IntTokens[1]); + pSel->pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER, + &sqlite3IntTokens[1], 0); pSel->iLimit = 0; pSel->selFlags &= ~SF_MultiValue; if( sqlite3Select(pParse, pSel, &dest) ){ @@ -2087,6 +2687,28 @@ int sqlite3CodeSubselect( } #endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( (pIn->flags & EP_xIsSelect) ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + sqlite3VectorErrorMsg(pParse, pIn->pLeft); + return 1; + } + return 0; +} +#endif + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code for an IN expression. @@ -2094,16 +2716,24 @@ int sqlite3CodeSubselect( ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) -** is an array of zero or more values. The expression is true if the LHS is -** contained within the RHS. The value of the expression is unknown (NULL) -** if the LHS is NULL or if the LHS is not contained within the RHS and the -** RHS contains one or more NULL values. +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. +** +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. ** ** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. */ static void sqlite3ExprCodeIN( Parse *pParse, /* Parsing and code generating context */ @@ -2112,36 +2742,83 @@ static void sqlite3ExprCodeIN( int destIfNull /* Jump here if the results are unknown due to NULLs */ ){ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ - char affinity; /* Comparison affinity to use */ int eType; /* Type of the RHS */ - int r1; /* Temporary use register */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ - /* Compute the RHS. After this step, the table with cursor - ** pExpr->iTable will contains the values that make up the RHS. - */ + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero( + pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 + ); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened ith cursor pExpr->iTable + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull); + destIfFalse==destIfNull ? 0 : &rRhsHasNull, aiMap); - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". + /* Code the LHS, the from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. */ sqlite3ExprCachePush(pParse); - r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, r1); + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + for(i=0; ix.pList; @@ -2153,7 +2830,7 @@ static void sqlite3ExprCodeIN( assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); @@ -2161,16 +2838,16 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + sqlite3VdbeAddOp4(v, OP_Eq, rLhs, labelOk, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverageIf(v, iinExpr-1); VdbeCoverageIf(v, ii==pList->nExpr-1); - sqlite3VdbeChangeP5(v, affinity); + sqlite3VdbeChangeP5(v, zAff[0]); }else{ assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + sqlite3VdbeAddOp4(v, OP_Ne, rLhs, destIfFalse, r2, (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); } sqlite3ReleaseTempReg(pParse, regToFree); } @@ -2180,78 +2857,113 @@ static void sqlite3ExprCodeIN( } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; }else{ - - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. - */ - if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + destStep2 = destStep6 = sqlite3VdbeMakeLabel(v); + } + for(i=0; ipLeft, i); + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); - if( rRhsHasNull==0 ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - VdbeCoverage(v); - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int addr1; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the answer is TRUE the presence of NULLs in the RHS does - ** not matter. If the LHS is not contained in the RHS, then the - ** answer is NULL if the RHS contains NULLs and the answer is - ** FALSE if the RHS is NULL-free. - */ - addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfFalse); - sqlite3VdbeJumpHere(v, addr1); - } } } - sqlite3ReleaseTempReg(pParse, r1); + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(v); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; iiTable, i, r3); + sqlite3VdbeAddOp4(v, OP_Ne, rLhs+i, destNotNull, r3, + (void*)pColl, P4_COLLSEQ); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r3); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + if( nVector>1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, pExpr->iTable, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -2295,35 +3007,38 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ const char *z = pExpr->u.zToken; assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); - if( c==0 || (c==2 && negFlag) ){ - if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); - }else{ + if( c==1 || (c==2 && !negFlag) || (negFlag && value==SMALLEST_INT64)){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else #ifndef SQLITE_OMIT_HEX_INTEGER if( sqlite3_strnicmp(z,"0x",2)==0 ){ - sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + sqlite3ErrorMsg(pParse, "hex literal too big: %s%s", negFlag?"-":"",z); }else #endif { codeReal(v, z, negFlag, iMem); } #endif + }else{ + if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); } } } /* -** Clear a cache entry. +** Erase column-cache entry number i */ -static void cacheEntryClear(Parse *pParse, struct yColCache *p){ - if( p->tempReg ){ +static void cacheEntryClear(Parse *pParse, int i){ + if( pParse->aColCache[i].tempReg ){ if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = p->iReg; + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } - p->tempReg = 0; + } + pParse->nColCache--; + if( inColCache ){ + pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; } } @@ -2354,43 +3069,33 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** that the object will never already be in cache. Verify this guarantee. */ #ifndef NDEBUG - for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + assert( p->iTable!=iTab || p->iColumn!=iCol ); } #endif - /* Find an empty slot and replace it */ - for(i=0, p=pParse->aColCache; iiReg==0 ){ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; + /* If the cache is already full, delete the least recently used entry */ + if( pParse->nColCache>=SQLITE_N_COLCACHE ){ + minLru = 0x7fffffff; + idxLru = -1; + for(i=0, p=pParse->aColCache; ilrulru; + } } + p = &pParse->aColCache[idxLru]; + }else{ + p = &pParse->aColCache[pParse->nColCache++]; } - /* Replace the last recently used */ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - if( ALWAYS(idxLru>=0) ){ - p = &pParse->aColCache[idxLru]; - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - return; - } + /* Add the new entry to the end of the cache */ + p->iLevel = pParse->iCacheLevel; + p->iTable = iTab; + p->iColumn = iCol; + p->iReg = iReg; + p->tempReg = 0; + p->lru = pParse->iCacheCnt++; } /* @@ -2398,14 +3103,13 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ ** Purge the range of registers from the column cache. */ void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i; - int iLast = iReg + nReg - 1; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iReg && r<=iLast ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + int i = 0; + while( inColCache ){ + struct yColCache *p = &pParse->aColCache[i]; + if( p->iReg >= iReg && p->iReg < iReg+nReg ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -2430,8 +3134,7 @@ void sqlite3ExprCachePush(Parse *pParse){ ** the cache to the state it was in prior the most recent Push. */ void sqlite3ExprCachePop(Parse *pParse){ - int i; - struct yColCache *p; + int i = 0; assert( pParse->iCacheLevel>=1 ); pParse->iCacheLevel--; #ifdef SQLITE_DEBUG @@ -2439,10 +3142,11 @@ void sqlite3ExprCachePop(Parse *pParse){ printf("POP to %d\n", pParse->iCacheLevel); } #endif - for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + while( inColCache ){ + if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ + cacheEntryClear(pParse, i); + }else{ + i++; } } } @@ -2456,7 +3160,7 @@ void sqlite3ExprCachePop(Parse *pParse){ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 0; } @@ -2500,7 +3204,7 @@ void sqlite3ExprCodeGetColumnOfTable( }else{ int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; int x = iCol; - if( !HasRowid(pTab) ){ + if( !HasRowid(pTab) && !IsVirtual(pTab) ){ x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); @@ -2534,8 +3238,8 @@ int sqlite3ExprCodeGetColumn( int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ + for(i=0, p=pParse->aColCache; inColCache; i++, p++){ + if( p->iTable==iTable && p->iColumn==iColumn ){ p->lru = pParse->iCacheCnt++; sqlite3ExprCachePinRegister(pParse, p->iReg); return p->iReg; @@ -2567,19 +3271,20 @@ void sqlite3ExprCodeGetColumnToReg( */ void sqlite3ExprCacheClear(Parse *pParse){ int i; - struct yColCache *p; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("CLEAR\n"); } #endif - for(i=0, p=pParse->aColCache; iiReg ){ - cacheEntryClear(pParse, p); - p->iReg = 0; + for(i=0; inColCache; i++){ + if( pParse->aColCache[i].tempReg + && pParse->nTempRegaTempReg) + ){ + pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; } } + pParse->nColCache = 0; } /* @@ -2611,7 +3316,7 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ int r = p->iReg; if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ } @@ -2619,8 +3324,11 @@ static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ + /* -** Convert an expression node to a TK_REGISTER +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. */ static void exprToRegister(Expr *p, int iReg){ p->op2 = p->op; @@ -2629,6 +3337,38 @@ static void exprToRegister(Expr *p, int iReg){ ExprClearProperty(p, EP_Skip); } +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in continguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ + iResult = sqlite3CodeSubselect(pParse, p, 0, 0); + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + + /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -2646,9 +3386,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ int inReg = target; /* Results stored in register inReg */ int regFree1 = 0; /* If non-zero free this temporary register */ int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2, r3, r4; /* Various register numbers */ - sqlite3 *db = pParse->db; /* The database connection */ + int r1, r2; /* Various register numbers */ Expr tempX; /* Temporary expression node */ + int p5 = 0; assert( target>0 && target<=pParse->nMem ); if( v==0 ){ @@ -2667,12 +3407,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ assert( pCol->iMem>0 ); - inReg = pCol->iMem; - break; + return pCol->iMem; }else if( pAggInfo->useSortingIdx ){ sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); - break; + return target; } /* Otherwise, fall thru into the TK_COLUMN case */ } @@ -2681,38 +3420,36 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( iTab<0 ){ if( pParse->ckBase>0 ){ /* Generating CHECK constraints or inserting into partial index */ - inReg = pExpr->iColumn + pParse->ckBase; - break; + return pExpr->iColumn + pParse->ckBase; }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ iTab = pParse->iSelfTab; } } - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, pExpr->iColumn, iTab, target, pExpr->op2); - break; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); - break; + return target; } #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeLoadString(v, target, pExpr->u.zToken); - break; + return target; } case TK_NULL: { sqlite3VdbeAddOp2(v, OP_Null, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { @@ -2727,7 +3464,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( z[n]=='\'' ); zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - break; + return target; } #endif case TK_VARIABLE: { @@ -2736,15 +3473,15 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ - assert( pExpr->u.zToken[0]=='?' - || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); - sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); + const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); + assert( pExpr->u.zToken[0]=='?' || strcmp(pExpr->u.zToken, z)==0 ); + pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ + sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); } - break; + return target; } case TK_REGISTER: { - inReg = pExpr->iTable; - break; + return pExpr->iTable; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { @@ -2758,42 +3495,37 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - break; + return inReg; } #endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2); - assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); - testcase( regFree1==0 ); - testcase( regFree2==0 ); + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pLeft, pExpr->pRight, op, + r1, r2, inReg, SQLITE_STOREP2 | p5); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } break; } case TK_AND: @@ -2831,10 +3563,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( pLeft ); if( pLeft->op==TK_INTEGER ){ codeInteger(pParse, pLeft, 1, target); + return target; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); + return target; #endif }else{ tempX.op = TK_INTEGER; @@ -2845,7 +3579,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); } - inReg = target; break; } case TK_BITNOT: @@ -2854,7 +3587,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - inReg = target; sqlite3VdbeAddOp2(v, op, r1, inReg); break; } @@ -2879,7 +3611,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); }else{ - inReg = pInfo->aFunc[pExpr->iAgg].iMem; + return pInfo->aFunc[pExpr->iAgg].iMem; } break; } @@ -2890,9 +3622,15 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ + if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + /* SQL functions can be expensive. So try to move constant functions + ** out of the inner loop, even if that means an extra OP_Copy. */ + return sqlite3ExprCodeAtInit(pParse, pExpr, -1); + } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); if( ExprHasProperty(pExpr, EP_TokenOnly) ){ pFarg = 0; @@ -2903,6 +3641,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif if( pDef==0 || pDef->xFinalize!=0 ){ sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); break; @@ -2933,10 +3676,25 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ */ if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ assert( nFarg>=1 ); - inReg = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); - break; + return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); } +#ifdef SQLITE_DEBUG + /* The AFFINITY() function evaluates to a string that describes + ** the type affinity of the argument. This is used for testing of + ** the SQLite type logic. + */ + if( pDef->funcFlags & SQLITE_FUNC_AFFINITY ){ + const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + char aff; + assert( nFarg==1 ); + aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + sqlite3VdbeLoadString(v, target, + aff ? azAff[aff-SQLITE_AFF_BLOB] : "none"); + return target; + } +#endif + for(i=0; ia[i].pExpr) ){ testcase( i==31 ); @@ -3009,16 +3767,35 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( nFarg && constMask==0 ){ sqlite3ReleaseTempRange(pParse, r1, nFarg); } - break; + return target; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: case TK_SELECT: { + int nCol; testcase( op==TK_EXISTS ); testcase( op==TK_SELECT ); - inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr, 0, 0); + } break; } + case TK_SELECT_COLUMN: { + int n; + if( pExpr->pLeft->iTable==0 ){ + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft, 0, 0); + } + assert( pExpr->iTable==0 || pExpr->pLeft->op==TK_SELECT ); + if( pExpr->iTable + && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) + ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pExpr->iTable, n); + } + return pExpr->pLeft->iTable + pExpr->iColumn; + } case TK_IN: { int destIfFalse = sqlite3VdbeMakeLabel(v); int destIfNull = sqlite3VdbeMakeLabel(v); @@ -3028,7 +3805,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeResolveLabel(v, destIfFalse); sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); sqlite3VdbeResolveLabel(v, destIfNull); - break; + return target; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -3045,35 +3822,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - Expr *pLeft = pExpr->pLeft; - struct ExprList_item *pLItem = pExpr->x.pList->a; - Expr *pRight = pLItem->pExpr; - - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - r3 = sqlite3GetTempReg(pParse); - r4 = sqlite3GetTempReg(pParse); - codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); - pLItem++; - pRight = pLItem->pExpr; - sqlite3ReleaseTempReg(pParse, regFree2); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree2==0 ); - codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); - sqlite3ReleaseTempReg(pParse, r3); - sqlite3ReleaseTempReg(pParse, r4); - break; + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; } case TK_SPAN: case TK_COLLATE: case TK_UPLUS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + return sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); } case TK_TRIGGER: { @@ -3132,6 +3887,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } /* ** Form A: @@ -3175,8 +3934,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ if( (pX = pExpr->pLeft)!=0 ){ tempX = *pX; testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); + exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; opCompare.pLeft = &tempX; pTest = &opCompare; @@ -3210,7 +3970,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( db->mallocFailed || pParse->nErr>0 + assert( pParse->db->mallocFailed || pParse->nErr>0 || pParse->iCacheLevel==iCacheLevel ); sqlite3VdbeResolveLabel(v, endLabel); break; @@ -3251,24 +4011,40 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ /* ** Factor out the code of the given expression to initialization time. +** +** If regDest>=0 then the result is always stored in that register and the +** result is not reusable. If regDest<0 then this routine is free to +** store the value whereever it wants. The register where the expression +** is stored is returned. When regDest<0, two identical expressions will +** code to the same register. */ -void sqlite3ExprCodeAtInit( +int sqlite3ExprCodeAtInit( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The expression to code when the VDBE initializes */ - int regDest, /* Store the value in this register */ - u8 reusable /* True if this expression is reusable */ + int regDest /* Store the value in this register */ ){ ExprList *p; assert( ConstFactorOk(pParse) ); p = pParse->pConstExpr; + if( regDest<0 && p ){ + struct ExprList_item *pItem; + int i; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->reusable && sqlite3ExprCompare(pItem->pExpr,pExpr,-1)==0 ){ + return pItem->u.iConstExprReg; + } + } + } pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); p = sqlite3ExprListAppend(pParse, p, pExpr); if( p ){ struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->reusable = regDest<0; + if( regDest<0 ) regDest = ++pParse->nMem; pItem->u.iConstExprReg = regDest; - pItem->reusable = reusable; } pParse->pConstExpr = p; + return regDest; } /* @@ -3291,19 +4067,8 @@ int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ && pExpr->op!=TK_REGISTER && sqlite3ExprIsConstantNotJoin(pExpr) ){ - ExprList *p = pParse->pConstExpr; - int i; *pReg = 0; - if( p ){ - struct ExprList_item *pItem; - for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ - if( pItem->reusable && sqlite3ExprCompare(pItem->pExpr,pExpr,-1)==0 ){ - return pItem->u.iConstExprReg; - } - } - } - r2 = ++pParse->nMem; - sqlite3ExprCodeAtInit(pParse, pExpr, r2, 1); + r2 = sqlite3ExprCodeAtInit(pParse, pExpr, -1); }else{ int r1 = sqlite3GetTempReg(pParse); r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); @@ -3357,7 +4122,7 @@ void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ */ void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target, 0); + sqlite3ExprCodeAtInit(pParse, pExpr, target); }else{ sqlite3ExprCode(pParse, pExpr, target); } @@ -3421,10 +4186,15 @@ int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ - sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ + if( flags & SQLITE_ECEL_OMITREF ){ + i--; + n--; + }else{ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + } }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); + sqlite3ExprCodeAtInit(pParse, pExpr, target+i); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ @@ -3455,20 +4225,33 @@ int sqlite3ExprCodeExprList( ** ** Code it as such, taking care to do the common subexpression ** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump here if the jump is taken */ - int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ + + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); exprX = *pExpr->pLeft; exprAnd.op = TK_AND; @@ -3480,23 +4263,30 @@ static void exprCodeBetween( compRight.op = TK_LE; compRight.pLeft = &exprX; compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); - if( jumpIfTrue ){ - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); + exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); }else{ - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_FromJoin bit. Bummer. */ + exprX.flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); } sqlite3ReleaseTempReg(pParse, regFree1); /* Ensure adequate test coverage */ - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); } /* @@ -3548,12 +4338,20 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -3563,23 +4361,12 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -3597,7 +4384,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -3611,6 +4398,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { + default_expr: if( exprAlwaysTrue(pExpr) ){ sqlite3VdbeGoto(v, dest); }else if( exprAlwaysFalse(pExpr) ){ @@ -3704,12 +4492,20 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); @@ -3719,23 +4515,12 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -3751,7 +4536,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -3767,6 +4552,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { + default_expr: if( exprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); }else if( exprAlwaysTrue(pExpr) ){ @@ -3894,6 +4680,17 @@ int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ return 0; } +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ + return sqlite3ExprCompare( + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), + iTab); +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might @@ -3924,15 +4721,69 @@ int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ ){ return 1; } - if( pE2->op==TK_NOTNULL - && sqlite3ExprCompare(pE1->pLeft, pE2->pLeft, iTab)==0 - && (pE1->op!=TK_ISNULL && pE1->op!=TK_IS) - ){ - return 1; + if( pE2->op==TK_NOTNULL && pE1->op!=TK_ISNULL && pE1->op!=TK_IS ){ + Expr *pX = sqlite3ExprSkipCollate(pE1->pLeft); + testcase( pX!=pE1->pLeft ); + if( sqlite3ExprCompare(pX, pE2->pLeft, iTab)==0 ) return 1; } return 0; } +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + /* ** An instance of the following structure is used by the tree walker ** to count references to table columns in the arguments of an @@ -4216,7 +5067,7 @@ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg && pParse->nTempRegaTempReg) ){ int i; struct yColCache *p; - for(i=0, p=pParse->aColCache; iaColCache; inColCache; i++, p++){ if( p->iReg==iReg ){ p->tempReg = 1; return; @@ -4227,10 +5078,11 @@ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ } /* -** Allocate or deallocate a block of nReg consecutive registers +** Allocate or deallocate a block of nReg consecutive registers. */ int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ @@ -4244,6 +5096,10 @@ int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; @@ -4258,3 +5114,26 @@ void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; pParse->nRangeReg = 0; } + +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ diff --git a/src/fault.c b/src/fault.c index c3028c4f93..5b41b60363 100644 --- a/src/fault.c +++ b/src/fault.c @@ -26,7 +26,7 @@ #include "sqliteInt.h" -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** Global variables. @@ -84,4 +84,4 @@ void sqlite3EndBenignMalloc(void){ } } -#endif /* #ifndef SQLITE_OMIT_BUILTIN_TEST */ +#endif /* #ifndef SQLITE_UNTESTABLE */ diff --git a/src/fkey.c b/src/fkey.c index 9a1a85fec0..cb78764f51 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -225,7 +225,7 @@ int sqlite3FkLocateIndex( } for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) && pIdx->pPartIdxWhere==0 ){ /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number ** of columns. If each indexed column corresponds to a foreign key ** column of pFKey, then this index is a winner. */ @@ -584,7 +584,7 @@ static void fkScanChildren( assert( iCol>=0 ); zCol = pFKey->pFrom->aCol[iCol].zName; pRight = sqlite3Expr(db, TK_ID, zCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); pWhere = sqlite3ExprAnd(db, pWhere, pEq); } @@ -606,7 +606,7 @@ static void fkScanChildren( if( HasRowid(pTab) ){ pLeft = exprTableRegister(pParse, pTab, regData, -1); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); - pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight); }else{ Expr *pEq, *pAll = 0; Index *pPk = sqlite3PrimaryKeyIndex(pTab); @@ -616,10 +616,10 @@ static void fkScanChildren( assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); pAll = sqlite3ExprAnd(db, pAll, pEq); } - pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0, 0); + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); } pWhere = sqlite3ExprAnd(db, pWhere, pNe); } @@ -871,7 +871,7 @@ void sqlite3FkCheck( if( (db->flags&SQLITE_ForeignKeys)==0 ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the ** child table (the table that the foreign key definition is part of). */ @@ -1007,7 +1007,7 @@ void sqlite3FkCheck( struct SrcList_item *pItem = pSrc->a; pItem->pTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nRef++; + pItem->pTab->nTabRef++; pItem->iCursor = pParse->nTab++; if( regNew!=0 ){ @@ -1165,7 +1165,6 @@ static Trigger *fkActionTrigger( if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ return 0; } - pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ @@ -1206,10 +1205,9 @@ static Trigger *fkActionTrigger( pEq = sqlite3PExpr(pParse, TK_EQ, sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tOld, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) - , 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) - , 0); + ); pWhere = sqlite3ExprAnd(db, pWhere, pEq); /* For ON UPDATE, construct the next term of the WHEN clause. @@ -1221,13 +1219,11 @@ static Trigger *fkActionTrigger( pEq = sqlite3PExpr(pParse, TK_IS, sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tOld, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), - 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tNew, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), - 0), - 0); + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) + ); pWhen = sqlite3ExprAnd(db, pWhen, pEq); } @@ -1236,17 +1232,16 @@ static Trigger *fkActionTrigger( if( action==OE_Cascade ){ pNew = sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tNew, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) - , 0); + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)); }else if( action==OE_SetDflt ){ Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } pList = sqlite3ExprListAppend(pParse, pList, pNew); sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); @@ -1293,7 +1288,7 @@ static Trigger *fkActionTrigger( pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); if( pWhen ){ - pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0, 0); + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0); pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); } } @@ -1373,7 +1368,8 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ - assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + assert( db==0 || IsVirtual(pTab) + || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ diff --git a/src/func.c b/src/func.c index 662a08f504..3d5a059a9f 100644 --- a/src/func.c +++ b/src/func.c @@ -200,23 +200,26 @@ static void instrFunc( if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; nHaystack = sqlite3_value_bytes(argv[0]); nNeedle = sqlite3_value_bytes(argv[1]); - if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ - zHaystack = sqlite3_value_blob(argv[0]); - zNeedle = sqlite3_value_blob(argv[1]); - isText = 0; - }else{ - zHaystack = sqlite3_value_text(argv[0]); - zNeedle = sqlite3_value_text(argv[1]); - isText = 1; + if( nNeedle>0 ){ + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else{ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; + while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; } - while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ - N++; - do{ - nHaystack--; - zHaystack++; - }while( isText && (zHaystack[0]&0xc0)==0x80 ); - } - if( nNeedle>nHaystack ) N = 0; sqlite3_result_int(context, N); } @@ -596,9 +599,19 @@ static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 }; static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* -** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" or "like" expression. Return true (1) if they -** are the same and false (0) if they are different. +** Possible error returns from patternMatch() +*/ +#define SQLITE_MATCH 0 +#define SQLITE_NOMATCH 1 +#define SQLITE_NOWILDCARDMATCH 2 + +/* +** Compare two UTF-8 strings for equality where the first string is +** a GLOB or LIKE expression. Return values: +** +** SQLITE_MATCH: Match +** SQLITE_NOMATCH: No match +** SQLITE_NOWILDCARDMATCH: No match in spite of having * or % wildcards. ** ** Globbing rules: ** @@ -649,30 +662,31 @@ static int patternCompare( ** single character of the input string for each "?" skipped */ while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ - return 0; + return SQLITE_NOWILDCARDMATCH; } } if( c==0 ){ - return 1; /* "*" at the end of the pattern matches */ + return SQLITE_MATCH; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOWILDCARDMATCH; }else{ /* "[...]" immediately follows the "*". We have to do a slow ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ - while( *zString - && patternCompare(&zPattern[-1],zString,pInfo,matchOther)==0 ){ + while( *zString ){ + int bMatch = patternCompare(&zPattern[-1],zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; SQLITE_SKIP_UTF8(zString); } - return *zString!=0; + return SQLITE_NOWILDCARDMATCH; } } /* At this point variable c contains the first character of the ** pattern string past the "*". Search in the input string for the - ** first matching character and recursively contine the match from + ** first matching character and recursively continue the match from ** that point. ** ** For a case-insensitive search, set variable cx to be the same as @@ -681,6 +695,7 @@ static int patternCompare( */ if( c<=0x80 ){ u32 cx; + int bMatch; if( noCase ){ cx = sqlite3Toupper(c); c = sqlite3Tolower(c); @@ -689,27 +704,30 @@ static int patternCompare( } while( (c2 = *(zString++))!=0 ){ if( c2!=c && c2!=cx ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; } }else{ + int bMatch; while( (c2 = Utf8Read(zString))!=0 ){ if( c2!=c ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; } } - return 0; + return SQLITE_NOWILDCARDMATCH; } if( c==matchOther ){ if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOMATCH; zEscaped = zPattern; }else{ u32 prior_c = 0; int seen = 0; int invert = 0; c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOMATCH; c2 = sqlite3Utf8Read(&zPattern); if( c2=='^' ){ invert = 1; @@ -733,34 +751,36 @@ static int patternCompare( c2 = sqlite3Utf8Read(&zPattern); } if( c2==0 || (seen ^ invert)==0 ){ - return 0; + return SQLITE_NOMATCH; } continue; } } c2 = Utf8Read(zString); if( c==c2 ) continue; - if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ continue; } if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; - return 0; + return SQLITE_NOMATCH; } - return *zString==0; + return *zString==0 ? SQLITE_MATCH : SQLITE_NOMATCH; } /* -** The sqlite3_strglob() interface. +** The sqlite3_strglob() interface. Return 0 on a match (like strcmp()) and +** non-zero if there is no match. */ int sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[')==0; + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); } /* -** The sqlite3_strlike() interface. +** The sqlite3_strlike() interface. Return 0 on a match and non-zero for +** a miss - like strcmp(). */ int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc)==0; + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); } /* @@ -841,7 +861,7 @@ static void likeFunc( #ifdef SQLITE_TEST sqlite3_like_count++; #endif - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); + sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); } } @@ -1316,6 +1336,26 @@ static void trimFunc( } +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + + /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE_SOUNDEX compile-time option is used ** when SQLite is built. @@ -1386,6 +1426,14 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3 *db = sqlite3_context_db_handle(context); char *zErrMsg = 0; + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + if( argc==2 ){ zProc = (const char *)sqlite3_value_text(argv[1]); }else{ @@ -1584,7 +1632,7 @@ static void groupConcatStep( zSep = ","; nSep = 1; } - if( nSep ) sqlite3StrAccumAppend(pAccum, zSep, nSep); + if( zSep ) sqlite3StrAccumAppend(pAccum, zSep, nSep); } zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); @@ -1725,6 +1773,9 @@ void sqlite3RegisterBuiltinFunctions(void){ FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), +#ifdef SQLITE_DEBUG + FUNCTION2(affinity, 1, 0, 0, noopFunc, SQLITE_FUNC_AFFINITY), +#endif FUNCTION(ltrim, 1, 1, 0, trimFunc ), FUNCTION(ltrim, 2, 1, 0, trimFunc ), FUNCTION(rtrim, 1, 2, 0, trimFunc ), @@ -1778,13 +1829,16 @@ void sqlite3RegisterBuiltinFunctions(void){ AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #ifdef SQLITE_CASE_SENSITIVE_LIKE +#ifdef SQLITE_CASE_SENSITIVE_LIKE LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), - #else +#else LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), - #endif +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + FUNCTION(unknown, -1, 0, 0, unknownFunc ), +#endif FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), diff --git a/src/global.c b/src/global.c index bd70b38273..a105135561 100644 --- a/src/global.c +++ b/src/global.c @@ -70,6 +70,7 @@ const unsigned char sqlite3UpperToLower[] = { ** isxdigit() 0x08 ** toupper() 0x20 ** SQLite identifier character 0x40 +** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper ** case. i.e. if the character is a lower-case ASCII character. @@ -78,16 +79,13 @@ const unsigned char sqlite3UpperToLower[] = { ** ** (x & ~(map[x]&0x20)) ** -** Standard function tolower() is implemented using the sqlite3UpperToLower[] +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** -** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an ** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. -** -** SQLite's versions are identical to the standard versions assuming a -** locale of "C". They are implemented as macros in sqliteInt.h. */ #ifdef SQLITE_ASCII const unsigned char sqlite3CtypeMap[256] = { @@ -95,7 +93,7 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ - 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ @@ -103,8 +101,8 @@ const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ - 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ - 0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ @@ -160,7 +158,7 @@ const unsigned char sqlite3CtypeMap[256] = { #endif /* Statement journals spill to disk when their size exceeds the following -** threashold (in bytes). 0 means that statement journals are created and +** threshold (in bytes). 0 means that statement journals are created and ** written to disk immediately (the default behavior for SQLite versions ** before 3.12.0). -1 means always keep the entire statement journal in ** memory. (The statement journal is also always held entirely in memory @@ -171,6 +169,19 @@ const unsigned char sqlite3CtypeMap[256] = { # define SQLITE_STMTJRNL_SPILL (64*1024) #endif +/* +** The default lookaside-configuration, the format "SZ,N". SZ is the +** number of bytes in each lookaside slot (should be a multiple of 8) +** and N is the number of slots. The lookaside-configuration can be +** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) +** or at run-time for an individual database connection using +** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); +*/ +#ifndef SQLITE_DEFAULT_LOOKASIDE +# define SQLITE_DEFAULT_LOOKASIDE 1200,100 +#endif + + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -183,8 +194,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ - 128, /* szLookaside */ - 500, /* nLookaside */ + SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ @@ -221,10 +231,11 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xVdbeBranch */ 0, /* pVbeBranchArg */ #endif -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif - 0 /* bLocaltimeFault */ + 0, /* bLocaltimeFault */ + 0x7ffffffe /* iOnceResetThreshold */ }; /* @@ -247,7 +258,7 @@ const Token sqlite3IntTokens[] = { ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts -** to read or write that page. The pending byte page is set assign +** to read or write that page. The pending byte page is set aside ** for use by the VFS layers as space for managing file locks. ** ** During testing, it is often desirable to move the pending byte to diff --git a/src/hash.c b/src/hash.c index b5886e0641..62b695a1bb 100644 --- a/src/hash.c +++ b/src/hash.c @@ -55,8 +55,12 @@ void sqlite3HashClear(Hash *pH){ static unsigned int strHash(const char *z){ unsigned int h = 0; unsigned char c; - while( (c = (unsigned char)*z++)!=0 ){ - h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). + ** 0x9e3779b1 is 2654435761 which is the closest prime number to + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ + h += sqlite3UpperToLower[c]; + h *= 0x9e3779b1; } return h; } @@ -148,7 +152,7 @@ static HashElem *findElementWithHash( int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - if( pH->ht ){ + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; h = strHash(pKey) % pH->htsize; pEntry = &pH->ht[h]; diff --git a/src/hash.h b/src/hash.h index 6dfa4e035c..90540eda50 100644 --- a/src/hash.h +++ b/src/hash.h @@ -12,8 +12,8 @@ ** This is the header file for the generic hash-table implementation ** used in SQLite. */ -#ifndef _SQLITE_HASH_H_ -#define _SQLITE_HASH_H_ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H /* Forward declarations of structures. */ typedef struct Hash Hash; @@ -93,4 +93,4 @@ void sqlite3HashClear(Hash*); */ /* #define sqliteHashCount(H) ((H)->count) // NOT USED */ -#endif /* _SQLITE_HASH_H_ */ +#endif /* SQLITE_HASH_H */ diff --git a/src/hwtime.h b/src/hwtime.h index b8bc5a295b..5b209db8af 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -13,8 +13,8 @@ ** This file contains inline asm code for retrieving "high-performance" ** counters for x86 class CPUs. */ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H /* ** The following routine only works on pentium-class (or newer) processors. @@ -82,4 +82,4 @@ #endif -#endif /* !defined(_HWTIME_H_) */ +#endif /* !defined(SQLITE_HWTIME_H) */ diff --git a/src/in-operator.md b/src/in-operator.md new file mode 100644 index 0000000000..e9ad2101aa --- /dev/null +++ b/src/in-operator.md @@ -0,0 +1,107 @@ +IN-Operator Implementation Notes +================================ + +## Definitions: + +An IN operator has one of the following formats: + +> + x IN (y1,y2,y3,...,yN) + x IN (subquery) + +The "x" is referred to as the LHS (left-hand side). The list or subquery +on the right is called the RHS (right-hand side). If the RHS is a list +it must be a non-empty list. But if the RHS is a subquery, it can be an +empty set. + +The LHS can be a scalar (a single quantity) or a vector (a list of +two or or more values) or a subquery that returns one or more columns. +We use the term "vector" to mean an actually list of values or a +subquery that returns two or more columns. An isolated value or +a subquery that returns a single columns is called a scalar. + +The RHS can be a subquery that returns a single column, a subquery +that returns two or more columns, or a list of scalars. It is not +currently support for the RHS to be a list of vectors. + +The number of columns for LHS must match the number of columns for +the RHS. If the RHS is a list of values, then the LHS must be a +scalar. If the RHS is a subquery returning N columns, then the LHS +must be a vector of size N. + +NULL values can occur in either or both of the LHS and RHS. +If the LHS contains only +NULL values then we say that it is a "total-NULL". If the LHS contains +some NULL values and some non-NULL values, then it is a "partial-NULL". +For a scalar, there is no difference between a partial-NULL and a total-NULL. +The RHS is a partial-NULL if any row contains a NULL value. The RHS is +a total-NULL if it contains one or more rows that contain only NULL values. +The LHS is called "non-NULL" if it contains no NULL values. The RHS is +called "non-NULL" if it contains no NULL values in any row. + +The result of an IN operator is one of TRUE, FALSE, or NULL. A NULL result +means that it cannot be determined if the LHS is contained in the RHS due +to the presence of NULL values. In some contexts (for example, when the IN +operator occurs in a WHERE clause) +the system only needs a binary result: TRUE or NOT-TRUE. One can also +to define a binary result of FALSE and NOT-FALSE, but +it turns out that no extra optimizations are possible in that case, so if +the FALSE/NOT-FALSE binary is needed, we have to compute the three-state +TRUE/FALSE/NULL result and then combine the TRUE and NULL values into +NOT-FALSE. + +A "NOT IN" operator is computed by first computing the equivalent IN +operator, then interchanging the TRUE and FALSE results. + +## Simple Full-Scan Algorithm + +The following algorithm always compute the correct answer. However, this +algorithm is suboptimal, especially if there are many rows on the RHS. + + 1. Set the null-flag to false + 2. For each row in the RHS: +
      +
    1. Compare the LHS against the RHS +
    2. If the LHS exactly matches the RHS, immediately return TRUE +
    3. If the comparison result is NULL, set the null-flag to true +
    + 3. If the null-flag is true, return NULL. + 4. Return FALSE + +## Optimized Algorithm + +The following procedure computes the same answer as the simple full-scan +algorithm, though it does so with less work in the common case. This +is the algorithm that is implemented in SQLite. + + 1. If the RHS is a constant list of length 1 or 2, then rewrite the + IN operator as a simple expression. Implement + + x IN (y1,y2) + + as if it were + + x=y1 OR x=y2 + + This is the INDEX_NOOP optimization and is only undertaken if the + IN operator is used for membership testing. If the IN operator is + driving a loop, then skip this step entirely. + + 2. Check the LHS to see if it is a partial-NULL and if it is, jump + ahead to step 5. + + 3. Do a binary search of the RHS using the LHS as a probe. If + an exact match is found, return TRUE. + + 4. If the RHS is non-NULL then return FALSE. + + 5. If we do not need to distinguish between FALSE and NULL, + then return FALSE. + + 6. For each row in the RHS, compare that row against the LHS and + if the result is NULL, immediately return NULL. In the case + of a scalar IN operator, we only need to look at the very first + row the RHS because for a scalar RHS, all NULLs will always come + first. If the RHS is empty, this step is a no-op. + + 7. Return FALSE. diff --git a/src/insert.c b/src/insert.c index 03fdf09acc..a1f5442515 100644 --- a/src/insert.c +++ b/src/insert.c @@ -200,7 +200,9 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) ** ** There is at most one AutoincInfo structure per table even if the ** same table is autoincremented multiple times due to inserts within @@ -223,7 +225,9 @@ static int autoIncBegin( Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ - if( pTab->tabFlags & TF_Autoincrement ){ + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->flags & SQLITE_Vacuum)==0 + ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; @@ -481,8 +485,7 @@ void sqlite3Insert( sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ char *zTab; /* Name of the table into which we are inserting */ - const char *zDb; /* Name of the database holding this table */ - int i, j, idx; /* Loop counters */ + int i, j; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ int nColumn; /* Number of columns in the data */ @@ -496,7 +499,6 @@ void sqlite3Insert( int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ - Db *pDb; /* The database containing table being inserted into */ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ @@ -546,9 +548,8 @@ void sqlite3Insert( } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - pDb = &db->aDb[iDb]; - zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ goto insert_cleanup; } withoutRowid = !HasRowid(pTab); @@ -791,8 +792,10 @@ void sqlite3Insert( if( aRegIdx==0 ){ goto insert_cleanup; } - for(i=0; ipIndex; ipNext, i++){ + assert( pIdx ); aRegIdx[i] = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; } } @@ -994,12 +997,26 @@ void sqlite3Insert( #endif { int isReplace; /* Set to true if constraints may cause a replace */ + int bUseSeek; /* True to use OPFLAG_SEEKRESULT */ sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0 ); sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + + /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE + ** constraints or (b) there are no triggers and this table is not a + ** parent table in a foreign key constraint. It is safe to set the + ** flag in the second case as if any REPLACE constraint is hit, an + ** OP_Delete or OP_IdxDelete instruction will be executed on each + ** cursor that is disturbed. And these instructions both clear the + ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT + ** functionality. */ + bUseSeek = (isReplace==0 || (pTrigger==0 && + ((db->flags & SQLITE_ForeignKeys)==0 || sqlite3FkReferences(pTab)==0) + )); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, - regIns, aRegIdx, 0, appendFlag, isReplace==0); + regIns, aRegIdx, 0, appendFlag, bUseSeek + ); } } @@ -1028,14 +1045,6 @@ void sqlite3Insert( sqlite3VdbeJumpHere(v, addrInsTop); } - if( !IsVirtual(pTab) && !isView ){ - /* Close all tables opened */ - if( iDataCurpIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp1(v, OP_Close, idx+iIdxCur); - } - } - insert_end: /* Update the sqlite_sequence table by storing the content of the ** maximum rowid counter values recorded while inserting into @@ -1242,7 +1251,6 @@ void sqlite3GenerateConstraintChecks( int ipkBottom = 0; /* Bottom of the rowid change constraint check */ u8 isUpdate; /* True if this is an UPDATE operation */ u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ - int regRowid = -1; /* Register holding ROWID value */ isUpdate = regOldData!=0; db = pParse->db; @@ -1297,8 +1305,9 @@ void sqlite3GenerateConstraintChecks( case OE_Fail: { char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, pTab->aCol[i].zName); - sqlite3VdbeAddOp4(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, - regNewData+1+i, zMsg, P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, + regNewData+1+i); + sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); VdbeCoverage(v); break; @@ -1362,7 +1371,7 @@ void sqlite3GenerateConstraintChecks( } if( isUpdate ){ - /* pkChng!=0 does not mean that the rowid has change, only that + /* pkChng!=0 does not mean that the rowid has changed, only that ** it might have changed. Skip the conflict logic below if the rowid ** is unchanged. */ sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); @@ -1431,9 +1440,18 @@ void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, - ONEPASS_SINGLE, -1); + regNewData, 1, 0, OE_Replace, 1, -1); }else{ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( HasRowid(pTab) ){ + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ if( pTab->pIndex ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); @@ -1488,7 +1506,7 @@ void sqlite3GenerateConstraintChecks( /* Create a record for this index entry as it should appear after ** the insert or update. Store that record in the aRegIdx[ix] register */ - regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); + regIdx = aRegIdx[ix]+1; for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; @@ -1499,9 +1517,7 @@ void sqlite3GenerateConstraintChecks( VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ if( iField==XN_ROWID || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; }else{ x = iField + regNewData + 1; } @@ -1511,7 +1527,9 @@ void sqlite3GenerateConstraintChecks( } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); - sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==2 ) sqlite3SetMakeRecordP5(v, pIdx->pTable); +#endif /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the @@ -1525,7 +1543,6 @@ void sqlite3GenerateConstraintChecks( /* Find out what action to take in case there is a uniqueness conflict */ onError = pIdx->onError; if( onError==OE_None ){ - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); sqlite3VdbeResolveLabel(v, addrUniqueOk); continue; /* pIdx is not a UNIQUE index */ } @@ -1534,7 +1551,26 @@ void sqlite3GenerateConstraintChecks( }else if( onError==OE_Default ){ onError = OE_Abort; } - + + /* Collision detection may be omitted if all of the following are true: + ** (1) The conflict resolution algorithm is REPLACE + ** (2) The table is a WITHOUT ROWID table + ** (3) There are no secondary indexes on the table + ** (4) No delete triggers need to be fired if there is a conflict + ** (5) No FK constraint counters need to be updated if a conflict occurs. + */ + if( (ix==0 && pIdx->pNext==0) /* Condition 3 */ + && pPk==pIdx /* Condition 2 */ + && onError==OE_Replace /* Condition 1 */ + && ( 0==(db->flags&SQLITE_RecTriggers) || /* Condition 4 */ + 0==sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0)) + && ( 0==(db->flags&SQLITE_ForeignKeys) || /* Condition 5 */ + (0==pTab->pFKey && 0==sqlite3FkReferences(pTab))) + ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } + /* Check to see if the new index entry will be unique */ sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, regIdx, pIdx->nKeyCol); VdbeCoverage(v); @@ -1618,13 +1654,12 @@ void sqlite3GenerateConstraintChecks( } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); seenReplace = 1; break; } } sqlite3VdbeResolveLabel(v, addrUniqueOk); - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); } if( ipkTop ){ @@ -1636,6 +1671,28 @@ void sqlite3GenerateConstraintChecks( VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } +#ifdef SQLITE_ENABLE_NULL_TRIM +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].pDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); +} +#endif + /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. @@ -1652,7 +1709,7 @@ void sqlite3CompleteInsertion( int iIdxCur, /* First index cursor */ int regNewData, /* Range of content */ int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int isUpdate, /* True for UPDATE, False for INSERT */ + int update_flags, /* True for UPDATE, False for INSERT */ int appendBias, /* True if this is likely to be an append */ int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ @@ -1664,6 +1721,11 @@ void sqlite3CompleteInsertion( int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + v = sqlite3GetVdbe(pParse); assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ @@ -1674,26 +1736,39 @@ void sqlite3CompleteInsertion( sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); } - sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]); - pik_flags = 0; - if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( update_flags==0 ){ + sqlite3VdbeAddOp4(v, OP_InsertInt, + iIdxCur+i, aRegIdx[i], 0, (char*)pTab, P4_TABLE + ); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + } +#endif } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; regData = regNewData + 1; regRec = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0); - sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); + } if( pParse->nested ){ pik_flags = 0; }else{ pik_flags = OPFLAG_NCHANGE; - pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); } if( appendBias ){ pik_flags |= OPFLAG_APPEND; @@ -1703,7 +1778,7 @@ void sqlite3CompleteInsertion( } sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -1768,15 +1843,15 @@ int sqlite3OpenTableAndIndices( for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + p5 = 0; + } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - } - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; - }else{ sqlite3VdbeChangeP5(v, p5); + VdbeComment((v, "%s", pIdx->zName)); } } if( iBase>pParse->nTab ) pParse->nTab = iBase; @@ -1904,7 +1979,7 @@ static int xferOptimization( return 0; /* tab1 must not have triggers */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->tabFlags & TF_Virtual ){ + if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ } #endif @@ -1966,7 +2041,7 @@ static int xferOptimization( return 0; /* source and destination must both be WITHOUT ROWID or not */ } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->tabFlags & TF_Virtual ){ + if( IsVirtual(pSrc) ){ return 0; /* tab2 must not be a virtual table */ } #endif @@ -2086,6 +2161,7 @@ static int xferOptimization( sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ + u8 insFlags; sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); if( pDest->iPKey>=0 ){ @@ -2101,10 +2177,17 @@ static int xferOptimization( addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + if( db->flags & SQLITE_Vacuum ){ + sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID| + OPFLAG_APPEND|OPFLAG_USESEEKRESULT; + }else{ + insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND; + } sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, - pDest->zName, 0); - sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + (char*)pDest, P4_TABLE); + sqlite3VdbeChangeP5(v, insFlags); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -2126,7 +2209,7 @@ static int xferOptimization( sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->flags & SQLITE_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use @@ -2144,8 +2227,6 @@ static int xferOptimization( ** sorted order. */ for(i=0; inColumn; i++){ const char *zColl = pSrcIdx->azColl[i]; - assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 - || sqlite3StrBINARY==zColl ); if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ @@ -2156,8 +2237,8 @@ static int xferOptimization( if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ idxInsFlags |= OPFLAG_NCHANGE; } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeChangeP5(v, idxInsFlags); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); + sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -2167,6 +2248,7 @@ static int xferOptimization( sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); diff --git a/src/legacy.c b/src/legacy.c index bd34512d31..df8e2780fb 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -73,7 +73,7 @@ int sqlite3_exec( (SQLITE_DONE==rc && !callbackIsInit && db->flags&SQLITE_NullCallback)) ){ if( !callbackIsInit ){ - azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); if( azCols==0 ){ goto exec_out; } @@ -94,6 +94,7 @@ int sqlite3_exec( goto exec_out; } } + azVals[i] = 0; } if( xCallback(pArg, nCol, azVals, azCols) ){ /* EVIDENCE-OF: R-38229-40159 If the callback function to diff --git a/src/loadext.c b/src/loadext.c index 3469fbb73d..1fee4260ab 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -18,10 +18,8 @@ #endif #include "sqlite3ext.h" #include "sqliteInt.h" -#include #ifndef SQLITE_OMIT_LOAD_EXTENSION - /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -91,7 +89,7 @@ # define sqlite3_enable_shared_cache 0 #endif -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) # define sqlite3_profile 0 # define sqlite3_trace 0 #endif @@ -111,6 +109,10 @@ #define sqlite3_blob_reopen 0 #endif +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -414,7 +416,14 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.10.0 and later */ sqlite3_status64, sqlite3_strlike, - sqlite3_db_cacheflush + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql, + /* Version 3.18.0 and later */ + sqlite3_set_last_insert_rowid }; /* @@ -437,13 +446,14 @@ static int sqlite3LoadExtension( ){ sqlite3_vfs *pVfs = db->pVfs; void *handle; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; u64 nMsg = 300 + sqlite3Strlen30(zFile); int ii; + int rc; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { @@ -462,8 +472,9 @@ static int sqlite3LoadExtension( /* Ticket #1863. To avoid a creating security problems for older ** applications that relink against newer versions of SQLite, the ** ability to run load_extension is turned off by default. One - ** must call sqlite3_enable_load_extension() to turn on extension - ** loading. Otherwise you get the following error. + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. */ if( (db->flags & SQLITE_LoadExtension)==0 ){ if( pzErrMsg ){ @@ -494,8 +505,7 @@ static int sqlite3LoadExtension( } return SQLITE_ERROR; } - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); /* If no entry point was specified and the default legacy ** entry point name "sqlite3_extension_init" was not found, then @@ -527,8 +537,7 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry+iEntry, "_init", 6); zEntry = zAltEntry; - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ @@ -545,7 +554,9 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } sqlite3_free(zAltEntry); - if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -602,26 +613,15 @@ void sqlite3CloseExtensions(sqlite3 *db){ int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ sqlite3_mutex_enter(db->mutex); if( onoff ){ - db->flags |= SQLITE_LoadExtension; + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~SQLITE_LoadExtension; + db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - -/* -** The auto-extension code added regardless of whether or not extension -** loading is supported. We need a dummy sqlite3Apis pointer for that -** code if regular extension loading is not available. This is that -** dummy pointer. -*/ -#ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; -#endif - +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ /* ** The following object holds the list of automatically loaded @@ -656,7 +656,9 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -int sqlite3_auto_extension(void (*xInit)(void)){ +int sqlite3_auto_extension( + void (*xInit)(void) +){ int rc = SQLITE_OK; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -701,7 +703,9 @@ int sqlite3_auto_extension(void (*xInit)(void)){ ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -int sqlite3_cancel_auto_extension(void (*xInit)(void)){ +int sqlite3_cancel_auto_extension( + void (*xInit)(void) +){ #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); #endif @@ -750,7 +754,7 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ u32 i; int go = 1; int rc; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; wsdAutoextInit; if( wsdAutoext.nExt==0 ){ @@ -761,18 +765,22 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ char *zErrmsg; #if SQLITE_THREADSAFE sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; #endif sqlite3_mutex_enter(mutex); if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - wsdAutoext.aExt[i]; + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; diff --git a/src/main.c b/src/main.c index 456219afa1..eed82b0e66 100644 --- a/src/main.c +++ b/src/main.c @@ -789,6 +789,11 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ int rc; va_start(ap, op); switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } case SQLITE_DBCONFIG_LOOKASIDE: { void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ @@ -811,6 +816,8 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, + { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -921,6 +928,21 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ return db->lastRowid; } +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + /* ** Return the number of changes in the most recent call to sqlite3_exec(). */ @@ -1039,6 +1061,9 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } /* Force xDisconnect calls on all virtual tables */ disconnectAllVtab(db); @@ -1565,7 +1590,7 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ */ void sqlite3_interrupt(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db) && (db==0 || db->magic!=SQLITE_MAGIC_ZOMBIE) ){ (void)SQLITE_MISUSE_BKPT; return; } @@ -1807,7 +1832,8 @@ int sqlite3_overload_function( ** trace is a pointer to a function that is invoked at the start of each ** SQL statement. */ -void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ +#ifndef SQLITE_OMIT_DEPRECATED +void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ void *pOld; #ifdef SQLITE_ENABLE_API_ARMOR @@ -1818,11 +1844,38 @@ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ #endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; - db->xTrace = xTrace; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED /* ** Register a profile function. The pArg from the previously registered ** profile function is returned. @@ -1851,6 +1904,7 @@ void *sqlite3_profile( sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ #endif /* SQLITE_OMIT_TRACE */ /* @@ -1929,6 +1983,27 @@ void *sqlite3_rollback_hook( return pRet; } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + #ifndef SQLITE_OMIT_WAL /* ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). @@ -2054,6 +2129,13 @@ int sqlite3_wal_checkpoint_v2( sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); + + /* If there are no active statements, clear the interrupt flag at this + ** point. */ + if( db->nVdbeActive==0 ){ + db->u1.isInterrupted = 0; + } + sqlite3_mutex_leave(db->mutex); return rc; #endif @@ -2247,6 +2329,9 @@ int sqlite3_extended_errcode(sqlite3 *db){ } return db->errCode; } +int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} /* ** Return a string that describes the kind of error specified in the @@ -2553,6 +2638,7 @@ int sqlite3ParseUri( assert( octet>=0 && octet<256 ); if( octet==0 ){ +#ifndef SQLITE_ENABLE_URI_00_ERROR /* This branch is taken when "%00" appears within the URI. In this ** case we ignore all text in the remainder of the path, name or ** value currently being parsed. So ignore the current character @@ -2565,6 +2651,12 @@ int sqlite3ParseUri( iIn++; } continue; +#else + /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ + *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); + rc = SQLITE_ERROR; + goto parse_uri_out; +#endif } c = octet; }else if( eState==1 && (c=='&' || c=='=') ){ @@ -2669,7 +2761,9 @@ int sqlite3ParseUri( }else{ zFile = sqlite3_malloc64(nUri+2); if( !zFile ) return SQLITE_NOMEM_BKPT; - memcpy(zFile, zUri, nUri); + if( nUri ){ + memcpy(zFile, zUri, nUri); + } zFile[nUri] = '\0'; zFile[nUri+1] = '\0'; flags &= ~SQLITE_OPEN_URI; @@ -2884,9 +2978,9 @@ static int openDatabase( /* The default safety_level for the main database is FULL; for the temp ** database it is OFF. This matches the pager layer defaults. */ - db->aDb[0].zName = "main"; + db->aDb[0].zDbSName = "main"; db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - db->aDb[1].zName = "temp"; + db->aDb[1].zDbSName = "temp"; db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; db->magic = SQLITE_MAGIC_OPEN; @@ -2900,11 +2994,20 @@ static int openDatabase( */ sqlite3Error(db, SQLITE_OK); sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + +#ifdef SQLITE_ENABLE_FTS5 + /* Register any built-in FTS5 module before loading the automatic + ** extensions. This allows automatic extensions to register FTS5 + ** tokenizers and auxiliary functions. */ + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif /* Load automatic extensions - extensions that have been registered ** using the sqlite3_automatic_extension() API. */ - rc = sqlite3_errcode(db); if( rc==SQLITE_OK ){ sqlite3AutoLoadExtensions(db); rc = sqlite3_errcode(db); @@ -2933,12 +3036,6 @@ static int openDatabase( } #endif -#ifdef SQLITE_ENABLE_FTS5 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts5Init(db); - } -#endif - #ifdef SQLITE_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); @@ -3347,8 +3444,7 @@ int sqlite3_table_column_metadata( ** explicitly declared column. Copy meta information from *pCol. */ if( pCol ){ - zDataType = sqlite3StrNext(pCol->zName); - if( zDataType[0]==0 ) zDataType = 0; + zDataType = sqlite3ColumnType(pCol,0); zCollSeq = pCol->zColl; notnull = pCol->notNull!=0; primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; @@ -3461,7 +3557,7 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ */ int sqlite3_test_control(int op, ...){ int rc = 0; -#ifdef SQLITE_OMIT_BUILTIN_TEST +#ifdef SQLITE_UNTESTABLE UNUSED_PARAMETER(op); #else va_list ap; @@ -3727,6 +3823,15 @@ int sqlite3_test_control(int op, ...){ break; } + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); ** @@ -3789,7 +3894,7 @@ int sqlite3_test_control(int op, ...){ } } va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ +#endif /* SQLITE_UNTESTABLE */ return rc; } @@ -3845,15 +3950,8 @@ sqlite3_int64 sqlite3_uri_int64( ** Return the Btree pointer identified by zDbName. Return NULL if not found. */ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int i; - for(i=0; inDb; i++){ - if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) - ){ - return db->aDb[i].pBt; - } - } - return 0; + int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; + return iDb<0 ? 0 : db->aDb[iDb].pBt; } /* @@ -3900,7 +3998,6 @@ int sqlite3_snapshot_get( ){ int rc = SQLITE_ERROR; #ifndef SQLITE_OMIT_WAL - int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -3909,13 +4006,15 @@ int sqlite3_snapshot_get( #endif sqlite3_mutex_enter(db->mutex); - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } } } } @@ -3962,6 +4061,38 @@ int sqlite3_snapshot_open( return rc; } +/* +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. +*/ +int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; + int iDb; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } + } + } + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + /* ** Free a snapshot handle obtained from sqlite3_snapshot_get(). */ diff --git a/src/malloc.c b/src/malloc.c index 462d78e68c..6d49107790 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -217,11 +217,26 @@ static void sqlite3MallocAlarm(int nByte){ ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. */ -static int mallocWithAlarm(int n, void **pp){ - int nFull; +static void mallocWithAlarm(int n, void **pp){ void *p; + int nFull; assert( sqlite3_mutex_held(mem0.mutex) ); + assert( n>0 ); + + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); + +#ifdef SQLITE_MAX_MEMORY + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ + *pp = 0; + return; + } +#endif + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); @@ -245,7 +260,6 @@ static int mallocWithAlarm(int n, void **pp){ sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; - return nFull; } /* @@ -411,7 +425,7 @@ int sqlite3MallocSize(void *p){ int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); if( db==0 || !isLookaside(db,p) ){ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -472,7 +486,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ } if( isLookaside(db, p) ){ LookasideSlot *pBuf = (LookasideSlot*)p; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* Trash all content in the buffer being freed */ memset(p, 0xaa, db->lookaside.sz); #endif @@ -519,7 +533,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + if( nDiff>0 && sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } @@ -726,9 +740,8 @@ char *sqlite3DbStrDup(sqlite3 *db, const char *z){ if( z==0 ){ return 0; } - n = sqlite3Strlen30(z) + 1; - assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRaw(db, (int)n); + n = strlen(z) + 1; + zNew = sqlite3DbMallocRaw(db, n); if( zNew ){ memcpy(zNew, z, n); } diff --git a/src/mem1.c b/src/mem1.c index b960ccfd47..512ab3747f 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -57,7 +57,9 @@ */ #include #include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE #include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); @@ -125,7 +127,9 @@ static malloc_zone_t* _sqliteZone_; */ static void *sqlite3MemMalloc(int nByte){ #ifdef SQLITE_MALLOCSIZE - void *p = SQLITE_MALLOC( nByte ); + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); if( p==0 ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); @@ -134,7 +138,7 @@ static void *sqlite3MemMalloc(int nByte){ #else sqlite3_int64 *p; assert( nByte>0 ); - nByte = ROUND8(nByte); + testcase( ROUND8(nByte)!=nByte ); p = SQLITE_MALLOC( nByte+8 ); if( p ){ p[0] = nByte; @@ -248,19 +252,10 @@ static int sqlite3MemInit(void *NotUsed){ }else{ /* only 1 core, use our own zone to contention over global locks, ** e.g. we have our own dedicated locks */ - bool success; - malloc_zone_t* newzone = malloc_create_zone(4096, 0); - malloc_set_zone_name(newzone, "Sqlite_Heap"); - do{ - success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, - (void * volatile *)&_sqliteZone_); - }while(!_sqliteZone_); - if( !success ){ - /* somebody registered a zone first */ - malloc_destroy_zone(newzone); - } + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); } -#endif +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ UNUSED_PARAMETER(NotUsed); return SQLITE_OK; } diff --git a/src/memjournal.c b/src/memjournal.c index b81682de85..cd8b87d8aa 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -13,6 +13,15 @@ ** This file contains code use to implement an in-memory rollback journal. ** The in-memory rollback journal is used to journal transactions for ** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** +** Update: The in-memory journal is also used to temporarily cache +** smaller journals that are not critical for power-loss recovery. +** For example, statement journals that are not too big will be held +** entirely in memory, thus reducing the number of file I/O calls, and +** more importantly, reducing temporary file creation events. If these +** journals become too large for memory, they are spilled to disk. But +** in the common case, they are usually small and no file I/O needs to +** occur. */ #include "sqliteInt.h" @@ -94,6 +103,7 @@ static int memjrnlRead( #endif assert( (iAmt+iOfst)<=p->endpoint.iOffset ); + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; for(pChunk=p->pFirst; @@ -104,6 +114,7 @@ static int memjrnlRead( } }else{ pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); } iChunkOffset = (int)(iOfst%p->nChunkSize); @@ -115,7 +126,7 @@ static int memjrnlRead( nRead -= iSpace; iChunkOffset = 0; } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); - p->readpoint.iOffset = iOfst+iAmt; + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; p->readpoint.pChunk = pChunk; return SQLITE_OK; @@ -403,5 +414,5 @@ int sqlite3JournalIsInMemory(sqlite3_file *p){ ** pVfs to create the underlying on-disk files. */ int sqlite3JournalSize(sqlite3_vfs *pVfs){ - return MAX(pVfs->szOsFile, sizeof(MemJournal)); + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); } diff --git a/src/msvc.h b/src/msvc.h index 01ebf2b46f..3914b05bcf 100644 --- a/src/msvc.h +++ b/src/msvc.h @@ -12,8 +12,8 @@ ** ** This file contains code that is specific to MSVC. */ -#ifndef _MSVC_H_ -#define _MSVC_H_ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H #if defined(_MSC_VER) #pragma warning(disable : 4054) @@ -33,4 +33,4 @@ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ -#endif /* _MSVC_H_ */ +#endif /* SQLITE_MSVC_H */ diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9570bdc0bf..9da93cf319 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -87,8 +87,7 @@ void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ - defined(_MSC_VER) && _MSC_VER>=1300 +#elif MSVC_VERSION>=1300 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); @@ -299,8 +298,8 @@ static void winMutexEnter(sqlite3_mutex *p){ p->owner = tid; p->nRef++; if( p->trace ){ - OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } @@ -342,8 +341,8 @@ static int winMutexTry(sqlite3_mutex *p){ #endif #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", - tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + OSTRACE(("TRY-MUTEX tid=%lu, mutex(%d)=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p->id, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -371,8 +370,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } diff --git a/src/os.c b/src/os.c index eed7828538..5cf0014794 100644 --- a/src/os.c +++ b/src/os.c @@ -13,9 +13,7 @@ ** This file contains OS interface code that is common to all ** architectures. */ -#define _SQLITE_OS_C_ 1 #include "sqliteInt.h" -#undef _SQLITE_OS_C_ /* ** If we compile with the SQLITE_TEST macro set, then the following block @@ -81,13 +79,11 @@ int sqlite3_memdebug_vfs_oom_test = 1; ** of this would be completely automatic if SQLite were coded using ** C++ instead of plain old C. */ -int sqlite3OsClose(sqlite3_file *pId){ - int rc = SQLITE_OK; +void sqlite3OsClose(sqlite3_file *pId){ if( pId->pMethods ){ - rc = pId->pMethods->xClose(pId); + pId->pMethods->xClose(pId); pId->pMethods = 0; } - return rc; } int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ DO_OS_MALLOC_TEST(id); @@ -262,6 +258,9 @@ int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } +int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() @@ -302,12 +301,10 @@ int sqlite3OsOpenMalloc( } return rc; } -int sqlite3OsCloseFree(sqlite3_file *pFile){ - int rc = SQLITE_OK; +void sqlite3OsCloseFree(sqlite3_file *pFile){ assert( pFile ); - rc = sqlite3OsClose(pFile); + sqlite3OsClose(pFile); sqlite3_free(pFile); - return rc; } /* diff --git a/src/os.h b/src/os.h index 2c1b86f913..947f88b36e 100644 --- a/src/os.h +++ b/src/os.h @@ -160,7 +160,7 @@ int sqlite3OsInit(void); /* ** Functions for accessing sqlite3_file methods */ -int sqlite3OsClose(sqlite3_file*); +void sqlite3OsClose(sqlite3_file*); int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); int sqlite3OsTruncate(sqlite3_file*, i64 size); @@ -197,6 +197,7 @@ void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ int sqlite3OsRandomness(sqlite3_vfs *, int, char *); int sqlite3OsSleep(sqlite3_vfs *, int); +int sqlite3OsGetLastError(sqlite3_vfs*); int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* @@ -204,6 +205,6 @@ int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); ** sqlite3_malloc() to obtain space for the file-handle structure. */ int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -int sqlite3OsCloseFree(sqlite3_file *); +void sqlite3OsCloseFree(sqlite3_file *); #endif /* _SQLITE_OS_H_ */ diff --git a/src/os_setup.h b/src/os_setup.h index 68de1446ed..08aaa1195a 100644 --- a/src/os_setup.h +++ b/src/os_setup.h @@ -13,8 +13,8 @@ ** This file contains pre-processor directives related to operating system ** detection and/or setup. */ -#ifndef _OS_SETUP_H_ -#define _OS_SETUP_H_ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H /* ** Figure out if we are dealing with Unix, Windows, or some other operating @@ -54,4 +54,4 @@ # endif #endif -#endif /* _OS_SETUP_H_ */ +#endif /* SQLITE_OS_SETUP_H */ diff --git a/src/os_unix.c b/src/os_unix.c index d6fbd80588..7f0ebdba6f 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -405,7 +405,7 @@ static struct unix_syscall { #else { "pread64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) +#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) @@ -423,7 +423,7 @@ static struct unix_syscall { #else { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off_t))\ +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, @@ -1061,7 +1061,14 @@ struct unixFileId { #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID for vxworks. */ #else - ino_t ino; /* Inode number */ + /* We are told that some versions of Android contain a bug that + ** sizes ino_t at only 32-bits instead of 64-bits. (See + ** https://android-review.googlesource.com/#/c/115351/3/dist/sqlite3.c) + ** To work around this, always allocate 64-bits for the inode number. + ** On small machines that only have 32-bit inodes, this wastes 4 bytes, + ** but that should not be a big deal. */ + /* WAS: ino_t ino; */ + u64 ino; /* Inode number */ #endif }; @@ -1306,7 +1313,7 @@ static int findInodeInfo( #if OS_VXWORKS fileId.pId = pFile->pId; #else - fileId.ino = statbuf.st_ino; + fileId.ino = (u64)statbuf.st_ino; #endif pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ @@ -1340,7 +1347,8 @@ static int fileHasMoved(unixFile *pFile){ #else struct stat buf; return pFile->pInode!=0 && - (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); + (osStat(pFile->zPath, &buf)!=0 + || (u64)buf.st_ino!=pFile->pInode->fileId.ino); #endif } @@ -1357,12 +1365,16 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); return; } - if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + if( buf.st_nlink==0 ){ sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); return; } @@ -1498,7 +1510,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** lock transitions in terms of the POSIX advisory shared and exclusive ** lock primitives (called read-locks and write-locks below, to avoid ** confusion with SQLite lock names). The algorithms are complicated - ** slightly in order to be compatible with windows systems simultaneously + ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved @@ -1506,8 +1518,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range', a range of 510 bytes at a well known offset. ** ** To obtain a SHARED lock, a read-lock is obtained on the 'pending - ** byte'. If this is successful, a random byte from the 'shared byte - ** range' is read-locked and the lock on the 'pending byte' released. + ** byte'. If this is successful, 'shared byte range' is read-locked + ** and the lock on the 'pending byte' released. (Legacy note: When + ** SQLite was first developed, Windows95 systems were still very common, + ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** single randomly selected by from the 'shared byte range' is locked. + ** Windows95 is now pretty much extinct, but this work-around for the + ** lack of shared-locks on Windows95 lives on, for backwards + ** compatibility.) ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the @@ -1526,11 +1544,6 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** range'. Since all other locks require a read-lock on one of the bytes ** within this range, this ensures that no other locks are held on the ** database. - ** - ** The reason a single byte cannot be used instead of the 'shared byte - ** range' is that some versions of windows do not support read-locks. By - ** locking a random byte from a range, concurrent SHARED locks may exist - ** even if the locking primitive used is always a write-lock. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -4283,10 +4296,12 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } } if( pInode->bProcessLock==0 ){ @@ -5405,20 +5420,24 @@ static const char *unixTempFileDir(void){ "/tmp", "." }; - unsigned int i; + unsigned int i = 0; struct stat buf; const char *zDir = sqlite3_temp_directory; if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - for(i=0; i=sizeof(azDirs)/sizeof(azDirs[0]) ) break; + zDir = azDirs[i++]; } - return zDir; + return 0; } /* @@ -5434,9 +5453,11 @@ static int unixGetTempname(int nBuf, char *zBuf){ ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ + zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; do{ u64 r; sqlite3_randomness(sizeof(r), &r); @@ -5499,7 +5520,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ unixEnterMutex(); pInode = inodeList; while( pInode && (pInode->fileId.dev!=sStat.st_dev - || pInode->fileId.ino!=sStat.st_ino) ){ + || pInode->fileId.ino!=(u64)sStat.st_ino) ){ pInode = pInode->pNext; } if( pInode ){ @@ -5516,6 +5537,27 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ return pUnused; } +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + /* ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned @@ -5551,7 +5593,6 @@ static int findCreateFileMode( if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ - struct stat sStat; /* Output of stat() on database file */ /* zPath is a path to a WAL or journal file. The following block derives ** the path to the associated database file from zPath. This block handles @@ -5582,15 +5623,18 @@ static int findCreateFileMode( memcpy(zDb, zPath, nDb); zDb[nDb] = '\0'; - if( 0==osStat(zDb, &sStat) ){ - *pMode = sStat.st_mode & 0777; - *pUid = sStat.st_uid; - *pGid = sStat.st_gid; - }else{ - rc = SQLITE_IOERR_FSTAT; - } + rc = getFileMode(zDb, pMode, pUid, pGid); }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } } return rc; } @@ -5803,9 +5847,6 @@ static int unixOpen( p->openFlags = openFlags; } #endif - - noLock = eType!=SQLITE_OPEN_MAIN_DB; - #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ @@ -5824,6 +5865,7 @@ static int unixOpen( /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -6262,23 +6304,18 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ # define unixCurrentTime 0 #endif -#ifndef SQLITE_OMIT_DEPRECATED /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return 0; + return errno; } -#else -# define unixGetLastError 0 -#endif /* diff --git a/src/os_win.c b/src/os_win.c index af6b1c814c..a87d7d0925 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -290,6 +290,17 @@ struct winFile { #endif }; +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + /* ** Allowed values for winFile.ctrlFlags */ @@ -341,22 +352,72 @@ struct winFile { ****************************************************************************** */ #ifndef SQLITE_WIN32_HEAP_CREATE -# define SQLITE_WIN32_HEAP_CREATE (TRUE) +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) +#endif + +/* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) #endif /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_DEFAULT_CACHE_SIZE) * \ - (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) #endif /* * The maximum size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_MAX_SIZE -# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) #endif /* @@ -364,7 +425,7 @@ struct winFile { * zero for the default behavior. */ #ifndef SQLITE_WIN32_HEAP_FLAGS -# define SQLITE_WIN32_HEAP_FLAGS (0) +# define SQLITE_WIN32_HEAP_FLAGS (0) #endif @@ -1248,8 +1309,8 @@ int sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); @@ -1294,6 +1355,12 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ assert( nMin==-1 || nMin==0 || nMin0 ){ memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); @@ -1619,147 +1686,244 @@ void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode. ** -** Space to hold the returned string is obtained from malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winUtf8ToUnicode(const char *zFilename){ +static LPWSTR winUtf8ToUnicode(const char *zText){ int nChar; - LPWSTR zWideFilename; + LPWSTR zWideText; - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); if( nChar==0 ){ return 0; } - zWideFilename = sqlite3MallocZero( nChar*sizeof(zWideFilename[0]) ); - if( zWideFilename==0 ){ + zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); + if( zWideText==0 ){ return 0; } - nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); if( nChar==0 ){ - sqlite3_free(zWideFilename); - zWideFilename = 0; + sqlite3_free(zWideText); + zWideText = 0; } - return zWideFilename; + return zWideText; } /* -** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is -** obtained from sqlite3_malloc(). +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ +static char *winUnicodeToUtf8(LPCWSTR zWideText){ int nByte; - char *zFilename; + char *zText; - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideFilename, -1, zFilename, nByte, + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert an ANSI string to Microsoft Unicode, based on the -** current codepage settings for file apis. +** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM +** code page. ** -** Space to hold the returned string is obtained -** from sqlite3_malloc. +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static LPWSTR winMbcsToUnicode(const char *zFilename){ +static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ int nByte; - LPWSTR zMbcsFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + LPWSTR zMbcsText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, NULL, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, 0)*sizeof(WCHAR); if( nByte==0 ){ return 0; } - zMbcsFilename = sqlite3MallocZero( nByte*sizeof(zMbcsFilename[0]) ); - if( zMbcsFilename==0 ){ + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zFilename, -1, zMbcsFilename, + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, nByte); if( nByte==0 ){ - sqlite3_free(zMbcsFilename); - zMbcsFilename = 0; + sqlite3_free(zMbcsText); + zMbcsText = 0; } - return zMbcsFilename; + return zMbcsText; } /* -** Convert Microsoft Unicode to multi-byte character string, based on the -** user's ANSI codepage. +** Convert a Microsoft Unicode string to a multi-byte character string, +** using the ANSI or OEM code page. ** -** Space to hold the returned string is obtained from -** sqlite3_malloc(). +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ +static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ int nByte; - char *zFilename; - int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; + char *zText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, 0, 0, 0, 0); + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); if( nByte == 0 ){ return 0; } - zFilename = sqlite3MallocZero( nByte ); - if( zFilename==0 ){ + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ return 0; } - nByte = osWideCharToMultiByte(codepage, 0, zWideFilename, -1, zFilename, + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, nByte, 0, 0); if( nByte == 0 ){ - sqlite3_free(zFilename); - zFilename = 0; + sqlite3_free(zText); + zText = 0; } - return zFilename; + return zText; } /* -** Convert multibyte character string to UTF-8. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a multi-byte character string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ - char *zFilenameUtf8; +static char *winMbcsToUtf8(const char *zText, int useAnsi){ + char *zTextUtf8; LPWSTR zTmpWide; - zTmpWide = winMbcsToUnicode(zFilename); + zTmpWide = winMbcsToUnicode(zText, useAnsi); if( zTmpWide==0 ){ return 0; } - zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); + zTextUtf8 = winUnicodeToUtf8(zTmpWide); sqlite3_free(zTmpWide); - return zFilenameUtf8; + return zTextUtf8; } /* -** Convert UTF-8 to multibyte character string. Space to hold the -** returned string is obtained from sqlite3_malloc(). +** Convert a UTF-8 string to a multi-byte character string. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). */ -char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ - char *zFilenameMbcs; +static char *winUtf8ToMbcs(const char *zText, int useAnsi){ + char *zTextMbcs; LPWSTR zTmpWide; - zTmpWide = winUtf8ToUnicode(zFilename); + zTmpWide = winUtf8ToUnicode(zText); if( zTmpWide==0 ){ return 0; } - zFilenameMbcs = winUnicodeToMbcs(zTmpWide); + zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); sqlite3_free(zTmpWide); - return zFilenameMbcs; + return zTextMbcs; +} + +/* +** This is a public wrapper for the winUtf8ToUnicode() function. +*/ +LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToUnicode(zText); +} + +/* +** This is a public wrapper for the winUnicodeToUtf8() function. +*/ +char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zWideText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUnicodeToUtf8(zWideText); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, useAnsi); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, useAnsi); } /* @@ -1861,7 +2025,7 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3EndBenignMalloc(); /* free the system buffer allocated by FormatMessage */ osLocalFree(zTemp); @@ -2003,16 +2167,17 @@ static void winLogIoerr(int nRetry, int lineno){ } } -#if SQLITE_OS_WINCE -/************************************************************************* -** This section contains code for WinCE only. -*/ -#if !defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API /* -** The MSVC CRT on Windows CE may not have a localtime() function. So -** create a substitute. +** This #if does not rely on the SQLITE_OS_WINCE define because the +** corresponding section in "date.c" cannot use it. */ -#include +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So define a substitute. +*/ +# include struct tm *__cdecl localtime(const time_t *t) { static struct tm y; @@ -2036,6 +2201,10 @@ struct tm *__cdecl localtime(const time_t *t) } #endif +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ #define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] /* @@ -2491,7 +2660,12 @@ static int winClose(sqlite3_file *id){ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 - winceDestroyLock(pFile); + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } if( pFile->zDeleteOnClose ){ int cnt = 0; while( @@ -3049,9 +3223,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( (pFile->locktype==NO_LOCK) - || ( (locktype==EXCLUSIVE_LOCK) - && (pFile->locktype==RESERVED_LOCK)) + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) ){ int cnt = 3; while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, @@ -3224,6 +3397,44 @@ static int winUnlock(sqlite3_file *id, int locktype){ return rc; } +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + /* ** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. @@ -3257,7 +3468,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { + case SQLITE_FCNTL_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -3315,6 +3526,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } #ifdef SQLITE_TEST case SQLITE_FCNTL_WIN32_SET_HANDLE: { LPHANDLE phFile = (LPHANDLE)pArg; @@ -3502,12 +3719,12 @@ struct winShm { /* ** Apply advisory locks for all n bytes beginning at ofst. */ -#define _SHM_UNLCK 1 -#define _SHM_RDLCK 2 -#define _SHM_WRLCK 3 +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 static int winShmSystemLock( winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ int ofst, /* Offset to first byte to be locked/unlocked */ int nByte /* Number of bytes to lock or unlock */ ){ @@ -3520,12 +3737,12 @@ static int winShmSystemLock( pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ - if( lockType==_SHM_UNLCK ){ + if( lockType==WINSHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); }else{ /* Initialize the locking parameters */ DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } @@ -3537,7 +3754,7 @@ static int winShmSystemLock( } OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; @@ -3645,10 +3862,12 @@ static int winOpenSharedMemory(winFile *pDbFd){ pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } } rc = winOpen(pDbFd->pVfs, @@ -3663,7 +3882,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ - if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ + if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); if( rc!=SQLITE_OK ){ rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), @@ -3671,8 +3890,8 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); } if( rc ) goto shm_open_err; } @@ -3701,7 +3920,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ /* Jump here on any error */ shm_open_err: - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ sqlite3_free(p); sqlite3_free(pNew); @@ -3790,7 +4009,7 @@ static int winShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -3818,7 +4037,7 @@ static int winShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -3843,7 +4062,7 @@ static int winShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -4286,6 +4505,44 @@ static const sqlite3_io_methods winIoMethod = { winUnfetch /* xUnfetch */ }; +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + /**************************************************************************** **************************** sqlite3_vfs methods **************************** ** @@ -4306,7 +4563,7 @@ static char *winConvertToUtf8Filename(const void *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_mbcs_to_utf8(zFilename); + zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -4327,7 +4584,7 @@ static void *winConvertFromUtf8Filename(const char *zFilename){ } #ifdef SQLITE_WIN32_HAS_ANSI else{ - zConverted = sqlite3_win32_utf8_to_mbcs(zFilename); + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); } #endif /* caller will handle out of memory */ @@ -4528,7 +4785,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ return winLogError(SQLITE_IOERR_GETTEMPPATH, osGetLastError(), "winGetTempname3", 0); } - zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); + zUtf8 = winMbcsToUtf8(zMbcsPath, osAreFileApisANSI()); if( zUtf8 ){ sqlite3_snprintf(nMax, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); @@ -4618,7 +4875,7 @@ static int winIsDir(const void *zConverted){ ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Used to get maximum path name length */ + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -4633,6 +4890,7 @@ static int winOpen( #if SQLITE_OS_WINCE int isTemp = 0; #endif + winVfsAppData *pAppData; winFile *pFile = (winFile*)id; void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ @@ -4854,15 +5112,20 @@ static int winOpen( "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + pAppData = (winVfsAppData*)pVfs->pAppData; + #if SQLITE_OS_WINCE - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK - ){ - osCloseHandle(h); - sqlite3_free(zConverted); - sqlite3_free(zTmpname); - OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); - return rc; + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -4873,7 +5136,7 @@ static int winOpen( } sqlite3_free(zTmpname); - pFile->pMethod = &winIoMethod; + pFile->pMethod = pAppData ? pAppData->pMethod : &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; if( isReadonly ){ @@ -5148,6 +5411,18 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:", where "X" is any alphabetic + ** character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + zRelative++; + } #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); @@ -5226,17 +5501,6 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; - void *zConverted; - char *zOut; - - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. - */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ - zRelative++; - } - /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -5306,7 +5570,7 @@ static int winFullPathname( "winFullPathname4", zRelative); } sqlite3_free(zConverted); - zOut = sqlite3_win32_mbcs_to_utf8(zTemp); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); sqlite3_free(zTemp); } #endif @@ -5424,8 +5688,8 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ EntropyGatherer e; UNUSED_PARAMETER(pVfs); memset(zBuf, 0, nBuf); -#if defined(_MSC_VER) && _MSC_VER>=1400 - rand_s((int*)zBuf); /* rand_s() is not available with MinGW */ +#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE + rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ #endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ e.a = (unsigned char*)zBuf; e.na = nBuf; @@ -5584,8 +5848,10 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; } /* @@ -5593,53 +5859,103 @@ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ */ int sqlite3_os_init(void){ static sqlite3_vfs winVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #if defined(SQLITE_WIN32_HAS_WIDE) static sqlite3_vfs winLongPathVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #endif @@ -5663,6 +5979,12 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + return SQLITE_OK; } diff --git a/src/os_win.h b/src/os_win.h index 17d6a2bef4..27714ed079 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -12,8 +12,8 @@ ** ** This file contains code that is specific to Windows. */ -#ifndef _OS_WIN_H_ -#define _OS_WIN_H_ +#ifndef SQLITE_OS_WIN_H +#define SQLITE_OS_WIN_H /* ** Include the primary Windows SDK header file. @@ -85,4 +85,4 @@ # define SQLITE_OS_WIN_THREADS 0 #endif -#endif /* _OS_WIN_H_ */ +#endif /* SQLITE_OS_WIN_H */ diff --git a/src/pager.c b/src/pager.c index c18b3a32f7..40c4dd9d88 100644 --- a/src/pager.c +++ b/src/pager.c @@ -693,6 +693,7 @@ struct Pager { int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ #ifdef SQLITE_HAS_CODEC void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ @@ -813,13 +814,20 @@ static const unsigned char aJournalMagic[] = { #define isOpen(pFd) ((pFd)->pMethods!=0) /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. +** Return true if this pager uses a write-ahead log to read page pgno. +** Return false if the pager reads pgno directly from the database. */ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ) +int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){ + u32 iRead = 0; + int rc; + if( pPager->pWal==0 ) return 0; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return rc || iRead; } +#endif +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 @@ -872,6 +880,7 @@ static int assert_pager_state(Pager *p){ ** state. */ if( MEMDB ){ + assert( !isOpen(p->fd) ); assert( p->noSync ); assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->journalMode==PAGER_JOURNALMODE_MEMORY @@ -958,7 +967,7 @@ static int assert_pager_state(Pager *p){ ** back to OPEN state. */ assert( pPager->errCode!=SQLITE_OK ); - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); break; } @@ -1017,6 +1026,33 @@ static char *print_pager_state(Pager *p){ } #endif +/* Forward references to the various page getters */ +static int getPageNormal(Pager*,Pgno,DbPage**,int); +static int getPageError(Pager*,Pgno,DbPage**,int); +#if SQLITE_MAX_MMAP_SIZE>0 +static int getPageMMap(Pager*,Pgno,DbPage**,int); +#endif + +/* +** Set the Pager.xGet method for the appropriate routine used to fetch +** content from the pager. +*/ +static void setGetterMethod(Pager *pPager){ + if( pPager->errCode ){ + pPager->xGet = getPageError; +#if SQLITE_MAX_MMAP_SIZE>0 + }else if( USEFETCH(pPager) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif + ){ + pPager->xGet = getPageMMap; +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ + pPager->xGet = getPageNormal; + } +} + /* ** Return true if it is necessary to write page *pPg into the sub-journal. ** A page needs to be written into the sub-journal if there exists one @@ -1170,6 +1206,8 @@ static int jrnlBufferSize(Pager *pPager){ return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); } +#else +# define jrnlBufferSize(x) 0 #endif /* @@ -1818,13 +1856,18 @@ static void pager_unlock(Pager *pPager){ ** it can safely move back to PAGER_OPEN state. This happens in both ** normal and exclusive-locking mode. */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); if( pPager->errCode ){ - assert( !MEMDB ); - pager_reset(pPager); - pPager->changeCountDone = pPager->tempFile; - pPager->eState = PAGER_OPEN; - pPager->errCode = SQLITE_OK; + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; + setGetterMethod(pPager); } pPager->journalOff = 0; @@ -1862,12 +1905,36 @@ static int pager_error(Pager *pPager, int rc){ if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); } return rc; } static int pager_truncate(Pager *pPager, Pgno nPage); +/* +** The write transaction open on pPager is being committed (bCommit==1) +** or rolled back (bCommit==0). +** +** Return TRUE if and only if all dirty pages should be flushed to disk. +** +** Rules: +** +** * For non-TEMP databases, always sync to disk. This is necessary +** for transactions to be durable. +** +** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing +** file has been created already (via a spill on pagerStress()) and +** when the number of dirty pages in memory exceeds 25% of the total +** cache size. +*/ +static int pagerFlushOnCommit(Pager *pPager, int bCommit){ + if( pPager->tempFile==0 ) return 1; + if( !bCommit ) return 0; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called @@ -1971,7 +2038,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ - rc = zeroJournalHdr(pPager, hasMaster); + rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if @@ -2006,8 +2073,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; - sqlite3PcacheCleanAll(pPager->pPCache); - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + if( rc==SQLITE_OK ){ + if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } if( pagerUseWal(pPager) ){ /* Drop the WAL write-lock, if any. Also, if the connection was in @@ -2291,7 +2364,7 @@ static int pager_playback_one_page( pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); - assert( pPager->eState!=PAGER_OPEN || pPg==0 ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") @@ -2341,7 +2414,6 @@ static int pager_playback_one_page( assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; if( rc!=SQLITE_OK ) return rc; - pPg->flags &= ~PGHDR_NEED_READ; sqlite3PcacheMakeDirty(pPg); } if( pPg ){ @@ -2355,29 +2427,10 @@ static int pager_playback_one_page( pData = pPg->pData; memcpy(pData, (u8*)aData, pPager->pageSize); pPager->xReiniter(pPg); - if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){ - /* If the contents of this page were just restored from the main - ** journal file, then its content must be as they were when the - ** transaction was first opened. In this case we can mark the page - ** as clean, since there will be no need to write it out to the - ** database. - ** - ** There is one exception to this rule. If the page is being rolled - ** back as part of a savepoint (or statement) rollback from an - ** unsynced portion of the main journal file, then it is not safe - ** to mark the page as clean. This is because marking the page as - ** clean will clear the PGHDR_NEED_SYNC flag. Since the page is - ** already in the journal file (recorded in Pager.pInJournal) and - ** the PGHDR_NEED_SYNC flag is cleared, if the page is written to - ** again within this transaction, it will be marked as dirty but - ** the PGHDR_NEED_SYNC flag will not be set. It could then potentially - ** be written out into the database file before its journal file - ** segment is synced. If a crash occurs during or following this, - ** database corruption may ensue. - */ - assert( !pagerUseWal(pPager) ); - sqlite3PcacheMakeClean(pPg); - } + /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But + ** that call was dangerous and had no detectable benefit since the cache + ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so + ** has been removed. */ pager_set_pagehash(pPg); /* If this was page 1, then restore the value of Pager.dbFileVers. @@ -3168,6 +3221,8 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the @@ -3175,14 +3230,11 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ - if( nPage==0 ){ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ i64 n = 0; /* Size of db file in bytes */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - int rc = sqlite3OsFileSize(pPager->fd, &n); - if( rc!=SQLITE_OK ){ - return rc; - } + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; } nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); } @@ -3421,6 +3473,7 @@ static void pagerFixMaplimit(Pager *pPager){ sqlite3_int64 sz; sz = pPager->szMmap; pPager->bUseFetch = (sz>0); + setGetterMethod(pPager); sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); } #endif @@ -3917,6 +3970,7 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Obtain a reference to a memory mapped page object for page number pgno. ** The new object will use the pointer pData, obtained from xFetch(). @@ -3939,7 +3993,8 @@ static int pagerAcquireMapPage( *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); + assert( pPager->nExtra>=8 ); + memset(p->pExtra, 0, 8); }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ @@ -3964,6 +4019,7 @@ static int pagerAcquireMapPage( return SQLITE_OK; } +#endif /* ** Release a reference to page pPg. pPg must have been returned by an @@ -4006,9 +4062,10 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** a hot journal may be left in the filesystem but no error is returned ** to the caller. */ -int sqlite3PagerClose(Pager *pPager){ +int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ u8 *pTmp = (u8 *)pPager->pTmpSpace; + assert( db || pagerUseWal(pPager)==0 ); assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); @@ -4016,7 +4073,10 @@ int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); + assert( db || pPager->pWal==0 ); + sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize, + (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) + ); pPager->pWal = 0; #endif pager_reset(pPager); @@ -4258,8 +4318,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch @@ -4534,7 +4595,9 @@ int sqlite3PagerFlush(Pager *pPager){ ** ** The nExtra parameter specifies the number of bytes of space allocated ** along with each page reference. This space is available to the user -** via the sqlite3PagerGetExtra() API. +** via the sqlite3PagerGetExtra() API. When a new page is allocated, the +** first 8 bytes of this space are zeroed but the remainder is uninitialized. +** (The extra space is used by btree as the MemPage object.) ** ** The flags argument is used to specify properties that affect the ** operation of the pager. It should be passed some bitwise combination @@ -4764,8 +4827,8 @@ act_like_temp_file: /* Initialize the PCache object. */ if( rc==SQLITE_OK ){ - assert( nExtra<1000 ); nExtra = ROUND8(nExtra); + assert( nExtra>=8 && nExtra<1000 ); rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } @@ -4830,6 +4893,7 @@ act_like_temp_file: /* pPager->xBusyHandler = 0; */ /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; + setGetterMethod(pPager); /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ @@ -4927,6 +4991,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ + assert( pPager->tempFile==0 ); rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ /* If the database is zero pages in size, that means that either (1) the @@ -5019,17 +5084,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either ** be OPEN or READER. READER is only possible if the pager is or was in - ** exclusive access mode. - */ + ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); - if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } + assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ int bHotJournal = 1; /* True if there exists a hot journal-file */ assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -5115,7 +5180,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ - rc = pager_playback(pPager, 1); + rc = pager_playback(pPager, !pPager->tempFile); pPager->eState = PAGER_OPEN; } }else if( !pPager->exclusiveMode ){ @@ -5211,7 +5276,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerBeginReadTransaction(pPager); } - if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ rc = pagerPagecount(pPager, &pPager->dbSize); } @@ -5242,10 +5307,17 @@ static void pagerUnlockIfUnused(Pager *pPager){ } /* -** Acquire a reference to page number pgno in pager pPager (a page -** reference has type DbPage*). If the requested reference is +** The page getter methods each try to acquire a reference to a +** page with page number pgno. If the requested reference is ** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. ** +** There are different implementations of the getter method depending +** on the current state of the pager. +** +** getPageNormal() -- The normal getter +** getPageError() -- Used if the pager is in an error state +** getPageMmap() -- Used if memory-mapped I/O is enabled +** ** If the requested page is already in the cache, it is returned. ** Otherwise, a new page object is allocated and populated with data ** read from the database file. In some cases, the pcache module may @@ -5257,14 +5329,14 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** already in the cache when this function is called, then the extra ** data is left as it was when the page object was last used. ** -** If the database image is smaller than the requested page or if a -** non-zero value is passed as the noContent parameter and the +** If the database image is smaller than the requested page or if +** the flags parameter contains the PAGER_GET_NOCONTENT bit and the ** requested page is not already stored in the cache, then no ** actual disk read occurs. In this case the memory image of the ** page is initialized to all zeros. ** -** If noContent is true, it means that we do not care about the contents -** of the page. This occurs in two scenarios: +** If PAGER_GET_NOCONTENT is true, it means that we do not care about +** the contents of the page. This occurs in two scenarios: ** ** a) When reading a free-list leaf page from the database, and ** @@ -5272,8 +5344,8 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** a new page into the cache to be filled with the data read ** from the savepoint journal. ** -** If noContent is true, then the data returned is zeroed instead of -** being read from the database. Additionally, the bits corresponding +** If PAGER_GET_NOCONTENT is true, then the data returned is zeroed instead +** of being read from the database. Additionally, the bits corresponding ** to pgno in Pager.pInJournal (bitvec of pages already written to the ** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open ** savepoints are set. This means if the page is made writable at any @@ -5291,106 +5363,39 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** Since Lookup() never goes to disk, it never has to deal with locks ** or journal files. */ -int sqlite3PagerGet( +static int getPageNormal( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ int flags /* PAGER_GET_XXX flags */ ){ int rc = SQLITE_OK; - PgHdr *pPg = 0; - u32 iFrame = 0; /* Frame to read from WAL file */ - const int noContent = (flags & PAGER_GET_NOCONTENT); + PgHdr *pPg; + u8 noContent; /* True if PAGER_GET_NOCONTENT is set */ + sqlite3_pcache_page *pBase; - /* It is acceptable to use a read-only (mmap) page for any page except - ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY - ** flag was specified by the caller. And so long as the db is not a - ** temporary or in-memory database. */ - const int bMmapOk = (pgno>1 && USEFETCH(pPager) - && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) -#ifdef SQLITE_HAS_CODEC - && pPager->xCodec==0 -#endif - ); - - /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here - ** allows the compiler optimizer to reuse the results of the "pgno>1" - ** test in the previous statement, and avoid testing pgno==0 in the - ** common case where pgno is large. */ - if( pgno<=1 && pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pPager->errCode==SQLITE_OK ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( noContent==0 || bMmapOk==0 ); - assert( pPager->hasHeldSharedLock==1 ); - /* If the pager is in the error state, return an error immediately. - ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK ){ - rc = pPager->errCode; - }else{ - if( bMmapOk && pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - } - - if( bMmapOk && iFrame==0 ){ - void *pData = 0; - - rc = sqlite3OsFetch(pPager->fd, - (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData - ); - - if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER ){ - pPg = sqlite3PagerLookup(pPager, pgno); - } - if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); - }else{ - sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); - } - if( pPg ){ - assert( rc==SQLITE_OK ); - *ppPage = pPg; - return SQLITE_OK; - } - } - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - } - - { - sqlite3_pcache_page *pBase; - pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); - if( pBase==0 ){ - rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pBase==0 ){ - pPg = *ppPage = 0; - rc = SQLITE_NOMEM_BKPT; - goto pager_acquire_err; - } - } - pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); - assert( pPg!=0 ); - } - } - - if( rc!=SQLITE_OK ){ - /* Either the call to sqlite3PcacheFetch() returned an error or the - ** pager was already in the error-state when this function was called. - ** Set pPg to 0 and jump to the exception handler. */ + if( pgno==0 ) return SQLITE_CORRUPT_BKPT; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ pPg = 0; - goto pager_acquire_err; + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto pager_acquire_err; + } } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); assert( pPg==(*ppPage) ); assert( pPg->pgno==pgno ); assert( pPg->pPager==pPager || pPg->pPager==0 ); + noContent = (flags & PAGER_GET_NOCONTENT)!=0; if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ @@ -5400,18 +5405,20 @@ int sqlite3PagerGet( }else{ /* The pager cache has created a new page. Its content needs to - ** be initialized. */ - - pPg->pPager = pPager; - - /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page - ** number greater than this, or the unused locking-page, is requested. */ + ** be initialized. But first some error checks: + ** + ** (1) The maximum page number is 2^31 + ** (2) Never try to fetch the locking page + */ if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ rc = SQLITE_CORRUPT_BKPT; goto pager_acquire_err; } - if( MEMDB || pPager->dbSizefd) ){ + pPg->pPager = pPager; + + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; @@ -5435,7 +5442,8 @@ int sqlite3PagerGet( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ - if( pagerUseWal(pPager) && bMmapOk==0 ){ + u32 iFrame = 0; /* Frame to read from WAL file */ + if( pagerUseWal(pPager) ){ rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); if( rc!=SQLITE_OK ) goto pager_acquire_err; } @@ -5448,7 +5456,6 @@ int sqlite3PagerGet( } pager_set_pagehash(pPg); } - return SQLITE_OK; pager_acquire_err: @@ -5457,11 +5464,109 @@ pager_acquire_err: sqlite3PcacheDrop(pPg); } pagerUnlockIfUnused(pPager); - *ppPage = 0; return rc; } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The page getter for when memory-mapped I/O is enabled */ +static int getPageMMap( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno>1 + && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) + ); + + assert( USEFETCH(pPager) ); +#ifdef SQLITE_HAS_CODEC + assert( pPager->xCodec==0 ); +#endif + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + assert( pPager->errCode==SQLITE_OK ); + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + if( bMmapOk && iFrame==0 ){ + void *pData = 0; + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ + pPg = sqlite3PagerLookup(pPager, pgno); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + return getPageNormal(pPager, pgno, ppPage, flags); +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* The page getter method for when the pager is an error state */ +static int getPageError( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + UNUSED_PARAMETER(pgno); + UNUSED_PARAMETER(flags); + assert( pPager->errCode!=SQLITE_OK ); + *ppPage = 0; + return pPager->errCode; +} + + +/* Dispatch all page fetch requests to the appropriate getter method. +*/ +int sqlite3PagerGet( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + return pPager->xGet(pPager, pgno, ppPage, flags); +} + /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, @@ -5553,24 +5658,24 @@ static int pager_open_journal(Pager *pPager){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3MemJournalOpen(pPager->jfd); }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); if( rc==SQLITE_OK ){ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif } } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); @@ -5935,12 +6040,13 @@ int sqlite3PagerWrite(PgHdr *pPg){ assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); - if( pPager->errCode ){ - return pPager->errCode; - }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; + }else if( pPager->errCode ){ + return pPager->errCode; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); }else{ return pager_write(pPg); @@ -5971,14 +6077,21 @@ int sqlite3PagerIswriteable(DbPage *pPg){ ** ** Tests show that this optimization can quadruple the speed of large ** DELETE operations. +** +** This optimization cannot be used with a temp-file, as the page may +** have been dirty at the start of the transaction. In that case, if +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the +** current transaction is rolled back. */ void sqlite3PagerDontWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; - if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; pPg->flags &= ~PGHDR_WRITEABLE; + testcase( pPg->flags & PGHDR_NEED_SYNC ); pager_set_pagehash(pPg); } } @@ -6173,17 +6286,21 @@ int sqlite3PagerCommitPhaseOne( /* If a prior error occurred, report that error again. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + /* Provide the ability to easily simulate an I/O error during testing */ + if( sqlite3FaultSim(400) ) return SQLITE_IOERR; + PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager, 1) ){ /* If this is an in-memory db, or no pages have been written to, or this ** function has already been called, it is mostly a no-op. However, any - ** backup in progress needs to be restarted. - */ + ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ if( pagerUseWal(pPager) ){ @@ -6422,6 +6539,7 @@ int sqlite3PagerRollback(Pager *pPager){ */ pPager->errCode = SQLITE_ABORT; pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); return rc; } }else{ @@ -6522,10 +6640,10 @@ void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ } /* -** Return true if this is an in-memory pager. +** Return true if this is an in-memory or temp-file backed pager. */ int sqlite3PagerIsMemdb(Pager *pPager){ - return MEMDB; + return pPager->tempFile; } /* @@ -6626,7 +6744,11 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** savepoint. If no errors occur, SQLITE_OK is returned. */ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; /* Return code */ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); @@ -6667,6 +6789,21 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } +#endif } return rc; @@ -6737,6 +6874,7 @@ void sqlite3PagerSetCodec( pPager->xCodecSizeChng = xCodecSizeChng; pPager->xCodecFree = xCodecFree; pPager->pCodec = pCodec; + setGetterMethod(pPager); pagerReportSize(pPager); } void *sqlite3PagerGetCodec(Pager *pPager){ @@ -6805,7 +6943,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ - if( MEMDB ){ + assert( pPager->tempFile || !MEMDB ); + if( pPager->tempFile ){ rc = sqlite3PagerWrite(pPg); if( rc ) return rc; } @@ -6862,7 +7001,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); - if( MEMDB ){ + if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ sqlite3PcacheMove(pPgOld, pPager->dbSize+1); @@ -6879,8 +7018,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** to exist, in case the transaction needs to roll back. Use pPgOld ** as the original page since it has already been allocated. */ - if( MEMDB ){ - assert( pPgOld ); + if( pPager->tempFile && pPgOld ){ sqlite3PcacheMove(pPgOld, origPgno); sqlite3PagerUnrefNotNull(pPgOld); } @@ -7132,10 +7270,12 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ ** Unless this is an in-memory or temporary database, clear the pager cache. */ void sqlite3PagerClearCache(Pager *pPager){ - if( !MEMDB && pPager->tempFile==0 ) pager_reset(pPager); + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); } #endif + #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -7144,10 +7284,16 @@ void sqlite3PagerClearCache(Pager *pPager){ ** ** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ +int sqlite3PagerCheckpoint( + Pager *pPager, /* Checkpoint on this pager */ + sqlite3 *db, /* Db handle used to check for interrupts */ + int eMode, /* Type of checkpoint */ + int *pnLog, /* OUT: Final number of frames in log */ + int *pnCkpt /* OUT: Final number of checkpointed frames */ +){ int rc = SQLITE_OK; if( pPager->pWal ){ - rc = sqlite3WalCheckpoint(pPager->pWal, eMode, + rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, @@ -7279,7 +7425,7 @@ int sqlite3PagerOpenWal( ** error (SQLITE_BUSY) is returned and the log connection is not closed. ** If successful, the EXCLUSIVE lock is not released before returning. */ -int sqlite3PagerCloseWal(Pager *pPager){ +int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ int rc = SQLITE_OK; assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); @@ -7307,10 +7453,11 @@ int sqlite3PagerCloseWal(Pager *pPager){ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + rc = sqlite3WalClose(pPager->pWal, db, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; @@ -7343,6 +7490,20 @@ int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ } return rc; } + +/* +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** is not a WAL database, return an error. +*/ +int sqlite3PagerSnapshotRecover(Pager *pPager){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotRecover(pPager->pWal); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} #endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ @@ -7360,5 +7521,4 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif - #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 38a4982150..585ef29497 100644 --- a/src/pager.h +++ b/src/pager.h @@ -14,8 +14,8 @@ ** at a time and provides a journal for rollback. */ -#ifndef _PAGER_H_ -#define _PAGER_H_ +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H /* ** Default maximum size for persistent journal files. A negative @@ -68,7 +68,11 @@ typedef struct PgHdr DbPage; #define PAGER_LOCKINGMODE_EXCLUSIVE 1 /* -** Numeric constants that encode the journalmode. +** Numeric constants that encode the journalmode. +** +** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) +** are exposed in the API via the "PRAGMA journal_mode" command and +** therefore cannot be changed without a compatibility break. */ #define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ #define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ @@ -86,6 +90,11 @@ typedef struct PgHdr DbPage; /* ** Flags for sqlite3PagerSetFlags() +** +** Value constraints (enforced via assert()): +** PAGER_FULLFSYNC == SQLITE_FullFSync +** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync +** PAGER_CACHE_SPILL == SQLITE_CacheSpill */ #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ @@ -113,7 +122,7 @@ int sqlite3PagerOpen( int, void(*)(DbPage*) ); -int sqlite3PagerClose(Pager *pPager); +int sqlite3PagerClose(Pager *pPager, sqlite3*); int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ @@ -164,15 +173,21 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); #ifndef SQLITE_OMIT_WAL - int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); + int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); - int sqlite3PagerCloseWal(Pager *pPager); + int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); +# ifdef SQLITE_DIRECT_OVERFLOW_READ + int sqlite3PagerUseWal(Pager *pPager, Pgno); +# endif # ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); + int sqlite3PagerSnapshotRecover(Pager *pPager); # endif +#else +# define sqlite3PagerUseWal(x,y) 0 #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -194,7 +209,7 @@ const char *sqlite3PagerJournalname(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); -void sqlite3PagerClearCache(Pager *); +void sqlite3PagerClearCache(Pager*); int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ @@ -221,4 +236,4 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif -#endif /* _PAGER_H_ */ +#endif /* SQLITE_PAGER_H */ diff --git a/src/parse.y b/src/parse.y index 20492edb80..ef9d3dd0ec 100644 --- a/src/parse.y +++ b/src/parse.y @@ -65,6 +65,19 @@ */ #define YYPARSEFREENEVERNULL 1 +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + /* ** Alternative datatype for the argument to the malloc() routine passed ** into sqlite3ParserAlloc(). The default is size_t. @@ -80,15 +93,6 @@ struct LimitVal { Expr *pOffset; /* The OFFSET expression. NULL if there is none */ }; -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int bNot; /* True if the NOT keyword is present */ -}; - /* ** An instance of the following structure describes the event of a ** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, @@ -100,11 +104,6 @@ struct LikeOp { */ struct TrigEvent { int a; IdList * b; }; -/* -** An instance of this structure holds the ATTACH key and the key type. -*/ -struct AttachKey { int type; Token key; }; - /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. @@ -194,28 +193,6 @@ columnlist ::= columnlist COMMA columnname carglist. columnlist ::= columnname carglist. columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} -// An IDENTIFIER can be a generic identifier, or one of several -// keywords. Any non-standard keyword can also be an identifier. -// -%token_class id ID|INDEXED. - -// The following directive causes tokens ABORT, AFTER, ASC, etc. to -// fallback to ID if they will not parse as their original value. -// This obviates the need for the "id" nonterminal. -// -%fallback ID - ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW - CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR - IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW - ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT -%ifdef SQLITE_OMIT_COMPOUND_SELECT - EXCEPT INTERSECT UNION -%endif SQLITE_OMIT_COMPOUND_SELECT - REINDEX RENAME CTIME_KW IF - . -%wildcard ANY. - // Define operator precedence early so that this is the first occurrence // of the operator tokens in the grammer. Keeping the operators together // causes them to be assigned integer values that are close together, @@ -240,6 +217,29 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);} %left COLLATE. %right BITNOT. +// An IDENTIFIER can be a generic identifier, or one of several +// keywords. Any non-standard keyword can also be an identifier. +// +%token_class id ID|INDEXED. + +// The following directive causes tokens ABORT, AFTER, ASC, etc. to +// fallback to ID if they will not parse as their original value. +// This obviates the need for the "id" nonterminal. +// +%fallback ID + ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW + CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR + IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT +%ifdef SQLITE_OMIT_COMPOUND_SELECT + EXCEPT INTERSECT UNION +%endif SQLITE_OMIT_COMPOUND_SELECT + REINDEX RENAME CTIME_KW IF + . +%wildcard ANY. + + // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. @@ -281,7 +281,7 @@ ccons ::= DEFAULT LP expr(X) RP. {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT PLUS term(X). {sqlite3AddDefaultValue(pParse,&X);} ccons ::= DEFAULT MINUS(A) term(X). { ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, X.pExpr, 0, 0); + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, X.pExpr, 0); v.zStart = A.z; v.zEnd = X.zEnd; sqlite3AddDefaultValue(pParse,&v); @@ -299,7 +299,8 @@ ccons ::= NULL onconf. ccons ::= NOT NULL onconf(R). {sqlite3AddNotNull(pParse, R);} ccons ::= PRIMARY KEY sortorder(Z) onconf(R) autoinc(I). {sqlite3AddPrimaryKey(pParse,0,R,I,Z);} -ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0);} +ccons ::= UNIQUE onconf(R). {sqlite3CreateIndex(pParse,0,0,0,0,R,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} ccons ::= CHECK LP expr(X) RP. {sqlite3AddCheckConstraint(pParse,X.pExpr);} ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R). {sqlite3CreateForeignKey(pParse,0,&T,TA,R);} @@ -348,7 +349,8 @@ tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP sortlist(X) RP onconf(R). - {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0);} + {sqlite3CreateIndex(pParse,0,0,0,X,R,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} tcons ::= CHECK LP expr(E) RP onconf. {sqlite3AddCheckConstraint(pParse,E.pExpr);} tcons ::= FOREIGN KEY LP eidlist(FA) RP @@ -553,10 +555,10 @@ selcollist(A) ::= sclp(A) STAR. { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); A = sqlite3ExprListAppend(pParse, A, p); } -selcollist(A) ::= sclp(A) nm(X) DOT STAR(Y). { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &Y); - Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); +selcollist(A) ::= sclp(A) nm(X) DOT STAR. { + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); A = sqlite3ExprListAppend(pParse,A, pDot); } @@ -786,10 +788,16 @@ setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). { A = sqlite3ExprListAppend(pParse, A, Y.pExpr); sqlite3ExprListSetName(pParse, A, &X, 1); } +setlist(A) ::= setlist(A) COMMA LP idlist(X) RP EQ expr(Y). { + A = sqlite3ExprListAppendVector(pParse, A, X, Y.pExpr); +} setlist(A) ::= nm(X) EQ expr(Y). { A = sqlite3ExprListAppend(pParse, 0, Y.pExpr); sqlite3ExprListSetName(pParse, A, &X, 1); } +setlist(A) ::= LP idlist(X) RP EQ expr(Y). { + A = sqlite3ExprListAppendVector(pParse, 0, X, Y.pExpr); +} ////////////////////////// The INSERT command ///////////////////////////////// // @@ -842,7 +850,24 @@ idlist(A) ::= nm(Y). ** that created the expression. */ static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, &t); + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = (u8)op; + p->flags = EP_Leaf; + p->iAgg = -1; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + if( p->u.zToken[0]=='"' ) p->flags |= EP_DblQuoted; + sqlite3Dequote(p->u.zToken); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + } + pOut->pExpr = p; pOut->zStart = t.z; pOut->zEnd = &t.z[t.n]; } @@ -855,38 +880,46 @@ term(A) ::= NULL(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} expr(A) ::= id(X). {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/} expr(A) ::= JOIN_KW(X). {spanExpr(&A,pParse,TK_ID,X); /*A-overwrites-X*/} expr(A) ::= nm(X) DOT nm(Y). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); spanSet(&A,&X,&Y); /*A-overwrites-X*/ - A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &X); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Y); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &Z); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &X, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &Y, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &Z, 1); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); spanSet(&A,&X,&Z); /*A-overwrites-X*/ - A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + A.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); +} +term(A) ::= FLOAT|BLOB(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} +term(A) ::= STRING(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} +term(A) ::= INTEGER(X). { + A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); + A.zStart = X.z; + A.zEnd = X.z + X.n; + if( A.pExpr ) A.pExpr->flags |= EP_Leaf|EP_Resolved; } -term(A) ::= INTEGER|FLOAT|BLOB(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} -term(A) ::= STRING(X). {spanExpr(&A,pParse,@X,X);/*A-overwrites-X*/} expr(A) ::= VARIABLE(X). { - Token t = X; /*A-overwrites-X*/ - if( t.n>=2 && t.z[0]=='#' && sqlite3Isdigit(t.z[1]) ){ + if( !(X.z[0]=='#' && sqlite3Isdigit(X.z[1])) ){ + u32 n = X.n; + spanExpr(&A, pParse, TK_VARIABLE, X); + sqlite3ExprAssignVarNumber(pParse, A.pExpr, n); + }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ + Token t = X; /*A-overwrites-X*/ + assert( t.n>=2 ); spanSet(&A, &t, &t); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); A.pExpr = 0; }else{ - A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &t); + A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); if( A.pExpr ) sqlite3GetInt32(&t.z[1], &A.pExpr->iTable); } - }else{ - spanExpr(&A, pParse, TK_VARIABLE, t); - sqlite3ExprAssignVarNumber(pParse, A.pExpr); } } expr(A) ::= expr(A) COLLATE ids(C). { @@ -896,7 +929,8 @@ expr(A) ::= expr(A) COLLATE ids(C). { %ifndef SQLITE_OMIT_CAST expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). { spanSet(&A,&X,&Y); /*A-overwrites-X*/ - A.pExpr = sqlite3PExpr(pParse, TK_CAST, E.pExpr, 0, &T); + A.pExpr = sqlite3ExprAlloc(pParse->db, TK_CAST, &T, 1); + sqlite3ExprAttachSubtrees(pParse->db, A.pExpr, E.pExpr, 0); } %endif SQLITE_OMIT_CAST expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). { @@ -928,7 +962,7 @@ term(A) ::= CTIME_KW(OP). { ExprSpan *pLeft, /* The left operand, and output */ ExprSpan *pRight /* The right operand */ ){ - pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); + pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr); pLeft->zEnd = pRight->zEnd; } @@ -937,11 +971,22 @@ term(A) ::= CTIME_KW(OP). { */ static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ if( doNot ){ - pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); + pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0); } } } +expr(A) ::= LP(L) nexprlist(X) COMMA expr(Y) RP(R). { + ExprList *pList = sqlite3ExprListAppend(pParse, X, Y.pExpr); + A.pExpr = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( A.pExpr ){ + A.pExpr->x.pList = pList; + spanSet(&A, &L, &R); + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + expr(A) ::= expr(A) AND(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} expr(A) ::= expr(A) OR(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} expr(A) ::= expr(A) LT|GT|GE|LE(OP) expr(Y). @@ -954,25 +999,29 @@ expr(A) ::= expr(A) PLUS|MINUS(OP) expr(Y). expr(A) ::= expr(A) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} expr(A) ::= expr(A) CONCAT(OP) expr(Y). {spanBinaryExpr(pParse,@OP,&A,&Y);} -%type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;/*A-overwrites-X*/} -likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} +%type likeop {Token} +likeop(A) ::= LIKE_KW|MATCH(A). +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A=X; A.n|=0x80000000; /*A-overwrite-X*/} expr(A) ::= expr(A) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; + int bNot = OP.n & 0x80000000; + OP.n &= 0x7fffffff; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, A.pExpr); - A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - exprNot(pParse, OP.bNot, &A); + A.pExpr = sqlite3ExprFunction(pParse, pList, &OP); + exprNot(pParse, bNot, &A); A.zEnd = Y.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { ExprList *pList; + int bNot = OP.n & 0x80000000; + OP.n &= 0x7fffffff; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, A.pExpr); pList = sqlite3ExprListAppend(pParse,pList, E.pExpr); - A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); - exprNot(pParse, OP.bNot, &A); + A.pExpr = sqlite3ExprFunction(pParse, pList, &OP); + exprNot(pParse, bNot, &A); A.zEnd = E.zEnd; if( A.pExpr ) A.pExpr->flags |= EP_InfixFunc; } @@ -986,7 +1035,7 @@ expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { ExprSpan *pOperand, /* The operand, and output */ Token *pPostOp /* The operand token for setting the span */ ){ - pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0); pOperand->zEnd = &pPostOp->z[pPostOp->n]; } } @@ -1033,7 +1082,7 @@ expr(A) ::= expr(A) IS NOT expr(Y). { Token *pPreOp /* The operand token for setting the span */ ){ pOut->zStart = pPreOp->z; - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); + pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0); pOut->zEnd = pOperand->zEnd; } } @@ -1055,7 +1104,7 @@ between_op(A) ::= NOT BETWEEN. {A = 1;} expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ExprList *pList = sqlite3ExprListAppend(pParse,0, X.pExpr); pList = sqlite3ExprListAppend(pParse,pList, Y.pExpr); - A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, A.pExpr, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, A.pExpr, 0); if( A.pExpr ){ A.pExpr->x.pList = pList; }else{ @@ -1079,7 +1128,7 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** regardless of the value of expr1. */ sqlite3ExprDelete(pParse->db, A.pExpr); - A.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[N]); + A.pExpr = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[N],1); }else if( Y->nExpr==1 ){ /* Expressions of the form: ** @@ -1106,9 +1155,9 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } - A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, A.pExpr, pRHS, 0); + A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, A.pExpr, pRHS); }else{ - A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0); if( A.pExpr ){ A.pExpr->x.pList = Y; sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); @@ -1121,58 +1170,36 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { } expr(A) ::= LP(B) select(X) RP(E). { spanSet(&A,&B,&E); /*A-overwrites-B*/ - A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = X; - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, X); - } + A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, A.pExpr, X); } expr(A) ::= expr(A) in_op(N) LP select(Y) RP(E). [IN] { - A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = Y; - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0); + sqlite3PExprAddSelect(pParse, A.pExpr, Y); exprNot(pParse, N, &A); A.zEnd = &E.z[E.n]; } - expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z). [IN] { + expr(A) ::= expr(A) in_op(N) nm(Y) dbnm(Z) paren_exprlist(E). [IN] { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&Y,&Z); - A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0, 0); - if( A.pExpr ){ - A.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); - }else{ - sqlite3SrcListDelete(pParse->db, pSrc); - } + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + if( E ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, E); + A.pExpr = sqlite3PExpr(pParse, TK_IN, A.pExpr, 0); + sqlite3PExprAddSelect(pParse, A.pExpr, pSelect); exprNot(pParse, N, &A); A.zEnd = Z.z ? &Z.z[Z.n] : &Y.z[Y.n]; } expr(A) ::= EXISTS(B) LP select(Y) RP(E). { Expr *p; spanSet(&A,&B,&E); /*A-overwrites-B*/ - p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - if( p ){ - p->x.pSelect = Y; - ExprSetProperty(p, EP_xIsSelect|EP_Subquery); - sqlite3ExprSetHeightAndFlags(pParse, p); - }else{ - sqlite3SelectDelete(pParse->db, Y); - } + p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, Y); } %endif SQLITE_OMIT_SUBQUERY /* CASE expressions */ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { spanSet(&A,&C,&E); /*A-overwrites-C*/ - A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0); if( A.pExpr ){ A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y; sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); @@ -1212,6 +1239,15 @@ nexprlist(A) ::= nexprlist(A) COMMA expr(Y). nexprlist(A) ::= expr(Y). {A = sqlite3ExprListAppend(pParse,0,Y.pExpr); /*A-overwrites-Y*/} +%ifndef SQLITE_OMIT_SUBQUERY +/* A paren_exprlist is an optional expression list contained inside +** of parenthesis */ +%type paren_exprlist {ExprList*} +%destructor paren_exprlist {sqlite3ExprListDelete(pParse->db, $$);} +paren_exprlist(A) ::= . {A = 0;} +paren_exprlist(A) ::= LP exprlist(X) RP. {A = X;} +%endif SQLITE_OMIT_SUBQUERY + ///////////////////////////// The CREATE INDEX command /////////////////////// // @@ -1219,7 +1255,7 @@ cmd ::= createkw(S) uniqueflag(U) INDEX ifnotexists(NE) nm(X) dbnm(D) ON nm(Y) LP sortlist(Z) RP where_opt(W). { sqlite3CreateIndex(pParse, &X, &D, sqlite3SrcListAppend(pParse->db,0,&Y,0), Z, U, - &S, W, SQLITE_SO_ASC, NE); + &S, W, SQLITE_SO_ASC, NE, SQLITE_IDXTYPE_APPDEF); } %type uniqueflag {int} @@ -1294,8 +1330,8 @@ cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} // %ifndef SQLITE_OMIT_VACUUM %ifndef SQLITE_OMIT_ATTACH -cmd ::= VACUUM. {sqlite3Vacuum(pParse);} -cmd ::= VACUUM nm. {sqlite3Vacuum(pParse);} +cmd ::= VACUUM. {sqlite3Vacuum(pParse,0);} +cmd ::= VACUUM nm(X). {sqlite3Vacuum(pParse,&X);} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM @@ -1423,14 +1459,14 @@ trigger_cmd(A) ::= select(X). // The special RAISE expression that may occur in trigger programs expr(A) ::= RAISE(X) LP IGNORE RP(Y). { spanSet(&A,&X,&Y); /*A-overwrites-X*/ - A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0); if( A.pExpr ){ A.pExpr->affinity = OE_Ignore; } } expr(A) ::= RAISE(X) LP raisetype(T) COMMA nm(Z) RP(Y). { spanSet(&A,&X,&Y); /*A-overwrites-X*/ - A.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &Z); + A.pExpr = sqlite3ExprAlloc(pParse->db, TK_RAISE, &Z, 1); if( A.pExpr ) { A.pExpr->affinity = (char)T; } diff --git a/src/pcache.c b/src/pcache.c index 61e7587422..dc7d00f306 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -14,7 +14,29 @@ #include "sqliteInt.h" /* -** A complete page cache is an instance of this structure. +** A complete page cache is an instance of this structure. Every +** entry in the cache holds a single page of the database file. The +** btree layer only operates on the cached copy of the database pages. +** +** A page cache entry is "clean" if it exactly matches what is currently +** on disk. A page is "dirty" if it has been modified and needs to be +** persisted to disk. +** +** pDirty, pDirtyTail, pSynced: +** All dirty pages are linked into the doubly linked list using +** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order +** such that p was added to the list more recently than p->pDirtyNext. +** PCache.pDirty points to the first (newest) element in the list and +** pDirtyTail to the last (oldest). +** +** The PCache.pSynced variable is used to optimize searching for a dirty +** page to eject from the cache mid-transaction. It is better to eject +** a page that does not require a journal sync than one that does. +** Therefore, pSynced is maintained to that it *almost* always points +** to either the oldest page in the pDirty/pDirtyTail list that has a +** clear PGHDR_NEED_SYNC flag or to a page that is older than this one +** (so that the right page to eject can be found by following pDirtyPrev +** pointers). */ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ @@ -31,6 +53,95 @@ struct PCache { sqlite3_pcache *pCache; /* Pluggable cache module */ }; +/********************************** Test and Debug Logic **********************/ +/* +** Debug tracing macros. Enable by by changing the "0" to "1" and +** recompiling. +** +** When sqlite3PcacheTrace is 1, single line trace messages are issued. +** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries +** is displayed for many operations, resulting in a lot of output. +*/ +#if defined(SQLITE_DEBUG) && 0 + int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ + int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ +# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} + void pcacheDump(PCache *pCache){ + int N; + int i, j; + sqlite3_pcache_page *pLower; + PgHdr *pPg; + unsigned char *a; + + if( sqlite3PcacheTrace<2 ) return; + if( pCache->pCache==0 ) return; + N = sqlite3PcachePagecount(pCache); + if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; + for(i=1; i<=N; i++){ + pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); + if( pLower==0 ) continue; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf("\n"); + if( pPg->pPage==0 ){ + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); + } + } + } + #else +# define pcacheTrace(X) +# define pcacheDump(X) +#endif + +/* +** Check invariants on a PgHdr entry. Return true if everything is OK. +** Return false if any invariant is violated. +** +** This routine is for use inside of assert() statements only. For +** example: +** +** assert( sqlite3PcachePageSanity(pPg) ); +*/ +#ifdef SQLITE_DEBUG +int sqlite3PcachePageSanity(PgHdr *pPg){ + PCache *pCache; + assert( pPg!=0 ); + assert( pPg->pgno>0 || pPg->pPager==0 ); /* Page number is 1 or more */ + pCache = pPg->pCache; + assert( pCache!=0 ); /* Every page has an associated PCache */ + if( pPg->flags & PGHDR_CLEAN ){ + assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ + assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ + assert( pCache->pDirtyTail!=pPg ); + } + /* WRITEABLE pages must also be DIRTY */ + if( pPg->flags & PGHDR_WRITEABLE ){ + assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ + } + /* NEED_SYNC can be set independently of WRITEABLE. This can happen, + ** for example, when using the sqlite3PagerDontWrite() optimization: + ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. + ** (2) Page X moved to freelist, WRITEABLE is cleared + ** (3) Page X reused, WRITEABLE is set again + ** If NEED_SYNC had been cleared in step 2, then it would not be reset + ** in step 3, and page might be written into the database without first + ** syncing the rollback journal, which might cause corruption on a power + ** loss. + ** + ** Another example is when the database page size is smaller than the + ** disk sector size. When any page of a sector is journalled, all pages + ** in that sector are marked NEED_SYNC even if they are still CLEAN, just + ** in case they are later modified, since all pages in the same sector + ** must be journalled and synced before any of those pages can be safely + ** written. + */ + return 1; +} +#endif /* SQLITE_DEBUG */ + + /********************************** Linked List Management ********************/ /* Allowed values for second argument to pcacheManageDirtyList() */ @@ -47,17 +158,16 @@ struct PCache { static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; + pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, + addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", + pPage->pgno)); if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); assert( pPage->pDirtyPrev || pPage==p->pDirty ); /* Update the PCache1.pSynced variable if necessary. */ if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; - } - p->pSynced = pSynced; + p->pSynced = pPage->pDirtyPrev; } if( pPage->pDirtyNext ){ @@ -69,10 +179,15 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( pPage->pDirtyPrev ){ pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; }else{ + /* If there are now no dirty pages in the cache, set eCreate to 2. + ** This is an optimization that allows sqlite3PcacheFetch() to skip + ** searching for a dirty page to eject from the cache when it might + ** otherwise have to. */ assert( pPage==p->pDirty ); p->pDirty = pPage->pDirtyNext; - if( p->pDirty==0 && p->bPurgeable ){ - assert( p->eCreate==1 ); + assert( p->bPurgeable || p->eCreate==2 ); + if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ + assert( p->bPurgeable==0 || p->eCreate==1 ); p->eCreate = 2; } } @@ -94,10 +209,19 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ } } p->pDirty = pPage; - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + + /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set + ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** optimization, as if pSynced points to a page with the NEED_SYNC + ** flag set sqlite3PcacheFetchStress() searches through all newer + ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ + if( !p->pSynced + && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ + ){ p->pSynced = pPage; } } + pcacheDump(p); } /* @@ -106,7 +230,9 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ */ static void pcacheUnpin(PgHdr *p){ if( p->pCache->bPurgeable ){ + pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + pcacheDump(p->pCache); } } @@ -158,6 +284,12 @@ int sqlite3PcacheSize(void){ return sizeof(PCache); } ** has already been allocated and is passed in as the p pointer. ** The caller discovers how much space needs to be allocated by ** calling sqlite3PcacheSize(). +** +** szExtra is some extra space allocated for each page. The first +** 8 bytes of the extra space will be zeroed as the page is allocated, +** but remaining content will be uninitialized. Though it is opaque +** to this module, the extra space really ends up being the MemPage +** structure in the pager. */ int sqlite3PcacheOpen( int szPage, /* Size of every page */ @@ -170,12 +302,14 @@ int sqlite3PcacheOpen( memset(p, 0, sizeof(PCache)); p->szPage = 1; p->szExtra = szExtra; + assert( szExtra>=8 ); /* First 8 bytes will be zeroed */ p->bPurgeable = bPurgeable; p->eCreate = 2; p->xStress = xStress; p->pStress = pStress; p->szCache = 100; p->szSpill = 1; + pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); return sqlite3PcacheSetPageSize(p, szPage); } @@ -198,6 +332,7 @@ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ } pCache->pCache = pNew; pCache->szPage = szPage; + pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); } return SQLITE_OK; } @@ -232,11 +367,12 @@ sqlite3_pcache_page *sqlite3PcacheFetch( int createFlag /* If true, create page if it does not exist already */ ){ int eCreate; + sqlite3_pcache_page *pRes; assert( pCache!=0 ); assert( pCache->pCache!=0 ); assert( createFlag==3 || createFlag==0 ); - assert( pgno>0 ); + assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) @@ -249,12 +385,15 @@ sqlite3_pcache_page *sqlite3PcacheFetch( assert( eCreate==0 || eCreate==1 || eCreate==2 ); assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); - return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + createFlag?" create":"",pRes)); + return pRes; } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new -** page because new clean pages are available for reuse and the cache +** page because no clean pages are available for reuse and the cache ** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to @@ -276,7 +415,11 @@ int sqlite3PcacheFetchStress( ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC ** cleared), but if that is not possible settle for any other ** unreferenced dirty page. - */ + ** + ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC + ** flag is currently referenced, then the following may leave pSynced + ** set incorrectly (pointing to other than the LRU page with NEED_SYNC + ** cleared). This is Ok, as pSynced is just an optimization. */ for(pPg=pCache->pSynced; pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev @@ -294,7 +437,9 @@ int sqlite3PcacheFetchStress( sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), numberOfCachePages(pCache)); #endif + pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); rc = pCache->xStress(pCache->pStress, pPg); + pcacheDump(pCache); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ return rc; } @@ -322,11 +467,11 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(pPgHdr, 0, sizeof(PgHdr)); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; - memset(pPgHdr->pExtra, 0, pCache->szExtra); + memset(pPgHdr->pExtra, 0, 8); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; pPgHdr->flags = PGHDR_CLEAN; @@ -354,6 +499,7 @@ PgHdr *sqlite3PcacheFetchFinish( } pCache->nRefSum++; pPgHdr->nRef++; + assert( sqlite3PcachePageSanity(pPgHdr) ); return pPgHdr; } @@ -367,8 +513,11 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); - }else if( p->pDirtyPrev!=0 ){ - /* Move the page to the head of the dirty list. */ + }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, + ** then page p is already at the head of the dirty list and the + ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE + ** tag above. */ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } @@ -379,6 +528,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ */ void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); + assert( sqlite3PcachePageSanity(p) ); p->nRef++; p->pCache->nRefSum++; } @@ -390,6 +540,7 @@ void sqlite3PcacheRef(PgHdr *p){ */ void sqlite3PcacheDrop(PgHdr *p){ assert( p->nRef==1 ); + assert( sqlite3PcachePageSanity(p) ); if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } @@ -403,13 +554,16 @@ void sqlite3PcacheDrop(PgHdr *p){ */ void sqlite3PcacheMakeDirty(PgHdr *p){ assert( p->nRef>0 ); - if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ + assert( sqlite3PcachePageSanity(p) ); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ p->flags &= ~PGHDR_DONT_WRITE; if( p->flags & PGHDR_CLEAN ){ p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); } + assert( sqlite3PcachePageSanity(p) ); } } @@ -418,11 +572,14 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ ** make it so. */ void sqlite3PcacheMakeClean(PgHdr *p){ - if( (p->flags & PGHDR_DIRTY) ){ + assert( sqlite3PcachePageSanity(p) ); + if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ assert( (p->flags & PGHDR_CLEAN)==0 ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); if( p->nRef==0 ){ pcacheUnpin(p); } @@ -434,11 +591,24 @@ void sqlite3PcacheMakeClean(PgHdr *p){ */ void sqlite3PcacheCleanAll(PCache *pCache){ PgHdr *p; + pcacheTrace(("%p.CLEAN-ALL\n",pCache)); while( (p = pCache->pDirty)!=0 ){ sqlite3PcacheMakeClean(p); } } +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + /* ** Clear the PGHDR_NEED_SYNC flag from all dirty pages. */ @@ -457,6 +627,8 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; assert( p->nRef>0 ); assert( newPgno>0 ); + assert( sqlite3PcachePageSanity(p) ); + pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ @@ -477,6 +649,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ if( pCache->pCache ){ PgHdr *p; PgHdr *pNext; + pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); for(p=pCache->pDirty; p; p=pNext){ pNext = p->pDirtyNext; /* This routine never gets call with a positive pgno except right @@ -484,7 +657,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** it must be that pgno==0. */ assert( p->pgno>0 ); - if( ALWAYS(p->pgno>pgno) ){ + if( p->pgno>pgno ){ assert( p->flags&PGHDR_DIRTY ); sqlite3PcacheMakeClean(p); } @@ -507,6 +680,7 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ */ void sqlite3PcacheClose(PCache *pCache){ assert( pCache->pCache!=0 ); + pcacheTrace(("%p.CLOSE\n",pCache)); sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } @@ -519,29 +693,31 @@ void sqlite3PcacheClear(PCache *pCache){ /* ** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pDirtyPrev pointers. +** Do not bother fixing the pDirtyPrev pointers. */ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ PgHdr result, *pTail; pTail = &result; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ if( pA->pgnopgno ){ pTail->pDirty = pA; pTail = pA; pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } }else{ pTail->pDirty = pB; pTail = pB; pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } } } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } return result.pDirty; } @@ -582,7 +758,8 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ } p = a[0]; for(i=1; ipDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; +} #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* diff --git a/src/pcache.h b/src/pcache.h index 475c04c061..9012edf22e 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -26,7 +26,7 @@ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ void *pData; /* Page data */ void *pExtra; /* Extra content */ - PgHdr *pDirty; /* Transient list of dirty pages */ + PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES @@ -51,11 +51,10 @@ struct PgHdr { #define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ #define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before ** writing this page to the database */ -#define PGHDR_NEED_READ 0x010 /* Content is unread */ -#define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ -#define PGHDR_MMAP 0x040 /* This is an mmap page object */ +#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ +#define PGHDR_MMAP 0x020 /* This is an mmap page object */ -#define PGHDR_WAL_APPEND 0x080 /* Appended to wal file */ +#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); @@ -99,6 +98,7 @@ void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +void sqlite3PcacheClearWritable(PCache*); /* Change a page number. Used by incr-vacuum. */ void sqlite3PcacheMove(PgHdr*, Pgno); @@ -137,6 +137,11 @@ int sqlite3PcachePagecount(PCache*); void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); #endif +#if defined(SQLITE_DEBUG) +/* Check invariants on a PgHdr object */ +int sqlite3PcachePageSanity(PgHdr*); +#endif + /* Set and get the suggested cache-size for the specified pager-cache. ** ** If no global maximum is configured, then the system attempts to limit @@ -173,4 +178,7 @@ void sqlite3PCacheSetDefault(void); int sqlite3HeaderSizePcache(void); int sqlite3HeaderSizePcache1(void); +/* Number of dirty pages as a percentage of the configured cache size */ +int sqlite3PCachePercentDirty(PCache*); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index 7147f6a7a8..fa0c6a79f2 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -279,14 +279,13 @@ static int pcache1InitBulk(PCache1 *pCache){ szBulk = -1024 * (i64)pcache1.nInitPage; } if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ - szBulk = pCache->szAlloc*pCache->nMax; + szBulk = pCache->szAlloc*(i64)pCache->nMax; } zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); sqlite3EndBenignMalloc(); if( zBulk ){ int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; - int i; - for(i=0; iszPage]; pX->page.pBuf = zBulk; pX->page.pExtra = &pX[1]; @@ -295,7 +294,7 @@ static int pcache1InitBulk(PCache1 *pCache){ pX->pNext = pCache->pFree; pCache->pFree = pX; zBulk += pCache->szAlloc; - } + }while( --nBulk ); } return pCache->pFree!=0; } @@ -348,7 +347,6 @@ static void *pcache1Alloc(int nByte){ ** Free an allocated buffer obtained from pcache1Alloc(). */ static void pcache1Free(void *p){ - int nFreed = 0; if( p==0 ) return; if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ PgFreeslot *pSlot; @@ -365,10 +363,13 @@ static void pcache1Free(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - nFreed = sqlite3MallocSize(p); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); - sqlite3_mutex_leave(pcache1.mutex); + { + int nFreed = 0; + nFreed = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); + sqlite3_mutex_leave(pcache1.mutex); + } #endif sqlite3_free(p); } @@ -630,12 +631,30 @@ static void pcache1TruncateUnsafe( PCache1 *pCache, /* The cache to truncate */ unsigned int iLimit /* Drop pages with this pgno or larger */ ){ - TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ - unsigned int h; + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - for(h=0; hnHash; h++){ - PgHdr1 **pp = &pCache->apHash[h]; + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; @@ -644,11 +663,13 @@ static void pcache1TruncateUnsafe( pcache1FreePage(pPage); }else{ pp = &pPage->pNext; - TESTONLY( nPage++; ) + TESTONLY( if( nPage>=0 ) nPage++; ) } } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; } - assert( pCache->nPage==nPage ); + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); } /******************************************************************************/ @@ -688,8 +709,8 @@ static int pcache1Init(void *NotUsed){ #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ - pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); - pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); } #endif if( pcache1.separateCache @@ -1125,7 +1146,7 @@ static void pcache1Destroy(sqlite3_pcache *p){ PGroup *pGroup = pCache->pGroup; assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); - pcache1TruncateUnsafe(pCache, 0); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); @@ -1189,7 +1210,7 @@ int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( sqlite3GlobalConfig.nPage==0 ){ + if( sqlite3GlobalConfig.pPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); while( (nReq<0 || nFreenPragCName; + sqlite3VdbeSetNumCols(v, n==0 ? 1 : n); + if( n==0 ){ + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, pPragma->zName, SQLITE_STATIC); + }else{ + int i, j; + for(i=0, j=pPragma->iPragCName; iupr ? 0 : &aPragmaName[mid]; +} + +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with the result +** held in register regResult. Decrement the result count and halt if +** the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v, int regResult){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); + return addr; +} + /* ** Process a pragma statement. ** @@ -305,12 +339,11 @@ void sqlite3Pragma( Token *pId; /* Pointer to token */ char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int iDb; /* Database index for */ - int lwr, upr, mid = 0; /* Binary search bounds */ int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ - const struct sPragmaNames *pPragma; + const PragmaName *pPragma; /* The pragma */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -338,7 +371,7 @@ void sqlite3Pragma( } assert( pId2 ); - zDb = pId2->n>0 ? pDb->zName : 0; + zDb = pId2->n>0 ? pDb->zDbSName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -365,7 +398,9 @@ void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - returnSingleText(v, "result", aFcntl[0]); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, aFcntl[0], SQLITE_TRANSIENT); + returnSingleText(v, aFcntl[0]); sqlite3_free(aFcntl[0]); goto pragma_out; } @@ -380,26 +415,21 @@ void sqlite3Pragma( } /* Locate the pragma in the lookup table */ - lwr = 0; - upr = ArraySize(aPragmaNames)-1; - while( lwr<=upr ){ - mid = (lwr+upr)/2; - rc = sqlite3_stricmp(zLeft, aPragmaNames[mid].zName); - if( rc==0 ) break; - if( rc<0 ){ - upr = mid - 1; - }else{ - lwr = mid + 1; - } - } - if( lwr>upr ) goto pragma_out; - pPragma = &aPragmaNames[mid]; + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ) goto pragma_out; /* Make sure the database schema is loaded if the pragma requires that */ - if( (pPragma->mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( (pPragma->mPragFlg & PragFlg_NeedSchema)!=0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; } + /* Register the result column names for pragmas that return results */ + if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 + && ((pPragma->mPragFlg & PragFlg_NoColumns1)==0 || zRight==0) + ){ + setPragmaResultColumnNames(v, pPragma); + } + /* Jump to the appropriate pragma handler */ switch( pPragma->ePragTyp ){ @@ -436,7 +466,6 @@ void sqlite3Pragma( VdbeOp *aOp; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - setOneColumnName(v, "cache_size"); pParse->nMem += 2; sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); @@ -471,7 +500,7 @@ void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(v, "page_size", size); + returnSingleInt(v, size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). @@ -506,7 +535,7 @@ void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(v, "secure_delete", b); + returnSingleInt(v, b); break; } @@ -538,8 +567,6 @@ void sqlite3Pragma( sqlite3AbsInt32(sqlite3Atoi(zRight))); } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); break; } @@ -585,7 +612,7 @@ void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - returnSingleText(v, "locking_mode", zRet); + returnSingleText(v, zRet); break; } @@ -598,7 +625,6 @@ void sqlite3Pragma( int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -644,7 +670,7 @@ void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(v, "journal_size_limit", iLimit); + returnSingleInt(v, iLimit); break; } @@ -662,7 +688,7 @@ void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -741,7 +767,7 @@ void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); + returnSingleInt(v, pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; @@ -775,7 +801,7 @@ void sqlite3Pragma( case PragTyp_CACHE_SPILL: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(v, "cache_spill", + returnSingleInt(v, (db->flags & SQLITE_CacheSpill)==0 ? 0 : sqlite3BtreeSetSpillSize(pDb->pBt,0)); }else{ @@ -829,7 +855,7 @@ void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(v, "mmap_size", sz); + returnSingleInt(v, sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -850,7 +876,7 @@ void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(v, "temp_store", db->temp_store); + returnSingleInt(v, db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -869,7 +895,7 @@ void sqlite3Pragma( */ case PragTyp_TEMP_STORE_DIRECTORY: { if( !zRight ){ - returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); + returnSingleText(v, sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -913,7 +939,7 @@ void sqlite3Pragma( */ case PragTyp_DATA_STORE_DIRECTORY: { if( !zRight ){ - returnSingleText(v, "data_store_directory", sqlite3_data_directory); + returnSingleText(v, sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -952,7 +978,7 @@ void sqlite3Pragma( sqlite3_file *pFile = sqlite3PagerFile(pPager); sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - returnSingleText(v, "lock_proxy_file", proxy_file_path); + returnSingleText(v, proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); @@ -984,12 +1010,12 @@ void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(v, "synchronous", pDb->safety_level-1); + returnSingleInt(v, pDb->safety_level-1); }else{ if( !db->autoCommit ){ sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); - }else{ + }else if( iDb!=1 ){ int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; @@ -1004,7 +1030,8 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); + setPragmaResultColumnNames(v, pPragma); + returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); }else{ int mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ @@ -1030,7 +1057,7 @@ void sqlite3Pragma( ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + sqlite3VdbeAddOp0(v, OP_Expire); setAllPagerFlags(db); } break; @@ -1053,21 +1080,16 @@ void sqlite3Pragma( */ case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - pTab = sqlite3FindTable(db, zRight, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "cid", "name", "type", "notnull", "dflt_value", "pk" - }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); pParse->nMem = 6; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - const char *zName; if( IsHiddenColumn(pCol) ){ nHidden++; continue; @@ -1080,11 +1102,10 @@ void sqlite3Pragma( for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); - zName = pCol->zName; sqlite3VdbeMultiLoad(v, 1, "issisi", i-nHidden, - zName, - sqlite3StrNext(zName), + pCol->zName, + sqlite3ColumnType(pCol,""), pCol->notNull ? 1 : 0, pCol->pDflt ? pCol->pDflt->u.zToken : 0, k); @@ -1094,41 +1115,39 @@ void sqlite3Pragma( } break; +#ifdef SQLITE_DEBUG case PragTyp_STATS: { - static const char *azCol[] = { "table", "index", "width", "height" }; Index *pIdx; HashElem *i; - v = sqlite3GetVdbe(pParse); - pParse->nMem = 4; + pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", + sqlite3VdbeMultiLoad(v, 1, "ssiii", pTab->zName, 0, pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pTab->nRowLogEst, + pTab->tabFlags); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", + sqlite3VdbeMultiLoad(v, 2, "siii", pIdx->zName, pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; +#endif case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); if( pIdx ){ - static const char *azCol[] = { - "seqno", "cid", "name", "desc", "coll", "key" - }; int i; int mx; if( pPragma->iArg ){ @@ -1142,8 +1161,7 @@ void sqlite3Pragma( } pTab = pIdx->pTable; sqlite3CodeVerifySchema(pParse, iDb); - assert( pParse->nMem<=ArraySize(azCol) ); - setAllColumnNames(v, pParse->nMem, azCol); + assert( pParse->nMem<=pPragma->nPragCName ); for(i=0; iaiColumn[i]; sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, @@ -1166,13 +1184,8 @@ void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "seq", "name", "unique", "origin", "partial" - }; - v = sqlite3GetVdbe(pParse); pParse->nMem = 5; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; sqlite3VdbeMultiLoad(v, 1, "isisi", @@ -1188,16 +1201,14 @@ void sqlite3Pragma( break; case PragTyp_DATABASE_LIST: { - static const char *azCol[] = { "seq", "name", "file" }; int i; pParse->nMem = 3; - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zName!=0 ); + assert( db->aDb[i].zDbSName!=0 ); sqlite3VdbeMultiLoad(v, 1, "iss", i, - db->aDb[i].zName, + db->aDb[i].zDbSName, sqlite3BtreeGetFilename(db->aDb[i].pBt)); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } @@ -1205,11 +1216,9 @@ void sqlite3Pragma( break; case PragTyp_COLLATION_LIST: { - static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; pParse->nMem = 2; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); @@ -1225,17 +1234,11 @@ void sqlite3Pragma( Table *pTab; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - v = sqlite3GetVdbe(pParse); pFK = pTab->pFKey; if( pFK ){ - static const char *azCol[] = { - "id", "seq", "table", "from", "to", "on_update", "on_delete", - "match" - }; int i = 0; pParse->nMem = 8; sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); while(pFK){ int j; for(j=0; jnCol; j++){ @@ -1276,14 +1279,11 @@ void sqlite3Pragma( int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ - static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; - v = sqlite3GetVdbe(pParse); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ @@ -1337,12 +1337,10 @@ void sqlite3Pragma( sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); sqlite3ColumnDefault(v, pTab, iKey, regRow); sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, - sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); } - sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, 0, regRow); VdbeCoverage(v); sqlite3VdbeGoto(v, addrOk); sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); }else{ @@ -1400,9 +1398,17 @@ void sqlite3Pragma( #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is reduced version of + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption - ** without most of the overhead of a full integrity-check. + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time wherease integrity_check is O(NlogN). */ case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; @@ -1424,7 +1430,6 @@ void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; @@ -1434,63 +1439,66 @@ void sqlite3Pragma( mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } } - sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ /* Do an integrity check on each database file */ for(i=0; inDb; i++){ HashElem *x; Hash *pTbls; + int *aRoot; int cnt = 0; + int mxIdx = 0; + int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); /* Do an integrity check of the B-Tree ** - ** Begin by filling registers 2, 3, ... with the root pages numbers + ** Begin by finding the root pages numbers ** for all tables and indices in the database. */ assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; - for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_Integer, pTab->tnum, 2+cnt); - VdbeComment((v, "%s", pTab->zName)); - cnt++; - } + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->tnum, 2+cnt); - VdbeComment((v, "%s", pIdx->zName)); - cnt++; + aRoot[cnt++] = pIdx->tnum; } } + aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, cnt+8 ); + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp3(v, OP_IntegrityCk, 2, cnt, 1); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), P4_DYNAMIC); sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); - sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + integrityCheckResultRow(v, 2); sqlite3VdbeJumpHere(v, addr); /* Make sure all the indices are constructed correctly. */ - for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; Index *pPrior = 0; @@ -1498,12 +1506,14 @@ void sqlite3Pragma( int iDataCur, iIdxCur; int r1 = -1; - if( pTab->pIndex==0 ) continue; + if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ + if( pTab->pCheck==0 + && (pTab->tabFlags & TF_HasNotNull)==0 + && (pTab->pIndex==0 || isQuick) + ){ + continue; /* No additional checks needed for this table */ + } pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); sqlite3ExprCacheClear(pParse); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); @@ -1511,30 +1521,49 @@ void sqlite3Pragma( for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - pParse->nMem = MAX(pParse->nMem, 8+j); + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; - int jmp2, jmp3; + int jmp2; if( j==pTab->iPKey ) continue; if( pTab->aCol[j].notNull==0 ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pTab->aCol[j].zName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeJumpHere(v, jmp3); + } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(v); + int addrCkOk = sqlite3VdbeMakeLabel(v); + ExprList *pCheck = pTab->pCheck; + char *zErr; + int k; + pParse->iSelfTab = iDataCur; + sqlite3ExprCachePush(pParse); + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v, 3); + sqlite3VdbeResolveLabel(v, addrCkOk); + sqlite3ExprCachePop(pParse); } /* Validate index entries for the current row */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + for(j=0, pIdx=pTab->pIndex; pIdx && !isQuick; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; int ckUniq = sqlite3VdbeMakeLabel(v); if( pPk==pIdx ) continue; @@ -1545,16 +1574,13 @@ void sqlite3Pragma( /* Verify that an index entry exists for the current table row */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); + jmp4 = integrityCheckResultRow(v, 3); sqlite3VdbeJumpHere(v, jmp2); /* For UNIQUE indexes, verify that only one entry exists with the ** current key. The entry is unique if (1) any column is NULL @@ -1575,7 +1601,6 @@ void sqlite3Pragma( sqlite3VdbeJumpHere(v, jmp6); sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, pIdx->nKeyCol); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); sqlite3VdbeGoto(v, jmp5); sqlite3VdbeResolveLabel(v, uniqOk); @@ -1586,19 +1611,18 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); #ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeLoadString(v, 3, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); - sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + if( !isQuick ){ + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeLoadString(v, 3, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); + integrityCheckResultRow(v, 7); + sqlite3VdbeJumpHere(v, addr); + } } #endif /* SQLITE_OMIT_BTREECOUNT */ } @@ -1607,7 +1631,7 @@ void sqlite3Pragma( static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 4, 0}, /* 1 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, /* 3 */ }; @@ -1615,7 +1639,7 @@ void sqlite3Pragma( aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); if( aOp ){ - aOp[0].p2 = -mxErr; + aOp[0].p2 = 1-mxErr; aOp[2].p4type = P4_STATIC; aOp[2].p4.z = "ok"; } @@ -1668,7 +1692,7 @@ void sqlite3Pragma( assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); + returnSingleText(v, encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value @@ -1703,7 +1727,9 @@ void sqlite3Pragma( ** PRAGMA [schema.]user_version ** PRAGMA [schema.]user_version = ** - ** PRAGMA [schema.]freelist_count = + ** PRAGMA [schema.]freelist_count + ** + ** PRAGMA [schema.]data_version ** ** PRAGMA [schema.]application_id ** PRAGMA [schema.]application_id = @@ -1729,7 +1755,7 @@ void sqlite3Pragma( case PragTyp_HEADER_VALUE: { int iCookie = pPragma->iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - if( zRight && (pPragma->mPragFlag & PragFlag_ReadOnly)==0 ){ + if( zRight && (pPragma->mPragFlg & PragFlg_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ @@ -1757,8 +1783,7 @@ void sqlite3Pragma( aOp[0].p1 = iDb; aOp[1].p1 = iDb; aOp[1].p3 = iCookie; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + sqlite3VdbeReusable(v); } } break; @@ -1775,11 +1800,11 @@ void sqlite3Pragma( int i = 0; const char *zOpt; pParse->nMem = 1; - setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } + sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ @@ -1791,7 +1816,6 @@ void sqlite3Pragma( ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { - static const char *azCol[] = { "busy", "log", "checkpointed" }; int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ @@ -1803,7 +1827,6 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); @@ -1822,7 +1845,7 @@ void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, "wal_autocheckpoint", + returnSingleInt(v, db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } @@ -1841,6 +1864,118 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all ** of the optimizations listed above except Debug Mode, including new + ** optimizations that have not yet been invented. If new optimizations are + ** ever added that should be off by default, those off-by-default + ** optimizations will have bitmasks of 0x10000 or larger. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (3) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xfffe; + } + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; /* Always analyze if any index lacks statistics */ + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + break; + } + /* ** PRAGMA busy_timeout ** PRAGMA busy_timeout = N @@ -1855,7 +1990,7 @@ void sqlite3Pragma( if( zRight ){ sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, "timeout", db->busyTimeout); + returnSingleInt(v, db->busyTimeout); break; } @@ -1875,7 +2010,7 @@ void sqlite3Pragma( if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, sqlite3_soft_heap_limit64(-1)); break; } @@ -1894,8 +2029,7 @@ void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(v, "threads", - sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + returnSingleInt(v, sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } @@ -1907,23 +2041,21 @@ void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; - static const char *azCol[] = { "database", "status" }; int i; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; - if( db->aDb[i].zName==0 ) continue; + if( db->aDb[i].zDbSName==0 ) continue; pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } break; @@ -1975,9 +2107,324 @@ void sqlite3Pragma( } /* End of the PRAGMA switch */ + /* The following block is a no-op unless SQLITE_DEBUG is defined. Its only + ** purpose is to execute assert() statements to verify that if the + ** PragFlg_NoColumns1 flag is set and the caller specified an argument + ** to the PRAGMA, the implementation has not added any OP_ResultRow + ** instructions to the VM. */ + if( (pPragma->mPragFlg & PragFlg_NoColumns1) && zRight ){ + sqlite3VdbeVerifyNoResultRow(v); + } + pragma_out: sqlite3DbFree(db, zLeft); sqlite3DbFree(db, zRight); } +#ifndef SQLITE_OMIT_VIRTUALTABLE +/***************************************************************************** +** Implementation of an eponymous virtual table that runs a pragma. +** +*/ +typedef struct PragmaVtab PragmaVtab; +typedef struct PragmaVtabCursor PragmaVtabCursor; +struct PragmaVtab { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection to which it belongs */ + const PragmaName *pName; /* Name of the pragma */ + u8 nHidden; /* Number of hidden columns */ + u8 iHidden; /* Index of the first hidden column */ +}; +struct PragmaVtabCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pPragma; /* The pragma statement to run */ + sqlite_int64 iRowid; /* Current rowid */ + char *azArg[2]; /* Value of the argument and schema */ +}; + +/* +** Pragma virtual table module xConnect method. +*/ +static int pragmaVtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + const PragmaName *pPragma = (const PragmaName*)pAux; + PragmaVtab *pTab = 0; + int rc; + int i, j; + char cSep = '('; + StrAccum acc; + char zBuf[200]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3StrAccumAppendAll(&acc, "CREATE TABLE x"); + for(i=0, j=pPragma->iPragCName; inPragCName; i++, j++){ + sqlite3XPrintf(&acc, "%c\"%s\"", cSep, pragCName[j]); + cSep = ','; + } + if( i==0 ){ + sqlite3XPrintf(&acc, "(\"%s\"", pPragma->zName); + cSep = ','; + i++; + } + j = 0; + if( pPragma->mPragFlg & PragFlg_Result1 ){ + sqlite3StrAccumAppendAll(&acc, ",arg HIDDEN"); + j++; + } + if( pPragma->mPragFlg & (PragFlg_SchemaOpt|PragFlg_SchemaReq) ){ + sqlite3StrAccumAppendAll(&acc, ",schema HIDDEN"); + j++; + } + sqlite3StrAccumAppend(&acc, ")", 1); + sqlite3StrAccumFinish(&acc); + assert( strlen(zBuf) < sizeof(zBuf)-1 ); + rc = sqlite3_declare_vtab(db, zBuf); + if( rc==SQLITE_OK ){ + pTab = (PragmaVtab*)sqlite3_malloc(sizeof(PragmaVtab)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(PragmaVtab)); + pTab->pName = pPragma; + pTab->db = db; + pTab->iHidden = i; + pTab->nHidden = j; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Pragma virtual table module xDisconnect method. +*/ +static int pragmaVtabDisconnect(sqlite3_vtab *pVtab){ + PragmaVtab *pTab = (PragmaVtab*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* Figure out the best index to use to search a pragma virtual table. +** +** There are not really any index choices. But we want to encourage the +** query planner to give == constraints on as many hidden parameters as +** possible, and especially on the first hidden parameter. So return a +** high cost if hidden parameters are unconstrained. +*/ +static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + PragmaVtab *pTab = (PragmaVtab*)tab; + const struct sqlite3_index_constraint *pConstraint; + int i, j; + int seen[2]; + + pIdxInfo->estimatedCost = (double)1; + if( pTab->nHidden==0 ){ return SQLITE_OK; } + pConstraint = pIdxInfo->aConstraint; + seen[0] = 0; + seen[1] = 0; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn < pTab->iHidden ) continue; + j = pConstraint->iColumn - pTab->iHidden; + assert( j < 2 ); + seen[j] = i+1; + } + if( seen[0]==0 ){ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + return SQLITE_OK; + } + j = seen[0]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 1; + pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1]==0 ) return SQLITE_OK; + pIdxInfo->estimatedCost = (double)20; + pIdxInfo->estimatedRows = 20; + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + return SQLITE_OK; +} + +/* Create a new cursor for the pragma virtual table */ +static int pragmaVtabOpen(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ + PragmaVtabCursor *pCsr; + pCsr = (PragmaVtabCursor*)sqlite3_malloc(sizeof(*pCsr)); + if( pCsr==0 ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(PragmaVtabCursor)); + pCsr->base.pVtab = pVtab; + *ppCursor = &pCsr->base; + return SQLITE_OK; +} + +/* Clear all content from pragma virtual table cursor. */ +static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ + int i; + sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + for(i=0; iazArg); i++){ + sqlite3_free(pCsr->azArg[i]); + pCsr->azArg[i] = 0; + } +} + +/* Close a pragma virtual table cursor */ +static int pragmaVtabClose(sqlite3_vtab_cursor *cur){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)cur; + pragmaVtabCursorClear(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* Advance the pragma virtual table cursor to the next row */ +static int pragmaVtabNext(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + int rc = SQLITE_OK; + + /* Increment the xRowid value */ + pCsr->iRowid++; + assert( pCsr->pPragma ); + if( SQLITE_ROW!=sqlite3_step(pCsr->pPragma) ){ + rc = sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + pragmaVtabCursorClear(pCsr); + } + return rc; +} + +/* +** Pragma virtual table module xFilter method. +*/ +static int pragmaVtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + int rc; + int i, j; + StrAccum acc; + char *zSql; + + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + pragmaVtabCursorClear(pCsr); + j = (pTab->pName->mPragFlg & PragFlg_Result1)!=0 ? 0 : 1; + for(i=0; iazArg) ); + pCsr->azArg[j] = sqlite3_mprintf("%s", sqlite3_value_text(argv[i])); + if( pCsr->azArg[j]==0 ){ + return SQLITE_NOMEM; + } + } + sqlite3StrAccumInit(&acc, 0, 0, 0, pTab->db->aLimit[SQLITE_LIMIT_SQL_LENGTH]); + sqlite3StrAccumAppendAll(&acc, "PRAGMA "); + if( pCsr->azArg[1] ){ + sqlite3XPrintf(&acc, "%Q.", pCsr->azArg[1]); + } + sqlite3StrAccumAppendAll(&acc, pTab->pName->zName); + if( pCsr->azArg[0] ){ + sqlite3XPrintf(&acc, "=%Q", pCsr->azArg[0]); + } + zSql = sqlite3StrAccumFinish(&acc); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pPragma, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + return rc; + } + return pragmaVtabNext(pVtabCursor); +} + +/* +** Pragma virtual table module xEof method. +*/ +static int pragmaVtabEof(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + return (pCsr->pPragma==0); +} + +/* The xColumn method simply returns the corresponding column from +** the PRAGMA. +*/ +static int pragmaVtabColumn( + sqlite3_vtab_cursor *pVtabCursor, + sqlite3_context *ctx, + int i +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + if( iiHidden ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pPragma, i)); + }else{ + sqlite3_result_text(ctx, pCsr->azArg[i-pTab->iHidden],-1,SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Pragma virtual table module xRowid method. +*/ +static int pragmaVtabRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *p){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + *p = pCsr->iRowid; + return SQLITE_OK; +} + +/* The pragma virtual table object */ +static const sqlite3_module pragmaVtabModule = { + 0, /* iVersion */ + 0, /* xCreate - create a table */ + pragmaVtabConnect, /* xConnect - connect to an existing table */ + pragmaVtabBestIndex, /* xBestIndex - Determine search strategy */ + pragmaVtabDisconnect, /* xDisconnect - Disconnect from a table */ + 0, /* xDestroy - Drop a table */ + pragmaVtabOpen, /* xOpen - open a cursor */ + pragmaVtabClose, /* xClose - close a cursor */ + pragmaVtabFilter, /* xFilter - configure scan constraints */ + pragmaVtabNext, /* xNext - advance a cursor */ + pragmaVtabEof, /* xEof */ + pragmaVtabColumn, /* xColumn - read data */ + pragmaVtabRowid, /* xRowid - read data */ + 0, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* +** Check to see if zTabName is really the name of a pragma. If it is, +** then register an eponymous virtual table for that pragma and return +** a pointer to the Module object for the new virtual table. +*/ +Module *sqlite3PragmaVtabRegister(sqlite3 *db, const char *zName){ + const PragmaName *pName; + assert( sqlite3_strnicmp(zName, "pragma_", 7)==0 ); + pName = pragmaLocate(zName+7); + if( pName==0 ) return 0; + if( (pName->mPragFlg & (PragFlg_Result0|PragFlg_Result1))==0 ) return 0; + assert( sqlite3HashFind(&db->aModule, zName)==0 ); + return sqlite3VtabCreateModule(db, zName, &pragmaVtabModule, (void*)pName, 0); +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ #endif /* SQLITE_OMIT_PRAGMA */ diff --git a/src/pragma.h b/src/pragma.h index 81779e9d4a..9b1c723b3e 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -3,6 +3,8 @@ ** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit ** that script and rerun it. */ + +/* The various pragma types */ #define PragTyp_HEADER_VALUE 0 #define PragTyp_AUTO_VACUUM 1 #define PragTyp_FLAG 2 @@ -28,11 +30,11 @@ #define PragTyp_LOCKING_MODE 22 #define PragTyp_PAGE_COUNT 23 #define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 +#define PragTyp_OPTIMIZE 25 +#define PragTyp_PAGE_SIZE 26 +#define PragTyp_SECURE_DELETE 27 +#define PragTyp_SHRINK_MEMORY 28 +#define PragTyp_SOFT_HEAP_LIMIT 29 #define PragTyp_SYNCHRONOUS 30 #define PragTyp_TABLE_INFO 31 #define PragTyp_TEMP_STORE 32 @@ -46,419 +48,569 @@ #define PragTyp_REKEY 40 #define PragTyp_LOCK_STATUS 41 #define PragTyp_PARSER_TRACE 42 -#define PragFlag_NeedSchema 0x01 -#define PragFlag_ReadOnly 0x02 -static const struct sPragmaNames { - const char *const zName; /* Name of pragma */ - u8 ePragTyp; /* PragTyp_XXX value */ - u8 mPragFlag; /* Zero or more PragFlag_XXX values */ - u32 iArg; /* Extra argument */ -} aPragmaNames[] = { +#define PragTyp_STATS 43 + +/* Property flags associated with various pragma. */ +#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ +#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ +#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ +#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ +#define PragFlg_Result0 0x10 /* Acts as query when no argument */ +#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ +#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ +#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ + +/* Names of columns for pragmas that return multi-column result +** or that return single-column results where the name of the +** result column is different from the name of the pragma +*/ +static const char *const pragCName[] = { + /* 0 */ "cache_size", /* Used by: default_cache_size */ + /* 1 */ "cid", /* Used by: table_info */ + /* 2 */ "name", + /* 3 */ "type", + /* 4 */ "notnull", + /* 5 */ "dflt_value", + /* 6 */ "pk", + /* 7 */ "tbl", /* Used by: stats */ + /* 8 */ "idx", + /* 9 */ "wdth", + /* 10 */ "hght", + /* 11 */ "flgs", + /* 12 */ "seqno", /* Used by: index_info */ + /* 13 */ "cid", + /* 14 */ "name", + /* 15 */ "seqno", /* Used by: index_xinfo */ + /* 16 */ "cid", + /* 17 */ "name", + /* 18 */ "desc", + /* 19 */ "coll", + /* 20 */ "key", + /* 21 */ "seq", /* Used by: index_list */ + /* 22 */ "name", + /* 23 */ "unique", + /* 24 */ "origin", + /* 25 */ "partial", + /* 26 */ "seq", /* Used by: database_list */ + /* 27 */ "name", + /* 28 */ "file", + /* 29 */ "seq", /* Used by: collation_list */ + /* 30 */ "name", + /* 31 */ "id", /* Used by: foreign_key_list */ + /* 32 */ "seq", + /* 33 */ "table", + /* 34 */ "from", + /* 35 */ "to", + /* 36 */ "on_update", + /* 37 */ "on_delete", + /* 38 */ "match", + /* 39 */ "table", /* Used by: foreign_key_check */ + /* 40 */ "rowid", + /* 41 */ "parent", + /* 42 */ "fkid", + /* 43 */ "busy", /* Used by: wal_checkpoint */ + /* 44 */ "log", + /* 45 */ "checkpointed", + /* 46 */ "timeout", /* Used by: busy_timeout */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", +}; + +/* Definitions of all built-in pragmas */ +typedef struct PragmaName { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlg; /* Zero or more PragFlg_XXX values */ + u8 iPragCName; /* Start of column names in pragCName[] */ + u8 nPragCName; /* Num of col names. 0 means use pragma name */ + u32 iArg; /* Extra argument */ +} PragmaName; +static const PragmaName aPragmaName[] = { #if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - { /* zName: */ "activate_extensions", - /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "application_id", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_APPLICATION_ID }, + {/* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) - { /* zName: */ "auto_vacuum", - /* ePragTyp: */ PragTyp_AUTO_VACUUM, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) - { /* zName: */ "automatic_index", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_AutoIndex }, + {/* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_AutoIndex }, #endif #endif - { /* zName: */ "busy_timeout", - /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 46, 1, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "cache_size", - /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_CACHE_SPILL, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_CACHE_SPILL, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif - { /* zName: */ "case_sensitive_like", - /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "cell_size_check", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CellSizeCk }, + {/* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "checkpoint_fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CkptFullFSync }, + {/* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CkptFullFSync }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "collation_list", - /* ePragTyp: */ PragTyp_COLLATION_LIST, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 29, 2, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) - { /* zName: */ "compile_options", - /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "count_changes", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CountRows }, + {/* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CountRows }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN - { /* zName: */ "data_store_directory", - /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "data_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ PragFlag_ReadOnly, - /* iArg: */ BTREE_DATA_VERSION }, + {/* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_DATA_VERSION }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "database_list", - /* ePragTyp: */ PragTyp_DATABASE_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, + /* ColNames: */ 26, 3, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) - { /* zName: */ "default_cache_size", - /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 1, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "defer_foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_DeferFKs }, + {/* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_DeferFKs }, #endif #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "empty_result_callbacks", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_NullCallback }, + {/* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NullCallback }, #endif #if !defined(SQLITE_OMIT_UTF16) - { /* zName: */ "encoding", - /* ePragTyp: */ PragTyp_ENCODING, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "foreign_key_check", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 39, 4, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) - { /* zName: */ "foreign_key_list", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 31, 8, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ForeignKeys }, + {/* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ForeignKeys }, #endif #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "freelist_count", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ PragFlag_ReadOnly, - /* iArg: */ BTREE_FREE_PAGE_COUNT }, + {/* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "full_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_FullColNames }, - { /* zName: */ "fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_FullFSync }, + {/* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullColNames }, + {/* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullFSync }, #endif #if defined(SQLITE_HAS_CODEC) - { /* zName: */ "hexkey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "hexrekey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "hexkey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "hexrekey", + /* ePragTyp: */ PragTyp_HEXKEY, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_CHECK) - { /* zName: */ "ignore_check_constraints", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_IgnoreChecks }, + {/* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_IgnoreChecks }, #endif #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) - { /* zName: */ "incremental_vacuum", - /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "index_info", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "index_list", - /* ePragTyp: */ PragTyp_INDEX_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "index_xinfo", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 1 }, + {/* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 12, 3, + /* iArg: */ 0 }, + {/* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 21, 5, + /* iArg: */ 0 }, + {/* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 15, 6, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - { /* zName: */ "integrity_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "journal_mode", - /* ePragTyp: */ PragTyp_JOURNAL_MODE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "journal_size_limit", - /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if defined(SQLITE_HAS_CODEC) - { /* zName: */ "key", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "key", + /* ePragTyp: */ PragTyp_KEY, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "legacy_file_format", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_LegacyFileFmt }, + {/* zName: */ "legacy_file_format", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_LegacyFileFmt }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE - { /* zName: */ "lock_proxy_file", - /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - { /* zName: */ "lock_status", - /* ePragTyp: */ PragTyp_LOCK_STATUS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 47, 2, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "locking_mode", - /* ePragTyp: */ PragTyp_LOCKING_MODE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "max_page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "mmap_size", - /* ePragTyp: */ PragTyp_MMAP_SIZE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "page_size", - /* ePragTyp: */ PragTyp_PAGE_SIZE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) - { /* zName: */ "parser_trace", - /* ePragTyp: */ PragTyp_PARSER_TRACE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_PARSER_TRACE, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "query_only", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_QueryOnly }, + {/* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_QueryOnly }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - { /* zName: */ "quick_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "read_uncommitted", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ReadUncommitted }, - { /* zName: */ "recursive_triggers", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_RecTriggers }, + {/* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReadUncommitted }, + {/* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_RecTriggers }, #endif #if defined(SQLITE_HAS_CODEC) - { /* zName: */ "rekey", - /* ePragTyp: */ PragTyp_REKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "rekey", + /* ePragTyp: */ PragTyp_REKEY, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "reverse_unordered_selects", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ReverseOrder }, + {/* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReverseOrder }, #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "schema_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_SCHEMA_VERSION }, + {/* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "secure_delete", - /* ePragTyp: */ PragTyp_SECURE_DELETE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "short_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ShortColNames }, + {/* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ShortColNames }, #endif - { /* zName: */ "shrink_memory", - /* ePragTyp: */ PragTyp_SHRINK_MEMORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "soft_heap_limit", - /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) - { /* zName: */ "sql_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_SqlTrace }, + {/* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "stats", - /* ePragTyp: */ PragTyp_STATS, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) + {/* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 7, 5, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "synchronous", - /* ePragTyp: */ PragTyp_SYNCHRONOUS, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "table_info", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 1, 6, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "temp_store", - /* ePragTyp: */ PragTyp_TEMP_STORE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "temp_store_directory", - /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif - { /* zName: */ "threads", - /* ePragTyp: */ PragTyp_THREADS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "user_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_USER_VERSION }, + {/* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) - { /* zName: */ "vdbe_addoptrace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeAddopTrace }, - { /* zName: */ "vdbe_debug", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, - { /* zName: */ "vdbe_eqp", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeEQP }, - { /* zName: */ "vdbe_listing", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeListing }, - { /* zName: */ "vdbe_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + {/* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeEQP }, + {/* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeListing }, + {/* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeTrace }, #endif #endif #if !defined(SQLITE_OMIT_WAL) - { /* zName: */ "wal_autocheckpoint", - /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "wal_checkpoint", - /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 43, 3, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "writable_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, + {/* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 60 on by default, 74 total. */ diff --git a/src/prepare.c b/src/prepare.c index 6685dfeaf7..74127bc76b 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -73,6 +73,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ ** structures that describe the table, index, or view. */ int rc; + u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ @@ -83,7 +84,8 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = 0; + db->init.iDb = saved_iDb; + assert( saved_iDb==0 || (db->flags & SQLITE_Vacuum)!=0 ); if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); @@ -107,7 +109,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); if( pIndex==0 ){ /* This can occur if there exists an index on a TEMP table which ** has the same name as another index on a permanent index. Since @@ -286,7 +288,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ char *zSql; zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", - db->aDb[iDb].zName, zMasterName); + db->aDb[iDb].zDbSName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { sqlite3_xauth xAuth; @@ -516,18 +518,14 @@ static int sqlite3Prepare( sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - Parse *pParse; /* Parsing context */ char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ + Parse sParse; /* Parsing context */ - /* Allocate the parsing context */ - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto end_prepare; - } - pParse->pReprepare = pReprepare; + memset(&sParse, 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); @@ -561,7 +559,7 @@ static int sqlite3Prepare( assert( sqlite3BtreeHoldsMutex(pBt) ); rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ - const char *zDb = db->aDb[i].zName; + const char *zDb = db->aDb[i].zDbSName; sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; @@ -571,8 +569,7 @@ static int sqlite3Prepare( sqlite3VtabUnlockList(db); - pParse->db = db; - pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ + sParse.db = db; if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -585,61 +582,61 @@ static int sqlite3Prepare( } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); - pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; sqlite3DbFree(db, zSqlCopy); }else{ - pParse->zTail = &zSql[nBytes]; + sParse.zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3RunParser(&sParse, zSql, &zErrMsg); } - assert( 0==pParse->nQueryLoop ); + assert( 0==sParse.nQueryLoop ); - if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; - if( pParse->checkSchema ){ - schemaIsValid(pParse); + if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; + if( sParse.checkSchema ){ + schemaIsValid(&sParse); } if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM_BKPT; + sParse.rc = SQLITE_NOMEM_BKPT; } if( pzTail ){ - *pzTail = pParse->zTail; + *pzTail = sParse.zTail; } - rc = pParse->rc; + rc = sParse.rc; #ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ + if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", "selectid", "order", "from", "detail" }; int iFirst, mx; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 4); + if( sParse.explain==2 ){ + sqlite3VdbeSetNumCols(sParse.pVdbe, 4); iFirst = 8; mx = 12; }else{ - sqlite3VdbeSetNumCols(pParse->pVdbe, 8); + sqlite3VdbeSetNumCols(sParse.pVdbe, 8); iFirst = 0; mx = 8; } for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, + sqlite3VdbeSetColName(sParse.pVdbe, i-iFirst, COLNAME_NAME, azColName[i], SQLITE_STATIC); } } #endif if( db->init.busy==0 ){ - Vdbe *pVdbe = pParse->pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + Vdbe *pVdbe = sParse.pVdbe; + sqlite3VdbeSetSql(pVdbe, zSql, (int)(sParse.zTail-zSql), saveSqlFlag); } - if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(pParse->pVdbe); + if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ + sqlite3VdbeFinalize(sParse.pVdbe); assert(!(*ppStmt)); }else{ - *ppStmt = (sqlite3_stmt*)pParse->pVdbe; + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; } if( zErrMsg ){ @@ -650,16 +647,15 @@ static int sqlite3Prepare( } /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( pParse->pTriggerPrg ){ - TriggerPrg *pT = pParse->pTriggerPrg; - pParse->pTriggerPrg = pT->pNext; + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; sqlite3DbFree(db, pT); } end_prepare: - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + sqlite3ParserReset(&sParse); rc = sqlite3ApiExit(db, rc); assert( (rc&db->errMask)==rc ); return rc; diff --git a/src/printf.c b/src/printf.c index 238997f37d..a14e658875 100644 --- a/src/printf.c +++ b/src/printf.c @@ -15,26 +15,27 @@ ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 1 /* Integer types. %d, %x, %o, and so forth */ -#define etFLOAT 2 /* Floating point. %f */ -#define etEXP 3 /* Exponentional notation. %e and %E */ -#define etGENERIC 4 /* Floating or exponential, depending on exponent. %g */ -#define etSIZE 5 /* Return number of characters processed so far. %n */ -#define etSTRING 6 /* Strings. %s */ -#define etDYNSTRING 7 /* Dynamically allocated strings. %z */ -#define etPERCENT 8 /* Percent symbol. %% */ -#define etCHARX 9 /* Characters. %c */ +#define etRADIX 0 /* non-decimal integer types. %x %o */ +#define etFLOAT 1 /* Floating point. %f */ +#define etEXP 2 /* Exponentional notation. %e and %E */ +#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 4 /* Return number of characters processed so far. %n */ +#define etSTRING 5 /* Strings. %s */ +#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ +#define etPERCENT 7 /* Percent symbol. %% */ +#define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 10 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 11 /* Strings with '\'' doubled and enclosed in '', +#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 12 /* a pointer to a Token structure */ -#define etSRCLIST 13 /* a pointer to a SrcList */ -#define etPOINTER 14 /* The %p conversion */ -#define etSQLESCAPE3 15 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 16 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCLIST 12 /* a pointer to a SrcList */ +#define etPOINTER 13 /* The %p conversion */ +#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 0 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -58,9 +59,8 @@ typedef struct et_info { /* Information about each format field */ /* ** Allowed values for et_info.flags */ -#define FLAG_SIGNED 1 /* True if the value to convert is signed */ -#define FLAG_INTERN 2 /* True if for internal use only */ -#define FLAG_STRING 4 /* Allow infinity precision */ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ /* @@ -70,7 +70,7 @@ typedef struct et_info { /* Information about each format field */ static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; static const et_info fmtinfo[] = { - { 'd', 10, 1, etRADIX, 0, 0 }, + { 'd', 10, 1, etDECIMAL, 0, 0 }, { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, { 'z', 0, 4, etDYNSTRING, 0, 0 }, @@ -79,7 +79,7 @@ static const et_info fmtinfo[] = { { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, { 'c', 0, 0, etCHARX, 0, 0 }, { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etRADIX, 0, 0 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, { 'x', 16, 0, etRADIX, 16, 1 }, { 'X', 16, 0, etRADIX, 0, 4 }, #ifndef SQLITE_OMIT_FLOATING_POINT @@ -88,16 +88,15 @@ static const et_info fmtinfo[] = { { 'E', 0, 1, etEXP, 14, 0 }, { 'G', 0, 1, etGENERIC, 14, 0 }, #endif - { 'i', 10, 1, etRADIX, 0, 0 }, + { 'i', 10, 1, etDECIMAL, 0, 0 }, { 'n', 0, 0, etSIZE, 0, 0 }, { '%', 0, 0, etPERCENT, 0, 0 }, { 'p', 16, 0, etPOINTER, 0, 1 }, -/* All the rest have the FLAG_INTERN bit set and are thus for internal -** use only */ - { 'T', 0, 2, etTOKEN, 0, 0 }, - { 'S', 0, 2, etSRCLIST, 0, 0 }, - { 'r', 10, 3, etORDINAL, 0, 0 }, + /* All the rest are undocumented and are for internal use only */ + { 'T', 0, 0, etTOKEN, 0, 0 }, + { 'S', 0, 0, etSRCLIST, 0, 0 }, + { 'r', 10, 1, etORDINAL, 0, 0 }, }; /* @@ -181,17 +180,15 @@ void sqlite3VXPrintf( int idx; /* A general purpose loop counter */ int width; /* Width of the current field */ etByte flag_leftjustify; /* True if "-" flag is present */ - etByte flag_plussign; /* True if "+" flag is present */ - etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ etByte flag_alternateform; /* True if "#" flag is present */ etByte flag_altform2; /* True if "!" flag is present */ etByte flag_zeropad; /* True if field width constant starts with zero */ - etByte flag_long; /* True if "l" flag is present */ - etByte flag_longlong; /* True if the "ll" flag is present */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ etByte done; /* Loop termination flag */ - etByte xtype = 0; /* Conversion paradigm */ + etByte cThousand; /* Thousands separator for %d and %u */ + etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ - u8 useIntern; /* Ok to use internal conversions (ex: %T) */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ LONGDOUBLE_TYPE realvalue; /* Value for real types */ @@ -210,13 +207,11 @@ void sqlite3VXPrintf( char buf[etBUFSIZE]; /* Conversion buffer */ bufpt = 0; - if( pAccum->printfFlags ){ - if( (bArgList = (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ - pArgList = va_arg(ap, PrintfArguments*); - } - useIntern = pAccum->printfFlags & SQLITE_PRINTF_INTERNAL; + if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ + pArgList = va_arg(ap, PrintfArguments*); + bArgList = 1; }else{ - bArgList = useIntern = 0; + bArgList = 0; } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ @@ -234,17 +229,18 @@ void sqlite3VXPrintf( break; } /* Find out what flags are present */ - flag_leftjustify = flag_plussign = flag_blanksign = + flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; do{ switch( c ){ case '-': flag_leftjustify = 1; break; - case '+': flag_plussign = 1; break; - case ' ': flag_blanksign = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; case '#': flag_alternateform = 1; break; case '!': flag_altform2 = 1; break; case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; default: done = 1; break; } }while( !done && (c=(*++fmt))!=0 ); @@ -314,13 +310,11 @@ void sqlite3VXPrintf( flag_long = 1; c = *++fmt; if( c=='l' ){ - flag_longlong = 1; + flag_long = 2; c = *++fmt; - }else{ - flag_longlong = 0; } }else{ - flag_long = flag_longlong = 0; + flag_long = 0; } /* Fetch the info entry for the field */ infop = &fmtinfo[0]; @@ -328,11 +322,7 @@ void sqlite3VXPrintf( for(idx=0; idxflags & FLAG_INTERN)==0 ){ - xtype = infop->type; - }else{ - return; - } + xtype = infop->type; break; } } @@ -342,15 +332,11 @@ void sqlite3VXPrintf( ** ** flag_alternateform TRUE if a '#' is present. ** flag_altform2 TRUE if a '!' is present. - ** flag_plussign TRUE if a '+' is present. + ** flag_prefix '+' or ' ' or zero ** flag_leftjustify TRUE if a '-' is present or if the ** field width was negative. ** flag_zeropad TRUE if the width began with 0. - ** flag_long TRUE if the letter 'l' (ell) prefixed - ** the conversion character. - ** flag_longlong TRUE if the letter 'll' (ell ell) prefixed - ** the conversion character. - ** flag_blanksign TRUE if a ' ' is present. + ** flag_long 1 for "l", 2 for "ll" ** width The specified field width. This is ** always non-negative. Zero is the default. ** precision The specified precision. The default @@ -360,19 +346,24 @@ void sqlite3VXPrintf( */ switch( xtype ){ case etPOINTER: - flag_longlong = sizeof(char*)==sizeof(i64); - flag_long = sizeof(char*)==sizeof(long int); + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; /* Fall through into the next case */ case etORDINAL: - case etRADIX: + case etRADIX: + cThousand = 0; + /* Fall through into the next case */ + case etDECIMAL: if( infop->flags & FLAG_SIGNED ){ i64 v; if( bArgList ){ v = getIntArg(pArgList); - }else if( flag_longlong ){ - v = va_arg(ap,i64); }else if( flag_long ){ - v = va_arg(ap,long int); + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } }else{ v = va_arg(ap,int); } @@ -385,17 +376,17 @@ void sqlite3VXPrintf( prefix = '-'; }else{ longvalue = v; - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } }else{ if( bArgList ){ longvalue = (u64)getIntArg(pArgList); - }else if( flag_longlong ){ - longvalue = va_arg(ap,u64); }else if( flag_long ){ - longvalue = va_arg(ap,unsigned long int); + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } }else{ longvalue = va_arg(ap,unsigned int); } @@ -405,16 +396,17 @@ void sqlite3VXPrintf( if( flag_zeropad && precision0 ); } length = (int)(&zOut[nOut-1]-bufpt); - for(idx=precision-length; idx>0; idx--){ + while( precision>length ){ *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } } if( prefix ) *(--bufpt) = prefix; /* Add sign */ if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ @@ -463,9 +470,7 @@ void sqlite3VXPrintf( realvalue = -realvalue; prefix = '-'; }else{ - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); @@ -701,7 +706,9 @@ void sqlite3VXPrintf( break; } case etTOKEN: { - Token *pToken = va_arg(ap, Token*); + Token *pToken; + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pToken = va_arg(ap, Token*); assert( bArgList==0 ); if( pToken && pToken->n ){ sqlite3StrAccumAppend(pAccum, (const char*)pToken->z, pToken->n); @@ -710,9 +717,13 @@ void sqlite3VXPrintf( break; } case etSRCLIST: { - SrcList *pSrc = va_arg(ap, SrcList*); - int k = va_arg(ap, int); - struct SrcList_item *pItem = &pSrc->a[k]; + SrcList *pSrc; + int k; + struct SrcList_item *pItem; + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pSrc = va_arg(ap, SrcList*); + k = va_arg(ap, int); + pItem = &pSrc->a[k]; assert( bArgList==0 ); assert( k>=0 && knSrc ); if( pItem->zDatabase ){ @@ -734,9 +745,13 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); + if( width>0 ){ + if( !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); + sqlite3StrAccumAppend(pAccum, bufpt, length); + if( flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); + }else{ + sqlite3StrAccumAppend(pAccum, bufpt, length); + } if( zExtra ){ sqlite3DbFree(pAccum->db, zExtra); @@ -841,7 +856,7 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); - }else{ + }else if( N ){ assert( p->zText ); p->nChar += N; memcpy(&p->zText[p->nChar-N], z, N); @@ -861,18 +876,23 @@ void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ ** Return a pointer to the resulting string. Return a NULL ** pointer if any kind of error was encountered. */ +static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + assert( p->mxAlloc>0 && !isMalloced(p) ); + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( p->zText ){ + memcpy(p->zText, p->zBase, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + setStrAccumError(p, STRACCUM_NOMEM); + } + return p->zText; +} char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; if( p->mxAlloc>0 && !isMalloced(p) ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - if( p->zText ){ - memcpy(p->zText, p->zBase, p->nChar+1); - p->printfFlags |= SQLITE_PRINTF_MALLOCED; - }else{ - setStrAccumError(p, STRACCUM_NOMEM); - } + return strAccumFinishRealloc(p); } } return p->zText; @@ -1012,7 +1032,8 @@ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ #endif sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); sqlite3VXPrintf(&acc, zFormat, ap); - return sqlite3StrAccumFinish(&acc); + zBuf[acc.nChar] = 0; + return zBuf; } char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ char *z; diff --git a/src/random.c b/src/random.c index 179d01bef2..d4ae77c435 100644 --- a/src/random.c +++ b/src/random.c @@ -106,7 +106,7 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_leave(mutex); } -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or @@ -131,4 +131,4 @@ void sqlite3PrngRestoreState(void){ sizeof(sqlite3Prng) ); } -#endif /* SQLITE_OMIT_BUILTIN_TEST */ +#endif /* SQLITE_UNTESTABLE */ diff --git a/src/resolve.c b/src/resolve.c index 8e290af123..6b3caa625a 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -15,8 +15,6 @@ ** table and column. */ #include "sqliteInt.h" -#include -#include /* ** Walk the expression tree pExpr and increase the aggregate function @@ -221,8 +219,8 @@ static int lookupName( zDb = 0; }else{ for(i=0; inDb; i++){ - assert( db->aDb[i].zName ); - if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ pSchema = db->aDb[i].pSchema; break; } @@ -231,7 +229,8 @@ static int lookupName( } /* Start at the inner-most context and move outward until a match is found */ - while( pNC && cnt==0 ){ + assert( pNC && cnt==0 ); + do{ ExprList *pEList; SrcList *pSrcList = pNC->pSrcList; @@ -400,6 +399,10 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); cnt = 1; pMatch = 0; @@ -412,11 +415,11 @@ static int lookupName( /* Advance to the next name context. The loop will exit when either ** we have a match (cnt>0) or when we run out of name contexts. */ - if( cnt==0 ){ - pNC = pNC->pNext; - nSubquery++; - } - } + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + /* ** If X and Y are NULL (in other words if only the column name Z is @@ -606,34 +609,38 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ - /* A lone identifier is the name of a column. - */ - case TK_ID: { - return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); - } - - /* A table name and column name: ID.ID + /* A column name: ID + ** Or table name and column name: ID.ID ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. */ + case TK_ID: case TK_DOT: { const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; - /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); - /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ + if( pExpr->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; + zTable = 0; + zColumn = pExpr->u.zToken; }else{ - assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + zTable = pExpr->pLeft->u.zToken; + zColumn = pRight->u.zToken; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pExpr->pLeft->u.zToken; + zTable = pRight->pLeft->u.zToken; + zColumn = pRight->pRight->u.zToken; + } } return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); } @@ -646,14 +653,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ - int auth; /* Authorization to use the function */ int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); @@ -690,15 +695,17 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ } } #ifndef SQLITE_OMIT_AUTHORIZATION - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; } - pExpr->op = TK_NULL; - return WRC_Prune; } #endif if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ @@ -711,14 +718,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* Date/time functions that use 'now', and other functions like ** sqlite_version() that might change over time cannot be used ** in an index. */ - notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + notValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx); } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 ){ + }else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); pNC->nErr++; }else if( wrong_num_args ){ @@ -763,6 +775,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); + pNC->ncFlags |= NC_VarSelect; } } break; @@ -771,6 +784,42 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_IS: + case TK_ISNOT: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + } + break; + } } return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; } diff --git a/src/rowset.c b/src/rowset.c index c2e73ed72e..aa81607b9f 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -57,8 +57,9 @@ ** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST ** primitives are constant time. The cost of DESTROY is O(N). ** -** There is an added cost of O(N) when switching between TEST and -** SMALLEST primitives. +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. */ #include "sqliteInt.h" @@ -179,7 +180,9 @@ void sqlite3RowSetClear(RowSet *p){ */ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); - if( p->nFresh==0 ){ + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ struct RowSetChunk *pNew; pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ @@ -213,7 +216,9 @@ void sqlite3RowSetInsert(RowSet *p, i64 rowid){ pEntry->pRight = 0; pLast = p->pLast; if( pLast ){ - if( (p->rsFlags & ROWSET_SORTED)!=0 && rowid<=pLast->v ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ p->rsFlags &= ~ROWSET_SORTED; } pLast->pRight = pEntry; @@ -237,28 +242,26 @@ static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pTail; pTail = &head; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ assert( pA->pRight==0 || pA->v<=pA->pRight->v ); assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->vv ){ - pTail->pRight = pA; + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; pA = pA->pRight; - pTail = pTail->pRight; - }else if( pB->vv ){ - pTail->pRight = pB; - pB = pB->pRight; - pTail = pTail->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } }else{ - pA = pA->pRight; + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } } } - if( pA ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - pTail->pRight = pA; - }else{ - assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); - pTail->pRight = pB; - } return head.pRight; } @@ -281,9 +284,10 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ aBucket[i] = pIn; pIn = pNext; } - pIn = 0; - for(i=0; i1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ p = *ppList; *ppList = p->pRight; p->pLeft = p->pRight = 0; - return p; } - pLeft = rowSetNDeepTree(ppList, iDepth-1); - p = *ppList; - if( p==0 ){ - return pLeft; - } - p->pLeft = pLeft; - *ppList = p->pRight; - p->pRight = rowSetNDeepTree(ppList, iDepth-1); return p; } @@ -378,59 +388,37 @@ static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ return p; } -/* -** Take all the entries on p->pEntry and on the trees in p->pForest and -** sort them all together into one big ordered list on p->pEntry. -** -** This routine should only be called once in the life of a RowSet. -*/ -static void rowSetToList(RowSet *p){ - - /* This routine is called only once */ - assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); - - if( (p->rsFlags & ROWSET_SORTED)==0 ){ - p->pEntry = rowSetEntrySort(p->pEntry); - } - - /* While this module could theoretically support it, sqlite3RowSetNext() - ** is never called after sqlite3RowSetText() for the same RowSet. So - ** there is never a forest to deal with. Should this change, simply - ** remove the assert() and the #if 0. */ - assert( p->pForest==0 ); -#if 0 - while( p->pForest ){ - struct RowSetEntry *pTree = p->pForest->pLeft; - if( pTree ){ - struct RowSetEntry *pHead, *pTail; - rowSetTreeToList(pTree, &pHead, &pTail); - p->pEntry = rowSetEntryMerge(p->pEntry, pHead); - } - p->pForest = p->pForest->pRight; - } -#endif - p->rsFlags |= ROWSET_NEXT; /* Verify this routine is never called again */ -} - /* ** Extract the smallest element from the RowSet. ** Write the element into *pRowid. Return 1 on success. Return ** 0 if the RowSet is already empty. ** ** After this routine has been called, the sqlite3RowSetInsert() -** routine may not be called again. +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. */ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ /* Merge the forest into a single sorted list on first call */ - if( (p->rsFlags & ROWSET_NEXT)==0 ) rowSetToList(p); + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } /* Return the next entry on the list */ if( p->pEntry ){ *pRowid = p->pEntry->v; p->pEntry = p->pEntry->pRight; - if( p->pEntry==0 ){ + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ sqlite3RowSetClear(p); } return 1; @@ -453,13 +441,15 @@ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ /* This routine is never called after sqlite3RowSetNext() */ assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); - /* Sort entries into the forest on the first test of a new batch + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. */ - if( iBatch!=pRowSet->iBatch ){ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ p = pRowSet->pEntry; if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; - if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entiries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ diff --git a/src/select.c b/src/select.c index c9bc389b2f..bb055c7894 100644 --- a/src/select.c +++ b/src/select.c @@ -56,6 +56,7 @@ struct SortCtx { int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ + u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ @@ -74,7 +75,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); - sqlite3WithDelete(db, p->pWith); + if( p->pWith ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFree(db, p); p = pPrior; bFree = 1; @@ -87,7 +88,7 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; pDest->iSDParm = iParm; - pDest->affSdst = 0; + pDest->zAffSdst = 0; pDest->iSdst = 0; pDest->nSdst = 0; } @@ -169,7 +170,7 @@ void sqlite3SelectSetName(Select *p, const char *zName){ ** Delete the given Select structure and all of its substructures. */ void sqlite3SelectDelete(sqlite3 *db, Select *p){ - clearSelect(db, p, 1); + if( p ) clearSelect(db, p, 1); } /* @@ -333,7 +334,7 @@ static void addWhereTerm( pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); - pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); if( pEq && isOuterJoin ){ ExprSetProperty(pEq, EP_FromJoin); assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); @@ -520,7 +521,7 @@ static void pushOntoSorter( int iLimit; /* LIMIT counter */ assert( bSeq==0 || bSeq==1 ); - assert( nData==1 || regData==regOrigData ); + assert( nData==1 || regData==regOrigData || regOrigData==0 ); if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); regBase = regData - nExpr - bSeq; @@ -532,11 +533,11 @@ static void pushOntoSorter( iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; pSort->labelDone = sqlite3VdbeMakeLabel(v); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, - SQLITE_ECEL_DUP|SQLITE_ECEL_REF); + SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } - if( nPrefixReg==0 ){ + if( nPrefixReg==0 && nData>0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -586,12 +587,34 @@ static void pushOntoSorter( }else{ op = OP_IdxInsert; } - sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); + sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, + regBase+nOBSat, nBase-nOBSat); if( iLimit ){ int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); + int r1 = 0; + /* Fill the sorter until it contains LIMIT+OFFSET entries. (The iLimit + ** register is initialized with value of LIMIT+OFFSET.) After the sorter + ** fills up, delete the least entry in the sorter after each insert. + ** Thus we never hold more than the LIMIT+OFFSET rows in memory at once */ + addr = sqlite3VdbeAddOp1(v, OP_IfNotZero, iLimit); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + r1 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Column, pSort->iECursor, nExpr, r1); + VdbeComment((v, "seq")); + } sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); + if( pSort->bOrderedInnerLoop ){ + /* If the inner loop is driven by an index such that values from + ** the same iteration of the inner loop are in sorted order, then + ** immediately jump to the next iteration of an inner loop if the + ** entry from the current iteration does not fit into the top + ** LIMIT+OFFSET entries of the sorter. */ + int iBrk = sqlite3VdbeCurrentAddr(v) + 2; + sqlite3VdbeAddOp3(v, OP_Eq, regBase+nExpr, iBrk, r1); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } sqlite3VdbeJumpHere(v, addr); } } @@ -633,34 +656,11 @@ static void codeDistinct( r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, iMem, N); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, r1); } -#ifndef SQLITE_OMIT_SUBQUERY -/* -** Generate an error message when a SELECT is used within a subexpression -** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error used to occur -** in multiple places. (The error only occurs in one place now, but we -** retain the subroutine to minimize code disruption.) -*/ -static int checkForMultiColumnSelectError( - Parse *pParse, /* Parse context. */ - SelectDest *pDest, /* Destination of SELECT results */ - int nExpr /* Number of result columns returned by SELECT */ -){ - int eDest = pDest->eDest; - if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ - sqlite3ErrorMsg(pParse, "only a single result allowed for " - "a SELECT that is part of an expression"); - return 1; - }else{ - return 0; - } -} -#endif - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -668,7 +668,7 @@ static int checkForMultiColumnSelectError( ** If srcTab is negative, then the pEList expressions ** are evaluated in order to get the data for this row. If srcTab is ** zero or more, then data is pulled from srcTab and pEList is used only -** to get number columns and the datatype for each column. +** to get the number of columns and the collation sequence for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ @@ -683,13 +683,20 @@ static void selectInnerLoop( ){ Vdbe *v = pParse->pVdbe; int i; - int hasDistinct; /* True if the DISTINCT keyword is present */ - int regResult; /* Start of memory holding result set */ + int hasDistinct; /* True if the DISTINCT keyword is present */ int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ int nPrefixReg = 0; /* Number of extra registers before regResult */ + /* Usually, regResult is the first cell in an array of memory cells + ** containing the current result row. In this case regOrig is set to the + ** same value. However, if the results are being sent to the sorter, the + ** values for any expressions that are also part of the sort-key are omitted + ** from this array. In this case regOrig is set to zero. */ + int regResult; /* Start of memory holding current results */ + int regOrig; /* Start of memory holding full result (or 0) */ + assert( v ); assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; @@ -720,7 +727,7 @@ static void selectInnerLoop( pParse->nMem += nResultCol; } pDest->nSdst = nResultCol; - regResult = pDest->iSdst; + regOrig = regResult = pDest->iSdst; if( srcTab>=0 ){ for(i=0; ipOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY + ** expression within the sort-key that pushOntoSorter() will generate. + ** This allows the pEList field to be omitted from the sorted record, + ** saving space and CPU cycles. */ + ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ + int j; + if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ + pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; + } + } + regOrig = 0; + assert( eDest==SRT_Set || eDest==SRT_Mem + || eDest==SRT_Coroutine || eDest==SRT_Output ); + } + nResultCol = sqlite3ExprCodeExprList(pParse,pEList,regResult,0,ecelFlags); } /* If the DISTINCT keyword was present on the SELECT statement @@ -810,7 +835,7 @@ static void selectInnerLoop( int r1; r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -847,7 +872,7 @@ static void selectInnerLoop( int addr = sqlite3VdbeCurrentAddr(v) + 4; sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol); assert( pSort==0 ); } #endif @@ -870,20 +895,20 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nResultCol==1 ); - pDest->affSdst = - sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regResult, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); sqlite3ReleaseTempReg(pParse, r1); } break; @@ -898,14 +923,16 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. */ case SRT_Mem: { - assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + assert( nResultCol<=pDest->nSdst ); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else{ + assert( nResultCol==pDest->nSdst ); assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } @@ -918,7 +945,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); @@ -968,7 +995,7 @@ static void selectInnerLoop( } sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempRange(pParse, r2, nKey+2); @@ -1006,7 +1033,7 @@ static void selectInnerLoop( */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1); - KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ p->aSortOrder = (u8*)&p->aColl[N+X]; p->nField = (u16)N; @@ -1028,7 +1055,7 @@ void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFree(0, p); + if( p->nRef==0 ) sqlite3DbFree(p->db, p); } } @@ -1203,14 +1230,13 @@ static void generateSortTail( int iParm = pDest->iSDParm; int regRow; int regRowid; + int iCol; int nKey; int iSortTab; /* Sorter cursor to read from */ int nSortData; /* Trailing values to read from sorter */ int i; int bSeq; /* True if sorter record includes seq. no. */ -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; -#endif assert( addrBreak<0 ); if( pSort->labelBkOut ){ @@ -1219,21 +1245,21 @@ static void generateSortTail( sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); - regRow = sqlite3GetTempReg(pParse); - nSortData = 1; + regRow = sqlite3GetTempRange(pParse, nColumn); + nSortData = nColumn; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; if( pSort->labelBkOut ){ - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); @@ -1248,11 +1274,18 @@ static void generateSortTail( iSortTab = iTab; bSeq = 1; } - for(i=0; iaffSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regRow, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); break; } case SRT_Mem: { - assert( nColumn==1 ); - sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } @@ -1289,7 +1320,11 @@ static void generateSortTail( } } if( regRowid ){ - sqlite3ReleaseTempReg(pParse, regRow); + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } sqlite3ReleaseTempReg(pParse, regRowid); } /* The bottom of the loop @@ -1430,19 +1465,19 @@ static const char *columnTypeImpl( zOrigCol = "rowid"; }else{ zOrigCol = pTab->aCol[iCol].zName; - zType = sqlite3StrNext(zOrigCol); + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; if( pNC->pParse ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOrigDb = pNC->pParse->db->aDb[iDb].zName; + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; } #else if( iCol<0 ){ zType = "INTEGER"; }else{ - zType = sqlite3StrNext(pTab->aCol[iCol].zName); + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); estWidth = pTab->aCol[iCol].szEst; } #endif @@ -1704,7 +1739,7 @@ int sqlite3ColumnsFromExprList( ** This routine requires that all identifiers in the SELECT ** statement be resolved. */ -static void selectAddColumnTypeAndCollation( +void sqlite3SelectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ Table *pTab, /* Add column type information to this table */ Select *pSelect /* SELECT used to determine types and collations */ @@ -1726,10 +1761,20 @@ static void selectAddColumnTypeAndCollation( sNC.pSrcList = pSelect->pSrc; a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + const char *zType; + int n, m; p = a[i].pExpr; - columnType(&sNC, p, 0, 0, 0, &pCol->szEst); + zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); szAll += pCol->szEst; pCol->affinity = sqlite3ExprAffinity(p); + if( zType && (m = sqlite3Strlen30(zType))>0 ){ + n = sqlite3Strlen30(pCol->zName); + pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); + if( pCol->zName ){ + memcpy(&pCol->zName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ @@ -1762,11 +1807,11 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside ** is disabled */ assert( db->lookaside.bDisable ); - pTab->nRef = 1; + pTab->nTabRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - selectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -1779,20 +1824,20 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. */ -Vdbe *sqlite3GetVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe; - if( v==0 ){ - v = pParse->pVdbe = sqlite3VdbeCreate(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Init); - if( pParse->pToplevel==0 - && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) - ){ - pParse->okConstFactor = 1; - } - +static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse); + if( v ) sqlite3VdbeAddOp2(v, OP_Init, 0, 1); + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; } return v; } +Vdbe *sqlite3GetVdbe(Parse *pParse){ + Vdbe *v = pParse->pVdbe; + return v ? v : allocVdbe(pParse); +} /* @@ -1993,6 +2038,7 @@ static void generateWithRecursiveQuery( /* Process the LIMIT and OFFSET clauses, if they exist */ addrBreak = sqlite3VdbeMakeLabel(v); + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, addrBreak); pLimit = p->pLimit; pOffset = p->pOffset; @@ -2462,7 +2508,7 @@ static int multiSelect( computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); + iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, @@ -2620,19 +2666,17 @@ static int generateOutputSubroutine( } #ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)" construct, - ** then there should be a single item on the stack. Write this - ** item into the set table with bogus data. + /* If we are creating a set for an "expr IN (SELECT ...)". */ case SRT_Set: { int r1; - assert( pIn->nSdst==1 || pParse->nErr>0 ); - pDest->affSdst = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); + testcase( pIn->nSdst>1 ); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, + pIn->iSdst, pIn->nSdst); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -3091,8 +3135,8 @@ static int multiSelectOrderBy( #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ -static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList*, int); +static void substExprList(Parse*, ExprList*, int, ExprList*); +static void substSelect(Parse*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to @@ -3108,36 +3152,46 @@ static void substSelect(sqlite3*, Select *, int, ExprList*, int); ** of the subquery rather the result set of the subquery. */ static Expr *substExpr( - sqlite3 *db, /* Report malloc errors to this connection */ + Parse *pParse, /* Report errors here */ Expr *pExpr, /* Expr in which substitution occurs */ int iTable, /* Table to be substituted */ ExprList *pEList /* Substitute expressions */ ){ + sqlite3 *db = pParse->db; if( pExpr==0 ) return 0; if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ if( pExpr->iColumn<0 ){ pExpr->op = TK_NULL; }else{ Expr *pNew; + Expr *pCopy = pEList->a[pExpr->iColumn].pExpr; assert( pEList!=0 && pExpr->iColumnnExpr ); assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - sqlite3ExprDelete(db, pExpr); - pExpr = pNew; + if( sqlite3ExprIsVector(pCopy) ){ + sqlite3VectorErrorMsg(pParse, pCopy); + }else{ + pNew = sqlite3ExprDup(db, pCopy, 0); + if( pNew && (pExpr->flags & EP_FromJoin) ){ + pNew->iRightJoinTable = pExpr->iRightJoinTable; + pNew->flags |= EP_FromJoin; + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; + } } }else{ - pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); - pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); + pExpr->pLeft = substExpr(pParse, pExpr->pLeft, iTable, pEList); + pExpr->pRight = substExpr(pParse, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); + substSelect(pParse, pExpr->x.pSelect, iTable, pEList, 1); }else{ - substExprList(db, pExpr->x.pList, iTable, pEList); + substExprList(pParse, pExpr->x.pList, iTable, pEList); } } return pExpr; } static void substExprList( - sqlite3 *db, /* Report malloc errors here */ + Parse *pParse, /* Report errors here */ ExprList *pList, /* List to scan and in which to make substitutes */ int iTable, /* Table to be substituted */ ExprList *pEList /* Substitute values */ @@ -3145,11 +3199,11 @@ static void substExprList( int i; if( pList==0 ) return; for(i=0; inExpr; i++){ - pList->a[i].pExpr = substExpr(db, pList->a[i].pExpr, iTable, pEList); + pList->a[i].pExpr = substExpr(pParse, pList->a[i].pExpr, iTable, pEList); } } static void substSelect( - sqlite3 *db, /* Report malloc errors here */ + Parse *pParse, /* Report errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ ExprList *pEList, /* Substitute values */ @@ -3160,17 +3214,17 @@ static void substSelect( int i; if( !p ) return; do{ - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + substExprList(pParse, p->pEList, iTable, pEList); + substExprList(pParse, p->pGroupBy, iTable, pEList); + substExprList(pParse, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(pParse, p->pHaving, iTable, pEList); + p->pWhere = substExpr(pParse, p->pWhere, iTable, pEList); pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList, 1); + substSelect(pParse, pItem->pSelect, iTable, pEList, 1); if( pItem->fg.isTabFunc ){ - substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + substExprList(pParse, pItem->u1.pFuncArg, iTable, pEList); } } }while( doPrior && (p = p->pPrior)!=0 ); @@ -3567,12 +3621,12 @@ static int flattenSubquery( */ if( ALWAYS(pSubitem->pTab!=0) ){ Table *pTabToDel = pSubitem->pTab; - if( pTabToDel->nRef==1 ){ + if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); pTabToDel->pNextZombie = pToplevel->pZombieTab; pToplevel->pZombieTab = pTabToDel; }else{ - pTabToDel->nRef--; + pTabToDel->nTabRef--; } pSubitem->pTab = 0; } @@ -3687,14 +3741,17 @@ static int flattenSubquery( assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, - sqlite3ExprDup(db, pSub->pHaving, 0)); + pParent->pHaving = sqlite3ExprAnd(db, + sqlite3ExprDup(db, pSub->pHaving, 0), pParent->pHaving + ); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); + } + if( db->mallocFailed==0 ){ + substSelect(pParse, pParent, iParent, pSub->pEList, 0); } - substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. @@ -3768,31 +3825,37 @@ static int flattenSubquery( ** terms are duplicated into the subquery. */ static int pushDownWhereTerms( - sqlite3 *db, /* The database connection (for malloc()) */ + Parse *pParse, /* Parse context (for malloc() and error reporting) */ Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ Expr *pWhere, /* The WHERE clause of the outer query */ int iCursor /* Cursor number of the subquery */ ){ Expr *pNew; int nChng = 0; + Select *pX; /* For looping over compound SELECTs in pSubq */ if( pWhere==0 ) return 0; - if( (pSubq->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - return 0; /* restrictions (1) and (2) */ + for(pX=pSubq; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ + testcase( pX->selFlags & SF_Aggregate ); + testcase( pX->selFlags & SF_Recursive ); + testcase( pX!=pSubq ); + return 0; /* restrictions (1) and (2) */ + } } if( pSubq->pLimit!=0 ){ - return 0; /* restriction (3) */ + return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, iCursor); pWhere = pWhere->pLeft; } if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ nChng++; while( pSubq ){ - pNew = sqlite3ExprDup(db, pWhere, 0); - pNew = substExpr(db, pNew, iCursor, pSubq->pEList); - pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); + pNew = sqlite3ExprDup(pParse->db, pWhere, 0); + pNew = substExpr(pParse, pNew, iCursor, pSubq->pEList); + pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); pSubq = pSubq->pPrior; } } @@ -4084,7 +4147,7 @@ static int withExpand( assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; - pTab->nRef = 1; + pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); @@ -4107,25 +4170,33 @@ static int withExpand( ){ pItem->pTab = pTab; pItem->fg.isRecursive = 1; - pTab->nRef++; + pTab->nTabRef++; pSel->selFlags |= SF_Recursive; } } } /* Only one recursive reference is permitted. */ - if( pTab->nRef>2 ){ + if( pTab->nTabRef>2 ){ sqlite3ErrorMsg( pParse, "multiple references to recursive table: %s", pCte->zName ); return SQLITE_ERROR; } - assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + assert( pTab->nTabRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nTabRef==2 )); pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; - sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + if( bMayRecursive ){ + Select *pPrior = pSel->pPrior; + assert( pPrior->pWith==0 ); + pPrior->pWith = pSel->pWith; + sqlite3WalkSelect(pWalker, pPrior); + pPrior->pWith = 0; + }else{ + sqlite3WalkSelect(pWalker, pSel); + } pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); @@ -4169,10 +4240,12 @@ static int withExpand( */ static void selectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - With *pWith = findRightmost(p)->pWith; - if( pWith!=0 ){ - assert( pParse->pWith==pWith ); - pParse->pWith = pWith->pOuter; + if( pParse->pWith && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } } } #else @@ -4222,8 +4295,8 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; - if( pWalker->xSelectCallback2==selectPopWith ){ - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( p->pWith ){ + sqlite3WithPush(pParse, p->pWith, 0); } /* Make sure cursor numbers have been assigned to all entries in @@ -4253,7 +4326,7 @@ static int selectExpander(Walker *pWalker, Select *p){ if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; - pTab->nRef = 1; + pTab->nTabRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); @@ -4266,13 +4339,13 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; - if( pTab->nRef==0xffff ){ + if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); pFrom->pTab = 0; return WRC_Abort; } - pTab->nRef++; + pTab->nTabRef++; if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ return WRC_Abort; } @@ -4376,7 +4449,7 @@ static int selectExpander(Walker *pWalker, Select *p){ continue; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; } for(j=0; jnCol; j++){ char *zName = pTab->aCol[j].zName; @@ -4422,10 +4495,10 @@ static int selectExpander(Walker *pWalker, Select *p){ if( longNames || pTabList->nSrc>1 ){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); - pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); if( zSchemaName ){ pLeft = sqlite3Expr(db, TK_ID, zSchemaName); - pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr); } if( longNames ){ zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); @@ -4510,9 +4583,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - if( (pSelect->selFlags & SF_MultiValue)==0 ){ - w.xSelectCallback2 = selectPopWith; - } + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } @@ -4550,7 +4621,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Select *pSel = pFrom->pSelect; if( pSel ){ while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } @@ -4662,8 +4733,8 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ ExprList *pList = pF->pExpr->x.pList; assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); - sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, - (void*)pF->pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } @@ -4714,8 +4785,8 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, - (void*)pF->pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp3(v, OP_AggStep0, 0, regAgg, pF->iMem); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -4859,16 +4930,6 @@ int sqlite3Select( } #endif - - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, p->pEList->nExpr) ){ - goto select_end; - } -#endif - /* Try to flatten subqueries in the FROM clause up into the main query */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -4959,7 +5020,7 @@ int sqlite3Select( ** inside the subquery. This can help the subquery to run more efficiently. */ if( (pItem->fg.jointype & JT_OUTER)==0 - && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor) ){ #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ @@ -4970,10 +5031,24 @@ int sqlite3Select( } /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if all of these are true: + ** (1) The subquery is guaranteed to be the outer loop (so that it + ** does not need to be computed more than once) + ** (2) The ALL keyword after SELECT is omitted. (Applications are + ** allowed to say "SELECT ALL" instead of just "SELECT" to disable + ** the use of co-routines.) + ** (3) Co-routines are not disabled using sqlite3_test_control() + ** with SQLITE_TESTCTRL_OPTIMIZATIONS. + ** + ** TODO: Are there other reasons beside (1) to use a co-routine + ** implementation? */ - if( pTabList->nSrc==1 - && (p->selFlags & SF_All)==0 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (p->selFlags & SF_All)==0 /* (2) */ + && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ ){ /* Implement a co-routine that will return a single row of the result ** set on each invocation. @@ -5009,7 +5084,7 @@ int sqlite3Select( /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); @@ -5068,6 +5143,13 @@ int sqlite3Select( ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif } /* If there is an ORDER BY clause, then create an ephemeral index to @@ -5100,7 +5182,9 @@ int sqlite3Select( /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = 320; /* 4 billion rows */ + if( (p->selFlags & SF_FixedLimit)==0 ){ + p->nSelectRow = 320; /* 4 billion rows */ + } computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); @@ -5139,6 +5223,7 @@ int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -5577,7 +5662,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax, 0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/shell.c b/src/shell.c index 675e2ffd19..e51694bc76 100644 --- a/src/shell.c +++ b/src/shell.c @@ -90,7 +90,7 @@ #else -# define shell_read_history(X) +# define shell_read_history(X) # define shell_write_history(X) # define shell_stifle_history(X) @@ -136,6 +136,16 @@ #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) +#if defined(_WIN32) || defined(WIN32) +#include + +/* string conversion routines only needed on Win32 */ +extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); +extern char *sqlite3_win32_mbcs_to_utf8_v2(const char *, int); +extern char *sqlite3_win32_utf8_to_mbcs_v2(const char *, int); +extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); +#endif + /* On Windows, we normally run with output mode of TEXT so that \n characters ** are automatically translated into \r\n. However, this behavior needs ** to be disabled in some cases (ex: when generating CSV output and when @@ -143,17 +153,17 @@ ** routines take care of that. */ #if defined(_WIN32) || defined(WIN32) -static void setBinaryMode(FILE *out){ - fflush(out); - _setmode(_fileno(out), _O_BINARY); +static void setBinaryMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_BINARY); } -static void setTextMode(FILE *out){ - fflush(out); - _setmode(_fileno(out), _O_TEXT); +static void setTextMode(FILE *file, int isOutput){ + if( isOutput ) fflush(file); + _setmode(_fileno(file), _O_TEXT); } #else -# define setBinaryMode(X) -# define setTextMode(X) +# define setBinaryMode(X,Y) +# define setTextMode(X,Y) #endif #include "shell_indexes.c" @@ -205,7 +215,7 @@ static void beginTimer(void){ /* Return the difference of two time_structs in seconds */ static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + (double)(pEnd->tv_sec - pStart->tv_sec); } @@ -230,8 +240,6 @@ static void endTimer(void){ #elif (defined(_WIN32) || defined(WIN32)) -#include - /* Saved resource information for the beginning of an operation */ static HANDLE hProcess; static FILETIME ftKernelBegin; @@ -262,7 +270,7 @@ static int hasTimer(void){ if( NULL != getProcessTimesAddr ){ return 1; } - FreeLibrary(hinstLib); + FreeLibrary(hinstLib); } } } @@ -308,7 +316,7 @@ static void endTimer(void){ #define HAS_TIMER hasTimer() #else -#define BEGIN_TIMER +#define BEGIN_TIMER #define END_TIMER #define HAS_TIMER 0 #endif @@ -362,6 +370,38 @@ static char *Argv0; static char mainPrompt[20]; /* First line prompt. default: "sqlite> "*/ static char continuePrompt[20]; /* Continuation prompt. default: " ...> " */ +/* +** Render output like fprintf(). Except, if the output is going to the +** console and if this is running on a Windows machine, translate the +** output from UTF-8 into MBCS. +*/ +#if defined(_WIN32) || defined(WIN32) +void utf8_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + if( stdout_is_console && (out==stdout || out==stderr) ){ + char *z1 = sqlite3_vmprintf(zFormat, ap); + char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); + sqlite3_free(z1); + fputs(z2, out); + sqlite3_free(z2); + }else{ + vfprintf(out, zFormat, ap); + } + va_end(ap); +} +#elif !defined(utf8_printf) +# define utf8_printf fprintf +#endif + +/* +** Render output like fprintf(). This should not be used on anything that +** includes string formatting (e.g. "%s"). +*/ +#if !defined(raw_printf) +# define raw_printf fprintf +#endif + /* ** Write I/O traces to the following stream. */ @@ -383,7 +423,7 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - fprintf(iotrace, "%s", z); + utf8_printf(iotrace, "%s", z); sqlite3_free(z); } #endif @@ -416,28 +456,6 @@ static int isNumber(const char *z, int *realnum){ return *z==0; } -/* -** A global char* and an SQL function to access its current value -** from within an SQL statement. This program used to use the -** sqlite_exec_printf() API to substitue a string into an SQL statement. -** The correct way to do this with sqlite3 is to use the bind API, but -** since the shell is built around the callback paradigm it would be a lot -** of work. Instead just use this hack, which is quite harmless. -*/ -static const char *zShellStatic = 0; -static void shellstaticFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - assert( 0==argc ); - assert( zShellStatic ); - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - sqlite3_result_text(context, zShellStatic, -1, SQLITE_STATIC); -} - - /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -484,11 +502,10 @@ static char *local_getline(char *zLine, FILE *in){ } } #if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the + /* For interactive input on Windows systems, translate the ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive ){ - extern char *sqlite3_win32_mbcs_to_utf8(const char*); - char *zTrans = sqlite3_win32_mbcs_to_utf8(zLine); + if( stdin_is_interactive && in==stdin ){ + char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ int nTrans = strlen30(zTrans)+1; if( nTrans>nLine ){ @@ -539,42 +556,795 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ } return zResult; } +/* +** A variable length string to which one can append text. +*/ +typedef struct ShellText ShellText; +struct ShellText { + char *z; + int n; + int nAlloc; +}; /* -** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, translate the -** output from UTF-8 into MBCS. +** Initialize and destroy a ShellText object */ -#if defined(_WIN32) || defined(WIN32) -void utf8_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) ){ - extern char *sqlite3_win32_utf8_to_mbcs(const char*); - char *z1 = sqlite3_vmprintf(zFormat, ap); - char *z2 = sqlite3_win32_utf8_to_mbcs(z1); - sqlite3_free(z1); - fputs(z2, out); - sqlite3_free(z2); - }else{ - vfprintf(out, zFormat, ap); - } - va_end(ap); +static void initText(ShellText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(ShellText *p){ + free(p->z); + initText(p); } -#elif !defined(utf8_printf) -# define utf8_printf fprintf -#endif -/* -** Render output like fprintf(). This should not be used on anything that -** includes string formatting (e.g. "%s"). +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. */ -#if !defined(raw_printf) -# define raw_printf fprintf +static void appendText(ShellText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = strlen30(zAppend); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + len + 20; + p->z = realloc(p->z, p->nAlloc); + if( p->z==0 ){ + memset(p, 0, sizeof(*p)); + return; + } + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + /* All SQLite keywords, in alphabetical order */ + static const char *azKeywords[] = { + "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS", + "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY", + "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT", + "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE", + "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE", + "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH", + "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN", + "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF", + "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER", + "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY", + "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL", + "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA", + "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP", + "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT", + "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP", + "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE", + "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE", + "WITH", "WITHOUT", + }; + int i, lwr, upr, mid, c; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + lwr = 0; + upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + c = sqlite3_stricmp(azKeywords[mid], zName); + if( c==0 ) return '"'; + if( c<0 ){ + lwr = mid+1; + }else{ + upr = mid-1; + } + } + return 0; +} + +/****************************************************************************** +** SHA3 hash implementation copied from ../ext/misc/shathree.c +*/ +typedef sqlite3_uint64 u64; +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + + +/* +** State structure for a SHA3 hash in progress +*/ +typedef struct SHA3Context SHA3Context; +struct SHA3Context { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(SHA3Context *p){ + int i; + u64 B0, B1, B2, B3, B4; + u64 C0, C1, C2, C3, C4; + u64 D0, D1, D2, D3, D4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void SHA3Init(SHA3Context *p, int iSize){ + memset(p, 0, sizeof(*p)); + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the SHA3Update function to add new content +** to the hash +*/ +static void SHA3Update( + SHA3Context *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke SHA3Final() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *SHA3Final(SHA3Context *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + SHA3Update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + SHA3Update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + SHA3Update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Implementation of the sha3(X,SIZE) function. +** +** Return a BLOB which is the SIZE-bit SHA3 hash of X. The default +** size is 256. If X is a BLOB, it is hashed as is. +** For all other non-NULL types of input, X is converted into a UTF-8 string +** and the string is hashed without the trailing 0x00 terminator. The hash +** of a NULL value is NULL. +*/ +static void sha3Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + SHA3Context cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + int iSize; + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( eType==SQLITE_NULL ) return; + SHA3Init(&cx, iSize); + if( eType==SQLITE_BLOB ){ + SHA3Update(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + SHA3Update(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} + +/* Compute a string using sqlite3_vsnprintf() with a maximum length +** of 50 bytes and add it to the hash. +*/ +static void hash_step_vformat( + SHA3Context *p, /* Add content to this context */ + const char *zFormat, + ... +){ + va_list ap; + int n; + char zBuf[50]; + va_start(ap, zFormat); + sqlite3_vsnprintf(sizeof(zBuf),zBuf,zFormat,ap); + va_end(ap); + n = (int)strlen(zBuf); + SHA3Update(p, (unsigned char*)zBuf, n); +} + +/* +** Implementation of the sha3_query(SQL,SIZE) function. +** +** This function compiles and runs the SQL statement(s) given in the +** argument. The results are hashed using a SIZE-bit SHA3. The default +** size is 256. +** +** The format of the byte stream that is hashed is summarized as follows: +** +** S: +** R +** N +** I +** F +** B: +** T: +** +** is the original SQL text for each statement run and is +** the size of that text. The SQL text is UTF-8. A single R character +** occurs before the start of each row. N means a NULL value. +** I mean an 8-byte little-endian integer . F is a floating point +** number with an 8-byte little-endian IEEE floating point value . +** B means blobs of bytes. T means text rendered as +** bytes of UTF-8. The and values are expressed as an ASCII +** text integers. +** +** For each SQL statement in the X input, there is one S segment. Each +** S segment is followed by zero or more R segments, one for each row in the +** result set. After each R, there are one or more N, I, F, B, or T segments, +** one for each column in the result set. Segments are concatentated directly +** with no delimiters of any kind. +*/ +static void sha3QueryFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + sqlite3_stmt *pStmt = 0; + int nCol; /* Number of columns in the result set */ + int i; /* Loop counter */ + int rc; + int n; + const char *z; + SHA3Context cx; + int iSize; + + if( argc==1 ){ + iSize = 256; + }else{ + iSize = sqlite3_value_int(argv[1]); + if( iSize!=224 && iSize!=256 && iSize!=384 && iSize!=512 ){ + sqlite3_result_error(context, "SHA3 size should be one of: 224 256 " + "384 512", -1); + return; + } + } + if( zSql==0 ) return; + SHA3Init(&cx, iSize); + while( zSql[0] ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc ){ + char *zMsg = sqlite3_mprintf("error SQL statement [%s]: %s", + zSql, sqlite3_errmsg(db)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + if( !sqlite3_stmt_readonly(pStmt) ){ + char *zMsg = sqlite3_mprintf("non-query: [%s]", sqlite3_sql(pStmt)); + sqlite3_finalize(pStmt); + sqlite3_result_error(context, zMsg, -1); + sqlite3_free(zMsg); + return; + } + nCol = sqlite3_column_count(pStmt); + z = sqlite3_sql(pStmt); + if( z==0 ){ + sqlite3_finalize(pStmt); + continue; + } + n = (int)strlen(z); + hash_step_vformat(&cx,"S%d:",n); + SHA3Update(&cx,(unsigned char*)z,n); + + /* Compute a hash over the result of the query */ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + SHA3Update(&cx,(const unsigned char*)"R",1); + for(i=0; i=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'I'; + SHA3Update(&cx, x, 9); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[9]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=8; j>=1; j--){ + x[j] = u & 0xff; + u >>= 8; + } + x[0] = 'F'; + SHA3Update(&cx,x,9); + break; + } + case SQLITE_TEXT: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_text(pStmt, i); + hash_step_vformat(&cx,"T%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + case SQLITE_BLOB: { + int n2 = sqlite3_column_bytes(pStmt, i); + const unsigned char *z2 = sqlite3_column_blob(pStmt, i); + hash_step_vformat(&cx,"B%d:",n2); + SHA3Update(&cx, z2, n2); + break; + } + } + } + } + sqlite3_finalize(pStmt); + } + sqlite3_result_blob(context, SHA3Final(&cx), iSize/8, SQLITE_TRANSIENT); +} +/* End of SHA3 hashing logic copy/pasted from ../ext/misc/shathree.c +********************************************************************************/ + +#if defined(SQLITE_ENABLE_SESSION) +/* +** State information for a single open session +*/ +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ +}; #endif /* -** Shell output mode information from before ".explain on", +** Shell output mode information from before ".explain on", ** saved so that it can be restored by ".explain off" */ typedef struct SavedModeInfo SavedModeInfo; @@ -592,14 +1362,11 @@ struct SavedModeInfo { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ - int echoOn; /* True to echo input commands */ int autoExplain; /* Automatically turn on .explain mode */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int bRecommend; /* Instead of sqlite3_exec(), recommend indexes */ int statsOn; /* True to display memory stats before each finalize */ int scanstatsOn; /* True to display scan stats before each finalize */ - int countChanges; /* True to display change counts */ - int backslashOn; /* Resolve C-style \x escapes in SQL input text */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -610,8 +1377,10 @@ struct ShellState { int normalMode; /* Output mode before ".explain on" */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ + int nCheck; /* Number of ".check" commands run */ unsigned shellFlgs; /* Various flags */ char *zDestTable; /* Name of destination table when MODE_Insert */ + char zTestcase[30]; /* Name of current test case */ char colSeparator[20]; /* Column separator character for several modes */ char rowSeparator[20]; /* Row separator character for MODE_Ascii */ int colWidth[100]; /* Requested width of each column when in column mode*/ @@ -627,14 +1396,29 @@ struct ShellState { int *aiIndent; /* Array of indents used in MODE_Explain */ int nIndent; /* Size of array aiIndent[] */ int iIndent; /* Index of current op in aiIndent[] */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif }; /* ** These are the allowed shellFlgs values */ -#define SHFLG_Scratch 0x00001 /* The --scratch option is used */ -#define SHFLG_Pagecache 0x00002 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00004 /* Lookaside memory is used */ +#define SHFLG_Scratch 0x00000001 /* The --scratch option is used */ +#define SHFLG_Pagecache 0x00000002 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000004 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000008 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000010 /* .dump preserves rowid values */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ + +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* ** These are the allowed modes. @@ -645,10 +1429,12 @@ struct ShellState { #define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ #define MODE_Html 4 /* Generate an XHTML table */ #define MODE_Insert 5 /* Generate SQL "insert" statements */ -#define MODE_Tcl 6 /* Generate ANSI-C or TCL quoted elements */ -#define MODE_Csv 7 /* Quote strings, numbers are plain */ -#define MODE_Explain 8 /* Like MODE_Column, but do not truncate data */ -#define MODE_Ascii 9 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Quote 6 /* Quote values as for SQL */ +#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ +#define MODE_Csv 8 /* Quote strings, numbers are plain */ +#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ +#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Pretty 11 /* Pretty-print schemas */ static const char *modeDescr[] = { "line", @@ -657,10 +1443,12 @@ static const char *modeDescr[] = { "semi", "html", "insert", + "quote", "tcl", "csv", "explain", "ascii", + "prettyprint", }; /* @@ -704,34 +1492,54 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ /* ** Output the given string as a quoted string using SQL quoting conventions. +** +** The "\n" and "\r" characters are converted to char(10) and char(13) +** to prevent them from being transformed by end-of-line translators. */ static void output_quoted_string(FILE *out, const char *z){ int i; - int nSingle = 0; - setBinaryMode(out); - for(i=0; z[i]; i++){ - if( z[i]=='\'' ) nSingle++; - } - if( nSingle==0 ){ + char c; + setBinaryMode(out, 1); + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ utf8_printf(out,"'%s'",z); }else{ - raw_printf(out,"'"); + int inQuote = 0; + int bStarted = 0; while( *z ){ - for(i=0; z[i] && z[i]!='\''; i++){} - if( i==0 ){ - raw_printf(out,"''"); - z++; - }else if( z[i]=='\'' ){ - utf8_printf(out,"%.*s''",i,z); - z += i+1; - }else{ - utf8_printf(out,"%s",z); + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + if( !inQuote ){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "'"); + inQuote = 1; + } + utf8_printf(out, "%.*s", i, z); + z += i; + bStarted = 1; + } + if( c=='\'' ){ + raw_printf(out, "'"); + continue; + } + if( inQuote ){ + raw_printf(out, "'"); + inQuote = 0; + } + if( c==0 ){ break; } + for(i=0; (c = z[i])=='\r' || c=='\n'; i++){ + if( bStarted ) raw_printf(out, "||"); + raw_printf(out, "char(%d)", c); + bStarted = 1; + } + z += i; } - raw_printf(out,"'"); + if( inQuote ) raw_printf(out, "'"); } - setTextMode(out); + setTextMode(out, 1); } /* @@ -773,11 +1581,11 @@ static void output_html_string(FILE *out, const char *z){ int i; if( z==0 ) z = ""; while( *z ){ - for(i=0; z[i] - && z[i]!='<' - && z[i]!='&' - && z[i]!='>' - && z[i]!='\"' + for(i=0; z[i] + && z[i]!='<' + && z[i]!='&' + && z[i]!='>' + && z[i]!='\"' && z[i]!='\''; i++){} if( i>0 ){ @@ -805,22 +1613,22 @@ static void output_html_string(FILE *out, const char *z){ ** array, then the string must be quoted for CSV. */ static const char needCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; /* @@ -837,8 +1645,8 @@ static void output_csv(ShellState *p, const char *z, int bSep){ int i; int nSep = strlen30(p->colSeparator); for(i=0; z[i]; i++){ - if( needCsvQuote[((unsigned char*)z)[i]] - || (z[i]==p->colSeparator[0] && + if( needCsvQuote[((unsigned char*)z)[i]] + || (z[i]==p->colSeparator[0] && (nSep==1 || memcmp(z, p->colSeparator, nSep)==0)) ){ i = 0; break; @@ -872,6 +1680,73 @@ static void interrupt_handler(int NotUsed){ } #endif +#ifndef SQLITE_OMIT_AUTHORIZATION +/* +** When the ".auth ON" is set, the following authorizer callback is +** invoked. It always returns SQLITE_OK. +*/ +static int shellAuth( + void *pClientData, + int op, + const char *zA1, + const char *zA2, + const char *zA3, + const char *zA4 +){ + ShellState *p = (ShellState*)pClientData; + static const char *azAction[] = { 0, + "CREATE_INDEX", "CREATE_TABLE", "CREATE_TEMP_INDEX", + "CREATE_TEMP_TABLE", "CREATE_TEMP_TRIGGER", "CREATE_TEMP_VIEW", + "CREATE_TRIGGER", "CREATE_VIEW", "DELETE", + "DROP_INDEX", "DROP_TABLE", "DROP_TEMP_INDEX", + "DROP_TEMP_TABLE", "DROP_TEMP_TRIGGER", "DROP_TEMP_VIEW", + "DROP_TRIGGER", "DROP_VIEW", "INSERT", + "PRAGMA", "READ", "SELECT", + "TRANSACTION", "UPDATE", "ATTACH", + "DETACH", "ALTER_TABLE", "REINDEX", + "ANALYZE", "CREATE_VTABLE", "DROP_VTABLE", + "FUNCTION", "SAVEPOINT", "RECURSIVE" + }; + int i; + const char *az[4]; + az[0] = zA1; + az[1] = zA2; + az[2] = zA3; + az[3] = zA4; + utf8_printf(p->out, "authorizer: %s", azAction[op]); + for(i=0; i<4; i++){ + raw_printf(p->out, " "); + if( az[i] ){ + output_c_string(p->out, az[i]); + }else{ + raw_printf(p->out, "NULL"); + } + } + raw_printf(p->out, "\n"); + return SQLITE_OK; +} +#endif + +/* +** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** +** This routine converts some CREATE TABLE statements for shadow tables +** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. +*/ +static void printSchemaLine(FILE *out, const char *z, const char *zTail){ + if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ + utf8_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + }else{ + utf8_printf(out, "%s%s", z, zTail); + } +} +static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ + char c = z[n]; + z[n] = 0; + printSchemaLine(out, z, zTail); + z[n] = c; +} + /* ** This is the callback routine that the shell ** invokes for each row of a query result. @@ -988,7 +1863,70 @@ static int shell_callback( } break; } - case MODE_Semi: + case MODE_Semi: { /* .schema and .fullschema output */ + printSchemaLine(p->out, azArg[0], ";\n"); + break; + } + case MODE_Pretty: { /* .schema and .fullschema with --indent */ + char *z; + int j; + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + assert( nArg==1 ); + if( azArg[0]==0 ) break; + if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 + ){ + utf8_printf(p->out, "%s;\n", azArg[0]); + break; + } + z = sqlite3_mprintf("%s", azArg[0]); + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 ){ + printSchemaLineN(p->out, z, j, "\n"); + j = 0; + } + } + z[j++] = c; + if( nParen==1 && (c=='(' || c==',' || c=='\n') ){ + if( c=='\n' ) j--; + printSchemaLineN(p->out, z, j, "\n "); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } + } + } + z[j] = 0; + } + printSchemaLine(p->out, z, ";\n"); + sqlite3_free(z); + break; + } case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s", z); if( iout, "%s", p->colSeparator); - }else if( p->cMode==MODE_Semi ){ - utf8_printf(p->out, ";%s", p->rowSeparator); }else{ utf8_printf(p->out, "%s", p->rowSeparator); } @@ -1048,7 +1984,7 @@ static int shell_callback( break; } case MODE_Csv: { - setBinaryMode(p->out); + setBinaryMode(p->out, 1); if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s", p->rowSeparator); } - setTextMode(p->out); + setTextMode(p->out, 1); break; } + case MODE_Quote: case MODE_Insert: { - p->cnt++; if( azArg==0 ) break; - utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); - if( p->showHeader ){ - raw_printf(p->out,"("); - for(i=0; i0 ? ",": ""; - utf8_printf(p->out, "%s%s", zSep, azCol[i]); + if( p->cMode==MODE_Insert ){ + utf8_printf(p->out,"INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + raw_printf(p->out,"("); + for(i=0; i0 ? ",": ""; + utf8_printf(p->out, "%s%s", zSep, azCol[i]); + } + raw_printf(p->out,")"); } - raw_printf(p->out,")"); + raw_printf(p->out," VALUES("); + }else if( p->cnt==0 && p->showHeader ){ + for(i=0; i0 ) raw_printf(p->out, ","); + output_quoted_string(p->out, azCol[i]); + } + raw_printf(p->out,"\n"); } - raw_printf(p->out," VALUES("); + p->cnt++; for(i=0; i0 ? ",": ""; if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ @@ -1084,9 +2029,13 @@ static int shell_callback( }else if( aiType && aiType[i]==SQLITE_TEXT ){ if( zSep[0] ) utf8_printf(p->out,"%s",zSep); output_quoted_string(p->out, azArg[i]); - }else if( aiType && (aiType[i]==SQLITE_INTEGER - || aiType[i]==SQLITE_FLOAT) ){ + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ utf8_printf(p->out,"%s%s",zSep, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + raw_printf(p->out, "%s%s", zSep, z); }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ const void *pBlob = sqlite3_column_blob(p->pStmt, i); int nBlob = sqlite3_column_bytes(p->pStmt, i); @@ -1099,7 +2048,7 @@ static int shell_callback( output_quoted_string(p->out, azArg[i]); } } - raw_printf(p->out,");\n"); + raw_printf(p->out,p->cMode==MODE_Quote?"\n":");\n"); break; } case MODE_Ascii: { @@ -1131,6 +2080,71 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ return shell_callback(pArg, nArg, azArg, azCol, NULL); } +/* +** This is the callback routine from sqlite3_exec() that appends all +** output onto the end of a ShellText object. +*/ +static int captureOutputCallback(void *pArg, int nArg, char **azArg, char **az){ + ShellText *p = (ShellText*)pArg; + int i; + UNUSED_PARAMETER(az); + if( p->n ) appendText(p, "|", 0); + for(i=0; idb, + "SAVEPOINT selftest_init;\n" + "CREATE TABLE IF NOT EXISTS selftest(\n" + " tno INTEGER PRIMARY KEY,\n" /* Test number */ + " op TEXT,\n" /* Operator: memo run */ + " cmd TEXT,\n" /* Command text */ + " ans TEXT\n" /* Desired answer */ + ");" + "CREATE TEMP TABLE [_shell$self](op,cmd,ans);\n" + "INSERT INTO [_shell$self](rowid,op,cmd)\n" + " VALUES(coalesce((SELECT (max(tno)+100)/10 FROM selftest),10),\n" + " 'memo','Tests generated by --init');\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run',\n" + " 'SELECT hex(sha3_query(''SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2'',224))',\n" + " hex(sha3_query('SELECT type,name,tbl_name,sql " + "FROM sqlite_master ORDER BY 2',224));\n" + "INSERT INTO [_shell$self]\n" + " SELECT 'run'," + " 'SELECT hex(sha3_query(''SELECT * FROM \"' ||" + " printf('%w',name) || '\" NOT INDEXED'',224))',\n" + " hex(sha3_query(printf('SELECT * FROM \"%w\" NOT INDEXED',name),224))\n" + " FROM (\n" + " SELECT name FROM sqlite_master\n" + " WHERE type='table'\n" + " AND name<>'selftest'\n" + " AND coalesce(rootpage,0)>0\n" + " )\n" + " ORDER BY name;\n" + "INSERT INTO [_shell$self]\n" + " VALUES('run','PRAGMA integrity_check','ok');\n" + "INSERT INTO selftest(tno,op,cmd,ans)" + " SELECT rowid*10,op,cmd,ans FROM [_shell$self];\n" + "DROP TABLE [_shell$self];" + ,0,0,&zErrMsg); + if( zErrMsg ){ + utf8_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_free(zErrMsg); + } + sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); +} + + /* ** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the @@ -1138,7 +2152,7 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ */ static void set_table_name(ShellState *p, const char *zName){ int i, n; - int needQuote; + int cQuote; char *z; if( p->zDestTable ){ @@ -1146,74 +2160,24 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - needQuote = !isalpha((unsigned char)*zName) && *zName!='_'; - for(i=n=0; zName[i]; i++, n++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ){ - needQuote = 1; - if( zName[i]=='\'' ) n++; - } - } - if( needQuote ) n += 2; + cQuote = quoteChar(zName); + n = strlen30(zName); + if( cQuote ) n += 2; z = p->zDestTable = malloc( n+1 ); if( z==0 ){ raw_printf(stderr,"Error: out of memory\n"); exit(1); } n = 0; - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; for(i=0; zName[i]; i++){ z[n++] = zName[i]; - if( zName[i]=='\'' ) z[n++] = '\''; + if( zName[i]==cQuote ) z[n++] = cQuote; } - if( needQuote ) z[n++] = '\''; + if( cQuote ) z[n++] = cQuote; z[n] = 0; } -/* zIn is either a pointer to a NULL-terminated string in memory obtained -** from malloc(), or a NULL pointer. The string pointed to by zAppend is -** added to zIn, and the result returned in memory obtained from malloc(). -** zIn, if it was not NULL, is freed. -** -** If the third argument, quote, is not '\0', then it is used as a -** quote character for zAppend. -*/ -static char *appendText(char *zIn, char const *zAppend, char quote){ - int len; - int i; - int nAppend = strlen30(zAppend); - int nIn = (zIn?strlen30(zIn):0); - - len = nAppend+nIn+1; - if( quote ){ - len += 2; - for(i=0; iout, "%s", z); - for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; @@ -1260,7 +2224,7 @@ static int run_table_dump_query( raw_printf(p->out, "\n;\n"); }else{ raw_printf(p->out, ";\n"); - } + } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); @@ -1313,15 +2277,40 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; i1 ){ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iCur, iHiwtr); + }else{ + sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); + } + raw_printf(p->out, "%-36s %s\n", zLabel, zLine); +} /* ** Display memory stats. @@ -1335,57 +2324,31 @@ static int display_stats( int iHiwtr; if( pArg && pArg->out ){ - - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Memory Used: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Memory Used:", + "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset); + displayStatLine(pArg, "Number of Outstanding Allocations:", + "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset); if( pArg->shellFlgs & SHFLG_Pagecache ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Pages Used: %d (max %d) pages\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Pages Used:", + "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Pcache Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_PAGECACHE_OVERFLOW, bReset); if( pArg->shellFlgs & SHFLG_Scratch ){ - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Allocations Used: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Number of Scratch Allocations Used:", + "%lld (max %lld)", SQLITE_STATUS_SCRATCH_USED, bReset); } - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, - "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", - iCur, iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Pcache Allocation: %d bytes\n", - iHiwtr); - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Largest Scratch Allocation: %d bytes\n", - iHiwtr); + displayStatLine(pArg, "Number of Scratch Overflow Bytes:", + "%lld (max %lld) bytes", SQLITE_STATUS_SCRATCH_OVERFLOW, bReset); + displayStatLine(pArg, "Largest Allocation:", + "%lld bytes", SQLITE_STATUS_MALLOC_SIZE, bReset); + displayStatLine(pArg, "Largest Pcache Allocation:", + "%lld bytes", SQLITE_STATUS_PAGECACHE_SIZE, bReset); + displayStatLine(pArg, "Largest Scratch Allocation:", + "%lld bytes", SQLITE_STATUS_SCRATCH_SIZE, bReset); #ifdef YYTRACKMAXSTACKDEPTH - iHiwtr = iCur = -1; - sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - raw_printf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", - iCur, iHiwtr); + displayStatLine(pArg, "Deepest Parser Stack:", + "%lld (max %lld)", SQLITE_STATUS_PARSER_STACK, bReset); #endif } @@ -1419,18 +2382,18 @@ static int display_stats( raw_printf(pArg->out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache misses: %d\n", iCur); + raw_printf(pArg->out, "Page cache misses: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - raw_printf(pArg->out, "Page cache writes: %d\n", iCur); + raw_printf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, "Schema Heap Usage: %d bytes\n", - iCur); + iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); raw_printf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", - iCur); + iCur); } if( pArg && pArg->out && db && pArg->pStmt ){ @@ -1492,7 +2455,7 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); utf8_printf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; - raw_printf(pArg->out, + raw_printf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); @@ -1519,7 +2482,7 @@ static int str_in_array(const char *zStr, const char **azArray){ /* ** If compiled statement pSql appears to be an EXPLAIN statement, allocate ** and populate the ShellState.aiIndent[] array with the number of -** spaces each opcode should be indented before it is output. +** spaces each opcode should be indented before it is output. ** ** The indenting rules are: ** @@ -1604,7 +2567,7 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ if( str_in_array(zOp, azGoto) && p2opnIndent && (abYield[p2op] || sqlite3_column_int(pSql, 2)) ){ - for(i=p2op+1; iaiIndent[i] += 2; + for(i=p2op; iaiIndent[i] += 2; } } @@ -1637,12 +2600,110 @@ static void recCommandOut(void *pCtx, const char *zLine){ } /* -** Execute a statement or set of statements. Print -** any result rows/columns depending on the current mode +** Disable and restore .wheretrace and .selecttrace settings. +*/ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) +extern int sqlite3SelectTrace; +static int savedSelectTrace; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) +extern int sqlite3WhereTrace; +static int savedWhereTrace; +#endif +static void disable_debug_trace_modes(void){ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + savedSelectTrace = sqlite3SelectTrace; + sqlite3SelectTrace = 0; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + savedWhereTrace = sqlite3WhereTrace; + sqlite3WhereTrace = 0; +#endif +} +static void restore_debug_trace_modes(void){ +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + sqlite3SelectTrace = savedSelectTrace; +#endif +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) + sqlite3WhereTrace = savedWhereTrace; +#endif +} + +/* +** Run a prepared statement +*/ +static void exec_prepared_stmt( + ShellState *pArg, /* Pointer to ShellState */ + sqlite3_stmt *pStmt, /* Statment to run */ + int (*xCallback)(void*,int,char**,char**,int*) /* Callback function */ +){ + int rc; + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* if we have a callback... */ + if( xCallback ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + rc = SQLITE_NOMEM; + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; icMode==MODE_Insert ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + } + }else{ + do{ + rc = sqlite3_step(pStmt); + } while( rc == SQLITE_ROW ); + } + } +} + +/* +** Execute a statement or set of statements. Print +** any result rows/columns depending on the current mode ** set via the supplied callback. ** -** This is very similar to SQLite's built-in sqlite3_exec() -** function except it takes a slightly different callback +** This is very similar to SQLite's built-in sqlite3_exec() +** function except it takes a slightly different callback ** and callback data argument. */ static int shell_exec( @@ -1670,6 +2731,7 @@ static int shell_exec( }else while( zSql[0] && (SQLITE_OK == rc) ){ + static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ if( pzErrMsg ){ @@ -1682,6 +2744,9 @@ static int shell_exec( while( IsSpace(zSql[0]) ) zSql++; continue; } + zStmtSql = sqlite3_sql(pStmt); + if( zStmtSql==0 ) zStmtSql = ""; + while( IsSpace(zStmtSql[0]) ) zStmtSql++; /* save off the prepared statment handle and reset row count */ if( pArg ){ @@ -1690,16 +2755,16 @@ static int shell_exec( } /* echo the sql statement if echo on */ - if( pArg && pArg->echoOn ){ - const char *zStmtSql = sqlite3_sql(pStmt); + if( pArg && ShellHasFlag(pArg, SHFLG_Echo) ){ utf8_printf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - if( pArg && pArg->autoEQP ){ + if( pArg && pArg->autoEQP && sqlite3_strlike("EXPLAIN%",zStmtSql,0)!=0 ){ sqlite3_stmt *pExplain; - char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", - sqlite3_sql(pStmt)); + char *zEQP; + disable_debug_trace_modes(); + zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ @@ -1711,17 +2776,31 @@ static int shell_exec( } sqlite3_finalize(pExplain); sqlite3_free(zEQP); + if( pArg->autoEQP>=2 ){ + /* Also do an EXPLAIN for ".eqp full" mode */ + zEQP = sqlite3_mprintf("EXPLAIN %s", zStmtSql); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + pArg->cMode = MODE_Explain; + explain_data_prepare(pArg, pExplain); + exec_prepared_stmt(pArg, pExplain, xCallback); + explain_data_delete(pArg); + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + } + restore_debug_trace_modes(); } if( pArg ){ pArg->cMode = pArg->mode; if( pArg->autoExplain && sqlite3_column_count(pStmt)==8 - && sqlite3_strlike("%EXPLAIN%", sqlite3_sql(pStmt),0)==0 + && sqlite3_strlike("EXPLAIN%", zStmtSql,0)==0 ){ pArg->cMode = MODE_Explain; } - + /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ if( pArg->cMode==MODE_Explain ){ @@ -1729,63 +2808,7 @@ static int shell_exec( } } - /* perform the first step. this will tell us if we - ** have a result set or not and how wide it is. - */ - rc = sqlite3_step(pStmt); - /* if we have a result set... */ - if( SQLITE_ROW == rc ){ - /* if we have a callback... */ - if( xCallback ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - rc = SQLITE_NOMEM; - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ - for(i=0; icMode==MODE_Insert ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ - - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( xCallback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } - } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - } - }else{ - do{ - rc = sqlite3_step(pStmt); - } while( rc == SQLITE_ROW ); - } - } - + exec_prepared_stmt(pArg, pStmt, xCallback); explain_data_delete(pArg); /* print usage stats if stats on */ @@ -1798,7 +2821,7 @@ static int shell_exec( display_scanstats(db, pArg); } - /* Finalize the statement just executed. If this fails, save a + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ rc2 = sqlite3_finalize(pStmt); @@ -1820,6 +2843,137 @@ static int shell_exec( return rc; } +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(ShellState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = ShellHasFlag(p, SHFLG_PreserveRowid); + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + nAlloc = nAlloc*2 + nCol + 10; + azCol = sqlite3_realloc(azCol, nAlloc*sizeof(azCol[0])); + if( azCol==0 ){ + raw_printf(stderr, "Error: out of memory\n"); + exit(1); + } + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + azCol[0] = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( preserveRowid && isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; +} + +/* +** Toggle the reverse_unordered_selects setting. +*/ +static void toggleSelectOrder(sqlite3 *db){ + sqlite3_stmt *pStmt = 0; + int iSetting = 0; + char zStmt[100]; + sqlite3_prepare_v2(db, "PRAGMA reverse_unordered_selects", -1, &pStmt, 0); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + iSetting = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + sqlite3_snprintf(sizeof(zStmt), zStmt, + "PRAGMA reverse_unordered_selects(%d)", !iSetting); + sqlite3_exec(db, zStmt, 0, 0, 0); +} /* ** This is a different callback routine used for dumping the database. @@ -1827,22 +2981,21 @@ static int shell_exec( ** the table type ("index" or "table") and SQL to create the table. ** This routine should print text sufficient to recreate the table. */ -static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ int rc; const char *zTable; const char *zType; const char *zSql; - const char *zPrepStmt = 0; ShellState *p = (ShellState *)pArg; - UNUSED_PARAMETER(azCol); + UNUSED_PARAMETER(azNotUsed); if( nArg!=3 ) return 1; zTable = azArg[0]; zType = azArg[1]; zSql = azArg[2]; - + if( strcmp(zTable, "sqlite_sequence")==0 ){ - zPrepStmt = "DELETE FROM sqlite_sequence;\n"; + raw_printf(p->out, "DELETE FROM sqlite_sequence;\n"); }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ raw_printf(p->out, "ANALYZE sqlite_master;\n"); }else if( strncmp(zTable, "sqlite_", 7)==0 ){ @@ -1861,62 +3014,74 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ sqlite3_free(zIns); return 0; }else{ - utf8_printf(p->out, "%s;\n", zSql); + printSchemaLine(p->out, zSql, ";\n"); } if( strcmp(zType, "table")==0 ){ - sqlite3_stmt *pTableInfo = 0; - char *zSelect = 0; - char *zTableInfo = 0; - char *zTmp = 0; - int nRow = 0; - - zTableInfo = appendText(zTableInfo, "PRAGMA table_info(", 0); - zTableInfo = appendText(zTableInfo, zTable, '"'); - zTableInfo = appendText(zTableInfo, ");", 0); + ShellText sSelect; + ShellText sTable; + char **azCol; + int i; + char *savedDestTable; + int savedMode; - rc = sqlite3_prepare_v2(p->db, zTableInfo, -1, &pTableInfo, 0); - free(zTableInfo); - if( rc!=SQLITE_OK || !pTableInfo ){ - return 1; + azCol = tableColumnList(p, zTable); + if( azCol==0 ){ + p->nErr++; + return 0; } - zSelect = appendText(zSelect, "SELECT 'INSERT INTO ' || ", 0); /* Always quote the table name, even if it appears to be pure ascii, ** in case it is a keyword. Ex: INSERT INTO "table" ... */ - zTmp = appendText(zTmp, zTable, '"'); - if( zTmp ){ - zSelect = appendText(zSelect, zTmp, '\''); - free(zTmp); - } - zSelect = appendText(zSelect, " || ' VALUES(' || ", 0); - rc = sqlite3_step(pTableInfo); - while( rc==SQLITE_ROW ){ - const char *zText = (const char *)sqlite3_column_text(pTableInfo, 1); - zSelect = appendText(zSelect, "quote(", 0); - zSelect = appendText(zSelect, zText, '"'); - rc = sqlite3_step(pTableInfo); - if( rc==SQLITE_ROW ){ - zSelect = appendText(zSelect, "), ", 0); - }else{ - zSelect = appendText(zSelect, ") ", 0); + initText(&sTable); + appendText(&sTable, zTable, quoteChar(zTable)); + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azCol[0], 0); + for(i=1; azCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azCol[i], quoteChar(azCol[i])); } - nRow++; + appendText(&sTable, ")", 0); } - rc = sqlite3_finalize(pTableInfo); - if( rc!=SQLITE_OK || nRow==0 ){ - free(zSelect); - return 1; - } - zSelect = appendText(zSelect, "|| ')' FROM ", 0); - zSelect = appendText(zSelect, zTable, '"'); - rc = run_table_dump_query(p, zSelect, zPrepStmt); - if( rc==SQLITE_CORRUPT ){ - zSelect = appendText(zSelect, " ORDER BY rowid DESC", 0); - run_table_dump_query(p, zSelect, 0); + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azCol[0] ){ + appendText(&sSelect, azCol[0], 0); + appendText(&sSelect, ",", 0); } - free(zSelect); + for(i=1; azCol[i]; i++){ + appendText(&sSelect, azCol[i], quoteChar(azCol[i])); + if( azCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + freeColumnList(azCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + savedDestTable = p->zDestTable; + savedMode = p->mode; + p->zDestTable = sTable.z; + p->mode = p->cMode = MODE_Insert; + rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0); + if( (rc&0xff)==SQLITE_CORRUPT ){ + raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n"); + toggleSelectOrder(p->db); + shell_exec(p->db, sSelect.z, shell_callback, p, 0); + toggleSelectOrder(p->db); + } + p->zDestTable = savedDestTable; + p->mode = savedMode; + freeText(&sTable); + freeText(&sSelect); + if( rc ) p->nErr++; } return 0; } @@ -1929,7 +3094,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ ** "ORDER BY rowid DESC" to the end. */ static int run_schema_dump_query( - ShellState *p, + ShellState *p, const char *zQuery ){ int rc; @@ -1963,10 +3128,14 @@ static int run_schema_dump_query( ** Text of a help message */ static char zHelp[] = +#ifndef SQLITE_OMIT_AUTHORIZATION + ".auth ON|OFF Show authorizer callbacks\n" +#endif ".backup ?DB? FILE Backup DB (default \"main\") to FILE\n" ".bail on|off Stop after hitting an error. Default OFF\n" ".binary on|off Turn binary output on or off. Default OFF\n" ".changes on|off Show number of rows changed by SQL\n" + ".check GLOB Fail if output since .testcase does not match\n" ".clone NEWDB Clone data into NEWDB from the existing database\n" ".databases List names and files of attached databases\n" ".dbinfo ?DB? Show status information about the database\n" @@ -1974,13 +3143,16 @@ static char zHelp[] = " If TABLE specified, only dump tables matching\n" " LIKE pattern TABLE.\n" ".echo on|off Turn command echo on or off\n" - ".eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN\n" + ".eqp on|off|full Enable or disable automatic EXPLAIN QUERY PLAN\n" ".exit Exit this program\n" ".explain ?on|off|auto? Turn EXPLAIN output mode on or off or to automatic\n" - ".fullschema Show schema and the content of sqlite_stat tables\n" + ".fullschema ?--indent? Show schema and the content of sqlite_stat tables\n" ".headers on|off Turn display of headers on or off\n" ".help Show this message\n" ".import FILE TABLE Import data from FILE into TABLE\n" +#ifndef SQLITE_OMIT_TEST_CONTROL + ".imposter INDEX TABLE Create imposter table TABLE on index INDEX\n" +#endif ".indexes ?TABLE? Show names of all indexes\n" " If TABLE specified, only show indexes for tables\n" " matching LIKE pattern TABLE.\n" @@ -1988,6 +3160,8 @@ static char zHelp[] = ".iotrace FILE Enable I/O diagnostic logging to FILE\n" #endif ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT\n" + ".lint OPTIONS Report potential schema issues. Options:\n" + " fkey-indexes Find missing foreign key indexes\n" #ifndef SQLITE_OMIT_LOAD_EXTENSION ".load FILE ?ENTRY? Load an extension library\n" #endif @@ -1999,12 +3173,14 @@ static char zHelp[] = " html HTML
    code\n" " insert SQL insert statements for TABLE\n" " line One value per line\n" - " list Values delimited by .separator strings\n" + " list Values delimited by \"|\"\n" + " quote Escape answers as for SQL\n" " tabs Tab-separated values\n" " tcl TCL list elements\n" ".nullvalue STRING Use STRING in place of NULL values\n" ".once FILENAME Output for the next SQL command only to FILENAME\n" - ".open ?FILENAME? Close existing database and reopen FILENAME\n" + ".open ?--new? ?FILE? Close existing database and reopen FILE\n" + " The --new starts with an empty file\n" ".output ?FILENAME? Send output to FILENAME or stdout\n" ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" @@ -2013,11 +3189,15 @@ static char zHelp[] = ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" ".save FILE Write in-memory database into FILE\n" ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" - ".schema ?TABLE? Show the CREATE statements\n" - " If TABLE specified, only show tables matching\n" - " LIKE pattern TABLE.\n" + ".schema ?PATTERN? Show the CREATE statements matching PATTERN\n" + " Add --indent for pretty-printing\n" + ".selftest ?--init? Run tests defined in the SELFTEST table\n" ".separator COL ?ROW? Change the column separator and optionally the row\n" " separator for both the output mode and .import\n" +#if defined(SQLITE_ENABLE_SESSION) + ".session CMD ... Create or control sessions\n" +#endif + ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" ".stats ?on|off? Show stats or turn stats on or off\n" @@ -2025,6 +3205,7 @@ static char zHelp[] = ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" + ".testcase NAME Begin redirecting output to 'testcase-out.txt'\n" ".timeout MS Try opening locked tables for MS milliseconds\n" ".timer on|off Turn SQL timer on or off\n" ".trace FILE|off Output each SQL statement as it is run\n" @@ -2035,8 +3216,70 @@ static char zHelp[] = " Negative values right-justify\n" ; +#if defined(SQLITE_ENABLE_SESSION) +/* +** Print help information for the ".sessions" command +*/ +void session_help(ShellState *p){ + raw_printf(p->out, + ".session ?NAME? SUBCOMMAND ?ARGS...?\n" + "If ?NAME? is omitted, the first defined session is used.\n" + "Subcommands:\n" + " attach TABLE Attach TABLE\n" + " changeset FILE Write a changeset into FILE\n" + " close Close one session\n" + " enable ?BOOLEAN? Set or query the enable bit\n" + " filter GLOB... Reject tables matching GLOBs\n" + " indirect ?BOOLEAN? Mark or query the indirect status\n" + " isempty Query whether the session is empty\n" + " list List currently open session names\n" + " open DB NAME Open a new session on DB\n" + " patchset FILE Write a patchset into FILE\n" + ); +} +#endif + + /* Forward reference */ static int process_input(ShellState *p, FILE *in); + +/* +** Read the content of file zName into memory obtained from sqlite3_malloc64() +** and return a pointer to the buffer. The caller is responsible for freeing +** the memory. +** +** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes +** read. +** +** For convenience, a nul-terminator byte is always appended to the data read +** from the file before the buffer is returned. This byte is not included in +** the final value of (*pnByte), if applicable. +** +** NULL is returned if any error is encountered. The final value of *pnByte +** is undefined in this case. +*/ +static char *readFile(const char *zName, int *pnByte){ + FILE *in = fopen(zName, "rb"); + long nIn; + size_t nRead; + char *pBuf; + if( in==0 ) return 0; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + if( pBuf==0 ) return 0; + nRead = fread(pBuf, nIn, 1, in); + fclose(in); + if( nRead!=1 ){ + sqlite3_free(pBuf); + return 0; + } + pBuf[nIn] = 0; + if( pnByte ) *pnByte = nIn; + return pBuf; +} + /* ** Implementation of the "readfile(X)" SQL function. The entire content ** of the file named X is read and returned as a BLOB. NULL is returned @@ -2048,25 +3291,14 @@ static void readfileFunc( sqlite3_value **argv ){ const char *zName; - FILE *in; - long nIn; void *pBuf; + int nBuf; UNUSED_PARAMETER(argc); zName = (const char*)sqlite3_value_text(argv[0]); if( zName==0 ) return; - in = fopen(zName, "rb"); - if( in==0 ) return; - fseek(in, 0, SEEK_END); - nIn = ftell(in); - rewind(in); - pBuf = sqlite3_malloc64( nIn ); - if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ - sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); - }else{ - sqlite3_free(pBuf); - } - fclose(in); + pBuf = readFile(zName, &nBuf); + if( pBuf ) sqlite3_result_blob(context, pBuf, nBuf, sqlite3_free); } /* @@ -2100,6 +3332,53 @@ static void writefileFunc( sqlite3_result_int64(context, rc); } +#if defined(SQLITE_ENABLE_SESSION) +/* +** Close a single OpenSession object and release all of its associated +** resources. +*/ +static void session_close(OpenSession *pSession){ + int i; + sqlite3session_delete(pSession->p); + sqlite3_free(pSession->zName); + for(i=0; inFilter; i++){ + sqlite3_free(pSession->azFilter[i]); + } + sqlite3_free(pSession->azFilter); + memset(pSession, 0, sizeof(OpenSession)); +} +#endif + +/* +** Close all OpenSession objects and release all associated resources. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static void session_close_all(ShellState *p){ + int i; + for(i=0; inSession; i++){ + session_close(&p->aSession[i]); + } + p->nSession = 0; +} +#else +# define session_close_all(X) +#endif + +/* +** Implementation of the xFilter function for an open session. Omit +** any tables named by ".session filter" but let all other table through. +*/ +#if defined(SQLITE_ENABLE_SESSION) +static int session_filter(void *pCtx, const char *zTab){ + OpenSession *pSession = (OpenSession*)pCtx; + int i; + for(i=0; inFilter; i++){ + if( sqlite3_strglob(pSession->azFilter[i], zTab)==0 ) return 0; + } + return 1; +} +#endif + /* ** Make sure the database is open. If it is not, then open it. If ** the database fails to open, print an error message and exit. @@ -2109,12 +3388,8 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); globalDb = p->db; - if( p->db && sqlite3_errcode(p->db)==SQLITE_OK ){ - sqlite3_create_function(p->db, "shellstatic", 0, SQLITE_UTF8, 0, - shellstaticFunc, 0, 0); - } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", + utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", p->zDbFilename, sqlite3_errmsg(p->db)); if( keepAlive ) return; exit(1); @@ -2126,6 +3401,14 @@ static void open_db(ShellState *p, int keepAlive){ readfileFunc, 0, 0); sqlite3_create_function(p->db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); + sqlite3_create_function(p->db, "sha3", 1, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3", 2, SQLITE_UTF8, 0, + sha3Func, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 1, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); + sqlite3_create_function(p->db, "sha3_query", 2, SQLITE_UTF8, 0, + sha3QueryFunc, 0, 0); } } @@ -2250,7 +3533,7 @@ static sqlite3_int64 integerValue(const char *zArg){ ** Interpret zArg as either an integer or a boolean value. Return 1 or 0 ** for TRUE and FALSE. Return the integer value if appropriate. */ -static int booleanValue(char *zArg){ +static int booleanValue(const char *zArg){ int i; if( zArg[0]=='0' && zArg[1]=='x' ){ for(i=2; hexDigitValue(zArg[i])>=0; i++){} @@ -2269,6 +3552,17 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Set or clear a shell flag according to a boolean value. +*/ +static void setOrClearFlag(ShellState *p, unsigned mFlag, const char *zArg){ + if( booleanValue(zArg) ){ + ShellSetFlag(p, mFlag); + }else{ + ShellClearFlag(p, mFlag); + } +} + /* ** Close an output file, assuming it is not stderr or stdout */ @@ -2278,7 +3572,7 @@ static void output_file_close(FILE *f){ /* ** Try to open an output file. The names "stdout" and "stderr" are -** recognized and do the right thing. NULL is returned if the output +** recognized and do the right thing. NULL is returned if the output ** filename is "off". */ static FILE *output_file_open(const char *zFile){ @@ -2298,17 +3592,30 @@ static FILE *output_file_open(const char *zFile){ return f; } +#if !defined(SQLITE_UNTESTABLE) +#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) /* ** A routine for handling output from sqlite3_trace(). */ -static void sql_trace_callback(void *pArg, const char *z){ +static int sql_trace_callback( + unsigned mType, + void *pArg, + void *pP, + void *pX +){ FILE *f = (FILE*)pArg; + UNUSED_PARAMETER(mType); + UNUSED_PARAMETER(pP); if( f ){ + const char *z = (const char*)pX; int i = (int)strlen(z); while( i>0 && z[i-1]==';' ){ i--; } utf8_printf(f, "%.*s;\n", i, z); } + return 0; } +#endif +#endif /* ** A no-op routine that runs with the ".breakpoint" doc-command. This is @@ -2467,7 +3774,7 @@ static void tryToCloneData( sqlite3 *newDb, const char *zTable ){ - sqlite3_stmt *pQuery = 0; + sqlite3_stmt *pQuery = 0; sqlite3_stmt *pInsert = 0; char *zQuery = 0; char *zInsert = 0; @@ -2704,10 +4011,10 @@ static int db_int(ShellState *p, const char *zSql){ /* ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ -unsigned int get2byteInt(unsigned char *a){ +static unsigned int get2byteInt(unsigned char *a){ return (a[0]<<8) + a[1]; } -unsigned int get4byteInt(unsigned char *a){ +static unsigned int get4byteInt(unsigned char *a){ return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; } @@ -2771,9 +4078,9 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ utf8_printf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) raw_printf(p->out, " (utf8)"); - if( val==2 ) raw_printf(p->out, " (utf16le)"); - if( val==3 ) raw_printf(p->out, " (utf16be)"); + if( val==1 ) raw_printf(p->out, " (utf8)"); + if( val==2 ) raw_printf(p->out, " (utf16le)"); + if( val==3 ) raw_printf(p->out, " (utf16be)"); } } raw_printf(p->out, "\n"); @@ -2812,6 +4119,377 @@ static int shellNomemError(void){ return 1; } +/* +** Compare the pattern in zGlob[] against the text in z[]. Return TRUE +** if they match and FALSE (0) if they do not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +** +** ' ' Any span of whitespace matches any other span of +** whitespace. +** +** Extra whitespace at the end of z[] is ignored. +*/ +static int testcase_glob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( IsSpace(c) ){ + if( !IsSpace(*z) ) return 0; + while( IsSpace(*zGlob) ) zGlob++; + while( IsSpace(*z) ) z++; + }else if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && testcase_glob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( testcase_glob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && IsDigit(z[1]) ) z++; + if( !IsDigit(z[0]) ) return 0; + z++; + while( IsDigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + while( IsSpace(*z) ){ z++; } + return *z==0; +} + + +/* +** Compare the string as a command-line option with either one or two +** initial "-" characters. +*/ +static int optionMatch(const char *zStr, const char *zOpt){ + if( zStr[0]!='-' ) return 0; + zStr++; + if( zStr[0]=='-' ) zStr++; + return strcmp(zStr, zOpt)==0; +} + +/* +** Delete a file. +*/ +int shellDeleteFile(const char *zFilename){ + int rc; +#ifdef _WIN32 + wchar_t *z = sqlite3_win32_utf8_to_unicode(zFilename); + rc = _wunlink(z); + sqlite3_free(z); +#else + rc = unlink(zFilename); +#endif + return rc; +} + + +/* +** The implementation of SQL scalar function fkey_collate_clause(), used +** by the ".lint fkey-indexes" command. This scalar function is always +** called with four arguments - the parent table name, the parent column name, +** the child table name and the child column name. +** +** fkey_collate_clause('parent-tab', 'parent-col', 'child-tab', 'child-col') +** +** If either of the named tables or columns do not exist, this function +** returns an empty string. An empty string is also returned if both tables +** and columns exist but have the same default collation sequence. Or, +** if both exist but the default collation sequences are different, this +** function returns the string " COLLATE ", where +** is the default collation sequence of the parent column. +*/ +static void shellFkeyCollateClause( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + const char *zParent; + const char *zParentCol; + const char *zParentSeq; + const char *zChild; + const char *zChildCol; + const char *zChildSeq = 0; /* Initialize to avoid false-positive warning */ + int rc; + + assert( nVal==4 ); + zParent = (const char*)sqlite3_value_text(apVal[0]); + zParentCol = (const char*)sqlite3_value_text(apVal[1]); + zChild = (const char*)sqlite3_value_text(apVal[2]); + zChildCol = (const char*)sqlite3_value_text(apVal[3]); + + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = sqlite3_table_column_metadata( + db, "main", zParent, zParentCol, 0, &zParentSeq, 0, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_table_column_metadata( + db, "main", zChild, zChildCol, 0, &zChildSeq, 0, 0, 0 + ); + } + + if( rc==SQLITE_OK && sqlite3_stricmp(zParentSeq, zChildSeq) ){ + char *z = sqlite3_mprintf(" COLLATE %s", zParentSeq); + sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); + sqlite3_free(z); + } +} + + +/* +** The implementation of dot-command ".lint fkey-indexes". +*/ +static int lintFkeyIndexes( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + sqlite3 *db = pState->db; /* Database handle to query "main" db of */ + FILE *out = pState->out; /* Stream to write non-error output to */ + int bVerbose = 0; /* If -verbose is present */ + int bGroupByParent = 0; /* If -groupbyparent is present */ + int i; /* To iterate through azArg[] */ + const char *zIndent = ""; /* How much to indent CREATE INDEX by */ + int rc; /* Return code */ + sqlite3_stmt *pSql = 0; /* Compiled version of SQL statement below */ + + /* + ** This SELECT statement returns one row for each foreign key constraint + ** in the schema of the main database. The column values are: + ** + ** 0. The text of an SQL statement similar to: + ** + ** "EXPLAIN QUERY PLAN SELECT rowid FROM child_table WHERE child_key=?" + ** + ** This is the same SELECT that the foreign keys implementation needs + ** to run internally on child tables. If there is an index that can + ** be used to optimize this query, then it can also be used by the FK + ** implementation to optimize DELETE or UPDATE statements on the parent + ** table. + ** + ** 1. A GLOB pattern suitable for sqlite3_strglob(). If the plan output by + ** the EXPLAIN QUERY PLAN command matches this pattern, then the schema + ** contains an index that can be used to optimize the query. + ** + ** 2. Human readable text that describes the child table and columns. e.g. + ** + ** "child_table(child_key1, child_key2)" + ** + ** 3. Human readable text that describes the parent table and columns. e.g. + ** + ** "parent_table(parent_key1, parent_key2)" + ** + ** 4. A full CREATE INDEX statement for an index that could be used to + ** optimize DELETE or UPDATE statements on the parent table. e.g. + ** + ** "CREATE INDEX child_table_child_key ON child_table(child_key)" + ** + ** 5. The name of the parent table. + ** + ** These six values are used by the C logic below to generate the report. + */ + const char *zSql = + "SELECT " + " 'EXPLAIN QUERY PLAN SELECT rowid FROM ' || quote(s.name) || ' WHERE '" + " || group_concat(quote(s.name) || '.' || quote(f.[from]) || '=?' " + " || fkey_collate_clause(f.[table], f.[to], s.name, f.[from]),' AND ')" + ", " + " 'SEARCH TABLE ' || s.name || ' USING COVERING INDEX*('" + " || group_concat('*=?', ' AND ') || ')'" + ", " + " s.name || '(' || group_concat(f.[from], ', ') || ')'" + ", " + " f.[table] || '(' || group_concat(COALESCE(f.[to], " + " (SELECT name FROM pragma_table_info(f.[table]) WHERE pk=seq+1)" + " )) || ')'" + ", " + " 'CREATE INDEX ' || quote(s.name ||'_'|| group_concat(f.[from], '_'))" + " || ' ON ' || quote(s.name) || '('" + " || group_concat(quote(f.[from]) ||" + " fkey_collate_clause(f.[table], f.[to], s.name, f.[from]), ', ')" + " || ');'" + ", " + " f.[table] " + + "FROM sqlite_master AS s, pragma_foreign_key_list(s.name) AS f " + "GROUP BY s.name, f.id " + "ORDER BY (CASE WHEN ? THEN f.[table] ELSE s.name END)" + ; + + for(i=2; i1 && sqlite3_strnicmp("-verbose", azArg[i], n)==0 ){ + bVerbose = 1; + } + else if( n>1 && sqlite3_strnicmp("-groupbyparent", azArg[i], n)==0 ){ + bGroupByParent = 1; + zIndent = " "; + } + else{ + raw_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", + azArg[0], azArg[1] + ); + return SQLITE_ERROR; + } + } + + /* Register the fkey_collate_clause() SQL function */ + rc = sqlite3_create_function(db, "fkey_collate_clause", 4, SQLITE_UTF8, + 0, shellFkeyCollateClause, 0, 0 + ); + + + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pSql, 1, bGroupByParent); + } + + if( rc==SQLITE_OK ){ + int rc2; + char *zPrev = 0; + while( SQLITE_ROW==sqlite3_step(pSql) ){ + int res = -1; + sqlite3_stmt *pExplain = 0; + const char *zEQP = (const char*)sqlite3_column_text(pSql, 0); + const char *zGlob = (const char*)sqlite3_column_text(pSql, 1); + const char *zFrom = (const char*)sqlite3_column_text(pSql, 2); + const char *zTarget = (const char*)sqlite3_column_text(pSql, 3); + const char *zCI = (const char*)sqlite3_column_text(pSql, 4); + const char *zParent = (const char*)sqlite3_column_text(pSql, 5); + + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc!=SQLITE_OK ) break; + if( SQLITE_ROW==sqlite3_step(pExplain) ){ + const char *zPlan = (const char*)sqlite3_column_text(pExplain, 3); + res = (0==sqlite3_strglob(zGlob, zPlan)); + } + rc = sqlite3_finalize(pExplain); + if( rc!=SQLITE_OK ) break; + + if( res<0 ){ + raw_printf(stderr, "Error: internal error"); + break; + }else{ + if( bGroupByParent + && (bVerbose || res==0) + && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) + ){ + raw_printf(out, "-- Parent table %s\n", zParent); + sqlite3_free(zPrev); + zPrev = sqlite3_mprintf("%s", zParent); + } + + if( res==0 ){ + raw_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + }else if( bVerbose ){ + raw_printf(out, "%s/* no extra indexes required for %s -> %s */\n", + zIndent, zFrom, zTarget + ); + } + } + } + sqlite3_free(zPrev); + + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + rc2 = sqlite3_finalize(pSql); + if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ + rc = rc2; + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + }else{ + raw_printf(stderr, "%s\n", sqlite3_errmsg(db)); + } + + return rc; +} + +/* +** Implementation of ".lint" dot command. +*/ +static int lintDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int n; + n = (nArg>=2 ? (int)strlen(azArg[1]) : 0); + if( n<1 || sqlite3_strnicmp(azArg[1], "fkey-indexes", n) ) goto usage; + return lintFkeyIndexes(pState, azArg, nArg); + + usage: + raw_printf(stderr, "Usage %s sub-command ?switches...?\n", azArg[0]); + raw_printf(stderr, "Where sub-commands are:\n"); + raw_printf(stderr, " fkey-indexes\n"); + return SQLITE_ERROR; +} + + /* ** If an input line begins with "." then invoke this routine to ** process that line. @@ -2833,9 +4511,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zLine[h]=='\'' || zLine[h]=='"' ){ int delim = zLine[h++]; azArg[nArg++] = &zLine[h]; - while( zLine[h] && zLine[h]!=delim ){ + while( zLine[h] && zLine[h]!=delim ){ if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; - h++; + h++; } if( zLine[h]==delim ){ zLine[h++] = 0; @@ -2854,6 +4532,23 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==0 ) return 0; /* no tokens, no error */ n = strlen30(azArg[0]); c = azArg[0][0]; + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( c=='a' && strncmp(azArg[0], "auth", n)==0 ){ + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .auth ON|OFF\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + if( booleanValue(azArg[1]) ){ + sqlite3_set_authorizer(p->db, shellAuth, p); + }else{ + sqlite3_set_authorizer(p->db, 0, 0); + } + }else +#endif + if( (c=='b' && n>=3 && strncmp(azArg[0], "backup", n)==0) || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) ){ @@ -2922,9 +4617,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='b' && n>=3 && strncmp(azArg[0], "binary", n)==0 ){ if( nArg==2 ){ if( booleanValue(azArg[1]) ){ - setBinaryMode(p->out); + setBinaryMode(p->out, 1); }else{ - setTextMode(p->out); + setTextMode(p->out, 1); } }else{ raw_printf(stderr, "Usage: .binary on|off\n"); @@ -2941,13 +4636,38 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='c' && n>=3 && strncmp(azArg[0], "changes", n)==0 ){ if( nArg==2 ){ - p->countChanges = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_CountChanges, azArg[1]); }else{ raw_printf(stderr, "Usage: .changes on|off\n"); rc = 1; } }else + /* Cancel output redirection, if it is currently set (by .testcase) + ** Then read the content of the testcase-out.txt file and compare against + ** azArg[1]. If there are differences, report an error and exit. + */ + if( c=='c' && n>=3 && strncmp(azArg[0], "check", n)==0 ){ + char *zRes = 0; + output_reset(p); + if( nArg!=2 ){ + raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); + rc = 2; + }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ + raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); + rc = 2; + }else if( testcase_glob(azArg[1],zRes)==0 ){ + utf8_printf(stderr, + "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); + rc = 2; + }else{ + utf8_printf(stdout, "testcase-%s ok\n", p->zTestcase); + p->nCheck++; + } + sqlite3_free(zRes); + }else + if( c=='c' && strncmp(azArg[0], "clone", n)==0 ){ if( nArg==2 ){ tryToClone(p, azArg[1]); @@ -2962,13 +4682,12 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); - data.showHeader = 1; - data.cMode = data.mode = MODE_Column; - data.colWidth[0] = 3; - data.colWidth[1] = 15; - data.colWidth[2] = 58; + data.showHeader = 0; + data.cMode = data.mode = MODE_List; + sqlite3_snprintf(sizeof(data.colSeparator),data.colSeparator,": "); data.cnt = 0; - sqlite3_exec(p->db, "PRAGMA database_list; ", callback, &data, &zErrMsg); + sqlite3_exec(p->db, "SELECT name, file FROM pragma_database_list", + callback, &data, &zErrMsg); if( zErrMsg ){ utf8_printf(stderr,"Error: %s\n", zErrMsg); sqlite3_free(zErrMsg); @@ -2981,26 +4700,54 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ + const char *zLike = 0; + int i; + ShellClearFlag(p, SHFLG_PreserveRowid); + for(i=1; iout, "PRAGMA foreign_keys=OFF;\n"); raw_printf(p->out, "BEGIN TRANSACTION;\n"); p->writableSchema = 0; + /* Set writable_schema=ON since doing so forces SQLite to initialize + ** as much of the schema as it can even if the sqlite_master table is + ** corrupt. */ sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0); p->nErr = 0; - if( nArg==1 ){ - run_schema_dump_query(p, + if( zLike==0 ){ + run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" ); - run_schema_dump_query(p, + run_schema_dump_query(p, "SELECT name, type, sql FROM sqlite_master " "WHERE name=='sqlite_sequence'" ); @@ -3009,21 +4756,20 @@ static int do_meta_command(char *zLine, ShellState *p){ "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 ); }else{ - int i; - for(i=1; iwritableSchema ){ raw_printf(p->out, "PRAGMA writable_schema=OFF;\n"); @@ -3036,7 +4782,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - p->echoOn = booleanValue(azArg[1]); + setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ raw_printf(stderr, "Usage: .echo on|off\n"); rc = 1; @@ -3045,11 +4791,15 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - p->autoEQP = booleanValue(azArg[1]); + if( strcmp(azArg[1],"full")==0 ){ + p->autoEQP = 2; + }else{ + p->autoEQP = booleanValue(azArg[1]); + } }else{ - raw_printf(stderr, "Usage: .eqp on|off\n"); + raw_printf(stderr, "Usage: .eqp on|off|full\n"); rc = 1; - } + } }else if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ @@ -3083,15 +4833,19 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellState data; char *zErrMsg = 0; int doStats = 0; + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + if( nArg==2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg = 1; + } if( nArg!=1 ){ - raw_printf(stderr, "Usage: .fullschema\n"); + raw_printf(stderr, "Usage: .fullschema ?--indent?\n"); rc = 1; goto meta_command_exit; } open_db(p, 0); - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; rc = sqlite3_exec(p->db, "SELECT sql FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" @@ -3341,51 +5095,78 @@ static int do_meta_command(char *zLine, ShellState *p){ if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); }else - if( c=='i' && (strncmp(azArg[0], "indices", n)==0 - || strncmp(azArg[0], "indexes", n)==0) ){ - ShellState data; - char *zErrMsg = 0; - open_db(p, 0); - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_List; - if( nArg==1 ){ - rc = sqlite3_exec(p->db, - "SELECT name FROM sqlite_master " - "WHERE type='index' AND name NOT LIKE 'sqlite_%' " - "UNION ALL " - "SELECT name FROM sqlite_temp_master " - "WHERE type='index' " - "ORDER BY 1", - callback, &data, &zErrMsg - ); - }else if( nArg==2 ){ - zShellStatic = azArg[1]; - rc = sqlite3_exec(p->db, - "SELECT name FROM sqlite_master " - "WHERE type='index' AND tbl_name LIKE shellstatic() " - "UNION ALL " - "SELECT name FROM sqlite_temp_master " - "WHERE type='index' AND tbl_name LIKE shellstatic() " - "ORDER BY 1", - callback, &data, &zErrMsg - ); - zShellStatic = 0; - }else{ - raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); +#ifndef SQLITE_UNTESTABLE + if( c=='i' && strncmp(azArg[0], "imposter", n)==0 ){ + char *zSql; + char *zCollist = 0; + sqlite3_stmt *pStmt; + int tnum = 0; + int i; + if( nArg!=3 ){ + utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n"); rc = 1; goto meta_command_exit; } - if( zErrMsg ){ - utf8_printf(stderr,"Error: %s\n", zErrMsg); - sqlite3_free(zErrMsg); + open_db(p, 0); + zSql = sqlite3_mprintf("SELECT rootpage FROM sqlite_master" + " WHERE name='%q' AND type='index'", azArg[1]); + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + tnum = sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + if( tnum==0 ){ + utf8_printf(stderr, "no such index: \"%s\"\n", azArg[1]); rc = 1; - }else if( rc != SQLITE_OK ){ - raw_printf(stderr, - "Error: querying sqlite_master and sqlite_temp_master\n"); + goto meta_command_exit; + } + zSql = sqlite3_mprintf("PRAGMA index_xinfo='%q'", azArg[1]); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + i = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char zLabel[20]; + const char *zCol = (const char*)sqlite3_column_text(pStmt,2); + i++; + if( zCol==0 ){ + if( sqlite3_column_int(pStmt,1)==-1 ){ + zCol = "_ROWID_"; + }else{ + sqlite3_snprintf(sizeof(zLabel),zLabel,"expr%d",i); + zCol = zLabel; + } + } + if( zCollist==0 ){ + zCollist = sqlite3_mprintf("\"%w\"", zCol); + }else{ + zCollist = sqlite3_mprintf("%z,\"%w\"", zCollist, zCol); + } + } + sqlite3_finalize(pStmt); + zSql = sqlite3_mprintf( + "CREATE TABLE \"%w\"(%s,PRIMARY KEY(%s))WITHOUT ROWID", + azArg[2], zCollist, zCollist); + sqlite3_free(zCollist); + rc = sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 1, tnum); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); + if( rc ){ + utf8_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); + }else{ + utf8_printf(stdout, "%s;\n", zSql); + raw_printf(stdout, + "WARNING: writing to an imposter table will corrupt the index!\n" + ); + } + }else{ + raw_printf(stderr, "SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); rc = 1; } + sqlite3_free(zSql); }else +#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ #ifdef SQLITE_ENABLE_IOTRACE if( c=='i' && strncmp(azArg[0], "iotrace", n)==0 ){ @@ -3409,6 +5190,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else #endif + if( c=='l' && n>=5 && strncmp(azArg[0], "limits", n)==0 ){ static const struct { const char *zLimitName; /* Name of a limit */ @@ -3431,7 +5213,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; idb, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -3468,6 +5250,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='l' && n>2 && strncmp(azArg[0], "lint", n)==0 ){ + open_db(p, 0); + lintDotCommand(p, azArg, nArg); + }else + #ifndef SQLITE_OMIT_LOAD_EXTENSION if( c=='l' && strncmp(azArg[0], "load", n)==0 ){ const char *zFile, *zProc; @@ -3506,15 +5293,20 @@ static int do_meta_command(char *zLine, ShellState *p){ int c2 = zMode[0]; if( c2=='l' && n2>2 && strncmp(azArg[1],"lines",n2)==0 ){ p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"columns",n2)==0 ){ p->mode = MODE_Column; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='l' && n2>2 && strncmp(azArg[1],"list",n2)==0 ){ p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='h' && strncmp(azArg[1],"html",n2)==0 ){ p->mode = MODE_Html; }else if( c2=='t' && strncmp(azArg[1],"tcl",n2)==0 ){ p->mode = MODE_Tcl; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); @@ -3525,13 +5317,15 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c2=='i' && strncmp(azArg[1],"insert",n2)==0 ){ p->mode = MODE_Insert; set_table_name(p, nArg>=3 ? azArg[2] : "table"); + }else if( c2=='q' && strncmp(azArg[1],"quote",n2)==0 ){ + p->mode = MODE_Quote; }else if( c2=='a' && strncmp(azArg[1],"ascii",n2)==0 ){ p->mode = MODE_Ascii; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); }else { raw_printf(stderr, "Error: mode should be one of: " - "ascii column csv html insert line list tabs tcl\n"); + "ascii column csv html insert line list quote tabs tcl\n"); rc = 1; } p->cMode = p->mode; @@ -3548,21 +5342,44 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='o' && strncmp(azArg[0], "open", n)==0 && n>=2 ){ - sqlite3 *savedDb = p->db; - const char *zSavedFilename = p->zDbFilename; - char *zNewFilename = 0; + char *zNewFilename; /* Name of the database file to open */ + int iName = 1; /* Index in azArg[] of the filename */ + int newFlag = 0; /* True to delete file before opening */ + /* Close the existing database */ + session_close_all(p); + sqlite3_close(p->db); p->db = 0; - if( nArg>=2 ) zNewFilename = sqlite3_mprintf("%s", azArg[1]); - p->zDbFilename = zNewFilename; - open_db(p, 1); - if( p->db!=0 ){ - sqlite3_close(savedDb); - sqlite3_free(p->zFreeOnClose); - p->zFreeOnClose = zNewFilename; - }else{ - sqlite3_free(zNewFilename); - p->db = savedDb; - p->zDbFilename = zSavedFilename; + p->zDbFilename = 0; + sqlite3_free(p->zFreeOnClose); + p->zFreeOnClose = 0; + /* Check for command-line arguments */ + for(iName=1; iNameiName ? sqlite3_mprintf("%s", azArg[iName]) : 0; + if( zNewFilename ){ + if( newFlag ) shellDeleteFile(zNewFilename); + p->zDbFilename = zNewFilename; + open_db(p, 1); + if( p->db==0 ){ + utf8_printf(stderr, "Error: cannot open '%s'\n", zNewFilename); + sqlite3_free(zNewFilename); + }else{ + p->zFreeOnClose = zNewFilename; + } + } + if( p->db==0 ){ + /* As a fall-back open a TEMP database */ + p->zDbFilename = 0; + open_db(p, 0); } }else @@ -3734,7 +5551,12 @@ static int do_meta_command(char *zLine, ShellState *p){ memcpy(&data, p, sizeof(data)); data.showHeader = 0; data.cMode = data.mode = MODE_Semi; - if( nArg==2 ){ + if( nArg>=2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; + nArg--; + if( nArg==2 ) azArg[1] = azArg[2]; + } + if( nArg==2 && azArg[1][0]!='-' ){ int i; for(i=0; azArg[1][i]; i++) azArg[1][i] = ToLower(azArg[1][i]); if( strcmp(azArg[1],"sqlite_master")==0 ){ @@ -3766,17 +5588,17 @@ static int do_meta_command(char *zLine, ShellState *p){ callback(&data, 1, new_argv, new_colv); rc = SQLITE_OK; }else{ - zShellStatic = azArg[1]; - rc = sqlite3_exec(p->db, + char *zSql; + zSql = sqlite3_mprintf( "SELECT sql FROM " " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE lower(tbl_name) LIKE shellstatic()" + "WHERE lower(tbl_name) LIKE %Q" " AND type!='meta' AND sql NOTNULL " - "ORDER BY rowid", - callback, &data, &zErrMsg); - zShellStatic = 0; + "ORDER BY rowid", azArg[1]); + rc = sqlite3_exec(p->db, zSql, callback, &data, &zErrMsg); + sqlite3_free(zSql); } }else if( nArg==1 ){ rc = sqlite3_exec(p->db, @@ -3789,7 +5611,7 @@ static int do_meta_command(char *zLine, ShellState *p){ callback, &data, &zErrMsg ); }else{ - raw_printf(stderr, "Usage: .schema ?LIKE-PATTERN?\n"); + raw_printf(stderr, "Usage: .schema ?--indent? ?LIKE-PATTERN?\n"); rc = 1; goto meta_command_exit; } @@ -3805,14 +5627,207 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ - extern int sqlite3SelectTrace; - sqlite3SelectTrace = integerValue(azArg[1]); + sqlite3SelectTrace = (int)integerValue(azArg[1]); }else #endif +#if defined(SQLITE_ENABLE_SESSION) + if( c=='s' && strncmp(azArg[0],"session",n)==0 && n>=3 ){ + OpenSession *pSession = &p->aSession[0]; + char **azCmd = &azArg[1]; + int iSes = 0; + int nCmd = nArg - 1; + int i; + if( nArg<=1 ) goto session_syntax_error; + open_db(p, 0); + if( nArg>=3 ){ + for(iSes=0; iSesnSession; iSes++){ + if( strcmp(p->aSession[iSes].zName, azArg[1])==0 ) break; + } + if( iSesnSession ){ + pSession = &p->aSession[iSes]; + azCmd++; + nCmd--; + }else{ + pSession = &p->aSession[0]; + iSes = 0; + } + } + + /* .session attach TABLE + ** Invoke the sqlite3session_attach() interface to attach a particular + ** table so that it is never filtered. + */ + if( strcmp(azCmd[0],"attach")==0 ){ + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ){ + session_not_open: + raw_printf(stderr, "ERROR: No sessions are open\n"); + }else{ + rc = sqlite3session_attach(pSession->p, azCmd[1]); + if( rc ){ + raw_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n", rc); + rc = 0; + } + } + }else + + /* .session changeset FILE + ** .session patchset FILE + ** Write a changeset or patchset into a file. The file is overwritten. + */ + if( strcmp(azCmd[0],"changeset")==0 || strcmp(azCmd[0],"patchset")==0 ){ + FILE *out = 0; + if( nCmd!=2 ) goto session_syntax_error; + if( pSession->p==0 ) goto session_not_open; + out = fopen(azCmd[1], "wb"); + if( out==0 ){ + utf8_printf(stderr, "ERROR: cannot open \"%s\" for writing\n", azCmd[1]); + }else{ + int szChng; + void *pChng; + if( azCmd[0][0]=='c' ){ + rc = sqlite3session_changeset(pSession->p, &szChng, &pChng); + }else{ + rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); + } + if( rc ){ + printf("Error: error code %d\n", rc); + rc = 0; + } + if( pChng + && fwrite(pChng, szChng, 1, out)!=1 ){ + raw_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", + szChng); + } + sqlite3_free(pChng); + fclose(out); + } + }else + + /* .session close + ** Close the identified session + */ + if( strcmp(azCmd[0], "close")==0 ){ + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + session_close(pSession); + p->aSession[iSes] = p->aSession[--p->nSession]; + } + }else + + /* .session enable ?BOOLEAN? + ** Query or set the enable flag + */ + if( strcmp(azCmd[0], "enable")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_enable(pSession->p, ii); + utf8_printf(p->out, "session %s enable flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session filter GLOB .... + ** Set a list of GLOB patterns of table names to be excluded. + */ + if( strcmp(azCmd[0], "filter")==0 ){ + int ii, nByte; + if( nCmd<2 ) goto session_syntax_error; + if( p->nSession ){ + for(ii=0; iinFilter; ii++){ + sqlite3_free(pSession->azFilter[ii]); + } + sqlite3_free(pSession->azFilter); + nByte = sizeof(pSession->azFilter[0])*(nCmd-1); + pSession->azFilter = sqlite3_malloc( nByte ); + if( pSession->azFilter==0 ){ + raw_printf(stderr, "Error: out or memory\n"); + exit(1); + } + for(ii=1; iiazFilter[ii-1] = sqlite3_mprintf("%s", azCmd[ii]); + } + pSession->nFilter = ii-1; + } + }else + + /* .session indirect ?BOOLEAN? + ** Query or set the indirect flag + */ + if( strcmp(azCmd[0], "indirect")==0 ){ + int ii; + if( nCmd>2 ) goto session_syntax_error; + ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); + if( p->nSession ){ + ii = sqlite3session_indirect(pSession->p, ii); + utf8_printf(p->out, "session %s indirect flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session isempty + ** Determine if the session is empty + */ + if( strcmp(azCmd[0], "isempty")==0 ){ + int ii; + if( nCmd!=1 ) goto session_syntax_error; + if( p->nSession ){ + ii = sqlite3session_isempty(pSession->p); + utf8_printf(p->out, "session %s isempty flag = %d\n", + pSession->zName, ii); + } + }else + + /* .session list + ** List all currently open sessions + */ + if( strcmp(azCmd[0],"list")==0 ){ + for(i=0; inSession; i++){ + utf8_printf(p->out, "%d %s\n", i, p->aSession[i].zName); + } + }else + + /* .session open DB NAME + ** Open a new session called NAME on the attached database DB. + ** DB is normally "main". + */ + if( strcmp(azCmd[0],"open")==0 ){ + char *zName; + if( nCmd!=3 ) goto session_syntax_error; + zName = azCmd[2]; + if( zName[0]==0 ) goto session_syntax_error; + for(i=0; inSession; i++){ + if( strcmp(p->aSession[i].zName,zName)==0 ){ + utf8_printf(stderr, "Session \"%s\" already exists\n", zName); + goto meta_command_exit; + } + } + if( p->nSession>=ArraySize(p->aSession) ){ + raw_printf(stderr, "Maximum of %d sessions\n", ArraySize(p->aSession)); + goto meta_command_exit; + } + pSession = &p->aSession[p->nSession]; + rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); + if( rc ){ + raw_printf(stderr, "Cannot open session: error code=%d\n", rc); + rc = 0; + goto meta_command_exit; + } + pSession->nFilter = 0; + sqlite3session_table_filter(pSession->p, session_filter, pSession); + p->nSession++; + pSession->zName = sqlite3_mprintf("%s", zName); + }else + /* If no command name matches, show a syntax error */ + session_syntax_error: + session_help(p); + }else +#endif #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change @@ -3837,6 +5852,119 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif + if( c=='s' && n>=4 && strncmp(azArg[0],"selftest",n)==0 ){ + int bIsInit = 0; /* True to initialize the SELFTEST table */ + int bVerbose = 0; /* Verbose output */ + int bSelftestExists; /* True if SELFTEST already exists */ + char **azTest = 0; /* Content of the SELFTEST table */ + int nRow = 0; /* Number of rows in the SELFTEST table */ + int nCol = 4; /* Number of columns in the SELFTEST table */ + int i; /* Loop counter */ + int nTest = 0; /* Number of tests runs */ + int nErr = 0; /* Number of errors seen */ + ShellText str; /* Answer for a query */ + static char *azDefaultTest[] = { + 0, 0, 0, 0, + "0", "memo", "Missing SELFTEST table - default checks only", "", + "1", "run", "PRAGMA integrity_check", "ok" + }; + static const int nDefaultRow = 2; + + open_db(p,0); + for(i=1; idb,"main","selftest",0,0,0,0,0,0) + != SQLITE_OK ){ + bSelftestExists = 0; + }else{ + bSelftestExists = 1; + } + if( bIsInit ){ + createSelftestTable(p); + bSelftestExists = 1; + } + if( bSelftestExists ){ + rc = sqlite3_get_table(p->db, + "SELECT tno,op,cmd,ans FROM selftest ORDER BY tno", + &azTest, &nRow, &nCol, 0); + if( rc ){ + raw_printf(stderr, "Error querying the selftest table\n"); + rc = 1; + sqlite3_free_table(azTest); + goto meta_command_exit; + }else if( nRow==0 ){ + sqlite3_free_table(azTest); + azTest = azDefaultTest; + nRow = nDefaultRow; + } + }else{ + azTest = azDefaultTest; + nRow = nDefaultRow; + } + initText(&str); + appendText(&str, "x", 0); + for(i=1; i<=nRow; i++){ + int tno = atoi(azTest[i*nCol]); + const char *zOp = azTest[i*nCol+1]; + const char *zSql = azTest[i*nCol+2]; + const char *zAns = azTest[i*nCol+3]; + + if( bVerbose>0 ){ + char *zQuote = sqlite3_mprintf("%q", zSql); + printf("%d: %s %s\n", tno, zOp, zSql); + sqlite3_free(zQuote); + } + if( strcmp(zOp,"memo")==0 ){ + utf8_printf(p->out, "%s\n", zSql); + }else + if( strcmp(zOp,"run")==0 ){ + char *zErrMsg = 0; + str.n = 0; + str.z[0] = 0; + rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); + nTest++; + if( bVerbose ){ + utf8_printf(p->out, "Result: %s\n", str.z); + } + if( rc || zErrMsg ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: error-code-%d: %s\n", tno, rc, zErrMsg); + sqlite3_free(zErrMsg); + }else if( strcmp(zAns,str.z)!=0 ){ + nErr++; + rc = 1; + utf8_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + utf8_printf(p->out, "%d: Got: [%s]\n", tno, str.z); + } + }else + { + utf8_printf(stderr, + "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); + rc = 1; + break; + } + } + freeText(&str); + if( azTest!=azDefaultTest ) sqlite3_free_table(azTest); + utf8_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + }else + if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ if( nArg<2 || nArg>3 ){ raw_printf(stderr, "Usage: .separator COL ?ROW?\n"); @@ -3852,6 +5980,122 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + if( c=='s' && n>=4 && strncmp(azArg[0],"sha3sum",n)==0 ){ + const char *zLike = 0; /* Which table to checksum. 0 means everything */ + int i; /* Loop counter */ + int bSchema = 0; /* Also hash the schema */ + int bSeparate = 0; /* Hash each table separately */ + int iSize = 224; /* Hash algorithm to use */ + int bDebug = 0; /* Only show the query that would have run */ + sqlite3_stmt *pStmt; /* For querying tables names */ + char *zSql; /* SQL to be run */ + char *zSep; /* Separator */ + ShellText sSql; /* Complete SQL for the query to run the hash */ + ShellText sQuery; /* Set of queries used to read all content */ + open_db(p, 0); + for(i=1; i1" + " UNION ALL SELECT 'sqlite_master'" + " ORDER BY 1 collate nocase"; + }else{ + zSql = "SELECT lower(name) FROM sqlite_master" + " WHERE type='table' AND coalesce(rootpage,0)>1" + " AND name NOT LIKE 'sqlite_%'" + " ORDER BY 1 collate nocase"; + } + sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + initText(&sQuery); + initText(&sSql); + appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); + zSep = "VALUES("; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zTab = (const char*)sqlite3_column_text(pStmt,0); + if( zLike && sqlite3_strlike(zLike, zTab, 0)!=0 ) continue; + if( strncmp(zTab, "sqlite_",7)!=0 ){ + appendText(&sQuery,"SELECT * FROM ", 0); + appendText(&sQuery,zTab,'"'); + appendText(&sQuery," NOT INDEXED;", 0); + }else if( strcmp(zTab, "sqlite_master")==0 ){ + appendText(&sQuery,"SELECT type,name,tbl_name,sql FROM sqlite_master" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_sequence")==0 ){ + appendText(&sQuery,"SELECT name,seq FROM sqlite_sequence" + " ORDER BY name;", 0); + }else if( strcmp(zTab, "sqlite_stat1")==0 ){ + appendText(&sQuery,"SELECT tbl,idx,stat FROM sqlite_stat1" + " ORDER BY tbl,idx;", 0); + }else if( strcmp(zTab, "sqlite_stat3")==0 + || strcmp(zTab, "sqlite_stat4")==0 ){ + appendText(&sQuery, "SELECT * FROM ", 0); + appendText(&sQuery, zTab, 0); + appendText(&sQuery, " ORDER BY tbl, idx, rowid;\n", 0); + } + appendText(&sSql, zSep, 0); + appendText(&sSql, sQuery.z, '\''); + sQuery.n = 0; + appendText(&sSql, ",", 0); + appendText(&sSql, zTab, '\''); + zSep = "),("; + } + sqlite3_finalize(pStmt); + if( bSeparate ){ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(a,%d))) AS hash, b AS label" + " FROM [sha3sum$query]", + sSql.z, iSize); + }else{ + zSql = sqlite3_mprintf( + "%s))" + " SELECT lower(hex(sha3_query(group_concat(a,''),%d))) AS hash" + " FROM [sha3sum$query]", + sSql.z, iSize); + } + freeText(&sQuery); + freeText(&sSql); + if( bDebug ){ + utf8_printf(p->out, "%s\n", zSql); + }else{ + shell_exec(p->db, zSql, shell_callback, p, 0); + } + sqlite3_free(zSql); + }else + if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -3873,17 +6117,19 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ + static const char *azBool[] = { "off", "on", "full", "unk" }; int i; if( nArg!=1 ){ raw_printf(stderr, "Usage: .show\n"); rc = 1; goto meta_command_exit; } - utf8_printf(p->out, "%12.12s: %s\n","echo", p->echoOn ? "on" : "off"); - utf8_printf(p->out, "%12.12s: %s\n","eqp", p->autoEQP ? "on" : "off"); + utf8_printf(p->out, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + utf8_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); utf8_printf(p->out, "%12.12s: %s\n","explain", p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - utf8_printf(p->out,"%12.12s: %s\n","headers", p->showHeader ? "on" : "off"); + utf8_printf(p->out,"%12.12s: %s\n","headers", azBool[p->showHeader!=0]); utf8_printf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); utf8_printf(p->out, "%12.12s: ", "nullvalue"); output_c_string(p->out, p->nullValue); @@ -3896,12 +6142,14 @@ static int do_meta_command(char *zLine, ShellState *p){ utf8_printf(p->out,"%12.12s: ", "rowseparator"); output_c_string(p->out, p->rowSeparator); raw_printf(p->out, "\n"); - utf8_printf(p->out, "%12.12s: %s\n","stats", p->statsOn ? "on" : "off"); + utf8_printf(p->out, "%12.12s: %s\n","stats", azBool[p->statsOn!=0]); utf8_printf(p->out, "%12.12s: ", "width"); for (i=0;i<(int)ArraySize(p->colWidth) && p->colWidth[i] != 0;i++) { raw_printf(p->out, "%d ", p->colWidth[i]); } raw_printf(p->out, "\n"); + utf8_printf(p->out, "%12.12s: %s\n", "filename", + p->zDbFilename ? p->zDbFilename : ""); }else if( c=='s' && strncmp(azArg[0], "stats", n)==0 ){ @@ -3915,7 +6163,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0 ){ + if( (c=='t' && n>1 && strncmp(azArg[0], "tables", n)==0) + || (c=='i' && (strncmp(azArg[0], "indices", n)==0 + || strncmp(azArg[0], "indexes", n)==0) ) + ){ sqlite3_stmt *pStmt; char **azResult; int nRow, nAlloc; @@ -3928,28 +6179,41 @@ static int do_meta_command(char *zLine, ShellState *p){ /* Create an SQL statement to query for the list of tables in the ** main and all attached databases where the table name matches the ** LIKE pattern bound to variable "?1". */ - zSql = sqlite3_mprintf( - "SELECT name FROM sqlite_master" - " WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite_%%'" - " AND name LIKE ?1"); - while( zSql && sqlite3_step(pStmt)==SQLITE_ROW ){ + if( c=='t' ){ + zSql = sqlite3_mprintf( + "SELECT name FROM sqlite_master" + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite_%%'" + " AND name LIKE ?1"); + }else if( nArg>2 ){ + /* It is an historical accident that the .indexes command shows an error + ** when called with the wrong number of arguments whereas the .tables + ** command does not. */ + raw_printf(stderr, "Usage: .indexes ?LIKE-PATTERN?\n"); + rc = 1; + goto meta_command_exit; + }else{ + zSql = sqlite3_mprintf( + "SELECT name FROM sqlite_master" + " WHERE type='index'" + " AND tbl_name LIKE ?1"); + } + for(ii=0; zSql && sqlite3_step(pStmt)==SQLITE_ROW; ii++){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); - if( zDbName==0 || strcmp(zDbName,"main")==0 ) continue; - if( strcmp(zDbName,"temp")==0 ){ - zSql = sqlite3_mprintf( - "%z UNION ALL " - "SELECT 'temp.' || name FROM sqlite_temp_master" - " WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite_%%'" - " AND name LIKE ?1", zSql); - }else{ + if( zDbName==0 || ii==0 ) continue; + if( c=='t' ){ zSql = sqlite3_mprintf( "%z UNION ALL " "SELECT '%q.' || name FROM \"%w\".sqlite_master" " WHERE type IN ('table','view')" " AND name NOT LIKE 'sqlite_%%'" " AND name LIKE ?1", zSql, zDbName, zDbName); + }else{ + zSql = sqlite3_mprintf( + "%z UNION ALL " + "SELECT '%q.' || name FROM \"%w\".sqlite_master" + " WHERE type='index'" + " AND tbl_name LIKE ?1", zSql, zDbName, zDbName); } } rc = sqlite3_finalize(pStmt); @@ -4019,6 +6283,21 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(azResult); }else + /* Begin redirecting output to the file "testcase-out.txt" */ + if( c=='t' && strcmp(azArg[0],"testcase")==0 ){ + output_reset(p); + p->out = output_file_open("testcase-out.txt"); + if( p->out==0 ){ + raw_printf(stderr, "Error: cannot open 'testcase-out.txt'\n"); + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); + } + }else + +#ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){ static const struct { const char *zCtrlName; /* Name of a test-control option */ @@ -4068,9 +6347,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, db, int) */ case SQLITE_TESTCTRL_OPTIMIZATIONS: - case SQLITE_TESTCTRL_RESERVE: + case SQLITE_TESTCTRL_RESERVE: if( nArg==3 ){ - int opt = (int)strtol(azArg[2], 0, 0); + int opt = (int)strtol(azArg[2], 0, 0); rc2 = sqlite3_test_control(testctrl, p->db, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4094,7 +6373,7 @@ static int do_meta_command(char *zLine, ShellState *p){ break; /* sqlite3_test_control(int, uint) */ - case SQLITE_TESTCTRL_PENDING_BYTE: + case SQLITE_TESTCTRL_PENDING_BYTE: if( nArg==3 ){ unsigned int opt = (unsigned int)integerValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); @@ -4104,13 +6383,13 @@ static int do_meta_command(char *zLine, ShellState *p){ " int option\n", azArg[1]); } break; - + /* sqlite3_test_control(int, int) */ - case SQLITE_TESTCTRL_ASSERT: - case SQLITE_TESTCTRL_ALWAYS: - case SQLITE_TESTCTRL_NEVER_CORRUPT: + case SQLITE_TESTCTRL_ASSERT: + case SQLITE_TESTCTRL_ALWAYS: + case SQLITE_TESTCTRL_NEVER_CORRUPT: if( nArg==3 ){ - int opt = booleanValue(azArg[2]); + int opt = booleanValue(azArg[2]); rc2 = sqlite3_test_control(testctrl, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4121,9 +6400,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* sqlite3_test_control(int, char *) */ #ifdef SQLITE_N_KEYWORD - case SQLITE_TESTCTRL_ISKEYWORD: + case SQLITE_TESTCTRL_ISKEYWORD: if( nArg==3 ){ - const char *opt = azArg[2]; + const char *opt = azArg[2]; rc2 = sqlite3_test_control(testctrl, opt); raw_printf(p->out, "%d (0x%08x)\n", rc2, rc2); } else { @@ -4136,7 +6415,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_IMPOSTER: if( nArg==5 ){ - rc2 = sqlite3_test_control(testctrl, p->db, + rc2 = sqlite3_test_control(testctrl, p->db, azArg[2], integerValue(azArg[3]), integerValue(azArg[4])); @@ -4146,10 +6425,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } break; - case SQLITE_TESTCTRL_BITVEC_TEST: - case SQLITE_TESTCTRL_FAULT_INSTALL: - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: - case SQLITE_TESTCTRL_SCRATCHMALLOC: + case SQLITE_TESTCTRL_BITVEC_TEST: + case SQLITE_TESTCTRL_FAULT_INSTALL: + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: + case SQLITE_TESTCTRL_SCRATCHMALLOC: default: utf8_printf(stderr, "Error: CLI support for testctrl %s not implemented\n", @@ -4158,12 +6437,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } } }else +#endif /* !defined(SQLITE_UNTESTABLE) */ if( c=='t' && n>4 && strncmp(azArg[0], "timeout", n)==0 ){ open_db(p, 0); sqlite3_busy_timeout(p->db, nArg>=2 ? (int)integerValue(azArg[1]) : 0); }else - + if( c=='t' && n>=5 && strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ enableTimer = booleanValue(azArg[1]); @@ -4176,7 +6456,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } }else - + if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ open_db(p, 0); if( nArg!=2 ){ @@ -4188,9 +6468,9 @@ static int do_meta_command(char *zLine, ShellState *p){ p->traceOut = output_file_open(azArg[1]); #if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ - sqlite3_trace(p->db, 0, 0); + sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace(p->db, sql_trace_callback, p->traceOut); + sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); } #endif }else @@ -4256,7 +6536,7 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "Usage: .user login|add|edit|delete ...\n"); rc = 1; goto meta_command_exit; - } + } }else #endif /* SQLITE_USER_AUTHENTICATION */ @@ -4267,7 +6547,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && strncmp(azArg[0], "vfsinfo", n)==0 ){ const char *zDbName = nArg==2 ? azArg[1] : "main"; - sqlite3_vfs *pVfs; + sqlite3_vfs *pVfs = 0; if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ @@ -4311,7 +6591,6 @@ static int do_meta_command(char *zLine, ShellState *p){ #if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_WHERETRACE) if( c=='w' && strncmp(azArg[0], "wheretrace", n)==0 ){ - extern int sqlite3WhereTrace; sqlite3WhereTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff; }else #endif @@ -4403,6 +6682,42 @@ static int line_is_complete(char *zSql, int nSql){ return rc; } +/* +** Run a single line of SQL +*/ +static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ + int rc; + char *zErrMsg = 0; + + open_db(p, 0); + if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); + BEGIN_TIMER; + rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg); + END_TIMER; + if( rc || zErrMsg ){ + char zPrefix[100]; + if( in!=0 || !stdin_is_interactive ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "Error: near line %d:", startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); + } + if( zErrMsg!=0 ){ + utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg); + sqlite3_free(zErrMsg); + zErrMsg = 0; + }else{ + utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); + } + return 1; + }else if( ShellHasFlag(p, SHFLG_CountChanges) ){ + raw_printf(p->out, "changes: %3d total_changes: %d\n", + sqlite3_changes(p->db), sqlite3_total_changes(p->db)); + } + return 0; +} + + /* ** Read input from *in and process it. If *in==0 then input ** is interactive - the user is typing it it. Otherwise, input @@ -4419,7 +6734,6 @@ static int process_input(ShellState *p, FILE *in){ int nSql = 0; /* Bytes of zSql[] used */ int nAlloc = 0; /* Allocated zSql[] space */ int nSqlPrior = 0; /* Bytes of zSql[] used by prior line */ - char *zErrMsg; /* Error message returned */ int rc; /* Error code */ int errCnt = 0; /* Number of errors seen */ int lineno = 0; /* Current line number */ @@ -4430,7 +6744,7 @@ static int process_input(ShellState *p, FILE *in){ zLine = one_input_line(in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( stdin_is_interactive ) printf("\n"); + if( in==0 && stdin_is_interactive ) printf("\n"); break; } if( seenInterrupt ){ @@ -4439,11 +6753,11 @@ static int process_input(ShellState *p, FILE *in){ } lineno++; if( nSql==0 && _all_whitespace(zLine) ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); continue; } if( zLine && zLine[0]=='.' && nSql==0 ){ - if( p->echoOn ) printf("%s\n", zLine); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zLine); rc = do_meta_command(zLine, p); if( rc==2 ){ /* exit requested */ break; @@ -4479,47 +6793,19 @@ static int process_input(ShellState *p, FILE *in){ } if( nSql && line_contains_semicolon(&zSql[nSqlPrior], nSql-nSqlPrior) && sqlite3_complete(zSql) ){ - p->cnt = 0; - open_db(p, 0); - if( p->backslashOn ) resolve_backslashes(zSql); - BEGIN_TIMER; - rc = shell_exec(p->db, zSql, shell_callback, p, &zErrMsg); - END_TIMER; - if( rc || zErrMsg ){ - char zPrefix[100]; - if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, - "Error: near line %d:", startline); - }else{ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, "Error:"); - } - if( zErrMsg!=0 ){ - utf8_printf(stderr, "%s %s\n", zPrefix, zErrMsg); - sqlite3_free(zErrMsg); - zErrMsg = 0; - }else{ - utf8_printf(stderr, "%s %s\n", zPrefix, sqlite3_errmsg(p->db)); - } - errCnt++; - }else if( p->countChanges ){ - raw_printf(p->out, "changes: %3d total_changes: %d\n", - sqlite3_changes(p->db), sqlite3_total_changes(p->db)); - } + errCnt += runOneSqlLine(p, zSql, in, startline); nSql = 0; if( p->outCount ){ output_reset(p); p->outCount = 0; } }else if( nSql && _all_whitespace(zSql) ){ - if( p->echoOn ) printf("%s\n", zSql); + if( ShellHasFlag(p, SHFLG_Echo) ) printf("%s\n", zSql); nSql = 0; } } - if( nSql ){ - if( !_all_whitespace(zSql) ){ - utf8_printf(stderr, "Error: incomplete SQL: %s\n", zSql); - errCnt++; - } + if( nSql && !_all_whitespace(zSql) ){ + runOneSqlLine(p, zSql, in, startline); } free(zSql); free(zLine); @@ -4530,8 +6816,13 @@ static int process_input(ShellState *p, FILE *in){ ** Return a pathname which is the user's home directory. A ** 0 return indicates an error of some kind. */ -static char *find_home_dir(void){ +static char *find_home_dir(int clearFlag){ static char *home_dir = NULL; + if( clearFlag ){ + free(home_dir); + home_dir = 0; + return 0; + } if( home_dir ) return home_dir; #if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ @@ -4606,7 +6897,7 @@ static void process_sqliterc( FILE *in = NULL; if (sqliterc == NULL) { - home_dir = find_home_dir(); + home_dir = find_home_dir(0); if( home_dir==0 ){ raw_printf(stderr, "-- warning: cannot find home directory;" " cannot read ~/.sqliterc\n"); @@ -4630,7 +6921,7 @@ static void process_sqliterc( /* ** Show available command line options */ -static const char zOptions[] = +static const char zOptions[] = " -ascii set output mode to 'ascii'\n" " -bail stop after hitting an error\n" " -batch force batch I/O\n" @@ -4667,7 +6958,7 @@ static const char zOptions[] = ; static void usage(int showDetail){ utf8_printf(stderr, - "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "Usage: %s [OPTIONS] FILENAME [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist.\n", Argv0); if( showDetail ){ @@ -4729,7 +7020,20 @@ static char *cmdline_option_value(int argc, char **argv, int i){ return argv[i]; } +#ifndef SQLITE_SHELL_IS_UTF8 +# if (defined(_WIN32) || defined(WIN32)) && defined(_MSC_VER) +# define SQLITE_SHELL_IS_UTF8 (0) +# else +# define SQLITE_SHELL_IS_UTF8 (1) +# endif +#endif + +#if SQLITE_SHELL_IS_UTF8 int SQLITE_CDECL main(int argc, char **argv){ +#else +int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +#endif char *zErrMsg = 0; ShellState data; const char *zInitFile = 0; @@ -4740,6 +7044,11 @@ int SQLITE_CDECL main(int argc, char **argv){ int nCmd = 0; char **azCmd = 0; + setBinaryMode(stdin, 0); + setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + stdin_is_interactive = isatty(0); + stdout_is_console = isatty(1); + #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ utf8_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", @@ -4747,12 +7056,24 @@ int SQLITE_CDECL main(int argc, char **argv){ exit(1); } #endif - setBinaryMode(stdin); - setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ - Argv0 = argv[0]; main_init(&data); - stdin_is_interactive = isatty(0); - stdout_is_console = isatty(1); +#if !SQLITE_SHELL_IS_UTF8 + sqlite3_initialize(); + argv = sqlite3_malloc64(sizeof(argv[0])*argc); + if( argv==0 ){ + raw_printf(stderr, "out of memory\n"); + exit(1); + } + for(i=0; i=1 && argv && argv[0] ); + Argv0 = argv[0]; /* Make sure we have a valid signal handler early, before anything ** else is done. @@ -4808,7 +7129,7 @@ int SQLITE_CDECL main(int argc, char **argv){ zInitFile = cmdline_option_value(argc, argv, ++i); }else if( strcmp(z,"-batch")==0 ){ /* Need to check for batch mode here to so we can avoid printing - ** informational messages (like from process_sqliterc) before + ** informational messages (like from process_sqliterc) before ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; @@ -4821,6 +7142,8 @@ int SQLITE_CDECL main(int argc, char **argv){ szHeap = integerValue(zSize); if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); +#else + (void)cmdline_option_value(argc, argv, ++i); #endif }else if( strcmp(z,"-scratch")==0 ){ int n, sz; @@ -4946,9 +7269,11 @@ int SQLITE_CDECL main(int argc, char **argv){ }else if( strcmp(z,"-noheader")==0 ){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ - data.echoOn = 1; + ShellSetFlag(&data, SHFLG_Echo); }else if( strcmp(z,"-eqp")==0 ){ data.autoEQP = 1; + }else if( strcmp(z,"-eqpfull")==0 ){ + data.autoEQP = 2; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( strcmp(z,"-scanstats")==0 ){ @@ -4959,7 +7284,7 @@ int SQLITE_CDECL main(int argc, char **argv){ ** prior to sending the SQL into SQLite. Useful for injecting ** crazy bytes in the middle of SQL statements for testing and debugging. */ - data.backslashOn = 1; + ShellSetFlag(&data, SHFLG_Backslash); }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ @@ -5063,7 +7388,7 @@ int SQLITE_CDECL main(int argc, char **argv){ printf(".\nUse \".open FILENAME\" to reopen on a " "persistent database.\n"); } - zHome = find_home_dir(); + zHome = find_home_dir(0); if( zHome ){ nHistory = strlen30(zHome) + 20; if( (zHistory = malloc(nHistory))!=0 ){ @@ -5083,8 +7408,14 @@ int SQLITE_CDECL main(int argc, char **argv){ } set_table_name(&data, 0); if( data.db ){ + session_close_all(&data); sqlite3_close(data.db); } - sqlite3_free(data.zFreeOnClose); + sqlite3_free(data.zFreeOnClose); + find_home_dir(1); +#if !SQLITE_SHELL_IS_UTF8 + for(i=0; i /* Needed for the definition of va_list */ /* @@ -54,8 +54,17 @@ extern "C" { #ifndef SQLITE_CDECL # define SQLITE_CDECL #endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif #ifndef SQLITE_STDCALL -# define SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI #endif /* @@ -99,13 +108,14 @@ extern "C" { ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** -** Since version 3.6.18, SQLite source code has been stored in the +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID -** string contains the date and time of the check-in (UTC) and an SHA1 -** hash of the entire source tree. +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], @@ -117,7 +127,7 @@ extern "C" { /* ** CAPI3REF: Run-Time Library Version Numbers -** KEYWORDS: sqlite3_version, sqlite3_sourceid +** KEYWORDS: sqlite3_version sqlite3_sourceid ** ** These interfaces provide the same information as the [SQLITE_VERSION], ** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros @@ -249,7 +259,11 @@ typedef struct sqlite3 sqlite3; */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; - typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif #elif defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 sqlite_int64; typedef unsigned __int64 sqlite_uint64; @@ -443,7 +457,8 @@ int sqlite3_exec( ** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to -** address this, newer versions of SQLite (version 3.3.8 and later) include +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include ** support for additional result codes that provide more detailed information ** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the @@ -506,6 +521,7 @@ int sqlite3_exec( #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -560,7 +576,7 @@ int sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. The +** flag indicates that a file cannot be deleted when open. The ** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on ** read-only media and cannot be changed even by processes with ** elevated privileges. @@ -710,6 +726,9 @@ struct sqlite3_file { **
  • [SQLITE_IOCAP_ATOMIC64K] **
  • [SQLITE_IOCAP_SAFE_APPEND] **
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of @@ -966,6 +985,12 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** +**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** **
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] ** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This ** opcode causes the xFileControl method to swap the file handle with the one @@ -1016,6 +1041,8 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 #define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 +#define SQLITE_FCNTL_PDB 30 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1035,6 +1062,16 @@ struct sqlite3_io_methods { */ typedef struct sqlite3_mutex sqlite3_mutex; +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + /* ** CAPI3REF: OS Interface Object ** @@ -1932,13 +1969,55 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back. ** +**
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    +**
    ^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argument to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
    +** +**
    SQLITE_DBCONFIG_MAINDBNAME
    +**
    ^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
    +** +**
    SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
    +**
    Usually, when a database in wal mode is closed or detached from a +** database handle, SQLite checks if this will mean that there are now no +** connections at all to the database. If so, it performs a checkpoint +** operation before closing the connection. This option may be used to +** override this behaviour. The first parameter passed to this operation +** is an integer - non-zero to disable checkpoints-on-close, or zero (the +** default) to enable them. The second parameter is a pointer to an integer +** into which is written 0 or 1 to indicate whether checkpoints-on-close +** have been disabled - 0 if they are not disabled, 1 if they are. +**
    +** ** */ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ #define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ -#define SQLITE_DBCONFIG_WHEREINFO 1004 /* xWhereInfo void* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ +#define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */ + +#define SQLITE_DBCONFIG_WHEREINFO 1007 /* xWhereInfo void* */ #define SQLITE_WHEREINFO_TABLE 1 #define SQLITE_WHEREINFO_EQUALS 2 @@ -1971,20 +2050,30 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the -** most recent successful [INSERT] into a rowid table or [virtual table] -** on database connection D. -** ^Inserts into [WITHOUT ROWID] tables are not recorded. -** ^If no successful [INSERT]s into rowid tables -** have ever occurred on the database connection D, -** then sqlite3_last_insert_rowid(D) returns zero. +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. ** -** ^(If an [INSERT] occurs within a trigger or within a [virtual table] -** method, then this routine will return the [rowid] of the inserted -** row as long as the trigger or virtual table method is running. -** But once the trigger or virtual table method ends, the value returned -** by this routine reverts to what it was before the trigger or virtual -** table method began.)^ +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2011,6 +2100,16 @@ int sqlite3_extended_result_codes(sqlite3*, int onoff); */ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); + /* ** CAPI3REF: Count The Number Of Rows Modified ** METHOD: sqlite3 @@ -2223,7 +2322,7 @@ int sqlite3_complete16(const void *sql); ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); /* ** CAPI3REF: Set A Busy Timeout @@ -2745,6 +2844,9 @@ int sqlite3_set_authorizer( ** CAPI3REF: Tracing And Profiling Functions ** METHOD: sqlite3 ** +** These routines are deprecated. Use the [sqlite3_trace_v2()] interface +** instead of the routines described here. +** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. ** @@ -2770,10 +2872,104 @@ int sqlite3_set_authorizer( ** sqlite3_profile() function is considered experimental and is ** subject to change in future versions of SQLite. */ -void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); -SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, +SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, + void(*xTrace)(void*,const char*), void*); +SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); +/* +** CAPI3REF: SQL Trace Event Codes +** KEYWORDS: SQLITE_TRACE +** +** These constants identify classes of events that can be monitored +** using the [sqlite3_trace_v2()] tracing logic. The third argument +** to [sqlite3_trace_v2()] is an OR-ed combination of one or more of +** the following constants. ^The first argument to the trace callback +** is one of the following constants. +** +** New tracing constants may be added in future releases. +** +** ^A trace callback has four arguments: xCallback(T,C,P,X). +** ^The T argument is one of the integer type codes above. +** ^The C argument is a copy of the context pointer passed in as the +** fourth argument to [sqlite3_trace_v2()]. +** The P and X arguments are pointers whose meanings depend on T. +** +**
    +** [[SQLITE_TRACE_STMT]]
    SQLITE_TRACE_STMT
    +**
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement +** first begins running and possibly at other times during the +** execution of the prepared statement, such as at the start of each +** trigger subprogram. ^The P argument is a pointer to the +** [prepared statement]. ^The X argument is a pointer to a string which +** is the unexpanded SQL text of the prepared statement or an SQL comment +** that indicates the invocation of a trigger. ^The callback can compute +** the same text that would have been returned by the legacy [sqlite3_trace()] +** interface by using the X argument when X begins with "--" and invoking +** [sqlite3_expanded_sql(P)] otherwise. +** +** [[SQLITE_TRACE_PROFILE]]
    SQLITE_TRACE_PROFILE
    +**
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same +** information as is provided by the [sqlite3_profile()] callback. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument points to a 64-bit integer which is the estimated of +** the number of nanosecond that the prepared statement took to run. +** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. +** +** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    +**
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared +** statement generates a single row of result. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument is unused. +** +** [[SQLITE_TRACE_CLOSE]]
    SQLITE_TRACE_CLOSE
    +**
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database +** connection closes. +** ^The P argument is a pointer to the [database connection] object +** and the X argument is unused. +**
    +*/ +#define SQLITE_TRACE_STMT 0x01 +#define SQLITE_TRACE_PROFILE 0x02 +#define SQLITE_TRACE_ROW 0x04 +#define SQLITE_TRACE_CLOSE 0x08 + +/* +** CAPI3REF: SQL Trace Hook +** METHOD: sqlite3 +** +** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback +** function X against [database connection] D, using property mask M +** and context pointer P. ^If the X callback is +** NULL or if the M mask is zero, then tracing is disabled. The +** M argument should be the bitwise OR-ed combination of +** zero or more [SQLITE_TRACE] constants. +** +** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides +** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** +** ^The X callback is invoked whenever any of the events identified by +** mask M occur. ^The integer return value from the callback is currently +** ignored, though this may change in future releases. Callback +** implementations should return zero to ensure future compatibility. +** +** ^A trace callback is invoked with four arguments: callback(T,C,P,X). +** ^The T argument is one of the [SQLITE_TRACE] +** constants to indicate why the callback was invoked. +** ^The C argument is a copy of the context pointer. +** The P and X arguments are pointers whose meanings depend on T. +** +** The sqlite3_trace_v2() interface is intended to replace the legacy +** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which +** are deprecated. +*/ +int sqlite3_trace_v2( + sqlite3*, + unsigned uMask, + int(*xCallback)(unsigned,void*,void*,void*), + void *pCtx +); + /* ** CAPI3REF: Query Progress Callbacks ** METHOD: sqlite3 @@ -3238,9 +3434,9 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_VDBE_OP]] ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement. This limit is not currently -** enforced, though that might be added in some future release of -** SQLite.
    )^ +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned.)^ ** ** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ @@ -3278,6 +3474,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 + /* ** CAPI3REF: Compiling An SQL Statement ** KEYWORDS: {SQL statement compiler} @@ -3392,11 +3589,35 @@ int sqlite3_prepare16_v2( ** CAPI3REF: Retrieving Statement SQL ** METHOD: sqlite3_stmt ** -** ^This interface can be used to retrieve a saved copy of the original -** SQL text used to create a [prepared statement] if that statement was -** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 +** SQL text used to create [prepared statement] P if P was +** created by either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. +** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 +** string containing the SQL text of prepared statement P with +** [bound parameters] expanded. +** +** ^(For example, if a prepared statement is created using the SQL +** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 +** and parameter :xyz is unbound, then sqlite3_sql() will return +** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() +** will return "SELECT 2345,NULL".)^ +** +** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory +** is available to hold the result, or if the result would exceed the +** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** +** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of +** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time +** option causes sqlite3_expanded_sql() to always return NULL. +** +** ^The string returned by sqlite3_sql(P) is managed by SQLite and is +** automatically freed when the prepared statement is finalized. +** ^The string returned by sqlite3_expanded_sql(P), on the other hand, +** is obtained from [sqlite3_malloc()] and must be free by the application +** by passing it to [sqlite3_free()]. */ const char *sqlite3_sql(sqlite3_stmt *pStmt); +char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database @@ -3427,6 +3648,10 @@ const char *sqlite3_sql(sqlite3_stmt *pStmt); ** sqlite3_stmt_readonly() to return true since, while those statements ** change the configuration of a database connection, they do not make ** changes to the content of the database files on disk. +** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since +** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and +** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so +** sqlite3_stmt_readonly() returns false for those commands. */ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); @@ -3709,8 +3934,12 @@ int sqlite3_clear_bindings(sqlite3_stmt*); ** METHOD: sqlite3_stmt ** ** ^Return the number of columns in the result set returned by the -** [prepared statement]. ^This routine returns 0 if pStmt is an SQL -** statement that does not return data (for example an [UPDATE]). +** [prepared statement]. ^If this routine returns 0, that means the +** [prepared statement] returns no data (for example an [UPDATE]). +** ^However, just because this routine returns a positive number does not +** mean that one or more rows of data will be returned. ^A SELECT statement +** will always have a positive sqlite3_column_count() but depending on the +** WHERE clause constraints and the table content, it might return no rows. ** ** See also: [sqlite3_data_count()] */ @@ -3891,7 +4120,8 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility ** break because any application that ever receives an SQLITE_MISUSE error @@ -4554,12 +4784,13 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** SQLite will invoke the destructor function X with parameter P exactly ** once, when the metadata is discarded. ** SQLite is free to discard the metadata at any time, including:
      -**
    • when the corresponding function parameter changes, or -**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the -** SQL statement, or -**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or -**
    • during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.
    )^ +**
  • ^(when the corresponding function parameter changes)^, or +**
  • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
  • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
  • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^ ** ** Note the last bullet in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the @@ -5196,7 +5427,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The sqlite3_update_hook() interface registers a callback function ** with the [database connection] identified by the first argument ** to be invoked whenever a row is updated, inserted or deleted in -** a rowid table. +** a [rowid table]. ** ^Any callback set by a previous call to this function ** for the same database connection is overridden. ** @@ -5217,7 +5448,7 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook -** is not invoked when duplication rows are deleted because of an +** is not invoked when conflicting rows are deleted because of an ** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook ** invoked when rows are deleted using the [truncate optimization]. ** The exceptions defined in this paragraph might change in a future @@ -5235,8 +5466,8 @@ void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); ** on the same [database connection] D, or NULL for ** the first call on D. ** -** See also the [sqlite3_commit_hook()] and [sqlite3_rollback_hook()] -** interfaces. +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. */ void *sqlite3_update_hook( sqlite3*, @@ -5253,7 +5484,8 @@ void *sqlite3_update_hook( ** and disabled if the argument is false.)^ ** ** ^Cache sharing is enabled and disabled for an entire process. -** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** ** ^(The cache sharing mode set by this interface effects all subsequent @@ -5347,7 +5579,8 @@ int sqlite3_db_release_memory(sqlite3*); ** from the heap. ** )^ ** -** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), +** the soft heap limit is enforced ** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] ** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], ** the soft heap limit is enforced on every memory allocation. Without @@ -5386,7 +5619,7 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** column exists. ^The sqlite3_table_column_metadata() interface returns ** SQLITE_ERROR and if the specified column does not exist. ** ^If the column-name parameter to sqlite3_table_column_metadata() is a -** NULL pointer, then this routine simply checks for the existance of the +** NULL pointer, then this routine simply checks for the existence of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it ** does not. ** @@ -5483,9 +5716,18 @@ int sqlite3_table_column_metadata( ** should free this memory by calling [sqlite3_free()]. ** ** ^Extension loading must be enabled using -** [sqlite3_enable_load_extension()] prior to calling this API, +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, ** otherwise an error will be returned. ** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** ** See also the [load_extension() SQL function]. */ int sqlite3_load_extension( @@ -5508,6 +5750,17 @@ int sqlite3_load_extension( ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); @@ -5521,7 +5774,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three -** arguments and expects and integer result as if the signature of the +** arguments and expects an integer result as if the signature of the ** entry point where as follows: ** ** 
    @@ -5547,7 +5800,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff);
 ** See also: [sqlite3_reset_auto_extension()]
 ** and [sqlite3_cancel_auto_extension()]
 */
-int sqlite3_auto_extension(void (*xEntryPoint)(void));
+int sqlite3_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Cancel Automatic Extension Loading
@@ -5559,7 +5812,7 @@ int sqlite3_auto_extension(void (*xEntryPoint)(void));
 ** unregistered and it returns 0 if X was not on the list of initialization
 ** routines.
 */
-int sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
+int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Reset Automatic Extension Loading
@@ -5721,13 +5974,15 @@ struct sqlite3_module {
 ** the xUpdate method are automatically rolled back by SQLite.
 **
 ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
-** structure for SQLite version 3.8.2. If a virtual table extension is
+** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). 
+** If a virtual table extension is
 ** used with an SQLite version earlier than 3.8.2, the results of attempting 
 ** to read or write the estimatedRows field are undefined (but are likely 
 ** to included crashing the application). The estimatedRows field should
 ** therefore only be used if [sqlite3_libversion_number()] returns a
 ** value greater than or equal to 3008002. Similarly, the idxFlags field
-** was added for version 3.9.0. It may therefore only be used if
+** was added for [version 3.9.0] ([dateof:3.9.0]). 
+** It may therefore only be used if
 ** sqlite3_libversion_number() returns a value greater than or equal to
 ** 3009000.
 */
@@ -5975,6 +6230,12 @@ typedef struct sqlite3_blob sqlite3_blob;
 ** [database connection] error code and message accessible via 
 ** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. 
 **
+** A BLOB referenced by sqlite3_blob_open() may be read using the
+** [sqlite3_blob_read()] interface and modified by using
+** [sqlite3_blob_write()].  The [BLOB handle] can be moved to a
+** different row of the same table using the [sqlite3_blob_reopen()]
+** interface.  However, the column, table, or database of a [BLOB handle]
+** cannot be changed after the [BLOB handle] is opened.
 **
 ** ^(If the row that a BLOB handle points to is modified by an
 ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects
@@ -5998,6 +6259,10 @@ typedef struct sqlite3_blob sqlite3_blob;
 **
 ** To avoid a resource leak, every open [BLOB handle] should eventually
 ** be released by a call to [sqlite3_blob_close()].
+**
+** See also: [sqlite3_blob_close()],
+** [sqlite3_blob_reopen()], [sqlite3_blob_read()],
+** [sqlite3_blob_bytes()], [sqlite3_blob_write()].
 */
 int sqlite3_blob_open(
   sqlite3*,
@@ -6013,11 +6278,11 @@ int sqlite3_blob_open(
 ** CAPI3REF: Move a BLOB Handle to a New Row
 ** METHOD: sqlite3_blob
 **
-** ^This function is used to move an existing blob handle so that it points
+** ^This function is used to move an existing [BLOB handle] so that it points
 ** to a different row of the same database table. ^The new row is identified
 ** by the rowid value passed as the second argument. Only the row can be
 ** changed. ^The database, table and column on which the blob handle is open
-** remain the same. Moving an existing blob handle to a new row can be
+** remain the same. Moving an existing [BLOB handle] to a new row is
 ** faster than closing the existing handle and opening a new one.
 **
 ** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] -
@@ -6425,7 +6690,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*);
 #define SQLITE_MUTEX_STATIC_MEM       3  /* sqlite3_malloc() */
 #define SQLITE_MUTEX_STATIC_MEM2      4  /* NOT USED */
 #define SQLITE_MUTEX_STATIC_OPEN      4  /* sqlite3BtreeOpen() */
-#define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_random() */
+#define SQLITE_MUTEX_STATIC_PRNG      5  /* sqlite3_randomness() */
 #define SQLITE_MUTEX_STATIC_LRU       6  /* lru page list */
 #define SQLITE_MUTEX_STATIC_LRU2      7  /* NOT USED */
 #define SQLITE_MUTEX_STATIC_PMEM      7  /* sqlite3PageMalloc() */
@@ -6529,6 +6794,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
 #define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
+#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD    19
 #define SQLITE_TESTCTRL_NEVER_CORRUPT           20
 #define SQLITE_TESTCTRL_VDBE_COVERAGE           21
 #define SQLITE_TESTCTRL_BYTEORDER               22
@@ -6735,6 +7001,18 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 ** memory used by all pager caches associated with the database connection.)^
 ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0.
 **
+** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] 
+** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    
+** 
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a
+** pager cache is shared between two or more connections the bytes of heap
+** memory used by that pager cache is divided evenly between the attached
+** connections.)^  In other words, if none of the pager caches associated
+** with the database connection are shared, this request returns the same
+** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are
+** shared, the value returned by this call will be smaller than that returned
+** by DBSTATUS_CACHE_USED. ^The highwater mark associated with
+** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0.
+**
 ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    
 ** 
    This parameter returns the approximate number of bytes of heap
 ** memory used to store the schema for all databases associated
@@ -6792,7 +7070,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg);
 #define SQLITE_DBSTATUS_CACHE_MISS           8
 #define SQLITE_DBSTATUS_CACHE_WRITE          9
 #define SQLITE_DBSTATUS_DEFERRED_FKS        10
-#define SQLITE_DBSTATUS_MAX                 10   /* Largest defined DBSTATUS */
+#define SQLITE_DBSTATUS_CACHE_USED_SHARED   11
+#define SQLITE_DBSTATUS_MAX                 11   /* Largest defined DBSTATUS */
 
 
 /*
@@ -7146,7 +7425,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with
 ** an error.
 **
-** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if 
+** ^A call to sqlite3_backup_init() will fail, returning NULL, if 
 ** there is already a read or read-write transaction open on the 
 ** destination database.
 **
@@ -7924,9 +8203,123 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*);
 */
 int sqlite3_db_cacheflush(sqlite3*);
 
+/*
+** CAPI3REF: The pre-update hook.
+**
+** ^These interfaces are only available if SQLite is compiled using the
+** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option.
+**
+** ^The [sqlite3_preupdate_hook()] interface registers a callback function
+** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation
+** on a database table.
+** ^At most one preupdate hook may be registered at a time on a single
+** [database connection]; each call to [sqlite3_preupdate_hook()] overrides
+** the previous setting.
+** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()]
+** with a NULL pointer as the second parameter.
+** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as
+** the first parameter to callbacks.
+**
+** ^The preupdate hook only fires for changes to real database tables; the
+** preupdate hook is not invoked for changes to [virtual tables] or to
+** system tables like sqlite_master or sqlite_stat1.
+**
+** ^The second parameter to the preupdate callback is a pointer to
+** the [database connection] that registered the preupdate hook.
+** ^The third parameter to the preupdate callback is one of the constants
+** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the
+** kind of update operation that is about to occur.
+** ^(The fourth parameter to the preupdate callback is the name of the
+** database within the database connection that is being modified.  This
+** will be "main" for the main database or "temp" for TEMP tables or 
+** the name given after the AS keyword in the [ATTACH] statement for attached
+** databases.)^
+** ^The fifth parameter to the preupdate callback is the name of the
+** table that is being modified.
+**
+** For an UPDATE or DELETE operation on a [rowid table], the sixth
+** parameter passed to the preupdate callback is the initial [rowid] of the 
+** row being modified or deleted. For an INSERT operation on a rowid table,
+** or any operation on a WITHOUT ROWID table, the value of the sixth 
+** parameter is undefined. For an INSERT or UPDATE on a rowid table the
+** seventh parameter is the final rowid value of the row being inserted
+** or updated. The value of the seventh parameter passed to the callback
+** function is not defined for operations on WITHOUT ROWID tables, or for
+** INSERT operations on rowid tables.
+**
+** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()],
+** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces
+** provide additional information about a preupdate event. These routines
+** may only be called from within a preupdate callback.  Invoking any of
+** these routines from outside of a preupdate callback or with a
+** [database connection] pointer that is different from the one supplied
+** to the preupdate callback results in undefined and probably undesirable
+** behavior.
+**
+** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns
+** in the row that is being inserted, updated, or deleted.
+**
+** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to
+** a [protected sqlite3_value] that contains the value of the Nth column of
+** the table row before it is updated.  The N parameter must be between 0
+** and one less than the number of columns or the behavior will be
+** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE
+** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the
+** behavior is undefined.  The [sqlite3_value] that P points to
+** will be destroyed when the preupdate callback returns.
+**
+** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to
+** a [protected sqlite3_value] that contains the value of the Nth column of
+** the table row after it is updated.  The N parameter must be between 0
+** and one less than the number of columns or the behavior will be
+** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE
+** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the
+** behavior is undefined.  The [sqlite3_value] that P points to
+** will be destroyed when the preupdate callback returns.
+**
+** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate
+** callback was invoked as a result of a direct insert, update, or delete
+** operation; or 1 for inserts, updates, or deletes invoked by top-level 
+** triggers; or 2 for changes resulting from triggers called by top-level
+** triggers; and so forth.
+**
+** See also:  [sqlite3_update_hook()]
+*/
+#if defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+void *sqlite3_preupdate_hook(
+  sqlite3 *db,
+  void(*xPreUpdate)(
+    void *pCtx,                   /* Copy of third arg to preupdate_hook() */
+    sqlite3 *db,                  /* Database handle */
+    int op,                       /* SQLITE_UPDATE, DELETE or INSERT */
+    char const *zDb,              /* Database name */
+    char const *zName,            /* Table name */
+    sqlite3_int64 iKey1,          /* Rowid of row about to be deleted/updated */
+    sqlite3_int64 iKey2           /* New rowid value (for a rowid UPDATE) */
+  ),
+  void*
+);
+int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **);
+int sqlite3_preupdate_count(sqlite3 *);
+int sqlite3_preupdate_depth(sqlite3 *);
+int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **);
+#endif
+
+/*
+** CAPI3REF: Low-level system error code
+**
+** ^Attempt to return the underlying operating system error code or error
+** number that caused the most recent I/O error or failure to open a file.
+** The return value is OS-dependent.  For example, on unix systems, after
+** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be
+** called to get back the underlying "errno" that caused the problem, such
+** as ENOSPC, EAUTH, EISDIR, and so forth.  
+*/
+int sqlite3_system_errno(sqlite3*);
+
 /*
 ** CAPI3REF: Database Snapshot
-** KEYWORDS: {snapshot}
+** KEYWORDS: {snapshot} {sqlite3_snapshot}
 ** EXPERIMENTAL
 **
 ** An instance of the snapshot object records the state of a [WAL mode]
@@ -7950,7 +8343,9 @@ int sqlite3_db_cacheflush(sqlite3*);
 ** to an historical snapshot (if possible).  The destructor for 
 ** sqlite3_snapshot objects is [sqlite3_snapshot_free()].
 */
-typedef struct sqlite3_snapshot sqlite3_snapshot;
+typedef struct sqlite3_snapshot {
+  unsigned char hidden[48];
+} sqlite3_snapshot;
 
 /*
 ** CAPI3REF: Record A Database Snapshot
@@ -7961,9 +8356,32 @@ typedef struct sqlite3_snapshot sqlite3_snapshot;
 ** schema S in database connection D.  ^On success, the
 ** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly
 ** created [sqlite3_snapshot] object into *P and returns SQLITE_OK.
-** ^If schema S of [database connection] D is not a [WAL mode] database
-** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)]
-** leaves the *P value unchanged and returns an appropriate [error code].
+** If there is not already a read-transaction open on schema S when
+** this function is called, one is opened automatically. 
+**
+** The following must be true for this function to succeed. If any of
+** the following statements are false when sqlite3_snapshot_get() is
+** called, SQLITE_ERROR is returned. The final value of *P is undefined
+** in this case. 
+**
+** 
      
+**   
    •  The database handle must be in [autocommit mode].
+**
+**   
    •  Schema S of [database connection] D must be a [WAL mode] database.
+**
+**   
    •  There must not be a write transaction open on schema S of database
+**        connection D.
+**
+**   
    •  One or more transactions must have been written to the current wal
+**        file since it was created on disk (by any connection). This means
+**        that a snapshot cannot be taken on a wal mode database with no wal 
+**        file immediately after it is first opened. At least one transaction
+**        must be written to it first.
+** 
    
+**
+** This function may also return SQLITE_NOMEM.  If it is called with the
+** database handle in autocommit mode but fails for some other reason, 
+** whether or not a read transaction is opened on schema S is undefined.
 **
 ** The [sqlite3_snapshot] object returned from a successful call to
 ** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()]
@@ -7982,20 +8400,29 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get(
 ** CAPI3REF: Start a read transaction on an historical snapshot
 ** EXPERIMENTAL
 **
-** ^The [sqlite3_snapshot_open(D,S,P)] interface attempts to move the
-** read transaction that is currently open on schema S of
-** [database connection] D so that it refers to historical [snapshot] P.
+** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a
+** read transaction for schema S of
+** [database connection] D such that the read transaction
+** refers to historical [snapshot] P, rather than the most
+** recent change to the database.
 ** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success
 ** or an appropriate [error code] if it fails.
 **
 ** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be
-** the first operation, apart from other sqlite3_snapshot_open() calls,
-** following the [BEGIN] that starts a new read transaction.
-** ^A [snapshot] will fail to open if it has been overwritten by a 
+** the first operation following the [BEGIN] that takes the schema S
+** out of [autocommit mode].
+** ^In other words, schema S must not currently be in
+** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the
+** database connection D must be out of [autocommit mode].
+** ^A [snapshot] will fail to open if it has been overwritten by a
 ** [checkpoint].
-** ^A [snapshot] will fail to open if the database connection D has not
-** previously completed at least one read operation against the database 
-** file.  (Hint: Run "[PRAGMA application_id]" against a newly opened
+** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the
+** database connection D does not know that the database file for
+** schema S is in [WAL mode].  A database connection might not know
+** that the database file is in [WAL mode] if there has been no prior
+** I/O on that database connection, or if the database entered [WAL mode] 
+** after the most recent I/O on the database connection.)^
+** (Hint: Run "[PRAGMA application_id]" against a newly opened
 ** database connection in order to make it ready to use snapshots.)
 **
 ** The [sqlite3_snapshot_open()] interface is only available when the
@@ -8020,6 +8447,55 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_open(
 */
 SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 
+/*
+** CAPI3REF: Compare the ages of two snapshot handles.
+** EXPERIMENTAL
+**
+** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages
+** of two valid snapshot handles. 
+**
+** If the two snapshot handles are not associated with the same database 
+** file, the result of the comparison is undefined. 
+**
+** Additionally, the result of the comparison is only valid if both of the
+** snapshot handles were obtained by calling sqlite3_snapshot_get() since the
+** last time the wal file was deleted. The wal file is deleted when the
+** database is changed back to rollback mode or when the number of database
+** clients drops to zero. If either snapshot handle was obtained before the 
+** wal file was last deleted, the value returned by this function 
+** is undefined.
+**
+** Otherwise, this API returns a negative value if P1 refers to an older
+** snapshot than P2, zero if the two handles refer to the same database
+** snapshot, and a positive value if P1 is a newer snapshot than P2.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp(
+  sqlite3_snapshot *p1,
+  sqlite3_snapshot *p2
+);
+
+/*
+** CAPI3REF: Recover snapshots from a wal file
+** EXPERIMENTAL
+**
+** If all connections disconnect from a database file but do not perform
+** a checkpoint, the existing wal file is opened along with the database
+** file the next time the database is opened. At this point it is only
+** possible to successfully call sqlite3_snapshot_open() to open the most
+** recent snapshot of the database (the one at the head of the wal file),
+** even though the wal file may contain other valid snapshots for which
+** clients have sqlite3_snapshot handles.
+**
+** This function attempts to scan the wal file associated with database zDb
+** of database handle db and make all valid snapshots available to
+** sqlite3_snapshot_open(). It is an error if there is already a read
+** transaction open on the database, or if the database is not a wal mode
+** database.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+*/
+SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb);
+
 /*
 ** Undo the hack that converts floating point types to integer for
 ** builds on processors without floating point support.
@@ -8031,4 +8507,4 @@ SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*);
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif
-#endif /* _SQLITE3_H_ */
+#endif /* SQLITE3_H */
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 2e1c764a52..f25084a1f5 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -15,12 +15,10 @@
 ** as extensions by SQLite should #include this file instead of 
 ** sqlite3.h.
 */
-#ifndef _SQLITE3EXT_H_
-#define _SQLITE3EXT_H_
+#ifndef SQLITE3EXT_H
+#define SQLITE3EXT_H
 #include "sqlite3.h"
 
-typedef struct sqlite3_api_routines sqlite3_api_routines;
-
 /*
 ** The following structure holds pointers to all of the SQLite API
 ** routines.
@@ -279,8 +277,25 @@ struct sqlite3_api_routines {
   int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
   int (*strlike)(const char*,const char*,unsigned int);
   int (*db_cacheflush)(sqlite3*);
+  /* Version 3.12.0 and later */
+  int (*system_errno)(sqlite3*);
+  /* Version 3.14.0 and later */
+  int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
+  char *(*expanded_sql)(sqlite3_stmt*);
+  /* Version 3.18.0 and later */
+  void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
 };
 
+/*
+** This is the function signature used for all extension entry points.  It
+** is also defined in the file "loadext.c".
+*/
+typedef int (*sqlite3_loadext_entry)(
+  sqlite3 *db,                       /* Handle to the database. */
+  char **pzErrMsg,                   /* Used to set error string on failure. */
+  const sqlite3_api_routines *pThunk /* Extension API function pointers. */
+);
+
 /*
 ** The following macros redefine the API routines so that they are
 ** redirected through the global sqlite3_api structure.
@@ -522,6 +537,13 @@ struct sqlite3_api_routines {
 #define sqlite3_status64               sqlite3_api->status64
 #define sqlite3_strlike                sqlite3_api->strlike
 #define sqlite3_db_cacheflush          sqlite3_api->db_cacheflush
+/* Version 3.12.0 and later */
+#define sqlite3_system_errno           sqlite3_api->system_errno
+/* Version 3.14.0 and later */
+#define sqlite3_trace_v2               sqlite3_api->trace_v2
+#define sqlite3_expanded_sql           sqlite3_api->expanded_sql
+/* Version 3.18.0 and later */
+#define sqlite3_set_last_insert_rowid  sqlite3_api->set_last_insert_rowid
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
@@ -539,4 +561,4 @@ struct sqlite3_api_routines {
 # define SQLITE_EXTENSION_INIT3     /*no-op*/
 #endif
 
-#endif /* _SQLITE3EXT_H_ */
+#endif /* SQLITE3EXT_H */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index e043203eec..0f6ffd5175 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -12,8 +12,43 @@
 ** Internal interface definitions for SQLite.
 **
 */
-#ifndef _SQLITEINT_H_
-#define _SQLITEINT_H_
+#ifndef SQLITEINT_H
+#define SQLITEINT_H
+
+/* Special Comments:
+**
+** Some comments have special meaning to the tools that measure test
+** coverage:
+**
+**    NO_TEST                     - The branches on this line are not
+**                                  measured by branch coverage.  This is
+**                                  used on lines of code that actually
+**                                  implement parts of coverage testing.
+**
+**    OPTIMIZATION-IF-TRUE        - This branch is allowed to alway be false
+**                                  and the correct answer is still obtained,
+**                                  though perhaps more slowly.
+**
+**    OPTIMIZATION-IF-FALSE       - This branch is allowed to alway be true
+**                                  and the correct answer is still obtained,
+**                                  though perhaps more slowly.
+**
+**    PREVENTS-HARMLESS-OVERREAD  - This branch prevents a buffer overread
+**                                  that would be harmless and undetectable
+**                                  if it did occur.  
+**
+** In all cases, the special comment must be enclosed in the usual
+** slash-asterisk...asterisk-slash comment marks, with no spaces between the 
+** asterisks and the comment text.
+*/
+
+/*
+** Make sure the Tcl calling convention macro is defined.  This macro is
+** only used by test code and Tcl integration code.
+*/
+#ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+#endif
 
 /*
 ** Make sure that rand_s() is available on Windows systems with MSVC 2005
@@ -68,12 +103,29 @@
 # define _LARGEFILE_SOURCE 1
 #endif
 
-/* What version of GCC is being used.  0 means GCC is not being used */
-#ifdef __GNUC__
+/* The GCC_VERSION and MSVC_VERSION macros are used to
+** conditionally include optimizations for each of these compilers.  A
+** value of 0 means that compiler is not being used.  The
+** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific
+** optimizations, and hence set all compiler macros to 0
+**
+** There was once also a CLANG_VERSION macro.  However, we learn that the
+** version numbers in clang are for "marketing" only and are inconsistent
+** and unreliable.  Fortunately, all versions of clang also recognize the
+** gcc version numbers and have reasonable settings for gcc version numbers,
+** so the GCC_VERSION macro will be set to a correct non-zero value even
+** when compiling with clang.
+*/
+#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC)
 # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__)
 #else
 # define GCC_VERSION 0
 #endif
+#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC)
+# define MSVC_VERSION _MSC_VER
+#else
+# define MSVC_VERSION 0
+#endif
 
 /* Needed for various definitions... */
 #if defined(__GNUC__) && !defined(_GNU_SOURCE)
@@ -181,21 +233,6 @@
 # define SQLITE_PTR_TO_INT(X)  ((int)(X))
 #endif
 
-/*
-** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to
-** something between S (inclusive) and E (exclusive).
-**
-** In other words, S is a buffer and E is a pointer to the first byte after
-** the end of buffer S.  This macro returns true if P points to something
-** contained within the buffer S.
-*/
-#if defined(HAVE_STDINT_H)
-# define SQLITE_WITHIN(P,S,E) \
-    ((uintptr_t)(P)>=(uintptr_t)(S) && (uintptr_t)(P)<(uintptr_t)(E))
-#else
-# define SQLITE_WITHIN(P,S,E) ((P)>=(S) && (P)<(E))
-#endif
-
 /*
 ** A macro to hint to the compiler that a function should not be
 ** inlined.
@@ -214,11 +251,12 @@
 ** the SQLITE_DISABLE_INTRINSIC define.
 */
 #if !defined(SQLITE_DISABLE_INTRINSIC)
-#  if defined(_MSC_VER) && _MSC_VER>=1300
+#  if defined(_MSC_VER) && _MSC_VER>=1400
 #    if !defined(_WIN32_WCE)
 #      include 
 #      pragma intrinsic(_byteswap_ushort)
 #      pragma intrinsic(_byteswap_ulong)
+#      pragma intrinsic(_byteswap_uint64)
 #      pragma intrinsic(_ReadWriteBarrier)
 #    else
 #      include 
@@ -399,7 +437,7 @@
 ** be true and false so that the unreachable code they specify will
 ** not be counted as untested code.
 */
-#if defined(SQLITE_COVERAGE_TEST)
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
 # define ALWAYS(X)      (1)
 # define NEVER(X)       (0)
 #elif !defined(NDEBUG)
@@ -483,6 +521,18 @@
 #include 
 #include 
 
+/*
+** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY.
+** This allows better measurements of where memcpy() is used when running
+** cachegrind.  But this macro version of memcpy() is very slow so it
+** should not be used in production.  This is a performance measurement
+** hack only.
+*/
+#ifdef SQLITE_INLINE_MEMCPY
+# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\
+                        int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);}
+#endif
+
 /*
 ** If compiling for a processor that lacks floating point support,
 ** substitute integer for floating-point
@@ -567,9 +617,12 @@
 ** pagecaches for each database connection.  A positive number is the
 ** number of pages.  A negative number N translations means that a buffer
 ** of -1024*N bytes is allocated and used for as many pages as it will hold.
+**
+** The default value of "20" was choosen to minimize the run-time of the
+** speedtest1 test program with options: --shrink-memory --reprepare
 */
 #ifndef SQLITE_DEFAULT_PCACHE_INITSZ
-# define SQLITE_DEFAULT_PCACHE_INITSZ 100
+# define SQLITE_DEFAULT_PCACHE_INITSZ 20
 #endif
 
 /*
@@ -583,8 +636,12 @@
 /*
 ** Macros to compute minimum and maximum of two numbers.
 */
-#define MIN(A,B) ((A)<(B)?(A):(B))
-#define MAX(A,B) ((A)>(B)?(A):(B))
+#ifndef MIN
+# define MIN(A,B) ((A)<(B)?(A):(B))
+#endif
+#ifndef MAX
+# define MAX(A,B) ((A)>(B)?(A):(B))
+#endif
 
 /*
 ** Swap two objects of type TYPE.
@@ -713,38 +770,62 @@ typedef INT16_TYPE LogEst;
 # endif
 #endif
 
+/* The uptr type is an unsigned integer large enough to hold a pointer
+*/
+#if defined(HAVE_STDINT_H)
+  typedef uintptr_t uptr;
+#elif SQLITE_PTRSIZE==4
+  typedef u32 uptr;
+#else
+  typedef u64 uptr;
+#endif
+
+/*
+** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to
+** something between S (inclusive) and E (exclusive).
+**
+** In other words, S is a buffer and E is a pointer to the first byte after
+** the end of buffer S.  This macro returns true if P points to something
+** contained within the buffer S.
+*/
+#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E)))
+
+
 /*
 ** Macros to determine whether the machine is big or little endian,
 ** and whether or not that determination is run-time or compile-time.
 **
 ** For best performance, an attempt is made to guess at the byte-order
 ** using C-preprocessor macros.  If that is unsuccessful, or if
-** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined
+** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined
 ** at run-time.
 */
-#if (defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
+#ifndef SQLITE_BYTEORDER
+# if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
      defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
      defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
-     defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER)
-# define SQLITE_BYTEORDER    1234
-# define SQLITE_BIGENDIAN    0
-# define SQLITE_LITTLEENDIAN 1
-# define SQLITE_UTF16NATIVE  SQLITE_UTF16LE
+     defined(__arm__)
+#   define SQLITE_BYTEORDER    1234
+# elif defined(sparc)    || defined(__ppc__)
+#   define SQLITE_BYTEORDER    4321
+# else
+#   define SQLITE_BYTEORDER 0
+# endif
 #endif
-#if (defined(sparc)    || defined(__ppc__))  \
-    && !defined(SQLITE_RUNTIME_BYTEORDER)
-# define SQLITE_BYTEORDER    4321
+#if SQLITE_BYTEORDER==4321
 # define SQLITE_BIGENDIAN    1
 # define SQLITE_LITTLEENDIAN 0
 # define SQLITE_UTF16NATIVE  SQLITE_UTF16BE
-#endif
-#if !defined(SQLITE_BYTEORDER)
+#elif SQLITE_BYTEORDER==1234
+# define SQLITE_BIGENDIAN    0
+# define SQLITE_LITTLEENDIAN 1
+# define SQLITE_UTF16NATIVE  SQLITE_UTF16LE
+#else
 # ifdef SQLITE_AMALGAMATION
   const int sqlite3one = 1;
 # else
   extern const int sqlite3one;
 # endif
-# define SQLITE_BYTEORDER    0     /* 0 means "unknown at compile-time" */
 # define SQLITE_BIGENDIAN    (*(char *)(&sqlite3one)==0)
 # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1)
 # define SQLITE_UTF16NATIVE  (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE)
@@ -978,6 +1059,7 @@ typedef struct LookasideSlot LookasideSlot;
 typedef struct Module Module;
 typedef struct NameContext NameContext;
 typedef struct Parse Parse;
+typedef struct PreUpdate PreUpdate;
 typedef struct PrintfArguments PrintfArguments;
 typedef struct RowSet RowSet;
 typedef struct Savepoint Savepoint;
@@ -1000,6 +1082,14 @@ typedef struct Walker Walker;
 typedef struct WhereInfo WhereInfo;
 typedef struct With With;
 
+/* A VList object records a mapping between parameters/variables/wildcards
+** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer
+** variable number associated with that parameter.  See the format description
+** on the sqlite3VListAdd() routine for more information.  A VList is really
+** just an array of integers.
+*/
+typedef int VList;
+
 /*
 ** Defer sourcing vdbe.h and btree.h until after the "u8" and
 ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque
@@ -1037,7 +1127,7 @@ typedef struct With With;
 ** and the one-based values are used internally.
 */
 #ifndef SQLITE_DEFAULT_SYNCHRONOUS
-# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1)
+# define SQLITE_DEFAULT_SYNCHRONOUS 2
 #endif
 #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS
 # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS
@@ -1051,7 +1141,7 @@ typedef struct With With;
 ** databases may be attached.
 */
 struct Db {
-  char *zName;         /* Name of this database */
+  char *zDbSName;      /* Name of this database. (schema name, not filename) */
   Btree *pBt;          /* The B*Tree structure for this database file */
   u8 safety_level;     /* How aggressive at syncing data to disk */
   u8 bSyncSet;         /* True if "PRAGMA synchronous=N" has been run */
@@ -1203,6 +1293,15 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**);
                                const char*);
 #endif
 
+#ifndef SQLITE_OMIT_DEPRECATED
+/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing
+** in the style of sqlite3_trace()
+*/
+#define SQLITE_TRACE_LEGACY  0x80
+#else
+#define SQLITE_TRACE_LEGACY  0
+#endif /* SQLITE_OMIT_DEPRECATED */
+
 
 /*
 ** Each database connection is an instance of the following structure.
@@ -1220,6 +1319,7 @@ struct sqlite3 {
   unsigned int openFlags;       /* Flags passed to sqlite3_vfs.xOpen() */
   int errCode;                  /* Most recent error code (SQLITE_*) */
   int errMask;                  /* & result codes with this before returning */
+  int iSysErrno;                /* Errno value from last system error */
   u16 dbOptFlags;               /* Flags to enable/disable optimizations */
   u8 enc;                       /* Text encoding */
   u8 autoCommit;                /* The auto-commit flag. */
@@ -1231,6 +1331,9 @@ struct sqlite3 {
   u8 suppressErr;               /* Do not issue error messages if true */
   u8 vtabOnConflict;            /* Value to return for s3_vtab_on_conflict() */
   u8 isTransactionSavepoint;    /* True if the outermost savepoint is a TS */
+  u8 mTrace;                    /* zero or more SQLITE_TRACE flags */
+  u8 skipBtreeMutex;            /* True if no shared-cache backends */
+  u8 nSqlExec;                  /* Number of pending OP_SqlExec opcodes */
   int nextPagesize;             /* Pagesize after VACUUM if >0 */
   u32 magic;                    /* Magic number for detect library misuse */
   int nChange;                  /* Value returned by sqlite3_changes() */
@@ -1251,7 +1354,7 @@ struct sqlite3 {
   int nVDestroy;                /* Number of active OP_VDestroy operations */
   int nExtension;               /* Number of loaded extensions */
   void **aExtension;            /* Array of shared library handles */
-  void (*xTrace)(void*,const char*);        /* Trace function */
+  int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
   void (*xProfile)(void*,const char*,u64);  /* Profiling function */
   void *pProfileArg;                        /* Argument to profile function */
@@ -1261,6 +1364,13 @@ struct sqlite3 {
   void (*xRollbackCallback)(void*); /* Invoked at every commit. */
   void *pUpdateArg;
   void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  void *pPreUpdateArg;          /* First argument to xPreUpdateCallback */
+  void (*xPreUpdateCallback)(   /* Registered using sqlite3_preupdate_hook() */
+    void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64
+  );
+  PreUpdate *pPreUpdate;        /* Context for active pre-update callback */
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
 #ifndef SQLITE_OMIT_WAL
   int (*xWalCallback)(void *, sqlite3 *, const char *, int);
   void *pWalArg;
@@ -1335,6 +1445,11 @@ struct sqlite3 {
 
 /*
 ** Possible values for the sqlite3.flags.
+**
+** Value constraints (enforced via assert()):
+**      SQLITE_FullFSync     == PAGER_FULLFSYNC
+**      SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC
+**      SQLITE_CacheSpill    == PAGER_CACHE_SPILL
 */
 #define SQLITE_VdbeTrace      0x00000001  /* True to trace VDBE execution */
 #define SQLITE_InternChanges  0x00000002  /* Uncommitted Hash table changes */
@@ -1362,13 +1477,15 @@ struct sqlite3 {
 #define SQLITE_AutoIndex      0x00100000  /* Enable automatic indexes */
 #define SQLITE_PreferBuiltin  0x00200000  /* Preference to built-in funcs */
 #define SQLITE_LoadExtension  0x00400000  /* Enable load_extension */
-#define SQLITE_EnableTrigger  0x00800000  /* True to enable triggers */
-#define SQLITE_DeferFKs       0x01000000  /* Defer all FK constraints */
-#define SQLITE_QueryOnly      0x02000000  /* Disable database changes */
-#define SQLITE_VdbeEQP        0x04000000  /* Debug EXPLAIN QUERY PLAN */
-#define SQLITE_Vacuum         0x08000000  /* Currently in a VACUUM */
-#define SQLITE_CellSizeCk     0x10000000  /* Check btree cell sizes on load */
-#define SQLITE_Fts3Tokenizer  0x20000000  /* Enable fts3_tokenizer(2) */
+#define SQLITE_LoadExtFunc    0x00800000  /* Enable load_extension() SQL func */
+#define SQLITE_EnableTrigger  0x01000000  /* True to enable triggers */
+#define SQLITE_DeferFKs       0x02000000  /* Defer all FK constraints */
+#define SQLITE_QueryOnly      0x04000000  /* Disable database changes */
+#define SQLITE_VdbeEQP        0x08000000  /* Debug EXPLAIN QUERY PLAN */
+#define SQLITE_Vacuum         0x10000000  /* Currently in a VACUUM */
+#define SQLITE_CellSizeCk     0x20000000  /* Check btree cell sizes on load */
+#define SQLITE_Fts3Tokenizer  0x40000000  /* Enable fts3_tokenizer(2) */
+#define SQLITE_NoCkptOnClose  0x80000000  /* No checkpoint on close()/DETACH */
 
 
 /*
@@ -1394,13 +1511,8 @@ struct sqlite3 {
 /*
 ** Macros for testing whether or not optimizations are enabled or disabled.
 */
-#ifndef SQLITE_OMIT_BUILTIN_TEST
 #define OptimizationDisabled(db, mask)  (((db)->dbOptFlags&(mask))!=0)
 #define OptimizationEnabled(db, mask)   (((db)->dbOptFlags&(mask))==0)
-#else
-#define OptimizationDisabled(db, mask)  0
-#define OptimizationEnabled(db, mask)   1
-#endif
 
 /*
 ** Return true if it OK to factor constant expressions into the initialization
@@ -1469,6 +1581,13 @@ struct FuncDestructor {
 ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  And
 ** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC.  There
 ** are assert() statements in the code to verify this.
+**
+** Value constraints (enforced via assert()):
+**     SQLITE_FUNC_MINMAX    ==  NC_MinMaxAgg      == SF_MinMaxAgg
+**     SQLITE_FUNC_LENGTH    ==  OPFLAG_LENGTHARG
+**     SQLITE_FUNC_TYPEOF    ==  OPFLAG_TYPEOFARG
+**     SQLITE_FUNC_CONSTANT  ==  SQLITE_DETERMINISTIC from the API
+**     SQLITE_FUNC_ENCMASK   depends on SQLITE_UTF* macros in the API
 */
 #define SQLITE_FUNC_ENCMASK  0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */
 #define SQLITE_FUNC_LIKE     0x0004 /* Candidate for the LIKE optimization */
@@ -1484,6 +1603,7 @@ struct FuncDestructor {
 #define SQLITE_FUNC_MINMAX   0x1000 /* True for min() and max() aggregates */
 #define SQLITE_FUNC_SLOCHNG  0x2000 /* "Slow Change". Value constant during a
                                     ** single query - might change over time */
+#define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */
 
 /*
 ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
@@ -1597,6 +1717,7 @@ struct Column {
 */
 #define COLFLAG_PRIMKEY  0x0001    /* Column is part of the primary key */
 #define COLFLAG_HIDDEN   0x0002    /* A hidden column in a virtual table */
+#define COLFLAG_HASTYPE  0x0004    /* Type name follows column name */
 
 /*
 ** A "Collating Sequence" is defined by an instance of the following
@@ -1659,6 +1780,7 @@ struct CollSeq {
 ** operator is NULL.  It is added to certain comparison operators to
 ** prove that the operands are always NOT NULL.
 */
+#define SQLITE_KEEPNULL     0x08  /* Used by vector == or <> */
 #define SQLITE_JUMPIFNULL   0x10  /* jumps if either operand is NULL */
 #define SQLITE_STOREP2      0x20  /* Store result in reg[P2] rather than jump */
 #define SQLITE_NULLEQ       0x80  /* NULL=NULL */
@@ -1730,15 +1852,15 @@ struct Table {
   ExprList *pCheck;    /* All CHECK constraints */
                        /*   ... also used as column name list in a VIEW */
   int tnum;            /* Root BTree page for this table */
+  u32 nTabRef;         /* Number of pointers to this Table */
+  u32 tabFlags;        /* Mask of TF_* values */
   i16 iPKey;           /* If not negative, use aCol[iPKey] as the rowid */
   i16 nCol;            /* Number of columns in this table */
-  u16 nRef;            /* Number of pointers to this Table */
   LogEst nRowLogEst;   /* Estimated rows in table - from sqlite_stat1 table */
   LogEst szTabRow;     /* Estimated size of each table row in bytes */
 #ifdef SQLITE_ENABLE_COSTMULT
   LogEst costMult;     /* Cost multiplier for using this table */
 #endif
-  u8 tabFlags;         /* Mask of TF_* values */
   u8 keyConf;          /* What to do in case of uniqueness conflict on iPKey */
 #ifndef SQLITE_OMIT_ALTERTABLE
   int addColOffset;    /* Offset in CREATE TABLE stmt to add a new column */
@@ -1762,15 +1884,17 @@ struct Table {
 ** the TF_OOOHidden attribute would apply in this case.  Such tables require
 ** special handling during INSERT processing.
 */
-#define TF_Readonly        0x01    /* Read-only system table */
-#define TF_Ephemeral       0x02    /* An ephemeral table */
-#define TF_HasPrimaryKey   0x04    /* Table has a primary key */
-#define TF_Autoincrement   0x08    /* Integer primary key is autoincrement */
-#define TF_Virtual         0x10    /* Is a virtual table */
-#define TF_WithoutRowid    0x20    /* No rowid.  PRIMARY KEY is the key */
-#define TF_NoVisibleRowid  0x40    /* No user-visible "rowid" column */
-#define TF_OOOHidden       0x80    /* Out-of-Order hidden columns */
-
+#define TF_Readonly        0x0001    /* Read-only system table */
+#define TF_Ephemeral       0x0002    /* An ephemeral table */
+#define TF_HasPrimaryKey   0x0004    /* Table has a primary key */
+#define TF_Autoincrement   0x0008    /* Integer primary key is autoincrement */
+#define TF_HasStat1        0x0010    /* nRowLogEst set from sqlite_stat1 */
+#define TF_WithoutRowid    0x0020    /* No rowid.  PRIMARY KEY is the key */
+#define TF_NoVisibleRowid  0x0040    /* No user-visible "rowid" column */
+#define TF_OOOHidden       0x0080    /* Out-of-Order hidden columns */
+#define TF_StatsUsed       0x0100    /* Query planner decisions affected by
+                                     ** Index.aiRowLogEst[] values */
+#define TF_HasNotNull      0x0200    /* Contains NOT NULL constraints */
 
 /*
 ** Test to see whether or not a table is a virtual table.  This is
@@ -1778,7 +1902,7 @@ struct Table {
 ** table support is omitted from the build.
 */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-#  define IsVirtual(X)      (((X)->tabFlags & TF_Virtual)!=0)
+#  define IsVirtual(X)      ((X)->nModuleArg)
 #else
 #  define IsVirtual(X)      0
 #endif
@@ -2013,6 +2137,7 @@ struct Index {
   unsigned isResized:1;    /* True if resizeIndexObject() has been called */
   unsigned isCovering:1;   /* True if this is a covering index */
   unsigned noSkipScan:1;   /* Do not try to use skip-scan if true */
+  unsigned hasStat1:1;     /* aiRowLogEst values come from sqlite_stat1 */
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   int nSample;             /* Number of elements in aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
@@ -2223,9 +2348,11 @@ struct Expr {
   int iTable;            /* TK_COLUMN: cursor number of table holding column
                          ** TK_REGISTER: register number
                          ** TK_TRIGGER: 1 -> new, 0 -> old
-                         ** EP_Unlikely:  134217728 times likelihood */
+                         ** EP_Unlikely:  134217728 times likelihood
+                         ** TK_SELECT: 1st register of result vector */
   ynVar iColumn;         /* TK_COLUMN: column index.  -1 for rowid.
-                         ** TK_VARIABLE: variable number (always >= 1). */
+                         ** TK_VARIABLE: variable number (always >= 1).
+                         ** TK_SELECT_COLUMN: column of the result vector */
   i16 iAgg;              /* Which entry in pAggInfo->aCol[] or ->aFunc[] */
   i16 iRightJoinTable;   /* If EP_FromJoin, the right table of the join */
   u8 op2;                /* TK_REGISTER: original value of Expr.op
@@ -2261,6 +2388,7 @@ struct Expr {
 #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */
 #define EP_Subquery  0x200000 /* Tree contains a TK_SELECT operator */
 #define EP_Alias     0x400000 /* Is an alias for a result set column */
+#define EP_Leaf      0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */
 
 /*
 ** Combinations of two or more EP_* flags
@@ -2320,7 +2448,7 @@ struct Expr {
 struct ExprList {
   int nExpr;             /* Number of expressions on the list */
   struct ExprList_item { /* For each expression in the list */
-    Expr *pExpr;            /* The list of expressions */
+    Expr *pExpr;            /* The parse tree for this expression */
     char *zName;            /* Token associated with this expression */
     char *zSpan;            /* Original text of the expression */
     u8 sortOrder;           /* 1 for DESC or 0 for ASC */
@@ -2429,7 +2557,7 @@ struct SrcList {
     int regReturn;    /* Register holding return address of addrFillSub */
     int regResult;    /* Registers holding results of a co-routine */
     struct {
-      u8 jointype;      /* Type of join between this able and the previous */
+      u8 jointype;      /* Type of join between this table and the previous */
       unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
       unsigned isIndexedBy :1;   /* True if there is an INDEXED BY clause */
       unsigned isTabFunc :1;     /* True if table-valued-function syntax */
@@ -2467,23 +2595,28 @@ struct SrcList {
 /*
 ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin()
 ** and the WhereInfo.wctrlFlags member.
+**
+** Value constraints (enforced via assert()):
+**     WHERE_USE_LIMIT  == SF_FixedLimit
 */
 #define WHERE_ORDERBY_NORMAL   0x0000 /* No-op */
 #define WHERE_ORDERBY_MIN      0x0001 /* ORDER BY processing for min() func */
 #define WHERE_ORDERBY_MAX      0x0002 /* ORDER BY processing for max() func */
 #define WHERE_ONEPASS_DESIRED  0x0004 /* Want to do one-pass UPDATE/DELETE */
-#define WHERE_DUPLICATES_OK    0x0008 /* Ok to return a row more than once */
-#define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
-#define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
-#define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
-#define WHERE_NO_AUTOINDEX     0x0080 /* Disallow automatic indexes */
-#define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
-#define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
-#define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
-#define WHERE_SORTBYGROUP      0x0800 /* Support sqlite3WhereIsSorted() */
-#define WHERE_REOPEN_IDX       0x1000 /* Try to use OP_ReopenIdx */
-#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */
-#define WHERE_USE_LIMIT        0x4000 /* There is a constant LIMIT clause */
+#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */
+#define WHERE_DUPLICATES_OK    0x0010 /* Ok to return a row more than once */
+#define WHERE_OR_SUBCLAUSE     0x0020 /* Processing a sub-WHERE as part of
+                                      ** the OR optimization  */
+#define WHERE_GROUPBY          0x0040 /* pOrderBy is really a GROUP BY */
+#define WHERE_DISTINCTBY       0x0080 /* pOrderby is really a DISTINCT clause */
+#define WHERE_WANT_DISTINCT    0x0100 /* All output needs to be distinct */
+#define WHERE_SORTBYGROUP      0x0200 /* Support sqlite3WhereIsSorted() */
+#define WHERE_SEEK_TABLE       0x0400 /* Do not defer seeks on main table */
+#define WHERE_ORDERBY_LIMIT    0x0800 /* ORDERBY+LIMIT on the inner loop */
+#define WHERE_SEEK_UNIQ_TABLE  0x1000 /* Do not defer seeks if unique */
+                        /*     0x2000    not currently used */
+#define WHERE_USE_LIMIT        0x4000 /* Use the LIMIT in cost estimates */
+                        /*     0x8000    not currently used */
 
 /* Allowed return values from sqlite3WhereIsDistinct()
 */
@@ -2527,16 +2660,18 @@ struct NameContext {
 /*
 ** Allowed values for the NameContext, ncFlags field.
 **
-** Note:  NC_MinMaxAgg must have the same value as SF_MinMaxAgg and
-** SQLITE_FUNC_MINMAX.
+** Value constraints (all checked via assert()):
+**    NC_HasAgg    == SF_HasAgg
+**    NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX
 **
 */
 #define NC_AllowAgg  0x0001  /* Aggregate functions are allowed here */
-#define NC_HasAgg    0x0002  /* One or more aggregate functions seen */
+#define NC_PartIdx   0x0002  /* True if resolving a partial index WHERE */
 #define NC_IsCheck   0x0004  /* True if resolving names in a CHECK constraint */
 #define NC_InAggFunc 0x0008  /* True if analyzing arguments to an agg func */
-#define NC_PartIdx   0x0010  /* True if resolving a partial index WHERE */
+#define NC_HasAgg    0x0010  /* One or more aggregate functions seen */
 #define NC_IdxExpr   0x0020  /* True if resolving columns of CREATE INDEX */
+#define NC_VarSelect 0x0040  /* A correlated subquery has been seen */
 #define NC_MinMaxAgg 0x1000  /* min/max aggregates seen.  See note above */
 
 /*
@@ -2584,24 +2719,30 @@ struct Select {
 /*
 ** Allowed values for Select.selFlags.  The "SF" prefix stands for
 ** "Select Flag".
+**
+** Value constraints (all checked via assert())
+**     SF_HasAgg     == NC_HasAgg
+**     SF_MinMaxAgg  == NC_MinMaxAgg     == SQLITE_FUNC_MINMAX
+**     SF_FixedLimit == WHERE_USE_LIMIT
 */
 #define SF_Distinct       0x00001  /* Output should be DISTINCT */
 #define SF_All            0x00002  /* Includes the ALL keyword */
 #define SF_Resolved       0x00004  /* Identifiers have been resolved */
-#define SF_Aggregate      0x00008  /* Contains aggregate functions */
-#define SF_UsesEphemeral  0x00010  /* Uses the OpenEphemeral opcode */
-#define SF_Expanded       0x00020  /* sqlite3SelectExpand() called on this */
-#define SF_HasTypeInfo    0x00040  /* FROM subqueries have Table metadata */
-#define SF_Compound       0x00080  /* Part of a compound query */
-#define SF_Values         0x00100  /* Synthesized from VALUES clause */
-#define SF_MultiValue     0x00200  /* Single VALUES term with multiple rows */
-#define SF_NestedFrom     0x00400  /* Part of a parenthesized FROM clause */
-#define SF_MaybeConvert   0x00800  /* Need convertCompoundSelectToSubquery() */
+#define SF_Aggregate      0x00008  /* Contains agg functions or a GROUP BY */
+#define SF_HasAgg         0x00010  /* Contains aggregate functions */
+#define SF_UsesEphemeral  0x00020  /* Uses the OpenEphemeral opcode */
+#define SF_Expanded       0x00040  /* sqlite3SelectExpand() called on this */
+#define SF_HasTypeInfo    0x00080  /* FROM subqueries have Table metadata */
+#define SF_Compound       0x00100  /* Part of a compound query */
+#define SF_Values         0x00200  /* Synthesized from VALUES clause */
+#define SF_MultiValue     0x00400  /* Single VALUES term with multiple rows */
+#define SF_NestedFrom     0x00800  /* Part of a parenthesized FROM clause */
 #define SF_MinMaxAgg      0x01000  /* Aggregate containing min() or max() */
 #define SF_Recursive      0x02000  /* The recursive part of a recursive CTE */
 #define SF_FixedLimit     0x04000  /* nSelectRow set by a constant LIMIT */
-#define SF_Converted      0x08000  /* By convertCompoundSelectToSubquery() */
-#define SF_IncludeHidden  0x10000  /* Include hidden columns in output */
+#define SF_MaybeConvert   0x08000  /* Need convertCompoundSelectToSubquery() */
+#define SF_Converted      0x10000  /* By convertCompoundSelectToSubquery() */
+#define SF_IncludeHidden  0x20000  /* Include hidden columns in output */
 
 
 /*
@@ -2693,7 +2834,7 @@ struct Select {
 */
 struct SelectDest {
   u8 eDest;            /* How to dispose of the results.  On of SRT_* above. */
-  char affSdst;        /* Affinity used when eDest==SRT_Set */
+  char *zAffSdst;      /* Affinity used when eDest==SRT_Set */
   int iSDParm;         /* A parameter used by the eDest disposal method */
   int iSdst;           /* Base register where results are written */
   int nSdst;           /* Number of registers allocated */
@@ -2798,36 +2939,24 @@ struct Parse {
   u8 hasCompound;      /* Need to invoke convertCompoundSelectToSubquery() */
   u8 okConstFactor;    /* OK to factor out constants */
   u8 disableLookaside; /* Number of times lookaside has been disabled */
-  int aTempReg[8];     /* Holding area for temporary registers */
+  u8 nColCache;        /* Number of entries in aColCache[] */
   int nRangeReg;       /* Size of the temporary register block */
   int iRangeReg;       /* First register in temporary register block */
   int nErr;            /* Number of errors seen */
   int nTab;            /* Number of previously allocated VDBE cursors */
   int nMem;            /* Number of memory cells used so far */
-  int nSet;            /* Number of sets used so far */
-  int nOnce;           /* Number of OP_Once instructions so far */
   int nOpAlloc;        /* Number of slots allocated for Vdbe.aOp[] */
   int szOpAlloc;       /* Bytes of memory space allocated for Vdbe.aOp[] */
-  int iFixedOp;        /* Never back out opcodes iFixedOp-1 or earlier */
   int ckBase;          /* Base register of data during check constraints */
   int iSelfTab;        /* Table of an index whose exprs are being coded */
   int iCacheLevel;     /* ColCache valid when aColCache[].iLevel<=iCacheLevel */
   int iCacheCnt;       /* Counter used to generate aColCache[].lru values */
   int nLabel;          /* Number of labels used */
   int *aLabel;         /* Space to hold the labels */
-  struct yColCache {
-    int iTable;           /* Table cursor number */
-    i16 iColumn;          /* Table column number */
-    u8 tempReg;           /* iReg is a temp register that needs to be freed */
-    int iLevel;           /* Nesting level */
-    int iReg;             /* Reg with value of this column. 0 means none. */
-    int lru;              /* Least recently used entry has the smallest value */
-  } aColCache[SQLITE_N_COLCACHE];  /* One for each column cache entry */
   ExprList *pConstExpr;/* Constant expressions */
   Token constraintName;/* Name of the constraint currently being parsed */
   yDbMask writeMask;   /* Start a write transaction on these databases */
   yDbMask cookieMask;  /* Bitmask of schema verified databases */
-  int cookieValue[SQLITE_MAX_ATTACHED+2];  /* Values of cookies to verify */
   int regRowid;        /* Register holding rowid of CREATE TABLE entry */
   int regRoot;         /* Register holding root page number for new objects */
   int nMaxArg;         /* Max args passed to user function by sub-program */
@@ -2840,8 +2969,6 @@ struct Parse {
   TableLock *aTableLock; /* Required table locks for shared-cache mode */
 #endif
   AutoincInfo *pAinc;  /* Information about AUTOINCREMENT counters */
-
-  /* Information used while coding trigger programs. */
   Parse *pToplevel;    /* Parse structure for main program (or NULL) */
   Table *pTriggerTab;  /* Table triggers are being coded for */
   int addrCrTab;       /* Address of OP_CreateTable opcode on CREATE TABLE */
@@ -2852,35 +2979,50 @@ struct Parse {
   u8 eOrconf;          /* Default ON CONFLICT policy for trigger steps */
   u8 disableTriggers;  /* True to disable triggers */
 
+  /**************************************************************************
+  ** Fields above must be initialized to zero.  The fields that follow,
+  ** down to the beginning of the recursive section, do not need to be
+  ** initialized as they will be set before being used.  The boundary is
+  ** determined by offsetof(Parse,aColCache).
+  **************************************************************************/
+
+  struct yColCache {
+    int iTable;           /* Table cursor number */
+    i16 iColumn;          /* Table column number */
+    u8 tempReg;           /* iReg is a temp register that needs to be freed */
+    int iLevel;           /* Nesting level */
+    int iReg;             /* Reg with value of this column. 0 means none. */
+    int lru;              /* Least recently used entry has the smallest value */
+  } aColCache[SQLITE_N_COLCACHE];  /* One for each column cache entry */
+  int aTempReg[8];        /* Holding area for temporary registers */
+  Token sNameToken;       /* Token with unqualified schema object name */
+
   /************************************************************************
   ** Above is constant between recursions.  Below is reset before and after
   ** each recursion.  The boundary between these two regions is determined
-  ** using offsetof(Parse,nVar) so the nVar field must be the first field
-  ** in the recursive region.
+  ** using offsetof(Parse,sLastToken) so the sLastToken field must be the
+  ** first field in the recursive region.
   ************************************************************************/
 
+  Token sLastToken;       /* The last token parsed */
   ynVar nVar;               /* Number of '?' variables seen in the SQL so far */
-  int nzVar;                /* Number of available slots in azVar[] */
   u8 iPkSortOrder;          /* ASC or DESC for INTEGER PRIMARY KEY */
   u8 explain;               /* True if the EXPLAIN flag is found on the query */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
   int nVtabLock;            /* Number of virtual tables to lock */
 #endif
-  int nAlias;               /* Number of aliased result set columns */
   int nHeight;              /* Expression tree height of current sub-select */
 #ifndef SQLITE_OMIT_EXPLAIN
   int iSelectId;            /* ID of current select for EXPLAIN output */
   int iNextSelectId;        /* Next available select ID for EXPLAIN output */
 #endif
-  char **azVar;             /* Pointers to names of parameters */
+  VList *pVList;            /* Mapping between variable names and numbers */
   Vdbe *pReprepare;         /* VM being reprepared (sqlite3Reprepare()) */
   const char *zTail;        /* All SQL text past the last semicolon parsed */
   Table *pNewTable;         /* A table being constructed by CREATE TABLE */
   Trigger *pNewTrigger;     /* Trigger under construct by a CREATE TRIGGER */
   const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */
-  Token sNameToken;         /* Token with unqualified schema object name */
-  Token sLastToken;         /* The last token parsed */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   Token sArg;               /* Complete text of a module argument */
   Table **apVtabLock;       /* Pointer to virtual tables needing locking */
@@ -2891,6 +3033,14 @@ struct Parse {
   With *pWithToFree;        /* Free this WITH object at the end of the parse */
 };
 
+/*
+** Sizes and pointers of various parts of the Parse object.
+*/
+#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/
+#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken)    /* Recursive part */
+#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */
+#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ)  /* Pointer to tail */
+
 /*
 ** Return true if currently inside an sqlite3_declare_vtab() call.
 */
@@ -2911,14 +3061,24 @@ struct AuthContext {
 
 /*
 ** Bitfield flags for P5 value in various opcodes.
+**
+** Value constraints (enforced via assert()):
+**    OPFLAG_LENGTHARG    == SQLITE_FUNC_LENGTH
+**    OPFLAG_TYPEOFARG    == SQLITE_FUNC_TYPEOF
+**    OPFLAG_BULKCSR      == BTREE_BULKLOAD
+**    OPFLAG_SEEKEQ       == BTREE_SEEK_EQ
+**    OPFLAG_FORDELETE    == BTREE_FORDELETE
+**    OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION
+**    OPFLAG_AUXDELETE    == BTREE_AUXDELETE
 */
 #define OPFLAG_NCHANGE       0x01    /* OP_Insert: Set to update db->nChange */
                                      /* Also used in P2 (not P5) of OP_Delete */
 #define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
-#define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
+#define OPFLAG_LASTROWID     0x20    /* Set to update db->lastRowid */
 #define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
 #define OPFLAG_APPEND        0x08    /* This is likely to be an append */
 #define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
+#define OPFLAG_ISNOOP        0x40    /* OP_Delete does pre-update-hook only */
 #define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
 #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
 #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
@@ -2926,7 +3086,7 @@ struct AuthContext {
 #define OPFLAG_FORDELETE     0x08    /* OP_Open should use BTREE_FORDELETE */
 #define OPFLAG_P2ISREG       0x10    /* P2 to OP_Open** is a register number */
 #define OPFLAG_PERMUTE       0x01    /* OP_Compare: use the permutation */
-#define OPFLAG_SAVEPOSITION  0x02    /* OP_Delete: keep cursor position */
+#define OPFLAG_SAVEPOSITION  0x02    /* OP_Delete/Insert: save cursor pos */
 #define OPFLAG_AUXDELETE     0x04    /* OP_Delete: index in a DELETE op */
 
 /*
@@ -3123,10 +3283,11 @@ struct Sqlite3Config {
   void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx);  /* Callback */
   void *pVdbeBranchArg;                                     /* 1st argument */
 #endif
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
   int (*xTestCallback)(int);        /* Invoked by sqlite3FaultSim() */
 #endif
   int bLocaltimeFault;              /* True to fail localtime() calls */
+  int iOnceResetThreshold;          /* When to reset OP_Once counters */
 };
 
 /*
@@ -3165,6 +3326,7 @@ struct Walker {
     struct SrcCount *pSrcCount;                /* Counting column references */
     struct CCurHint *pCCurHint;                /* Used by codeCursorHint() */
     int *aiCol;                                /* array of column indexes */
+    struct IdxCover *pIdxCover;                /* Check for index coverage */
   } u;
 };
 
@@ -3282,6 +3444,7 @@ int sqlite3CantopenError(int);
 # define sqlite3Isdigit(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
 # define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
 # define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
+# define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
 #else
 # define sqlite3Toupper(x)   toupper((unsigned char)(x))
 # define sqlite3Isspace(x)   isspace((unsigned char)(x))
@@ -3290,6 +3453,7 @@ int sqlite3CantopenError(int);
 # define sqlite3Isdigit(x)   isdigit((unsigned char)(x))
 # define sqlite3Isxdigit(x)  isxdigit((unsigned char)(x))
 # define sqlite3Tolower(x)   tolower((unsigned char)(x))
+# define sqlite3Isquote(x)   ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`')
 #endif
 #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS
 int sqlite3IsIdChar(u8);
@@ -3300,7 +3464,7 @@ int sqlite3IsIdChar(u8);
 */
 int sqlite3StrICmp(const char*,const char*);
 int sqlite3Strlen30(const char*);
-const char *sqlite3StrNext(const char*);
+char *sqlite3ColumnType(Column*,char*);
 #define sqlite3StrNICmp sqlite3_strnicmp
 
 int sqlite3MallocInit(void);
@@ -3323,7 +3487,7 @@ void sqlite3ScratchFree(void*);
 void *sqlite3PageMalloc(int);
 void sqlite3PageFree(void*);
 void sqlite3MemSetDefault(void);
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
 void sqlite3BenignMallocHooks(void (*)(void), void (*)(void));
 #endif
 int sqlite3HeapNearlyFull(void);
@@ -3346,11 +3510,15 @@ int sqlite3HeapNearlyFull(void);
 # define sqlite3StackFree(D,P)       sqlite3DbFree(D,P)
 #endif
 
-#ifdef SQLITE_ENABLE_MEMSYS3
-const sqlite3_mem_methods *sqlite3MemGetMemsys3(void);
-#endif
+/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together.  If they
+** are, disable MEMSYS3
+*/
 #ifdef SQLITE_ENABLE_MEMSYS5
 const sqlite3_mem_methods *sqlite3MemGetMemsys5(void);
+#undef SQLITE_ENABLE_MEMSYS3
+#endif
+#ifdef SQLITE_ENABLE_MEMSYS3
+const sqlite3_mem_methods *sqlite3MemGetMemsys3(void);
 #endif
 
 
@@ -3405,6 +3573,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
 
 #if defined(SQLITE_DEBUG)
   void sqlite3TreeViewExpr(TreeView*, const Expr*, u8);
+  void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*);
   void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*);
   void sqlite3TreeViewSelect(TreeView*, const Select*, u8);
   void sqlite3TreeViewWith(TreeView*, const With*, u8);
@@ -3413,7 +3582,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
 
 void sqlite3SetString(char **, sqlite3*, const char*);
 void sqlite3ErrorMsg(Parse*, const char*, ...);
-int sqlite3Dequote(char*);
+void sqlite3Dequote(char*);
 void sqlite3TokenInit(Token*,char*);
 int sqlite3KeywordCode(const unsigned char*, int);
 int sqlite3RunParser(Parse*, const char*, char **);
@@ -3423,15 +3592,20 @@ void sqlite3ReleaseTempReg(Parse*,int);
 int sqlite3GetTempRange(Parse*,int);
 void sqlite3ReleaseTempRange(Parse*,int,int);
 void sqlite3ClearTempRegCache(Parse*);
+#ifdef SQLITE_DEBUG
+int sqlite3NoTempsInRange(Parse*,int,int);
+#endif
 Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int);
 Expr *sqlite3Expr(sqlite3*,int,const char*);
 void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*);
-Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*);
+Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*);
+void sqlite3PExprAddSelect(Parse*, Expr*, Select*);
 Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*);
 Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*);
-void sqlite3ExprAssignVarNumber(Parse*, Expr*);
+void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32);
 void sqlite3ExprDelete(sqlite3*, Expr*);
 ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
+ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*);
 void sqlite3ExprListSetSortOrder(ExprList*,int);
 void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*);
@@ -3440,12 +3614,16 @@ u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3Init(sqlite3*, char**);
 int sqlite3InitCallback(void*, int, char**, char**);
 void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName);
+#endif
 void sqlite3ResetAllSchemasOfConnection(sqlite3*);
 void sqlite3ResetOneSchema(sqlite3*,int);
 void sqlite3CollapseDatabaseArray(sqlite3*);
 void sqlite3CommitInternalChanges(sqlite3*);
 void sqlite3DeleteColumnNames(sqlite3*,Table*);
 int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**);
+void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*);
 Table *sqlite3ResultSetOfSelect(Parse*,Select*);
 void sqlite3OpenMasterTable(Parse *, int);
 Index *sqlite3PrimaryKeyIndex(Table*);
@@ -3466,9 +3644,8 @@ void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*);
 int sqlite3ParseUri(const char*,const char*,unsigned int*,
                     sqlite3_vfs**,char**,char **);
 Btree *sqlite3DbNameToBtree(sqlite3*,const char*);
-int sqlite3CodeOnce(Parse *);
 
-#ifdef SQLITE_OMIT_BUILTIN_TEST
+#ifdef SQLITE_UNTESTABLE
 # define sqlite3FaultSim(X) SQLITE_OK
 #else
   int sqlite3FaultSim(int);
@@ -3481,7 +3658,7 @@ int sqlite3BitvecSet(Bitvec*, u32);
 void sqlite3BitvecClear(Bitvec*, u32, void*);
 void sqlite3BitvecDestroy(Bitvec*);
 u32 sqlite3BitvecSize(Bitvec*);
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
 int sqlite3BitvecBuiltinTest(int,int*);
 #endif
 
@@ -3528,8 +3705,8 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*);
 void sqlite3IdListDelete(sqlite3*, IdList*);
 void sqlite3SrcListDelete(sqlite3*, SrcList*);
 Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**);
-Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
-                          Expr*, int, int);
+void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*,
+                          Expr*, int, int, u8);
 void sqlite3DropIndex(Parse*, SrcList*, int);
 int sqlite3Select(Parse*, Select*, SelectDest*);
 Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*,
@@ -3548,6 +3725,7 @@ void sqlite3WhereEnd(WhereInfo*);
 LogEst sqlite3WhereOutputRowCount(WhereInfo*);
 int sqlite3WhereIsDistinct(WhereInfo*);
 int sqlite3WhereIsOrdered(WhereInfo*);
+int sqlite3WhereOrderedInnerLoop(WhereInfo*);
 int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);
@@ -3569,7 +3747,7 @@ void sqlite3ExprCacheAffinityChange(Parse*, int, int);
 void sqlite3ExprCode(Parse*, Expr*, int);
 void sqlite3ExprCodeCopy(Parse*, Expr*, int);
 void sqlite3ExprCodeFactorable(Parse*, Expr*, int);
-void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8);
+int sqlite3ExprCodeAtInit(Parse*, Expr*, int);
 int sqlite3ExprCodeTemp(Parse*, Expr*, int*);
 int sqlite3ExprCodeTarget(Parse*, Expr*, int);
 void sqlite3ExprCodeAndCache(Parse*, Expr*, int);
@@ -3577,26 +3755,31 @@ int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8);
 #define SQLITE_ECEL_DUP      0x01  /* Deep, not shallow copies */
 #define SQLITE_ECEL_FACTOR   0x02  /* Factor out constant terms */
 #define SQLITE_ECEL_REF      0x04  /* Use ExprList.u.x.iOrderByCol */
+#define SQLITE_ECEL_OMITREF  0x08  /* Omit if ExprList.u.x.iOrderByCol */
 void sqlite3ExprIfTrue(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalse(Parse*, Expr*, int, int);
 void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int);
 Table *sqlite3FindTable(sqlite3*,const char*, const char*);
-Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*);
-Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *);
+#define LOCATE_VIEW    0x01
+#define LOCATE_NOERR   0x02
+Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*);
+Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *);
 Index *sqlite3FindIndex(sqlite3*,const char*, const char*);
 void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*);
 void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*);
-void sqlite3Vacuum(Parse*);
-int sqlite3RunVacuum(char**, sqlite3*);
+void sqlite3Vacuum(Parse*,Token*);
+int sqlite3RunVacuum(char**, sqlite3*, int);
 char *sqlite3NameFromToken(sqlite3*, Token*);
 int sqlite3ExprCompare(Expr*, Expr*, int);
+int sqlite3ExprCompareSkip(Expr*, Expr*, int);
 int sqlite3ExprListCompare(ExprList*, ExprList*, int);
 int sqlite3ExprImpliesExpr(Expr*, Expr*, int);
 void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*);
 void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*);
+int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx);
 int sqlite3FunctionUsesThisSrc(Expr*, SrcList*);
 Vdbe *sqlite3GetVdbe(Parse*);
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
 void sqlite3PrngSaveState(void);
 void sqlite3PrngRestoreState(void);
 #endif
@@ -3627,6 +3810,11 @@ int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
 void sqlite3ResolvePartIdxLabel(Parse*,int);
 void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
                                      u8,u8,int,int*,int*);
+#ifdef SQLITE_ENABLE_NULL_TRIM
+  void sqlite3SetMakeRecordP5(Vdbe*,Table*);
+#else
+# define sqlite3SetMakeRecordP5(A,B)
+#endif
 void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int);
 int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*);
 void sqlite3BeginWriteOperation(Parse*, int, int);
@@ -3733,6 +3921,9 @@ LogEst sqlite3LogEstFromDouble(double);
     defined(SQLITE_EXPLAIN_ESTIMATED_ROWS)
 u64 sqlite3LogEstToInt(LogEst);
 #endif
+VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int);
+const char *sqlite3VListNumToName(VList*,int);
+int sqlite3VListNameToNum(VList*,const char*,int);
 
 /*
 ** Routines to read and write variable-length integers.  These used to
@@ -3762,11 +3953,13 @@ const char *sqlite3IndexAffinityStr(sqlite3*, Index*);
 void sqlite3TableAffinity(Vdbe*, Table*, int);
 char sqlite3CompareAffinity(Expr *pExpr, char aff2);
 int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity);
+char sqlite3TableColumnAffinity(Table*,int);
 char sqlite3ExprAffinity(Expr *pExpr);
 int sqlite3Atoi64(const char*, i64*, int, u8);
 int sqlite3DecOrHexToI64(const char*, i64*);
 void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...);
 void sqlite3Error(sqlite3*,int);
+void sqlite3SystemError(sqlite3*,int);
 void *sqlite3HexToBlob(sqlite3*, const char *z, int n);
 u8 sqlite3HexToInt(int h);
 int sqlite3TwoPartName(Parse *, Token *, Token *, Token **);
@@ -3826,7 +4019,7 @@ void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
 int sqlite3GetToken(const unsigned char *, int *);
 void sqlite3NestedParse(Parse*, const char*, ...);
 void sqlite3ExpirePreparedStatements(sqlite3*);
-int sqlite3CodeSubselect(Parse *, Expr *, int, int);
+int sqlite3CodeSubselect(Parse*, Expr *, int, int);
 void sqlite3SelectPrep(Parse*, Select*, NameContext*);
 void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p);
 int sqlite3MatchSpanName(const char*, const char*, const char*, const char*);
@@ -3881,19 +4074,29 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int);
 void sqlite3BackupRestart(sqlite3_backup *);
 void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *);
 
+#ifndef SQLITE_OMIT_SUBQUERY
+int sqlite3ExprCheckIN(Parse*, Expr*);
+#else
+# define sqlite3ExprCheckIN(x,y) SQLITE_OK
+#endif
+
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
 void sqlite3AnalyzeFunctions(void);
-int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*);
+int sqlite3Stat4ProbeSetValue(
+    Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*);
 int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**);
 void sqlite3Stat4ProbeFree(UnpackedRecord*);
 int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**);
+char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
 #endif
 
 /*
 ** The interface to the LEMON-generated parser
 */
-void *sqlite3ParserAlloc(void*(*)(u64));
-void sqlite3ParserFree(void*, void(*)(void*));
+#ifndef SQLITE_AMALGAMATION
+  void *sqlite3ParserAlloc(void*(*)(u64));
+  void sqlite3ParserFree(void*, void(*)(void*));
+#endif
 void sqlite3Parser(void*, int, Token, Parse*);
 #ifdef YYTRACKMAXSTACKDEPTH
   int sqlite3ParserStackPeak(void*);
@@ -3939,6 +4142,13 @@ void sqlite3AutoLoadExtensions(sqlite3*);
    int sqlite3VtabSavepoint(sqlite3 *, int, int);
    void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*);
    VTable *sqlite3GetVTable(sqlite3*, Table*);
+   Module *sqlite3VtabCreateModule(
+     sqlite3*,
+     const char*,
+     const sqlite3_module*,
+     void*,
+     void(*)(void*)
+   );
 #  define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0)
 #endif
 int sqlite3VtabEponymousTableInit(Parse*,Module*);
@@ -3996,6 +4206,7 @@ const char *sqlite3JournalModename(int);
   #define sqlite3FkDropTable(a,b,c)
   #define sqlite3FkOldmask(a,b)         0
   #define sqlite3FkRequired(a,b,c,d)    0
+  #define sqlite3FkReferences(a)        0
 #endif
 #ifndef SQLITE_OMIT_FOREIGN_KEY
   void sqlite3FkDelete(sqlite3 *, Table*);
@@ -4014,10 +4225,10 @@ const char *sqlite3JournalModename(int);
 
 /*
 ** The interface to the code in fault.c used for identifying "benign"
-** malloc failures. This is only present if SQLITE_OMIT_BUILTIN_TEST
+** malloc failures. This is only present if SQLITE_UNTESTABLE
 ** is not defined.
 */
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
   void sqlite3BeginBenignMalloc(void);
   void sqlite3EndBenignMalloc(void);
 #else
@@ -4039,7 +4250,7 @@ const char *sqlite3JournalModename(int);
 #define IN_INDEX_NOOP_OK     0x0001  /* OK to return IN_INDEX_NOOP */
 #define IN_INDEX_MEMBERSHIP  0x0002  /* IN operator used for membership test */
 #define IN_INDEX_LOOP        0x0004  /* IN operator used as a loop */
-int sqlite3FindInIndex(Parse *, Expr *, u32, int*);
+int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*);
 
 int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int);
 int sqlite3JournalSize(sqlite3_vfs *);
@@ -4144,4 +4355,10 @@ int sqlite3ThreadJoin(SQLiteThread*, void**);
 int sqlite3DbstatRegister(sqlite3*);
 #endif
 
-#endif /* _SQLITEINT_H_ */
+int sqlite3ExprVectorSize(Expr *pExpr);
+int sqlite3ExprIsVector(Expr *pExpr);
+Expr *sqlite3VectorFieldSubexpr(Expr*, int);
+Expr *sqlite3ExprForVectorField(Parse*,Expr*,int);
+void sqlite3VectorErrorMsg(Parse*, Expr*);
+
+#endif /* SQLITEINT_H */
diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h
index 4b5ddaadec..28e7a41cc3 100644
--- a/src/sqliteLimit.h
+++ b/src/sqliteLimit.h
@@ -87,7 +87,7 @@
 ** Not currently enforced.
 */
 #ifndef SQLITE_MAX_VDBE_OP
-# define SQLITE_MAX_VDBE_OP 25000
+# define SQLITE_MAX_VDBE_OP 250000000
 #endif
 
 /*
@@ -101,8 +101,8 @@
 ** The suggested maximum number of in-memory pages to use for
 ** the main database table and for temporary tables.
 **
-** IMPLEMENTATION-OF: R-31093-59126 The default suggested cache size
-** is 2000*1024 bytes.
+** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000,
+** which means the cache size is limited to 2048000 bytes of memory.
 ** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be
 ** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options.
 */
diff --git a/src/status.c b/src/status.c
index 69f92ff7c6..24dcad4572 100644
--- a/src/status.c
+++ b/src/status.c
@@ -158,7 +158,7 @@ int sqlite3_status64(
   return SQLITE_OK;
 }
 int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){
-  sqlite3_int64 iCur, iHwtr;
+  sqlite3_int64 iCur = 0, iHwtr = 0;
   int rc;
 #ifdef SQLITE_ENABLE_API_ARMOR
   if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT;
@@ -219,6 +219,7 @@ int sqlite3_db_status(
     ** by all pagers associated with the given database connection.  The
     ** highwater mark is meaningless and is returned as zero.
     */
+    case SQLITE_DBSTATUS_CACHE_USED_SHARED:
     case SQLITE_DBSTATUS_CACHE_USED: {
       int totalUsed = 0;
       int i;
@@ -227,7 +228,11 @@ int sqlite3_db_status(
         Btree *pBt = db->aDb[i].pBt;
         if( pBt ){
           Pager *pPager = sqlite3BtreePager(pBt);
-          totalUsed += sqlite3PagerMemUsed(pPager);
+          int nByte = sqlite3PagerMemUsed(pPager);
+          if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){
+            nByte = nByte / sqlite3BtreeConnectionCount(pBt);
+          }
+          totalUsed += nByte;
         }
       }
       sqlite3BtreeLeaveAll(db);
diff --git a/src/table.c b/src/table.c
index a50d83ce63..c79255f990 100644
--- a/src/table.c
+++ b/src/table.c
@@ -17,8 +17,6 @@
 ** if they are not used.
 */
 #include "sqliteInt.h"
-#include 
-#include 
 
 #ifndef SQLITE_OMIT_GET_TABLE
 
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 4e19f92644..9df023b45c 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -30,10 +30,17 @@
 ** If requested, include the SQLite compiler options file for MSVC.
 */
 #if defined(INCLUDE_MSVC_H)
-#include "msvc.h"
+# include "msvc.h"
 #endif
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+#  define SQLITE_TCLAPI
+# endif
+#endif
 #include 
 
 /*
@@ -133,6 +140,7 @@ struct SqliteDb {
   char *zBusy;               /* The busy callback routine */
   char *zCommit;             /* The commit hook callback routine */
   char *zTrace;              /* The trace callback routine */
+  char *zTraceV2;            /* The trace_v2 callback routine */
   char *zProfile;            /* The profile callback routine */
   char *zProgress;           /* The progress callback routine */
   char *zAuth;               /* The authorization callback routine */
@@ -140,6 +148,7 @@ struct SqliteDb {
   char *zNull;               /* Text to substitute for an SQL NULL value */
   SqlFunc *pFunc;            /* List of SQL functions */
   Tcl_Obj *pUpdateHook;      /* Update hook script (if any) */
+  Tcl_Obj *pPreUpdateHook;   /* Pre-update hook script (if any) */
   Tcl_Obj *pRollbackHook;    /* Rollback hook script (if any) */
   Tcl_Obj *pWalHook;         /* WAL hook script (if any) */
   Tcl_Obj *pUnlockNotify;    /* Unlock notify script (if any) */
@@ -191,7 +200,7 @@ static void closeIncrblobChannels(SqliteDb *pDb){
   for(p=pDb->pIncrblob; p; p=pNext){
     pNext = p->pNext;
 
-    /* Note: Calling unregister here call Tcl_Close on the incrblob channel, 
+    /* Note: Calling unregister here call Tcl_Close on the incrblob channel,
     ** which deletes the IncrblobChannel structure at *p. So do not
     ** call Tcl_Free() here.
     */
@@ -202,7 +211,10 @@ static void closeIncrblobChannels(SqliteDb *pDb){
 /*
 ** Close an incremental blob channel.
 */
-static int incrblobClose(ClientData instanceData, Tcl_Interp *interp){
+static int SQLITE_TCLAPI incrblobClose(
+  ClientData instanceData,
+  Tcl_Interp *interp
+){
   IncrblobChannel *p = (IncrblobChannel *)instanceData;
   int rc = sqlite3_blob_close(p->pBlob);
   sqlite3 *db = p->pDb->db;
@@ -231,9 +243,9 @@ static int incrblobClose(ClientData instanceData, Tcl_Interp *interp){
 /*
 ** Read data from an incremental blob channel.
 */
-static int incrblobInput(
-  ClientData instanceData, 
-  char *buf, 
+static int SQLITE_TCLAPI incrblobInput(
+  ClientData instanceData,
+  char *buf,
   int bufSize,
   int *errorCodePtr
 ){
@@ -263,9 +275,9 @@ static int incrblobInput(
 /*
 ** Write data to an incremental blob channel.
 */
-static int incrblobOutput(
-  ClientData instanceData, 
-  CONST char *buf, 
+static int SQLITE_TCLAPI incrblobOutput(
+  ClientData instanceData,
+  CONST char *buf,
   int toWrite,
   int *errorCodePtr
 ){
@@ -296,8 +308,8 @@ static int incrblobOutput(
 /*
 ** Seek an incremental blob channel.
 */
-static int incrblobSeek(
-  ClientData instanceData, 
+static int SQLITE_TCLAPI incrblobSeek(
+  ClientData instanceData,
   long offset,
   int seekMode,
   int *errorCodePtr
@@ -322,10 +334,17 @@ static int incrblobSeek(
 }
 
 
-static void incrblobWatch(ClientData instanceData, int mode){ 
-  /* NO-OP */ 
+static void SQLITE_TCLAPI incrblobWatch(
+  ClientData instanceData,
+  int mode
+){
+  /* NO-OP */
 }
-static int incrblobHandle(ClientData instanceData, int dir, ClientData *hPtr){
+static int SQLITE_TCLAPI incrblobHandle(
+  ClientData instanceData,
+  int dir,
+  ClientData *hPtr
+){
   return TCL_ERROR;
 }
 
@@ -351,11 +370,11 @@ static Tcl_ChannelType IncrblobChannelType = {
 ** Create a new incrblob channel.
 */
 static int createIncrblobChannel(
-  Tcl_Interp *interp, 
-  SqliteDb *pDb, 
+  Tcl_Interp *interp,
+  SqliteDb *pDb,
   const char *zDb,
-  const char *zTable, 
-  const char *zColumn, 
+  const char *zTable,
+  const char *zColumn,
   sqlite_int64 iRow,
   int isReadonly
 ){
@@ -437,7 +456,7 @@ static SqlFunc *findSqlFunc(SqliteDb *pDb, const char *zName){
   pNew = (SqlFunc*)Tcl_Alloc( sizeof(*pNew) + nName + 1 );
   pNew->zName = (char*)&pNew[1];
   memcpy(pNew->zName, zName, nName+1);
-  for(p=pDb->pFunc; p; p=p->pNext){ 
+  for(p=pDb->pFunc; p; p=p->pNext){
     if( sqlite3_stricmp(p->zName, pNew->zName)==0 ){
       Tcl_Free((char*)pNew);
       return p;
@@ -484,7 +503,7 @@ static void flushStmtCache(SqliteDb *pDb){
 ** TCL calls this procedure when an sqlite3 database command is
 ** deleted.
 */
-static void DbDeleteCmd(void *db){
+static void SQLITE_TCLAPI DbDeleteCmd(void *db){
   SqliteDb *pDb = (SqliteDb*)db;
   flushStmtCache(pDb);
   closeIncrblobChannels(pDb);
@@ -507,6 +526,9 @@ static void DbDeleteCmd(void *db){
   if( pDb->zTrace ){
     Tcl_Free(pDb->zTrace);
   }
+  if( pDb->zTraceV2 ){
+    Tcl_Free(pDb->zTraceV2);
+  }
   if( pDb->zProfile ){
     Tcl_Free(pDb->zProfile);
   }
@@ -519,6 +541,9 @@ static void DbDeleteCmd(void *db){
   if( pDb->pUpdateHook ){
     Tcl_DecrRefCount(pDb->pUpdateHook);
   }
+  if( pDb->pPreUpdateHook ){
+    Tcl_DecrRefCount(pDb->pPreUpdateHook);
+  }
   if( pDb->pRollbackHook ){
     Tcl_DecrRefCount(pDb->pRollbackHook);
   }
@@ -565,7 +590,8 @@ static int DbProgressHandler(void *cd){
 }
 #endif
 
-#ifndef SQLITE_OMIT_TRACE
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
 /*
 ** This routine is called by the SQLite trace handler whenever a new
 ** block of SQL is executed.  The TCL script in pDb->zTrace is executed.
@@ -585,6 +611,83 @@ static void DbTraceHandler(void *cd, const char *zSql){
 
 #ifndef SQLITE_OMIT_TRACE
 /*
+** This routine is called by the SQLite trace_v2 handler whenever a new
+** supported event is generated.  Unsupported event types are ignored.
+** The TCL script in pDb->zTraceV2 is executed, with the arguments for
+** the event appended to it (as list elements).
+*/
+static int DbTraceV2Handler(
+  unsigned type, /* One of the SQLITE_TRACE_* event types. */
+  void *cd,      /* The original context data pointer. */
+  void *pd,      /* Primary event data, depends on event type. */
+  void *xd       /* Extra event data, depends on event type. */
+){
+  SqliteDb *pDb = (SqliteDb*)cd;
+  Tcl_Obj *pCmd;
+
+  switch( type ){
+    case SQLITE_TRACE_STMT: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+      char *zSql = (char *)xd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewStringObj(zSql, -1));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_PROFILE: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+      sqlite3_int64 ns = (sqlite3_int64)xd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)ns));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_ROW: {
+      sqlite3_stmt *pStmt = (sqlite3_stmt *)pd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)pStmt));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+    case SQLITE_TRACE_CLOSE: {
+      sqlite3 *db = (sqlite3 *)pd;
+
+      pCmd = Tcl_NewStringObj(pDb->zTraceV2, -1);
+      Tcl_IncrRefCount(pCmd);
+      Tcl_ListObjAppendElement(pDb->interp, pCmd,
+                               Tcl_NewWideIntObj((Tcl_WideInt)db));
+      Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+      Tcl_DecrRefCount(pCmd);
+      Tcl_ResetResult(pDb->interp);
+      break;
+    }
+  }
+  return SQLITE_OK;
+}
+#endif
+
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
+/*
 ** This routine is called by the SQLite profile handler after a statement
 ** SQL has executed.  The TCL script in pDb->zProfile is evaluated.
 */
@@ -633,9 +736,9 @@ static void DbRollbackHandler(void *clientData){
 ** This procedure handles wal_hook callbacks.
 */
 static int DbWalHandler(
-  void *clientData, 
-  sqlite3 *db, 
-  const char *zDb, 
+  void *clientData,
+  sqlite3 *db,
+  const char *zDb,
   int nEntry
 ){
   int ret = SQLITE_OK;
@@ -649,7 +752,7 @@ static int DbWalHandler(
   Tcl_IncrRefCount(p);
   Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1));
   Tcl_ListObjAppendElement(interp, p, Tcl_NewIntObj(nEntry));
-  if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0) 
+  if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0)
    || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &ret)
   ){
     Tcl_BackgroundError(interp);
@@ -686,23 +789,63 @@ static void DbUnlockNotify(void **apArg, int nArg){
 }
 #endif
 
-static void DbUpdateHandler(
-  void *p, 
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Pre-update hook callback.
+*/
+static void DbPreUpdateHandler(
+  void *p,
+  sqlite3 *db,
   int op,
-  const char *zDb, 
-  const char *zTbl, 
+  const char *zDb,
+  const char *zTbl,
+  sqlite_int64 iKey1,
+  sqlite_int64 iKey2
+){
+  SqliteDb *pDb = (SqliteDb *)p;
+  Tcl_Obj *pCmd;
+  static const char *azStr[] = {"DELETE", "INSERT", "UPDATE"};
+
+  assert( (SQLITE_DELETE-1)/9 == 0 );
+  assert( (SQLITE_INSERT-1)/9 == 1 );
+  assert( (SQLITE_UPDATE-1)/9 == 2 );
+  assert( pDb->pPreUpdateHook );
+  assert( db==pDb->db );
+  assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
+
+  pCmd = Tcl_DuplicateObj(pDb->pPreUpdateHook);
+  Tcl_IncrRefCount(pCmd);
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(azStr[(op-1)/9], -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zDb, -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zTbl, -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(iKey1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(iKey2));
+  Tcl_EvalObjEx(pDb->interp, pCmd, TCL_EVAL_DIRECT);
+  Tcl_DecrRefCount(pCmd);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+static void DbUpdateHandler(
+  void *p,
+  int op,
+  const char *zDb,
+  const char *zTbl,
   sqlite_int64 rowid
 ){
   SqliteDb *pDb = (SqliteDb *)p;
   Tcl_Obj *pCmd;
+  static const char *azStr[] = {"DELETE", "INSERT", "UPDATE"};
+
+  assert( (SQLITE_DELETE-1)/9 == 0 );
+  assert( (SQLITE_INSERT-1)/9 == 1 );
+  assert( (SQLITE_UPDATE-1)/9 == 2 );
 
   assert( pDb->pUpdateHook );
   assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
 
   pCmd = Tcl_DuplicateObj(pDb->pUpdateHook);
   Tcl_IncrRefCount(pCmd);
-  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(
-    ( (op==SQLITE_INSERT)?"INSERT":(op==SQLITE_UPDATE)?"UPDATE":"DELETE"), -1));
+  Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(azStr[(op-1)/9], -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zDb, -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewStringObj(zTbl, -1));
   Tcl_ListObjAppendElement(0, pCmd, Tcl_NewWideIntObj(rowid));
@@ -771,7 +914,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     ** script object, lappend the arguments, then evaluate the copy.
     **
     ** By "shallow" copy, we mean only the outer list Tcl_Obj is duplicated.
-    ** The new Tcl_Obj contains pointers to the original list elements. 
+    ** The new Tcl_Obj contains pointers to the original list elements.
     ** That way, when Tcl_EvalObjv() is run and shimmers the first element
     ** of the list to tclCmdNameType, that alternate representation will
     ** be preserved and reused on the next invocation.
@@ -779,15 +922,15 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
     Tcl_Obj **aArg;
     int nArg;
     if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){
-      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
       return;
-    }     
+    }
     pCmd = Tcl_NewListObj(nArg, aArg);
     Tcl_IncrRefCount(pCmd);
     for(i=0; iinterp, pCmd, pVal);
       if( rc ){
         Tcl_DecrRefCount(pCmd);
-        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
         return;
       }
     }
@@ -837,7 +980,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){
   }
 
   if( rc && rc!=TCL_RETURN ){
-    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); 
+    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
   }else{
     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
     int n;
@@ -939,7 +1082,7 @@ static int auth_callback(
   Tcl_DStringAppendElement(&str, zArg4 ? zArg4 : "");
 #ifdef SQLITE_USER_AUTHENTICATION
   Tcl_DStringAppendElement(&str, zArg5 ? zArg5 : "");
-#endif  
+#endif
   rc = Tcl_GlobalEval(pDb->interp, Tcl_DStringValue(&str));
   Tcl_DStringFree(&str);
   zReply = rc==TCL_OK ? Tcl_GetStringResult(pDb->interp) : "SQLITE_DENY";
@@ -1010,7 +1153,7 @@ static char *local_getline(char *zPrompt, FILE *in){
 ** It is invoked after evaluating the script SCRIPT to commit or rollback
 ** the transaction or savepoint opened by the [transaction] command.
 */
-static int DbTransPostCmd(
+static int SQLITE_TCLAPI DbTransPostCmd(
   ClientData data[],                   /* data[0] is the Sqlite3Db* for $db */
   Tcl_Interp *interp,                  /* Tcl interpreter */
   int result                           /* Result of evaluating SCRIPT */
@@ -1031,12 +1174,12 @@ static int DbTransPostCmd(
   pDb->disableAuth++;
   if( sqlite3_exec(pDb->db, zEnd, 0, 0, 0) ){
       /* This is a tricky scenario to handle. The most likely cause of an
-      ** error is that the exec() above was an attempt to commit the 
+      ** error is that the exec() above was an attempt to commit the
       ** top-level transaction that returned SQLITE_BUSY. Or, less likely,
       ** that an IO-error has occurred. In either case, throw a Tcl exception
       ** and try to rollback the transaction.
       **
-      ** But it could also be that the user executed one or more BEGIN, 
+      ** But it could also be that the user executed one or more BEGIN,
       ** COMMIT, SAVEPOINT, RELEASE or ROLLBACK commands that are confusing
       ** this method's logic. Not clear how this would be best handled.
       */
@@ -1055,7 +1198,7 @@ static int DbTransPostCmd(
 ** Unless SQLITE_TEST is defined, this function is a simple wrapper around
 ** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either
 ** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending
-** on whether or not the [db_use_legacy_prepare] command has been used to 
+** on whether or not the [db_use_legacy_prepare] command has been used to
 ** configure the connection.
 */
 static int dbPrepare(
@@ -1111,7 +1254,7 @@ static int dbPrepareAndBind(
 
   for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pPreStmt->pNext){
     int n = pPreStmt->nSql;
-    if( nSql>=n 
+    if( nSql>=n
         && memcmp(pPreStmt->zSql, zSql, n)==0
         && (zSql[n]==0 || zSql[n-1]==';')
     ){
@@ -1137,7 +1280,7 @@ static int dbPrepareAndBind(
       break;
     }
   }
-  
+
   /* If no prepared statement was found. Compile the SQL text. Also allocate
   ** a new SqlPreparedStmt structure.  */
   if( pPreStmt==0 ){
@@ -1183,7 +1326,7 @@ static int dbPrepareAndBind(
   assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql );
   assert( 0==memcmp(pPreStmt->zSql, zSql, pPreStmt->nSql) );
 
-  /* Bind values to parameters that begin with $ or : */  
+  /* Bind values to parameters that begin with $ or : */
   for(i=1; i<=nVar; i++){
     const char *zVar = sqlite3_bind_parameter_name(pStmt, i);
     if( zVar!=0 && (zVar[0]=='$' || zVar[0]==':' || zVar[0]=='@') ){
@@ -1271,8 +1414,8 @@ static void dbReleaseStmt(
       assert( pDb->nStmt>0 );
     }
     pDb->nStmt++;
-   
-    /* If we have too many statement in cache, remove the surplus from 
+
+    /* If we have too many statement in cache, remove the surplus from
     ** the end of the cache list.  */
     while( pDb->nStmt>pDb->maxStmt ){
       SqlPreparedStmt *pLast = pDb->stmtLast;
@@ -1326,8 +1469,8 @@ static void dbReleaseColumnNames(DbEvalContext *p){
 ** If pArray is not NULL, then it contains the name of a Tcl array
 ** variable. The "*" member of this array is set to a list containing
 ** the names of the columns returned by the statement as part of each
-** call to dbEvalStep(), in order from left to right. e.g. if the names 
-** of the returned columns are a, b and c, it does the equivalent of the 
+** call to dbEvalStep(), in order from left to right. e.g. if the names
+** of the returned columns are a, b and c, it does the equivalent of the
 ** tcl command:
 **
 **     set ${pArray}(*) {a b c}
@@ -1448,7 +1591,7 @@ static int dbEvalStep(DbEvalContext *p){
 #if SQLITE_TEST
         if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){
           /* If the runtime error was an SQLITE_SCHEMA, and the database
-          ** handle is configured to use the legacy sqlite3_prepare() 
+          ** handle is configured to use the legacy sqlite3_prepare()
           ** interface, retry prepare()/step() on the same SQL statement.
           ** This only happens once. If there is a second SQLITE_SCHEMA
           ** error, the error will be returned to the caller. */
@@ -1536,11 +1679,11 @@ static int DbUseNre(void){
   return( (major==8 && minor>=6) || major>8 );
 }
 #else
-/* 
+/*
 ** Compiling using headers earlier than 8.6. In this case NR cannot be
 ** used, so DbUseNre() to always return zero. Add #defines for the other
 ** Tcl_NRxxx() functions to prevent them from causing compilation errors,
-** even though the only invocations of them are within conditional blocks 
+** even though the only invocations of them are within conditional blocks
 ** of the form:
 **
 **   if( DbUseNre() ) { ... }
@@ -1557,7 +1700,7 @@ static int DbUseNre(void){
 **
 **   $db eval SQL ?ARRAYNAME? SCRIPT
 */
-static int DbEvalNextCmd(
+static int SQLITE_TCLAPI DbEvalNextCmd(
   ClientData data[],                   /* data[0] is the (DbEvalContext*) */
   Tcl_Interp *interp,                  /* Tcl interpreter */
   int result                           /* Result so far */
@@ -1586,11 +1729,11 @@ static int DbEvalNextCmd(
       }
     }
 
-    /* The required interpreter variables are now populated with the data 
+    /* The required interpreter variables are now populated with the data
     ** from the current row. If using NRE, schedule callbacks to evaluate
     ** script pScript, then to invoke this function again to fetch the next
     ** row (or clean up if there is no next row or the script throws an
-    ** exception). After scheduling the callbacks, return control to the 
+    ** exception). After scheduling the callbacks, return control to the
     ** caller.
     **
     ** If not using NRE, evaluate pScript directly and continue with the
@@ -1614,6 +1757,46 @@ static int DbEvalNextCmd(
   return rc;
 }
 
+/*
+** This function is used by the implementations of the following database
+** handle sub-commands:
+**
+**   $db update_hook ?SCRIPT?
+**   $db wal_hook ?SCRIPT?
+**   $db commit_hook ?SCRIPT?
+**   $db preupdate hook ?SCRIPT?
+*/
+static void DbHookCmd(
+  Tcl_Interp *interp,             /* Tcl interpreter */
+  SqliteDb *pDb,                  /* Database handle */
+  Tcl_Obj *pArg,                  /* SCRIPT argument (or NULL) */
+  Tcl_Obj **ppHook                /* Pointer to member of SqliteDb */
+){
+  sqlite3 *db = pDb->db;
+
+  if( *ppHook ){
+    Tcl_SetObjResult(interp, *ppHook);
+    if( pArg ){
+      Tcl_DecrRefCount(*ppHook);
+      *ppHook = 0;
+    }
+  }
+  if( pArg ){
+    assert( !(*ppHook) );
+    if( Tcl_GetCharLength(pArg)>0 ){
+      *ppHook = pArg;
+      Tcl_IncrRefCount(*ppHook);
+    }
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  sqlite3_preupdate_hook(db, (pDb->pPreUpdateHook?DbPreUpdateHandler:0), pDb);
+#endif
+  sqlite3_update_hook(db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb);
+  sqlite3_rollback_hook(db, (pDb->pRollbackHook?DbRollbackHandler:0), pDb);
+  sqlite3_wal_hook(db, (pDb->pWalHook?DbWalHandler:0), pDb);
+}
+
 /*
 ** The "sqlite" command below creates a new Tcl command for each
 ** connection it opens to an SQLite database.  This routine is invoked
@@ -1627,7 +1810,12 @@ static int DbEvalNextCmd(
 ** and calls that connection "db1".  The second command causes this
 ** subroutine to be invoked.
 */
-static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
+static int SQLITE_TCLAPI DbObjCmd(
+  void *cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *const*objv
+){
   SqliteDb *pDb = (SqliteDb*)cd;
   int choice;
   int rc = TCL_OK;
@@ -1639,11 +1827,13 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     "errorcode",          "eval",              "exists",
     "function",           "incrblob",          "interrupt",
     "last_insert_rowid",  "nullvalue",         "onecolumn",
-    "profile",            "progress",          "rekey",
-    "restore",            "rollback_hook",     "status",
-    "timeout",            "total_changes",     "trace",
-    "transaction",        "unlock_notify",     "update_hook",
-    "version",            "wal_hook",          0
+    "preupdate",          "profile",           "progress",
+    "rekey",              "restore",           "rollback_hook",
+    "status",             "timeout",           "total_changes",
+    "trace",              "trace_v2",          "transaction",
+    "unlock_notify",      "update_hook",       "version",
+    "wal_hook",
+    0
   };
   enum DB_enum {
     DB_AUTHORIZER,        DB_BACKUP,           DB_BUSY,
@@ -1653,11 +1843,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     DB_ERRORCODE,         DB_EVAL,             DB_EXISTS,
     DB_FUNCTION,          DB_INCRBLOB,         DB_INTERRUPT,
     DB_LAST_INSERT_ROWID, DB_NULLVALUE,        DB_ONECOLUMN,
-    DB_PROFILE,           DB_PROGRESS,         DB_REKEY,
-    DB_RESTORE,           DB_ROLLBACK_HOOK,    DB_STATUS,
-    DB_TIMEOUT,           DB_TOTAL_CHANGES,    DB_TRACE,
-    DB_TRANSACTION,       DB_UNLOCK_NOTIFY,    DB_UPDATE_HOOK,
-    DB_VERSION,           DB_WAL_HOOK
+    DB_PREUPDATE,         DB_PROFILE,          DB_PROGRESS,
+    DB_REKEY,             DB_RESTORE,          DB_ROLLBACK_HOOK,
+    DB_STATUS,            DB_TIMEOUT,          DB_TOTAL_CHANGES,
+    DB_TRACE,             DB_TRACE_V2,         DB_TRANSACTION,
+    DB_UNLOCK_NOTIFY,     DB_UPDATE_HOOK,      DB_VERSION,
+    DB_WAL_HOOK,
   };
   /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */
 
@@ -1843,7 +2034,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         return TCL_ERROR;
       }else{
         if( TCL_ERROR==Tcl_GetIntFromObj(interp, objv[3], &n) ){
-          Tcl_AppendResult( interp, "cannot convert \"", 
+          Tcl_AppendResult( interp, "cannot convert \"",
                Tcl_GetStringFromObj(objv[3],0), "\" to integer", (char*)0);
           return TCL_ERROR;
         }else{
@@ -1857,7 +2048,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         }
       }
     }else{
-      Tcl_AppendResult( interp, "bad option \"", 
+      Tcl_AppendResult( interp, "bad option \"",
           Tcl_GetStringFromObj(objv[2],0), "\": must be flush or size",
           (char*)0);
       return TCL_ERROR;
@@ -1868,7 +2059,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   /*     $db changes
   **
   ** Return the number of rows that were modified, inserted, or deleted by
-  ** the most recent INSERT, UPDATE or DELETE statement, not including 
+  ** the most recent INSERT, UPDATE or DELETE statement, not including
   ** any changes made by trigger programs.
   */
   case DB_CHANGES: {
@@ -1915,7 +2106,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     pCollate->zScript = (char*)&pCollate[1];
     pDb->pCollate = pCollate;
     memcpy(pCollate->zScript, zScript, nScript+1);
-    if( sqlite3_create_collation(pDb->db, zName, SQLITE_UTF8, 
+    if( sqlite3_create_collation(pDb->db, zName, SQLITE_UTF8,
         pCollate, tclSqlCollate) ){
       Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE);
       return TCL_ERROR;
@@ -2041,7 +2232,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     const char *zSep;
     const char *zNull;
     if( objc<5 || objc>7 ){
-      Tcl_WrongNumArgs(interp, 2, objv, 
+      Tcl_WrongNumArgs(interp, 2, objv,
          "CONFLICT-ALGORITHM TABLE FILENAME ?SEPARATOR? ?NULLINDICATOR?");
       return TCL_ERROR;
     }
@@ -2070,7 +2261,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
        strcmp(zConflict, "fail"    ) != 0 &&
        strcmp(zConflict, "ignore"  ) != 0 &&
        strcmp(zConflict, "replace" ) != 0 ) {
-      Tcl_AppendResult(interp, "Error: \"", zConflict, 
+      Tcl_AppendResult(interp, "Error: \"", zConflict,
             "\", conflict-algorithm must be one of: rollback, "
             "abort, fail, ignore, or replace", (char*)0);
       return TCL_ERROR;
@@ -2116,7 +2307,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     }
     in = fopen(zFile, "rb");
     if( in==0 ){
-      Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, NULL);
+      Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0);
       sqlite3_finalize(pStmt);
       return TCL_ERROR;
     }
@@ -2159,7 +2350,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       for(i=0; i0 && strcmp(azCol[i], zNull)==0)
-          || strlen30(azCol[i])==0 
+          || strlen30(azCol[i])==0
         ){
           sqlite3_bind_null(pStmt, i+1);
         }else{
@@ -2238,8 +2429,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   ** The onecolumn method is the equivalent of:
   **     lindex [$db eval $sql] 0
   */
-  case DB_EXISTS: 
+  case DB_EXISTS:
   case DB_ONECOLUMN: {
+    Tcl_Obj *pResult = 0;
     DbEvalContext sEval;
     if( objc!=3 ){
       Tcl_WrongNumArgs(interp, 2, objv, "SQL");
@@ -2250,21 +2442,22 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     rc = dbEvalStep(&sEval);
     if( choice==DB_ONECOLUMN ){
       if( rc==TCL_OK ){
-        Tcl_SetObjResult(interp, dbEvalColumnValue(&sEval, 0));
+        pResult = dbEvalColumnValue(&sEval, 0);
       }else if( rc==TCL_BREAK ){
         Tcl_ResetResult(interp);
       }
     }else if( rc==TCL_BREAK || rc==TCL_OK ){
-      Tcl_SetObjResult(interp, Tcl_NewBooleanObj(rc==TCL_OK));
+      pResult = Tcl_NewBooleanObj(rc==TCL_OK);
     }
     dbEvalFinalize(&sEval);
+    if( pResult ) Tcl_SetObjResult(interp, pResult);
 
     if( rc==TCL_BREAK ){
       rc = TCL_OK;
     }
     break;
   }
-   
+
   /*
   **    $db eval $sql ?array? ?{  ...code... }?
   **
@@ -2310,7 +2503,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }
       pScript = objv[objc-1];
       Tcl_IncrRefCount(pScript);
-      
+
       p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext));
       dbEvalInit(p, pDb, objv[2], pArray);
 
@@ -2343,7 +2536,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       int n = strlen30(z);
       if( n>2 && strncmp(z, "-argcount",n)==0 ){
         if( i==(objc-2) ){
-          Tcl_AppendResult(interp, "option requires an argument: ", z, 0);
+          Tcl_AppendResult(interp, "option requires an argument: ", z,(char*)0);
           return TCL_ERROR;
         }
         if( Tcl_GetIntFromObj(interp, objv[i+1], &nArg) ) return TCL_ERROR;
@@ -2357,8 +2550,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       if( n>2 && strncmp(z, "-deterministic",n)==0 ){
         flags |= SQLITE_DETERMINISTIC;
       }else{
-        Tcl_AppendResult(interp, "bad option \"", z, 
-            "\": must be -argcount or -deterministic", 0
+        Tcl_AppendResult(interp, "bad option \"", z,
+            "\": must be -argcount or -deterministic", (char*)0
         );
         return TCL_ERROR;
       }
@@ -2466,7 +2659,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   }
 
   /*
-  **     $db last_insert_rowid 
+  **     $db last_insert_rowid
   **
   ** Return an integer which is the ROWID for the most recent insert.
   */
@@ -2488,7 +2681,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   */
 
   /*    $db progress ?N CALLBACK?
-  ** 
+  **
   ** Invoke the given callback every N virtual machine opcodes while executing
   ** queries.
   */
@@ -2556,7 +2749,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }else{
         pDb->zProfile = 0;
       }
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
       if( pDb->zProfile ){
         pDb->interp = interp;
         sqlite3_profile(pDb->db, DbProfileHandler, pDb);
@@ -2574,7 +2768,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   ** Change the encryption key on the currently open database.
   */
   case DB_REKEY: {
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
     int nKey;
     void *pKey;
 #endif
@@ -2582,7 +2776,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       Tcl_WrongNumArgs(interp, 2, objv, "KEY");
       return TCL_ERROR;
     }
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
     pKey = Tcl_GetByteArrayFromObj(objv[2], &nKey);
     rc = sqlite3_rekey(pDb->db, pKey, nKey);
     if( rc ){
@@ -2595,7 +2789,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
 
   /*    $db restore ?DATABASE? FILENAME
   **
-  ** Open a database file named FILENAME.  Transfer the content 
+  ** Open a database file named FILENAME.  Transfer the content
   ** of FILENAME into the local database DATABASE (default: "main").
   */
   case DB_RESTORE: {
@@ -2656,7 +2850,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   /*
   **     $db status (step|sort|autoindex)
   **
-  ** Display SQLITE_STMTSTATUS_FULLSCAN_STEP or 
+  ** Display SQLITE_STMTSTATUS_FULLSCAN_STEP or
   ** SQLITE_STMTSTATUS_SORT for the most recent eval.
   */
   case DB_STATUS: {
@@ -2674,15 +2868,15 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     }else if( strcmp(zOp, "autoindex")==0 ){
       v = pDb->nIndex;
     }else{
-      Tcl_AppendResult(interp, 
-            "bad argument: should be autoindex, step, or sort", 
+      Tcl_AppendResult(interp,
+            "bad argument: should be autoindex, step, or sort",
             (char*)0);
       return TCL_ERROR;
     }
     Tcl_SetObjResult(interp, Tcl_NewIntObj(v));
     break;
   }
-  
+
   /*
   **     $db timeout MILLESECONDS
   **
@@ -2698,11 +2892,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     sqlite3_busy_timeout(pDb->db, ms);
     break;
   }
-  
+
   /*
   **     $db total_changes
   **
-  ** Return the number of rows that were modified, inserted, or deleted 
+  ** Return the number of rows that were modified, inserted, or deleted
   ** since the database handle was created.
   */
   case DB_TOTAL_CHANGES: {
@@ -2743,7 +2937,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       }else{
         pDb->zTrace = 0;
       }
-#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) && \
+    !defined(SQLITE_OMIT_DEPRECATED)
       if( pDb->zTrace ){
         pDb->interp = interp;
         sqlite3_trace(pDb->db, DbTraceHandler, pDb);
@@ -2755,6 +2950,88 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     break;
   }
 
+  /*    $db trace_v2 ?CALLBACK? ?MASK?
+  **
+  ** Make arrangements to invoke the CALLBACK routine for each trace event
+  ** matching the mask that is generated.  The parameters are appended to
+  ** CALLBACK before it is executed.
+  */
+  case DB_TRACE_V2: {
+    if( objc>4 ){
+      Tcl_WrongNumArgs(interp, 2, objv, "?CALLBACK? ?MASK?");
+      return TCL_ERROR;
+    }else if( objc==2 ){
+      if( pDb->zTraceV2 ){
+        Tcl_AppendResult(interp, pDb->zTraceV2, (char*)0);
+      }
+    }else{
+      char *zTraceV2;
+      int len;
+      Tcl_WideInt wMask = 0;
+      if( objc==4 ){
+        static const char *TTYPE_strs[] = {
+          "statement", "profile", "row", "close", 0
+        };
+        enum TTYPE_enum {
+          TTYPE_STMT, TTYPE_PROFILE, TTYPE_ROW, TTYPE_CLOSE
+        };
+        int i;
+        if( TCL_OK!=Tcl_ListObjLength(interp, objv[3], &len) ){
+          return TCL_ERROR;
+        }
+        for(i=0; izTraceV2 ){
+        Tcl_Free(pDb->zTraceV2);
+      }
+      zTraceV2 = Tcl_GetStringFromObj(objv[2], &len);
+      if( zTraceV2 && len>0 ){
+        pDb->zTraceV2 = Tcl_Alloc( len + 1 );
+        memcpy(pDb->zTraceV2, zTraceV2, len+1);
+      }else{
+        pDb->zTraceV2 = 0;
+      }
+#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT)
+      if( pDb->zTraceV2 ){
+        pDb->interp = interp;
+        sqlite3_trace_v2(pDb->db, (unsigned)wMask, DbTraceV2Handler, pDb);
+      }else{
+        sqlite3_trace_v2(pDb->db, 0, 0, 0);
+      }
+#endif
+    }
+    break;
+  }
+
   /*    $db transaction [-deferred|-immediate|-exclusive] SCRIPT
   **
   ** Start a new transaction (if we are not already in the midst of a
@@ -2807,7 +3084,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     /* If using NRE, schedule a callback to invoke the script pScript, then
     ** a second callback to commit (or rollback) the transaction or savepoint
     ** opened above. If not using NRE, evaluate the script directly, then
-    ** call function DbTransPostCmd() to commit (or rollback) the transaction 
+    ** call function DbTransPostCmd() to commit (or rollback) the transaction
     ** or savepoint.  */
     if( DbUseNre() ){
       Tcl_NRAddCallback(interp, DbTransPostCmd, cd, 0, 0, 0);
@@ -2838,14 +3115,14 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
         Tcl_DecrRefCount(pDb->pUnlockNotify);
         pDb->pUnlockNotify = 0;
       }
-  
+
       if( objc==3 ){
         xNotify = DbUnlockNotify;
         pNotifyArg = (void *)pDb;
         pDb->pUnlockNotify = objv[2];
         Tcl_IncrRefCount(pDb->pUnlockNotify);
       }
-  
+
       if( sqlite3_unlock_notify(pDb->db, xNotify, pNotifyArg) ){
         Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0);
         rc = TCL_ERROR;
@@ -2855,50 +3132,112 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
     break;
   }
 
+  /*
+  **    $db preupdate_hook count
+  **    $db preupdate_hook hook ?SCRIPT?
+  **    $db preupdate_hook new INDEX
+  **    $db preupdate_hook old INDEX
+  */
+  case DB_PREUPDATE: {
+#ifndef SQLITE_ENABLE_PREUPDATE_HOOK
+    Tcl_AppendResult(interp, "preupdate_hook was omitted at compile-time", 
+                     (char*)0);
+    rc = TCL_ERROR;
+#else
+    static const char *azSub[] = {"count", "depth", "hook", "new", "old", 0};
+    enum DbPreupdateSubCmd {
+      PRE_COUNT, PRE_DEPTH, PRE_HOOK, PRE_NEW, PRE_OLD
+    };
+    int iSub;
+
+    if( objc<3 ){
+      Tcl_WrongNumArgs(interp, 2, objv, "SUB-COMMAND ?ARGS?");
+    }
+    if( Tcl_GetIndexFromObj(interp, objv[2], azSub, "sub-command", 0, &iSub) ){
+      return TCL_ERROR;
+    }
+
+    switch( (enum DbPreupdateSubCmd)iSub ){
+      case PRE_COUNT: {
+        int nCol = sqlite3_preupdate_count(pDb->db);
+        Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
+        break;
+      }
+
+      case PRE_HOOK: {
+        if( objc>4 ){
+          Tcl_WrongNumArgs(interp, 2, objv, "hook ?SCRIPT?");
+          return TCL_ERROR;
+        }
+        DbHookCmd(interp, pDb, (objc==4 ? objv[3] : 0), &pDb->pPreUpdateHook);
+        break;
+      }
+
+      case PRE_DEPTH: {
+        Tcl_Obj *pRet;
+        if( objc!=3 ){
+          Tcl_WrongNumArgs(interp, 3, objv, "");
+          return TCL_ERROR;
+        }
+        pRet = Tcl_NewIntObj(sqlite3_preupdate_depth(pDb->db));
+        Tcl_SetObjResult(interp, pRet);
+        break;
+      }
+
+      case PRE_NEW:
+      case PRE_OLD: {
+        int iIdx;
+        sqlite3_value *pValue;
+        if( objc!=4 ){
+          Tcl_WrongNumArgs(interp, 3, objv, "INDEX");
+          return TCL_ERROR;
+        }
+        if( Tcl_GetIntFromObj(interp, objv[3], &iIdx) ){
+          return TCL_ERROR;
+        }
+
+        if( iSub==PRE_OLD ){
+          rc = sqlite3_preupdate_old(pDb->db, iIdx, &pValue);
+        }else{
+          assert( iSub==PRE_NEW );
+          rc = sqlite3_preupdate_new(pDb->db, iIdx, &pValue);
+        }
+
+        if( rc==SQLITE_OK ){
+          Tcl_Obj *pObj;
+          pObj = Tcl_NewStringObj((char*)sqlite3_value_text(pValue), -1);
+          Tcl_SetObjResult(interp, pObj);
+        }else{
+          Tcl_AppendResult(interp, sqlite3_errmsg(pDb->db), (char*)0);
+          return TCL_ERROR;
+        }
+      }
+    }
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+    break;
+  }
+
   /*
   **    $db wal_hook ?script?
   **    $db update_hook ?script?
   **    $db rollback_hook ?script?
   */
-  case DB_WAL_HOOK: 
-  case DB_UPDATE_HOOK: 
+  case DB_WAL_HOOK:
+  case DB_UPDATE_HOOK:
   case DB_ROLLBACK_HOOK: {
-
-    /* set ppHook to point at pUpdateHook or pRollbackHook, depending on 
+    /* set ppHook to point at pUpdateHook or pRollbackHook, depending on
     ** whether [$db update_hook] or [$db rollback_hook] was invoked.
     */
-    Tcl_Obj **ppHook; 
-    if( choice==DB_UPDATE_HOOK ){
-      ppHook = &pDb->pUpdateHook;
-    }else if( choice==DB_WAL_HOOK ){
-      ppHook = &pDb->pWalHook;
-    }else{
-      ppHook = &pDb->pRollbackHook;
-    }
-
-    if( objc!=2 && objc!=3 ){
+    Tcl_Obj **ppHook = 0;
+    if( choice==DB_WAL_HOOK ) ppHook = &pDb->pWalHook;
+    if( choice==DB_UPDATE_HOOK ) ppHook = &pDb->pUpdateHook;
+    if( choice==DB_ROLLBACK_HOOK ) ppHook = &pDb->pRollbackHook;
+    if( objc>3 ){
        Tcl_WrongNumArgs(interp, 2, objv, "?SCRIPT?");
        return TCL_ERROR;
     }
-    if( *ppHook ){
-      Tcl_SetObjResult(interp, *ppHook);
-      if( objc==3 ){
-        Tcl_DecrRefCount(*ppHook);
-        *ppHook = 0;
-      }
-    }
-    if( objc==3 ){
-      assert( !(*ppHook) );
-      if( Tcl_GetCharLength(objv[2])>0 ){
-        *ppHook = objv[2];
-        Tcl_IncrRefCount(*ppHook);
-      }
-    }
-
-    sqlite3_update_hook(pDb->db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb);
-    sqlite3_rollback_hook(pDb->db,(pDb->pRollbackHook?DbRollbackHandler:0),pDb);
-    sqlite3_wal_hook(pDb->db,(pDb->pWalHook?DbWalHandler:0),pDb);
 
+    DbHookCmd(interp, pDb, (objc==3 ? objv[2] : 0), ppHook);
     break;
   }
 
@@ -2921,7 +3260,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
 ** Adaptor that provides an objCmd interface to the NRE-enabled
 ** interface implementation.
 */
-static int DbObjCmdAdaptor(
+static int SQLITE_TCLAPI DbObjCmdAdaptor(
   void *cd,
   Tcl_Interp *interp,
   int objc,
@@ -2946,7 +3285,12 @@ static int DbObjCmdAdaptor(
 ** The second argument is the name of the database file.
 **
 */
-static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
+static int SQLITE_TCLAPI DbMain(
+  void *cd,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *const*objv
+){
   SqliteDb *p;
   const char *zArg;
   char *zErrMsg;
@@ -2955,7 +3299,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   const char *zVfs = 0;
   int flags;
   Tcl_DString translatedFilename;
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
   void *pKey = 0;
   int nKey = 0;
 #endif
@@ -2984,7 +3328,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
       return TCL_OK;
     }
     if( strcmp(zArg,"-has-codec")==0 ){
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
       Tcl_AppendResult(interp,"1",(char*)0);
 #else
       Tcl_AppendResult(interp,"0",(char*)0);
@@ -2995,7 +3339,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
   for(i=3; i+1db ){
     sqlite3_key(p->db, pKey, nKey);
   }
@@ -3342,7 +3682,7 @@ static void MD5Init(MD5Context *ctx){
  * Update context to reflect the concatenation of another buffer full
  * of bytes.
  */
-static 
+static
 void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
         uint32 t;
 
@@ -3388,7 +3728,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){
 }
 
 /*
- * Final wrapup - pad to 64-byte boundary with the bit pattern 
+ * Final wrapup - pad to 64-byte boundary with the bit pattern
  * 1 0* (64-bit count of bits processed, MSB-first)
  */
 static void MD5Final(unsigned char digest[16], MD5Context *ctx){
@@ -3464,16 +3804,21 @@ static void MD5DigestToBase10x8(unsigned char digest[16], char zDigest[50]){
 
 /*
 ** A TCL command for md5.  The argument is the text to be hashed.  The
-** Result is the hash in base64.  
+** Result is the hash in base64.
 */
-static int md5_cmd(void*cd, Tcl_Interp *interp, int argc, const char **argv){
+static int SQLITE_TCLAPI md5_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
   MD5Context ctx;
   unsigned char digest[16];
   char zBuf[50];
   void (*converter)(unsigned char*, char*);
 
   if( argc!=2 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
         " TEXT\"", (char*)0);
     return TCL_ERROR;
   }
@@ -3490,7 +3835,12 @@ static int md5_cmd(void*cd, Tcl_Interp *interp, int argc, const char **argv){
 ** A TCL command to take the md5 hash of a file.  The argument is the
 ** name of the file.
 */
-static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){
+static int SQLITE_TCLAPI md5file_cmd(
+  void*cd,
+  Tcl_Interp *interp,
+  int argc,
+  const char **argv
+){
   FILE *in;
   MD5Context ctx;
   void (*converter)(unsigned char*, char*);
@@ -3498,13 +3848,13 @@ static int md5file_cmd(void*cd, Tcl_Interp*interp, int argc, const char **argv){
   char zBuf[10240];
 
   if( argc!=2 ){
-    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0], 
+    Tcl_AppendResult(interp,"wrong # args: should be \"", argv[0],
         " FILENAME\"", (char*)0);
     return TCL_ERROR;
   }
   in = fopen(argv[1],"rb");
   if( in==0 ){
-    Tcl_AppendResult(interp,"unable to open file \"", argv[1], 
+    Tcl_AppendResult(interp,"unable to open file \"", argv[1],
          "\" for reading", (char*)0);
     return TCL_ERROR;
   }
@@ -3570,8 +3920,12 @@ static void md5finalize(sqlite3_context *context){
   MD5DigestToBase16(digest, zBuf);
   sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
 }
-int Md5_Register(sqlite3 *db){
-  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0, 
+int Md5_Register(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pThunk
+){
+  int rc = sqlite3_create_function(db, "md5sum", -1, SQLITE_UTF8, 0, 0,
                                  md5step, md5finalize);
   sqlite3_overload_function(db, "md5sum", -1);  /* To exercise this API */
   return rc;
@@ -3618,7 +3972,7 @@ static const char *tclsh_main_loop(void);
 
 #ifdef SQLITE_TEST
 static void init_all(Tcl_Interp *);
-static int init_all_cmd(
+static int SQLITE_TCLAPI init_all_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3648,7 +4002,7 @@ static int init_all_cmd(
 **   to use the sqlite3_prepare_v2() function to prepare statements. If it
 **   is false, sqlite3_prepare().
 */
-static int db_use_legacy_prepare_cmd(
+static int SQLITE_TCLAPI db_use_legacy_prepare_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3685,7 +4039,7 @@ static int db_use_legacy_prepare_cmd(
 **   return the text representation of the most recently used statement
 **   handle.
 */
-static int db_last_stmt_ptr(
+static int SQLITE_TCLAPI db_last_stmt_ptr(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -3722,7 +4076,7 @@ static int db_last_stmt_ptr(
 ** Configure the interpreter passed as the first argument to have access
 ** to the commands and linked variables that make up:
 **
-**   * the [sqlite3] extension itself, 
+**   * the [sqlite3] extension itself,
 **
 **   * If SQLITE_TCLMD5 or SQLITE_TEST is defined, the Md5 commands, and
 **
@@ -3772,6 +4126,9 @@ static void init_all(Tcl_Interp *interp){
     extern int Sqlitemultiplex_Init(Tcl_Interp*);
     extern int SqliteSuperlock_Init(Tcl_Interp*);
     extern int SqlitetestSyscall_Init(Tcl_Interp*);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    extern int TestSession_Init(Tcl_Interp*);
+#endif
     extern int Fts5tcl_Init(Tcl_Interp *);
     extern int SqliteRbu_Init(Tcl_Interp*);
     extern int Sqlitetesttcl_Init(Tcl_Interp*);
@@ -3807,7 +4164,7 @@ static void init_all(Tcl_Interp *interp){
     Sqlitetesttclvar_Init(interp);
     Sqlitetestfs_Init(interp);
     SqlitetestThread_Init(interp);
-    SqlitetestOnefile_Init(interp);
+    SqlitetestOnefile_Init();
     SqlitetestOsinst_Init(interp);
     Sqlitetestbackup_Init(interp);
     Sqlitetestintarray_Init(interp);
@@ -3817,6 +4174,9 @@ static void init_all(Tcl_Interp *interp){
     Sqlitemultiplex_Init(interp);
     SqliteSuperlock_Init(interp);
     SqlitetestSyscall_Init(interp);
+#if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)
+    TestSession_Init(interp);
+#endif
     Fts5tcl_Init(interp);
     SqliteRbu_Init(interp);
     Sqlitetesttcl_Init(interp);
@@ -3848,7 +4208,7 @@ static void init_all(Tcl_Interp *interp){
 #endif
 
 #define TCLSH_MAIN main   /* Needed to fake out mktclapp */
-int TCLSH_MAIN(int argc, char **argv){
+int SQLITE_CDECL TCLSH_MAIN(int argc, char **argv){
   Tcl_Interp *interp;
 
 #if !defined(_WIN32_WCE)
diff --git a/src/test1.c b/src/test1.c
index 744b400b2a..d17fc1022a 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -19,7 +19,11 @@
 #endif
 
 #include "vdbeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -75,7 +79,7 @@ void *sqlite3TestTextToPtr(const char *z){
 ** for an sqlite connection instance.  Bad things happen if the
 ** input is not an sqlite connection.
 */
-static int get_sqlite_pointer(
+static int SQLITE_TCLAPI get_sqlite_pointer(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -221,7 +225,7 @@ static void io_trace_callback(const char *zFormat, ...){
 ** I/O tracing begins going into FILENAME. If FILENAME is an empty
 ** string, I/O tracing is turned off.
 */
-static int test_io_trace(
+static int SQLITE_TCLAPI test_io_trace(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -262,7 +266,7 @@ static int test_io_trace(
 **
 ** Also return true if the OMIT_MISUSE environment variable exists.
 */
-static int clang_sanitize_address(
+static int SQLITE_TCLAPI clang_sanitize_address(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -289,7 +293,7 @@ static int clang_sanitize_address(
 ** DB.  The SQL is the string FORMAT.  The format string should contain
 ** one %s or %q.  STRING is the value inserted into %s or %q.
 */
-static int test_exec_printf(
+static int SQLITE_TCLAPI test_exec_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -327,7 +331,7 @@ static int test_exec_printf(
 ** HEX into ASCII.  Most characters are translated as is.  %HH becomes
 ** a hex character.
 */
-static int test_exec_hex(
+static int SQLITE_TCLAPI test_exec_hex(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -373,7 +377,7 @@ static int test_exec_hex(
 **
 ** Enter or leave the mutex on a database connection.
 */
-static int db_enter(
+static int SQLITE_TCLAPI db_enter(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -389,7 +393,7 @@ static int db_enter(
   sqlite3_mutex_enter(db->mutex);
   return TCL_OK;
 }
-static int db_leave(
+static int SQLITE_TCLAPI db_leave(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -411,7 +415,7 @@ static int db_leave(
 **
 ** Invoke the sqlite3_exec interface using the open database DB
 */
-static int test_exec(
+static int SQLITE_TCLAPI test_exec(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -458,7 +462,7 @@ static int test_exec(
 ** Invoke the sqlite3_exec interface using the open database DB.  Discard
 ** all results
 */
-static int test_exec_nr(
+static int SQLITE_TCLAPI test_exec_nr(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -485,7 +489,7 @@ static int test_exec_nr(
 ** concatenate arg0 through argn using separator as the separator.
 ** Return the result.
 */
-static int test_mprintf_z(
+static int SQLITE_TCLAPI test_mprintf_z(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -508,7 +512,7 @@ static int test_mprintf_z(
 ** Test the %n format of sqlite_mprintf().  Return the length of the
 ** input string.
 */
-static int test_mprintf_n(
+static int SQLITE_TCLAPI test_mprintf_n(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -532,7 +536,7 @@ static int test_mprintf_n(
 ** You pass in a format string that requires more than one argument,
 ** bad things will happen.
 */
-static int test_snprintf_int(
+static int SQLITE_TCLAPI test_snprintf_int(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -558,7 +562,7 @@ static int test_snprintf_int(
 ** DB.  The SQL is the string FORMAT.  The format string should contain
 ** one %s or %q.  STRING is the value inserted into %s or %q.
 */
-static int test_get_table_printf(
+static int SQLITE_TCLAPI test_get_table_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -621,7 +625,7 @@ static int test_get_table_printf(
 **
 ** Returns the integer ROWID of the most recent insert.
 */
-static int test_last_rowid(
+static int SQLITE_TCLAPI test_last_rowid(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -645,13 +649,13 @@ static int test_last_rowid(
 **
 ** Set the codec key.
 */
-static int test_key(
+static int SQLITE_TCLAPI test_key(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   char **argv            /* Text of each argument */
 ){
-#ifdef SQLITE_HAS_CODEC
+#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_CODEC_FROM_TCL)
   sqlite3 *db;
   const char *zKey;
   int nKey;
@@ -673,7 +677,7 @@ static int test_key(
 **
 ** Change the codec key.
 */
-static int test_rekey(
+static int SQLITE_TCLAPI test_rekey(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -701,7 +705,7 @@ static int test_rekey(
 **
 ** Closes the database opened by sqlite3_open.
 */
-static int sqlite_test_close(
+static int SQLITE_TCLAPI sqlite_test_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -725,7 +729,7 @@ static int sqlite_test_close(
 **
 ** Closes the database opened by sqlite3_open.
 */
-static int sqlite_test_close_v2(
+static int SQLITE_TCLAPI sqlite_test_close_v2(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1009,7 +1013,7 @@ static void nondeterministicFunction(
 ** sqlite3_create_function function while a query is in progress in order
 ** to test the SQLITE_MISUSE detection logic.
 */
-static int test_create_function(
+static int SQLITE_TCLAPI test_create_function(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1166,7 +1170,7 @@ static void legacyCountFinalize(sqlite3_context *context){
 ** "legacy_count()" with the supplied database handle. This is used
 ** to test the deprecated sqlite3_aggregate_count() API.
 */
-static int test_create_aggregate(
+static int SQLITE_TCLAPI test_create_aggregate(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1207,7 +1211,7 @@ static int test_create_aggregate(
 ** Puts uses a separate buffer and debugging statements will be out of
 ** sequence if it is used.
 */
-static int test_printf(
+static int SQLITE_TCLAPI test_printf(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1229,7 +1233,7 @@ static int test_printf(
 **
 ** Call mprintf with three integer arguments
 */
-static int sqlite3_mprintf_int(
+static int SQLITE_TCLAPI sqlite3_mprintf_int(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1256,7 +1260,7 @@ static int sqlite3_mprintf_int(
 **
 ** Call mprintf with three 64-bit integer arguments
 */
-static int sqlite3_mprintf_int64(
+static int SQLITE_TCLAPI sqlite3_mprintf_int64(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1271,7 +1275,7 @@ static int sqlite3_mprintf_int64(
     return TCL_ERROR;
   }
   for(i=2; i<5; i++){
-    if( sqlite3Atoi64(argv[i], &a[i-2], 1000000, SQLITE_UTF8) ){
+    if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){
       Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0);
       return TCL_ERROR;
     }
@@ -1289,7 +1293,7 @@ static int sqlite3_mprintf_int64(
 ** same as sqlite3_mprintf_int or sqlite3_mprintf_int64, depending on
 ** platform.
 */
-static int sqlite3_mprintf_long(
+static int SQLITE_TCLAPI sqlite3_mprintf_long(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1320,7 +1324,7 @@ static int sqlite3_mprintf_long(
 **
 ** Call mprintf with two integer arguments and one string argument
 */
-static int sqlite3_mprintf_str(
+static int SQLITE_TCLAPI sqlite3_mprintf_str(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1347,7 +1351,7 @@ static int sqlite3_mprintf_str(
 **
 ** Call mprintf with two integer arguments and one string argument
 */
-static int sqlite3_snprintf_str(
+static int SQLITE_TCLAPI sqlite3_snprintf_str(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1381,7 +1385,7 @@ static int sqlite3_snprintf_str(
 **
 ** Call mprintf with two integer arguments and one double argument
 */
-static int sqlite3_mprintf_double(
+static int SQLITE_TCLAPI sqlite3_mprintf_double(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1412,7 +1416,7 @@ static int sqlite3_mprintf_double(
 ** two arguments given above.  This is used to generate overflow and underflow
 ** doubles to test that they are converted properly.
 */
-static int sqlite3_mprintf_scaled(
+static int SQLITE_TCLAPI sqlite3_mprintf_scaled(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1442,7 +1446,7 @@ static int sqlite3_mprintf_scaled(
 ** two arguments given above.  This is used to generate overflow and underflow
 ** doubles to test that they are converted properly.
 */
-static int sqlite3_mprintf_stronly(
+static int SQLITE_TCLAPI sqlite3_mprintf_stronly(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1466,7 +1470,7 @@ static int sqlite3_mprintf_stronly(
 ** Call mprintf with a single double argument which is derived from the
 ** hexadecimal encoding of an IEEE double.
 */
-static int sqlite3_mprintf_hexdouble(
+static int SQLITE_TCLAPI sqlite3_mprintf_hexdouble(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -1499,7 +1503,7 @@ static int sqlite3_mprintf_hexdouble(
 **
 */
 #if !defined(SQLITE_OMIT_SHARED_CACHE)
-static int test_enable_shared(
+static int SQLITE_TCLAPI test_enable_shared(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1536,7 +1540,7 @@ static int test_enable_shared(
 ** Usage: sqlite3_extended_result_codes   DB    BOOLEAN
 **
 */
-static int test_extended_result_codes(
+static int SQLITE_TCLAPI test_extended_result_codes(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1559,7 +1563,7 @@ static int test_extended_result_codes(
 ** Usage: sqlite3_libversion_number
 **
 */
-static int test_libversion_number(
+static int SQLITE_TCLAPI test_libversion_number(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1573,7 +1577,7 @@ static int test_libversion_number(
 ** Usage: sqlite3_table_column_metadata DB dbname tblname colname
 **
 */
-static int test_table_column_metadata(
+static int SQLITE_TCLAPI test_table_column_metadata(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1624,7 +1628,7 @@ static int test_table_column_metadata(
 
 #ifndef SQLITE_OMIT_INCRBLOB
 
-static int blobHandleFromObj(
+static int SQLITE_TCLAPI blobHandleFromObj(
   Tcl_Interp *interp, 
   Tcl_Obj *pObj,
   sqlite3_blob **ppBlob
@@ -1653,7 +1657,7 @@ static int blobHandleFromObj(
   return TCL_OK;
 }
 
-static int test_blob_reopen(
+static int SQLITE_TCLAPI test_blob_reopen(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1729,7 +1733,7 @@ static int testCreateCollationCmp(
 
   return iRes;
 }
-static int test_create_collation_v2(
+static int SQLITE_TCLAPI test_create_collation_v2(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1804,7 +1808,7 @@ static void cf2Destroy(void *pUser){
   if( p->pDestroy ) Tcl_DecrRefCount(p->pDestroy); 
   sqlite3_free(p);
 }
-static int test_create_function_v2(
+static int SQLITE_TCLAPI test_create_function_v2(
   ClientData clientData,          /* Not used */
   Tcl_Interp *interp,             /* The invoking TCL interpreter */
   int objc,                       /* Number of arguments */
@@ -1892,7 +1896,7 @@ static int test_create_function_v2(
 /*
 ** Usage: sqlite3_load_extension DB-HANDLE FILE ?PROC?
 */
-static int test_load_extension(
+static int SQLITE_TCLAPI test_load_extension(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1931,6 +1935,8 @@ static int test_load_extension(
 #ifdef SQLITE_OMIT_LOAD_EXTENSION
   rc = SQLITE_ERROR;
   zErr = sqlite3_mprintf("this build omits sqlite3_load_extension()");
+  (void)zProc;
+  (void)zFile;
 #else
   rc = sqlite3_load_extension(db, zFile, zProc, &zErr);
 #endif
@@ -1948,7 +1954,7 @@ static int test_load_extension(
 /*
 ** Usage: sqlite3_enable_load_extension DB-HANDLE ONOFF
 */
-static int test_enable_load(
+static int SQLITE_TCLAPI test_enable_load(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1994,7 +2000,7 @@ static int test_enable_load(
 ** This command is used to test the recoverability of a database in
 ** the event of a program crash.
 */
-static int sqlite_abort(
+static int SQLITE_TCLAPI sqlite_abort(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2053,7 +2059,7 @@ error_out:
 **
 ** Register the test SQL function on the database DB under the name NAME.
 */
-static int test_register_func(
+static int SQLITE_TCLAPI test_register_func(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2082,7 +2088,7 @@ static int test_register_func(
 **
 ** Finalize a statement handle.
 */
-static int test_finalize(
+static int SQLITE_TCLAPI test_finalize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2114,7 +2120,7 @@ static int test_finalize(
 **
 ** Get the value of a status counter from a statement.
 */
-static int test_stmt_status(
+static int SQLITE_TCLAPI test_stmt_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2159,7 +2165,7 @@ static int test_stmt_status(
 /*
 ** Usage:  sqlite3_stmt_scanstatus STMT IDX
 */
-static int test_stmt_scanstatus(
+static int SQLITE_TCLAPI test_stmt_scanstatus(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2209,7 +2215,7 @@ static int test_stmt_scanstatus(
 /*
 ** Usage:  sqlite3_stmt_scanstatus_reset  STMT
 */
-static int test_stmt_scanstatus_reset(
+static int SQLITE_TCLAPI test_stmt_scanstatus_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2232,7 +2238,7 @@ static int test_stmt_scanstatus_reset(
 **
 ** Zero the SQLITE_CONFIG_SQLLOG configuration
 */
-static int test_config_sqllog(
+static int SQLITE_TCLAPI test_config_sqllog(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2252,7 +2258,7 @@ static int test_config_sqllog(
 **
 ** Return the value returned by the default VFS's xCurrentTimeInt64 method.
 */
-static int vfsCurrentTimeInt64(
+static int SQLITE_TCLAPI vfsCurrentTimeInt64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2273,7 +2279,7 @@ static int vfsCurrentTimeInt64(
 /*
 ** Usage: sqlite3_snapshot_get DB DBNAME
 */
-static int test_snapshot_get(
+static int SQLITE_TCLAPI test_snapshot_get(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2304,11 +2310,43 @@ static int test_snapshot_get(
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Usage: sqlite3_snapshot_recover DB DBNAME
+*/
+static int SQLITE_TCLAPI test_snapshot_recover(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_snapshot_recover(db, zName);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }else{
+    Tcl_ResetResult(interp);
+  }
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
 #ifdef SQLITE_ENABLE_SNAPSHOT
 /*
 ** Usage: sqlite3_snapshot_open DB DBNAME SNAPSHOT
 */
-static int test_snapshot_open(
+static int SQLITE_TCLAPI test_snapshot_open(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2340,7 +2378,7 @@ static int test_snapshot_open(
 /*
 ** Usage: sqlite3_snapshot_free SNAPSHOT
 */
-static int test_snapshot_free(
+static int SQLITE_TCLAPI test_snapshot_free(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2357,12 +2395,167 @@ static int test_snapshot_free(
 }
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Usage: sqlite3_snapshot_cmp SNAPSHOT1 SNAPSHOT2
+*/
+static int SQLITE_TCLAPI test_snapshot_cmp(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int res;
+  sqlite3_snapshot *p1;
+  sqlite3_snapshot *p2;
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2");
+    return TCL_ERROR;
+  }
+  p1 = (sqlite3_snapshot*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
+  p2 = (sqlite3_snapshot*)sqlite3TestTextToPtr(Tcl_GetString(objv[2]));
+  res = sqlite3_snapshot_cmp(p1, p2);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(res));
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Usage: sqlite3_snapshot_get_blob DB DBNAME
+*/
+static int SQLITE_TCLAPI test_snapshot_get_blob(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+  sqlite3_snapshot *pSnapshot = 0;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+
+  rc = sqlite3_snapshot_get(db, zName, &pSnapshot);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }else{
+    Tcl_SetObjResult(interp, 
+        Tcl_NewByteArrayObj((unsigned char*)pSnapshot, sizeof(sqlite3_snapshot))
+    );
+    sqlite3_snapshot_free(pSnapshot);
+  }
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
+#ifdef SQLITE_ENABLE_SNAPSHOT
+  /*
+  ** Usage: sqlite3_snapshot_open_blob DB DBNAME SNAPSHOT
+*/
+static int SQLITE_TCLAPI test_snapshot_open_blob(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  char *zName;
+  unsigned char *pBlob;
+  int nBlob;
+
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SNAPSHOT");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zName = Tcl_GetString(objv[2]);
+  pBlob = Tcl_GetByteArrayFromObj(objv[3], &nBlob);
+  if( nBlob!=sizeof(sqlite3_snapshot) ){
+    Tcl_AppendResult(interp, "bad SNAPSHOT", 0);
+    return TCL_ERROR;
+  }
+  rc = sqlite3_snapshot_open(db, zName, (sqlite3_snapshot*)pBlob);
+  if( rc!=SQLITE_OK ){
+    Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+    return TCL_ERROR;
+  }
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Usage: sqlite3_snapshot_cmp_blob SNAPSHOT1 SNAPSHOT2
+*/
+static int SQLITE_TCLAPI test_snapshot_cmp_blob(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int res;
+  unsigned char *p1;
+  unsigned char *p2;
+  int n1;
+  int n2;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2");
+    return TCL_ERROR;
+  }
+
+  p1 = Tcl_GetByteArrayFromObj(objv[1], &n1);
+  p2 = Tcl_GetByteArrayFromObj(objv[2], &n2);
+
+  if( n1!=sizeof(sqlite3_snapshot) || n1!=n2 ){
+    Tcl_AppendResult(interp, "bad SNAPSHOT", 0);
+    return TCL_ERROR;
+  }
+
+  res = sqlite3_snapshot_cmp((sqlite3_snapshot*)p1, (sqlite3_snapshot*)p2);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(res));
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
+/*
+** Usage: sqlite3_delete_database FILENAME
+*/
+int sqlite3_delete_database(const char*);   /* in test_delete.c */
+static int SQLITE_TCLAPI test_delete_database(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  const char *zFile;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "FILE");
+    return TCL_ERROR;
+  }
+  zFile = (const char*)Tcl_GetString(objv[1]);
+  rc = sqlite3_delete_database(zFile);
+
+  Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_next_stmt  DB  STMT
 **
 ** Return the next statment in sequence after STMT.
 */
-static int test_next_stmt(
+static int SQLITE_TCLAPI test_next_stmt(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2394,7 +2587,7 @@ static int test_next_stmt(
 ** Return true if STMT is a NULL pointer or a pointer to a statement
 ** that is guaranteed to leave the database unmodified.
 */
-static int test_stmt_readonly(
+static int SQLITE_TCLAPI test_stmt_readonly(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2421,7 +2614,7 @@ static int test_stmt_readonly(
 ** Return true if STMT is a non-NULL pointer to a statement
 ** that has been stepped but not to completion.
 */
-static int test_stmt_busy(
+static int SQLITE_TCLAPI test_stmt_busy(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2447,7 +2640,7 @@ static int test_stmt_busy(
 **
 ** Return true if STMT uses a statement journal.
 */
-static int uses_stmt_journal(
+static int SQLITE_TCLAPI uses_stmt_journal(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2473,7 +2666,7 @@ static int uses_stmt_journal(
 **
 ** Reset a statement handle.
 */
-static int test_reset(
+static int SQLITE_TCLAPI test_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2508,7 +2701,7 @@ static int test_reset(
 **
 ** Return TRUE if a recompilation of the statement is recommended.
 */
-static int test_expired(
+static int SQLITE_TCLAPI test_expired(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2532,7 +2725,7 @@ static int test_expired(
 **
 ** Transfer all bindings from FROMSTMT over to TOSTMT
 */
-static int test_transfer_bind(
+static int SQLITE_TCLAPI test_transfer_bind(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2559,7 +2752,7 @@ static int test_transfer_bind(
 ** Return the number of changes made to the database by the last SQL
 ** execution.
 */
-static int test_changes(
+static int SQLITE_TCLAPI test_changes(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2594,7 +2787,7 @@ static int sqlite_static_bind_nbyte = 0;
 ** of the VALUE is made.  If FLAGS=="blob10" then a VALUE is ignored
 ** an a 10-byte blob "abc\000xyz\000pq" is inserted.
 */
-static int test_bind(
+static int SQLITE_TCLAPI test_bind(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -2716,7 +2909,7 @@ static int test_collate_func(
   Tcl_GetIntFromObj(i, Tcl_GetObjResult(i), &res);
   return res;
 }
-static int test_collate(
+static int SQLITE_TCLAPI test_collate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2790,7 +2983,7 @@ static int test_utf16bin_collate_func(
   if( res==0 ) res = nA - nB;
   return res;
 }
-static int test_utf16bin_collate(
+static int SQLITE_TCLAPI test_utf16bin_collate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2847,7 +3040,7 @@ static void test_collate_needed_cb(
 /*
 ** Usage: add_test_collate_needed DB
 */
-static int test_collate_needed(
+static int SQLITE_TCLAPI test_collate_needed(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -2898,7 +3091,7 @@ static int alignmentCollFunc(
   }
   return rc;
 }
-static int add_alignment_test_collations(
+static int SQLITE_TCLAPI add_alignment_test_collations(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3016,7 +3209,7 @@ static void test_function_utf16be(
   sqlite3ValueFree(pVal);
 }
 #endif /* SQLITE_OMIT_UTF16 */
-static int test_function(
+static int SQLITE_TCLAPI test_function(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3061,7 +3254,7 @@ bad_args:
 ** The result is a list of two elements, the string representation of the
 ** error code and the english language explanation.
 */
-static int test_errstr(
+static int SQLITE_TCLAPI test_errstr(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3094,7 +3287,7 @@ static int test_errstr(
 ** Then run testfixture in the debugger and wait for the breakpoint to
 ** fire.  Then additional breakpoints can be set to trace down the bug.
 */
-static int test_breakpoint(
+static int SQLITE_TCLAPI test_breakpoint(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -3110,7 +3303,7 @@ static int test_breakpoint(
 ** IDX is the index of a wildcard in the prepared statement.  This command
 ** binds a N-byte zero-filled BLOB to the wildcard.
 */
-static int test_bind_zeroblob(
+static int SQLITE_TCLAPI test_bind_zeroblob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3146,7 +3339,7 @@ static int test_bind_zeroblob(
 ** IDX is the index of a wildcard in the prepared statement.  This command
 ** binds a N-byte zero-filled BLOB to the wildcard.
 */
-static int test_bind_zeroblob64(
+static int SQLITE_TCLAPI test_bind_zeroblob64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3183,7 +3376,7 @@ static int test_bind_zeroblob64(
 ** N is the index of a wildcard in the prepared statement.  This command
 ** binds a 32-bit integer VALUE to that wildcard.
 */
-static int test_bind_int(
+static int SQLITE_TCLAPI test_bind_int(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3214,6 +3407,145 @@ static int test_bind_int(
 }
 
 
+/*
+** Usage:   intarray_addr  INT  ...
+**
+** Return the address of a C-language array of 32-bit integers.
+**
+** Space to hold the array is obtained from malloc().  Call this procedure once
+** with no arguments in order to release memory.  Each call to this procedure
+** overwrites the previous array.
+*/
+static int SQLITE_TCLAPI test_intarray_addr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int i;
+  static int *p = 0;
+
+  sqlite3_free(p);
+  p = 0;
+  if( objc>1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; i1 ){
+    p = sqlite3_malloc( sizeof(p[0])*(objc-1) );
+    if( p==0 ) return TCL_ERROR;
+    for(i=0; ilen ){
+    char zBuf[200];
+    sqlite3_snprintf(sizeof(zBuf), zBuf,
+                     "cannot use %d blob bytes, have %d", bytes, len);
+    Tcl_AppendResult(interp, zBuf, -1);
+    return TCL_ERROR;
+  }
+
   rc = sqlite3_bind_blob(pStmt, idx, value, bytes, xDestructor);
   if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR;
   if( rc!=SQLITE_OK ){
@@ -3503,7 +3844,7 @@ static int test_bind_blob(
 **
 ** Return the number of wildcards in the given statement.
 */
-static int test_bind_parameter_count(
+static int SQLITE_TCLAPI test_bind_parameter_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3527,7 +3868,7 @@ static int test_bind_parameter_count(
 ** An empty string is returned if N is out of range or if the wildcard
 ** is nameless.
 */
-static int test_bind_parameter_name(
+static int SQLITE_TCLAPI test_bind_parameter_name(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3554,7 +3895,7 @@ static int test_bind_parameter_name(
 ** Return the index of the wildcard called NAME.  Return 0 if there is
 ** no such wildcard.
 */
-static int test_bind_parameter_index(
+static int SQLITE_TCLAPI test_bind_parameter_index(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3579,7 +3920,7 @@ static int test_bind_parameter_index(
 ** Usage:   sqlite3_clear_bindings STMT
 **
 */
-static int test_clear_bindings(
+static int SQLITE_TCLAPI test_clear_bindings(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3599,7 +3940,7 @@ static int test_clear_bindings(
 /*
 ** Usage:   sqlite3_sleep MILLISECONDS
 */
-static int test_sleep(
+static int SQLITE_TCLAPI test_sleep(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3624,7 +3965,7 @@ static int test_sleep(
 ** Return the string representation of the most recent sqlite3_* API
 ** error code. e.g. "SQLITE_ERROR".
 */
-static int test_ex_errcode(
+static int SQLITE_TCLAPI test_ex_errcode(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3651,7 +3992,7 @@ static int test_ex_errcode(
 ** Return the string representation of the most recent sqlite3_* API
 ** error code. e.g. "SQLITE_ERROR".
 */
-static int test_errcode(
+static int SQLITE_TCLAPI test_errcode(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3677,7 +4018,7 @@ static int test_errcode(
 ** Returns the UTF-8 representation of the error message string for the
 ** most recent sqlite3_* API call.
 */
-static int test_errmsg(
+static int SQLITE_TCLAPI test_errmsg(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3706,7 +4047,7 @@ static int test_errmsg(
 ** level, and it includes the 0x00 0x00 terminator bytes at the end of the
 ** UTF-16 string.
 */
-static int test_errmsg16(
+static int SQLITE_TCLAPI test_errmsg16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3743,7 +4084,7 @@ static int test_errmsg16(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare(
+static int SQLITE_TCLAPI test_prepare(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3800,7 +4141,7 @@ static int test_prepare(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare_v2(
+static int SQLITE_TCLAPI test_prepare_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3868,7 +4209,7 @@ static int test_prepare_v2(
 ** Generate a prepared statement for a zero-byte string as a test
 ** for ticket #3134.  The string should be preceded by a zero byte.
 */
-static int test_prepare_tkt3134(
+static int SQLITE_TCLAPI test_prepare_tkt3134(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3911,7 +4252,7 @@ static int test_prepare_tkt3134(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare16(
+static int SQLITE_TCLAPI test_prepare16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -3971,7 +4312,7 @@ static int test_prepare16(
 ** variable that is set to the unused portion of  (if any). A
 ** STMT handle is returned.
 */
-static int test_prepare16_v2(
+static int SQLITE_TCLAPI test_prepare16_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4026,7 +4367,7 @@ static int test_prepare16_v2(
 /*
 ** Usage: sqlite3_open filename ?options-list?
 */
-static int test_open(
+static int SQLITE_TCLAPI test_open(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4053,7 +4394,7 @@ static int test_open(
 /*
 ** Usage: sqlite3_open_v2 FILENAME FLAGS VFS
 */
-static int test_open_v2(
+static int SQLITE_TCLAPI test_open_v2(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4123,7 +4464,7 @@ static int test_open_v2(
 /*
 ** Usage: sqlite3_open16 filename options
 */
-static int test_open16(
+static int SQLITE_TCLAPI test_open16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4155,7 +4496,7 @@ static int test_open16(
 ** Return 1 if the supplied argument is a complete SQL statement, or zero
 ** otherwise.
 */
-static int test_complete16(
+static int SQLITE_TCLAPI test_complete16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4180,7 +4521,7 @@ static int test_complete16(
 **
 ** Advance the statement to the next row.
 */
-static int test_step(
+static int SQLITE_TCLAPI test_step(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4203,7 +4544,7 @@ static int test_step(
   return TCL_OK;
 }
 
-static int test_sql(
+static int SQLITE_TCLAPI test_sql(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4220,13 +4561,33 @@ static int test_sql(
   Tcl_SetResult(interp, (char *)sqlite3_sql(pStmt), TCL_VOLATILE);
   return TCL_OK;
 }
+static int SQLITE_TCLAPI test_ex_sql(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+  char *z;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "STMT");
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  z = sqlite3_expanded_sql(pStmt);
+  Tcl_SetResult(interp, z, TCL_VOLATILE);
+  sqlite3_free(z);
+  return TCL_OK;
+}
 
 /*
 ** Usage: sqlite3_column_count STMT 
 **
 ** Return the number of columns returned by the sql statement STMT.
 */
-static int test_column_count(
+static int SQLITE_TCLAPI test_column_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4251,7 +4612,7 @@ static int test_column_count(
 **
 ** Return the type of the data in column 'column' of the current row.
 */
-static int test_column_type(
+static int SQLITE_TCLAPI test_column_type(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4300,7 +4661,7 @@ static int test_column_type(
 ** Return the data in column 'column' of the current row cast as an
 ** wide (64-bit) integer.
 */
-static int test_column_int64(
+static int SQLITE_TCLAPI test_column_int64(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4327,7 +4688,7 @@ static int test_column_int64(
 /*
 ** Usage: sqlite3_column_blob STMT column
 */
-static int test_column_blob(
+static int SQLITE_TCLAPI test_column_blob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4359,7 +4720,7 @@ static int test_column_blob(
 **
 ** Return the data in column 'column' of the current row cast as a double.
 */
-static int test_column_double(
+static int SQLITE_TCLAPI test_column_double(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4388,7 +4749,7 @@ static int test_column_double(
 **
 ** Return the number of columns returned by the sql statement STMT.
 */
-static int test_data_count(
+static int SQLITE_TCLAPI test_data_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4415,7 +4776,7 @@ static int test_data_count(
 **
 ** Usage: sqlite3_column_name STMT column
 */
-static int test_stmt_utf8(
+static int SQLITE_TCLAPI test_stmt_utf8(
   void * clientData,        /* Pointer to SQLite API function to be invoke */
   Tcl_Interp *interp,
   int objc,
@@ -4442,7 +4803,7 @@ static int test_stmt_utf8(
   return TCL_OK;
 }
 
-static int test_global_recover(
+static int SQLITE_TCLAPI test_global_recover(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4467,7 +4828,7 @@ static int test_global_recover(
 **
 ** Usage: sqlite3_column_name STMT column
 */
-static int test_stmt_utf16(
+static int SQLITE_TCLAPI test_stmt_utf16(
   void * clientData,     /* Pointer to SQLite API function to be invoked */
   Tcl_Interp *interp,
   int objc,
@@ -4511,7 +4872,7 @@ static int test_stmt_utf16(
 ** Usage: sqlite3_column_bytes16 STMT column
 **
 */
-static int test_stmt_int(
+static int SQLITE_TCLAPI test_stmt_int(
   void * clientData,    /* Pointer to SQLite API function to be invoked */
   Tcl_Interp *interp,
   int objc,
@@ -4540,7 +4901,7 @@ static int test_stmt_int(
 **
 ** Set the db->magic value.  This is used to test error recovery logic.
 */
-static int sqlite_set_magic(
+static int SQLITE_TCLAPI sqlite_set_magic(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4572,7 +4933,7 @@ static int sqlite_set_magic(
 **
 ** Trigger an interrupt on DB
 */
-static int test_interrupt(
+static int SQLITE_TCLAPI test_interrupt(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4613,7 +4974,7 @@ u64 sqlite3StackDepth(void){
 **
 ** Try to measure the amount of stack space used by a call to sqlite3_exec
 */
-static int test_stack_used(
+static int SQLITE_TCLAPI test_stack_used(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4641,7 +5002,7 @@ static int test_stack_used(
 ** is assumed that the user function was created as UTF8, any number of
 ** arguments (the way the TCL interface does it).
 */
-static int delete_function(
+static int SQLITE_TCLAPI delete_function(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4667,7 +5028,7 @@ static int delete_function(
 ** DB. It is assumed that the collation sequence was created as UTF8 (the 
 ** way the TCL interface does it).
 */
-static int delete_collation(
+static int SQLITE_TCLAPI delete_collation(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4692,7 +5053,7 @@ static int delete_collation(
 ** Return true if the database DB is currently in auto-commit mode.
 ** Return false if not.
 */
-static int get_autocommit(
+static int SQLITE_TCLAPI get_autocommit(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4718,7 +5079,7 @@ static int get_autocommit(
 ** method of the TCL interface.  But we need a way to test the case
 ** where it returns SQLITE_MISUSE.
 */
-static int test_busy_timeout(
+static int SQLITE_TCLAPI test_busy_timeout(
   void * clientData,
   Tcl_Interp *interp,
   int argc,
@@ -4744,7 +5105,7 @@ static int test_busy_timeout(
 ** Return the name of the internal representation for the
 ** value of the given variable.
 */
-static int tcl_variable_type(
+static int SQLITE_TCLAPI tcl_variable_type(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4770,7 +5131,7 @@ static int tcl_variable_type(
 ** The integer N is the number of bytes we are trying to release.  The 
 ** return value is the amount of memory actually released.
 */
-static int test_release_memory(
+static int SQLITE_TCLAPI test_release_memory(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4801,7 +5162,7 @@ static int test_release_memory(
 ** Attempt to release memory currently held by database DB.  Return the
 ** result code (which in the current implementation is always zero).
 */
-static int test_db_release_memory(
+static int SQLITE_TCLAPI test_db_release_memory(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4824,7 +5185,7 @@ static int test_db_release_memory(
 **
 ** Attempt to flush any dirty pages to disk.
 */
-static int test_db_cacheflush(
+static int SQLITE_TCLAPI test_db_cacheflush(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4847,12 +5208,35 @@ static int test_db_cacheflush(
   return TCL_OK;
 }
 
+/*
+** Usage:  sqlite3_system_errno DB
+**
+** Return the low-level system errno value.
+*/
+static int SQLITE_TCLAPI test_system_errno(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  int iErrno;
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  iErrno = sqlite3_system_errno(db);
+  Tcl_SetObjResult(interp, Tcl_NewIntObj(iErrno));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_db_filename DB DBNAME
 **
 ** Return the name of a file associated with a database.
 */
-static int test_db_filename(
+static int SQLITE_TCLAPI test_db_filename(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4876,7 +5260,7 @@ static int test_db_filename(
 ** Return 1 or 0 if DBNAME is readonly or not.  Return -1 if DBNAME does
 ** not exist.
 */
-static int test_db_readonly(
+static int SQLITE_TCLAPI test_db_readonly(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4901,7 +5285,7 @@ static int test_db_readonly(
 ** limit is only changed if the N is present.  The previous limit
 ** is returned.
 */
-static int test_soft_heap_limit(
+static int SQLITE_TCLAPI test_soft_heap_limit(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4926,7 +5310,7 @@ static int test_soft_heap_limit(
 **
 ** Call the sqlite3_thread_cleanup API.
 */
-static int test_thread_cleanup(
+static int SQLITE_TCLAPI test_thread_cleanup(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4944,7 +5328,7 @@ static int test_thread_cleanup(
 ** Return a list of numbers which are the PagerRefcount for all
 ** pagers on each database connection.
 */
-static int test_pager_refcounts(
+static int SQLITE_TCLAPI test_pager_refcounts(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -4990,7 +5374,7 @@ static int test_pager_refcounts(
 ** and that the errors they are seeing in the test scripts might be
 ** a result of their defective TCL rather than problems in SQLite.
 */
-static int working_64bit_int(
+static int SQLITE_TCLAPI working_64bit_int(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5015,7 +5399,7 @@ static int working_64bit_int(
 ** VFS when none are previously registered, and the ability to 
 ** unregister the only available VFS.  Ticket #2738
 */
-static int vfs_unlink_test(
+static int SQLITE_TCLAPI vfs_unlink_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5117,7 +5501,7 @@ static int vfs_unlink_test(
 ** This TCL command attempts to vfs_find and vfs_register when the
 ** sqlite3_initialize() interface is failing.  All calls should fail.
 */
-static int vfs_initfail_test(
+static int SQLITE_TCLAPI vfs_initfail_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5145,7 +5529,7 @@ static int nVfs = 0;
 **
 ** Unregister all VFSes.
 */
-static int vfs_unregister_all(
+static int SQLITE_TCLAPI vfs_unregister_all(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5163,17 +5547,19 @@ static int vfs_unregister_all(
 /*
 ** tclcmd:   vfs_reregister_all
 **
-** Restore all VFSes that were removed using vfs_unregister_all
+** Restore all VFSes that were removed using vfs_unregister_all. Taking
+** care to put the linked list back together in the same order as it was
+** in before vfs_unregister_all was invoked.
 */
-static int vfs_reregister_all(
+static int SQLITE_TCLAPI vfs_reregister_all(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
   Tcl_Obj *CONST objv[]  /* Command arguments */
 ){
   int i;
-  for(i=0; i=0; i--){
+    sqlite3_vfs_register(apVfs[i], 1);
   }
   return TCL_OK;
 }
@@ -5185,7 +5571,7 @@ static int vfs_reregister_all(
 ** This TCL command runs the sqlite3_file_control interface and
 ** verifies correct operation of the same.
 */
-static int file_control_test(
+static int SQLITE_TCLAPI file_control_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5220,7 +5606,7 @@ static int file_control_test(
 ** This TCL command runs the sqlite3_file_control interface and
 ** verifies correct operation of the SQLITE_LAST_ERRNO verb.
 */
-static int file_control_lasterrno_test(
+static int SQLITE_TCLAPI file_control_lasterrno_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5258,7 +5644,7 @@ static int file_control_lasterrno_test(
 ** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and
 ** SQLITE_SET_LOCKPROXYFILE verbs.
 */
-static int file_control_chunksize_test(
+static int SQLITE_TCLAPI file_control_chunksize_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5295,7 +5681,7 @@ static int file_control_chunksize_test(
 ** This TCL command runs the sqlite3_file_control interface 
 ** with SQLITE_FCNTL_SIZE_HINT
 */
-static int file_control_sizehint_test(
+static int SQLITE_TCLAPI file_control_sizehint_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5333,7 +5719,7 @@ static int file_control_sizehint_test(
 ** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and
 ** SQLITE_SET_LOCKPROXYFILE verbs.
 */
-static int file_control_lockproxy_test(
+static int SQLITE_TCLAPI file_control_lockproxy_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5403,7 +5789,7 @@ static int file_control_lockproxy_test(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_WIN32_AV_RETRY opcode.
 */
-static int file_control_win32_av_retry(
+static int SQLITE_TCLAPI file_control_win32_av_retry(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5430,13 +5816,45 @@ static int file_control_win32_av_retry(
   return TCL_OK;  
 }
 
+/*
+** tclcmd:   file_control_win32_get_handle DB
+**
+** This TCL command runs the sqlite3_file_control interface with
+** the SQLITE_FCNTL_WIN32_GET_HANDLE opcode.
+*/
+static int file_control_win32_get_handle(
+  ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
+  Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
+  int objc,              /* Number of arguments */
+  Tcl_Obj *CONST objv[]  /* Command arguments */
+){
+  sqlite3 *db;
+  int rc;
+  HANDLE hFile = NULL;
+  char z[100];
+
+  if( objc!=2 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"",
+        Tcl_GetStringFromObj(objv[0], 0), " DB", 0);
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){
+    return TCL_ERROR;
+  }
+  rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_GET_HANDLE,
+                            (void*)&hFile);
+  sqlite3_snprintf(sizeof(z), z, "%d %p", rc, (void*)hFile);
+  Tcl_AppendResult(interp, z, (char*)0);
+  return TCL_OK;
+}
+
 /*
 ** tclcmd:   file_control_win32_set_handle DB HANDLE
 **
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_WIN32_SET_HANDLE opcode.
 */
-static int file_control_win32_set_handle(
+static int SQLITE_TCLAPI file_control_win32_set_handle(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5472,7 +5890,7 @@ static int file_control_win32_set_handle(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_PERSIST_WAL opcode.
 */
-static int file_control_persist_wal(
+static int SQLITE_TCLAPI file_control_persist_wal(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5504,7 +5922,7 @@ static int file_control_persist_wal(
 ** This TCL command runs the sqlite3_file_control interface with
 ** the SQLITE_FCNTL_POWERSAFE_OVERWRITE opcode.
 */
-static int file_control_powersafe_overwrite(
+static int SQLITE_TCLAPI file_control_powersafe_overwrite(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5536,7 +5954,7 @@ static int file_control_powersafe_overwrite(
 **
 ** Return a string that describes the stack of VFSes.
 */
-static int file_control_vfsname(
+static int SQLITE_TCLAPI file_control_vfsname(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5568,7 +5986,7 @@ static int file_control_vfsname(
 **
 ** Return a string that is a temporary filename
 */
-static int file_control_tempfilename(
+static int SQLITE_TCLAPI file_control_tempfilename(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5601,7 +6019,7 @@ static int file_control_tempfilename(
 **
 **   Return a tcl list containing the names of all registered vfs's.
 */
-static int vfs_list(
+static int SQLITE_TCLAPI vfs_list(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5626,7 +6044,7 @@ static int vfs_list(
 ** This TCL command runs the sqlite3_limit interface and
 ** verifies correct operation of the same.
 */
-static int test_limit(
+static int SQLITE_TCLAPI test_limit(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5689,7 +6107,7 @@ static int test_limit(
 ** At the same time, verify that sqlite3_test_control works even when
 ** called with an out-of-range opcode.
 */
-static int save_prng_state(
+static int SQLITE_TCLAPI save_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5705,7 +6123,7 @@ static int save_prng_state(
 /*
 ** tclcmd:  restore_prng_state
 */
-static int restore_prng_state(
+static int SQLITE_TCLAPI restore_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5717,7 +6135,7 @@ static int restore_prng_state(
 /*
 ** tclcmd:  reset_prng_state
 */
-static int reset_prng_state(
+static int SQLITE_TCLAPI reset_prng_state(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5733,7 +6151,7 @@ static int reset_prng_state(
 ** Indicate that database files might be corrupt.  In other words, set the normal
 ** state of operation.
 */
-static int database_may_be_corrupt(
+static int SQLITE_TCLAPI database_may_be_corrupt(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5748,7 +6166,7 @@ static int database_may_be_corrupt(
 ** Indicate that database files are always well-formed.  This enables extra assert()
 ** statements that test conditions that are always true for well-formed databases.
 */
-static int database_never_corrupt(
+static int SQLITE_TCLAPI database_never_corrupt(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5761,7 +6179,7 @@ static int database_never_corrupt(
 /*
 ** tclcmd:  pcache_stats
 */
-static int test_pcache_stats(
+static int SQLITE_TCLAPI test_pcache_stats(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5803,7 +6221,7 @@ static void test_unlock_notify_cb(void **aArg, int nArg){
 ** tclcmd:  sqlite3_unlock_notify db
 */
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
-static int test_unlock_notify(
+static int SQLITE_TCLAPI test_unlock_notify(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5829,7 +6247,7 @@ static int test_unlock_notify(
 /*
 ** tclcmd:  sqlite3_wal_checkpoint db ?NAME?
 */
-static int test_wal_checkpoint(
+static int SQLITE_TCLAPI test_wal_checkpoint(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5873,7 +6291,7 @@ static int test_wal_checkpoint(
 ** the number of frames in the log and the number of frames in the log
 ** that have been checkpointed.
 */
-static int test_wal_checkpoint_v2(
+static int SQLITE_TCLAPI test_wal_checkpoint_v2(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5929,7 +6347,7 @@ static int test_wal_checkpoint_v2(
 /*
 ** tclcmd:  sqlite3_wal_autocheckpoint db VALUE
 */
-static int test_wal_autocheckpoint(
+static int SQLITE_TCLAPI test_wal_autocheckpoint(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -5980,7 +6398,7 @@ static void xLogcallback(void *unused, int err, char *zMsg){
   Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT);
   Tcl_DecrRefCount(pNew);
 }
-static int test_sqlite3_log(
+static int SQLITE_TCLAPI test_sqlite3_log(
   ClientData clientData,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6011,7 +6429,7 @@ static int test_sqlite3_log(
 ** Run a TCL command using its objProc interface.  Throw an error if
 ** the command has no objProc interface.
 */
-static int runAsObjProc(
+static int SQLITE_TCLAPI runAsObjProc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6074,7 +6492,7 @@ int printExplainQueryPlan(sqlite3_stmt *pStmt){
   return sqlite3_finalize(pExplain);
 }
 
-static int test_print_eqp(
+static int SQLITE_TCLAPI test_print_eqp(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6102,7 +6520,7 @@ static int test_print_eqp(
 /*
 ** sqlite3_test_control VERB ARGS...
 */
-static int test_test_control(
+static int SQLITE_TCLAPI test_test_control(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6181,7 +6599,7 @@ static int test_test_control(
 #include 
 #include 
 
-static int test_getrusage(
+static int SQLITE_TCLAPI test_getrusage(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6224,7 +6642,7 @@ struct win32FileLocker {
 /*
 ** The background thread that does file locking.
 */
-static void win32_file_locker(void *pAppData){
+static void SQLITE_CDECL win32_file_locker(void *pAppData){
   struct win32FileLocker *p = (struct win32FileLocker*)pAppData;
   if( p->evName ){
     HANDLE ev = OpenEvent(EVENT_MODIFY_STATE, FALSE, p->evName);
@@ -6255,7 +6673,7 @@ static void win32_file_locker(void *pAppData){
 ** Get an exclusive manditory lock on file for DELAY2 milliseconds.
 ** Wait DELAY1 milliseconds before acquiring the lock.
 */
-static int win32_file_lock(
+static int SQLITE_TCLAPI win32_file_lock(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6319,7 +6737,7 @@ static int win32_file_lock(
 ** Returns non-zero if the specified path exists, whose fully qualified name
 ** may exceed 260 characters if it is prefixed with "\\?\".
 */
-static int win32_exists_path(
+static int SQLITE_TCLAPI win32_exists_path(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6341,7 +6759,7 @@ static int win32_exists_path(
 ** whose fully qualified name may exceed 248 characters if it is prefixed with
 ** "\\?\".
 */
-static int win32_find_file(
+static int SQLITE_TCLAPI win32_find_file(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6386,7 +6804,7 @@ static int win32_find_file(
 ** Deletes the specified file, whose fully qualified name may exceed 260
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_delete_file(
+static int SQLITE_TCLAPI win32_delete_file(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6410,7 +6828,7 @@ static int win32_delete_file(
 ** Creates the specified directory, whose fully qualified name may exceed 248
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_mkdir(
+static int SQLITE_TCLAPI win32_mkdir(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6434,7 +6852,7 @@ static int win32_mkdir(
 ** Removes the specified directory, whose fully qualified name may exceed 248
 ** characters if it is prefixed with "\\?\".
 */
-static int win32_rmdir(
+static int SQLITE_TCLAPI win32_rmdir(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6461,7 +6879,7 @@ static int win32_rmdir(
 ** interface.  Disable if BOOLEAN is false and enable if BOOLEAN is true.
 ** OPT is the name of the optimization to be disabled.
 */
-static int optimization_control(
+static int SQLITE_TCLAPI optimization_control(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6518,20 +6936,21 @@ static int optimization_control(
   return TCL_OK;
 }
 
-typedef struct sqlite3_api_routines sqlite3_api_routines;
 /*
 **     load_static_extension DB NAME ...
 **
 ** Load one or more statically linked extensions.
 */
-static int tclLoadStaticExtensionCmd(
+static int SQLITE_TCLAPI tclLoadStaticExtensionCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
   extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_carray_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_csv_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6539,6 +6958,7 @@ static int tclLoadStaticExtensionCmd(
   extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_percentile_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*);
+  extern int sqlite3_remember_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*);
   extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*);
@@ -6548,7 +6968,9 @@ static int tclLoadStaticExtensionCmd(
     int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*);
   } aExtension[] = {
     { "amatch",                sqlite3_amatch_init               },
+    { "carray",                sqlite3_carray_init               },
     { "closure",               sqlite3_closure_init              },
+    { "csv",                   sqlite3_csv_init                  },
     { "eval",                  sqlite3_eval_init                 },
     { "fileio",                sqlite3_fileio_init               },
     { "fuzzer",                sqlite3_fuzzer_init               },
@@ -6556,6 +6978,7 @@ static int tclLoadStaticExtensionCmd(
     { "nextchar",              sqlite3_nextchar_init             },
     { "percentile",            sqlite3_percentile_init           },
     { "regexp",                sqlite3_regexp_init               },
+    { "remember",              sqlite3_remember_init             },
     { "series",                sqlite3_series_init               },
     { "spellfix",              sqlite3_spellfix_init             },
     { "totype",                sqlite3_totype_init               },
@@ -6598,7 +7021,7 @@ static int tclLoadStaticExtensionCmd(
 **     sorter_test_fakeheap BOOL
 **
 */
-static int sorter_test_fakeheap(
+static int SQLITE_TCLAPI sorter_test_fakeheap(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6638,7 +7061,7 @@ static int sorter_test_fakeheap(
 ** Then execute statement $SQL2. Check that the statement returns the same
 ** set of integers in the same order as in the previous step (using $SQL1).
 */
-static int sorter_test_sort4_helper(
+static int SQLITE_TCLAPI sorter_test_sort4_helper(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6648,8 +7071,8 @@ static int sorter_test_sort4_helper(
   const char *zSql2;
   int nStep; 
   int iStep; 
-  int iCksum1 = 0; 
-  int iCksum2 = 0; 
+  unsigned int iCksum1 = 0; 
+  unsigned int iCksum2 = 0; 
   int rc;
   int iB;
   sqlite3 *db;
@@ -6676,7 +7099,7 @@ static int sorter_test_sort4_helper(
       return TCL_ERROR;
     }
 
-    iCksum1 += (iCksum1 << 3) + a;
+    iCksum1 += (iCksum1 << 3) + (unsigned int)a;
   }
   rc = sqlite3_finalize(pStmt);
   if( rc!=SQLITE_OK ) goto sql_error;
@@ -6685,7 +7108,7 @@ static int sorter_test_sort4_helper(
   if( rc!=SQLITE_OK ) goto sql_error;
   for(iStep=0; SQLITE_ROW==sqlite3_step(pStmt); iStep++){
     int a = sqlite3_column_int(pStmt, 0);
-    iCksum2 += (iCksum2 << 3) + a;
+    iCksum2 += (iCksum2 << 3) + (unsigned int)a;
   }
   rc = sqlite3_finalize(pStmt);
   if( rc!=SQLITE_OK ) goto sql_error;
@@ -6707,7 +7130,7 @@ static int sorter_test_sort4_helper(
 /*
 ** tclcmd:  sqlite3_user_authenticate DB USERNAME PASSWORD
 */
-static int test_user_authenticate(
+static int SQLITE_TCLAPI test_user_authenticate(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6738,7 +7161,7 @@ static int test_user_authenticate(
 /*
 ** tclcmd:  sqlite3_user_add DB USERNAME PASSWORD ISADMIN
 */
-static int test_user_add(
+static int SQLITE_TCLAPI test_user_add(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6771,7 +7194,7 @@ static int test_user_add(
 /*
 ** tclcmd:  sqlite3_user_change DB USERNAME PASSWORD ISADMIN
 */
-static int test_user_change(
+static int SQLITE_TCLAPI test_user_change(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6804,7 +7227,7 @@ static int test_user_change(
 /*
 ** tclcmd:  sqlite3_user_delete DB USERNAME
 */
-static int test_user_delete(
+static int SQLITE_TCLAPI test_user_delete(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6841,7 +7264,7 @@ static int test_user_delete(
 **       3          Read after free
 **       4          Panic
 */
-static int test_bad_behavior(
+static int SQLITE_TCLAPI test_bad_behavior(
   ClientData clientData, /* Pointer to an integer containing zero */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -6891,7 +7314,7 @@ static int test_bad_behavior(
 **
 ** Cause the dbstat virtual table to be available on the connection DB
 */
-static int test_register_dbstat_vtab(
+static int SQLITE_TCLAPI test_register_dbstat_vtab(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6925,7 +7348,7 @@ static int test_register_dbstat_vtab(
 **
 ** Invoke sqlite3_db_config() for one of the setting values.
 */
-static int test_sqlite3_db_config(
+static int SQLITE_TCLAPI test_sqlite3_db_config(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
@@ -6938,6 +7361,8 @@ static int test_sqlite3_db_config(
     { "FKEY",            SQLITE_DBCONFIG_ENABLE_FKEY },
     { "TRIGGER",         SQLITE_DBCONFIG_ENABLE_TRIGGER },
     { "FTS3_TOKENIZER",  SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER },
+    { "LOAD_EXTENSION",  SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION },
+    { "NO_CKPT_ON_CLOSE",SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE },
   };
   int i;
   int v;
@@ -6967,6 +7392,29 @@ static int test_sqlite3_db_config(
   return TCL_OK;
 }
 
+/*
+** Change the name of the main database schema from "main" to "icecube".
+*/
+static int SQLITE_TCLAPI test_dbconfig_maindbname_icecube(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  int rc;
+  sqlite3 *db;
+  extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**);
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }else{
+    if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_MAINDBNAME, "icecube");
+    Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
+    return TCL_OK;
+  }
+}
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -6981,7 +7429,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
   extern int sqlite3_hostid_num;
 #endif
   extern int sqlite3_max_blobsize;
-  extern int sqlite3BtreeSharedCacheReport(void*,
+  extern int SQLITE_TCLAPI sqlite3BtreeSharedCacheReport(void*,
                                           Tcl_Interp*,int,Tcl_Obj*CONST*);
   static int iZero = 0;
   static struct {
@@ -7040,6 +7488,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "bad_behavior",                  test_bad_behavior,  (void*)&iZero },
      { "register_dbstat_vtab",          test_register_dbstat_vtab  },
      { "sqlite3_connection_pointer",    get_sqlite_pointer, 0 },
+     { "intarray_addr",                 test_intarray_addr, 0 },
+     { "int64array_addr",               test_int64array_addr, 0 },
+     { "doublearray_addr",              test_doublearray_addr, 0 },
+     { "textarray_addr",                test_textarray_addr, 0 },
      { "sqlite3_bind_int",              test_bind_int,      0 },
      { "sqlite3_bind_zeroblob",         test_bind_zeroblob, 0 },
      { "sqlite3_bind_zeroblob64",       test_bind_zeroblob64, 0 },
@@ -7076,6 +7528,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_changes",               test_changes       ,0 },
      { "sqlite3_step",                  test_step          ,0 },
      { "sqlite3_sql",                   test_sql           ,0 },
+     { "sqlite3_expanded_sql",          test_ex_sql        ,0 },
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
@@ -7084,6 +7537,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_release_memory",        test_release_memory,     0},
      { "sqlite3_db_release_memory",     test_db_release_memory,  0},
      { "sqlite3_db_cacheflush",         test_db_cacheflush,      0},
+     { "sqlite3_system_errno",          test_system_errno,       0},
      { "sqlite3_db_filename",           test_db_filename,        0},
      { "sqlite3_db_readonly",           test_db_readonly,        0},
      { "sqlite3_soft_heap_limit",       test_soft_heap_limit,    0},
@@ -7094,6 +7548,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_enable_load_extension", test_enable_load,        0},
      { "sqlite3_extended_result_codes", test_extended_result_codes, 0},
      { "sqlite3_limit",                 test_limit,                 0},
+     { "dbconfig_maindbname_icecube",   test_dbconfig_maindbname_icecube },
 
      { "save_prng_state",               save_prng_state,    0 },
      { "restore_prng_state",            restore_prng_state, 0 },
@@ -7160,6 +7615,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "file_control_sizehint_test",  file_control_sizehint_test,   0   },
 #if SQLITE_OS_WIN
      { "file_control_win32_av_retry", file_control_win32_av_retry,  0   },
+     { "file_control_win32_get_handle", file_control_win32_get_handle, 0  },
      { "file_control_win32_set_handle", file_control_win32_set_handle, 0  },
 #endif
      { "file_control_persist_wal",    file_control_persist_wal,     0   },
@@ -7223,7 +7679,13 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_snapshot_get", test_snapshot_get, 0 },
      { "sqlite3_snapshot_open", test_snapshot_open, 0 },
      { "sqlite3_snapshot_free", test_snapshot_free, 0 },
+     { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 },
+     { "sqlite3_snapshot_recover", test_snapshot_recover, 0 },
+     { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 },
+     { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 },
+     { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 },
 #endif
+     { "sqlite3_delete_database", test_delete_database, 0 },
   };
   static int bitmask_size = sizeof(Bitmask)*8;
   static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
diff --git a/src/test2.c b/src/test2.c
index d16ba5016a..850e1e1a04 100644
--- a/src/test2.c
+++ b/src/test2.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -38,7 +42,7 @@ static void pager_test_reiniter(DbPage *pNotUsed){
 **
 ** Open a new pager
 */
-static int pager_open(
+static int SQLITE_TCLAPI pager_open(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -75,7 +79,7 @@ static int pager_open(
 **
 ** Close the given pager.
 */
-static int pager_close(
+static int SQLITE_TCLAPI pager_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -89,7 +93,7 @@ static int pager_close(
     return TCL_ERROR;
   }
   pPager = sqlite3TestTextToPtr(argv[1]);
-  rc = sqlite3PagerClose(pPager);
+  rc = sqlite3PagerClose(pPager, 0);
   if( rc!=SQLITE_OK ){
     Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
     return TCL_ERROR;
@@ -102,7 +106,7 @@ static int pager_close(
 **
 ** Rollback changes
 */
-static int pager_rollback(
+static int SQLITE_TCLAPI pager_rollback(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -129,7 +133,7 @@ static int pager_rollback(
 **
 ** Commit all changes
 */
-static int pager_commit(
+static int SQLITE_TCLAPI pager_commit(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -161,7 +165,7 @@ static int pager_commit(
 **
 ** Start a new checkpoint.
 */
-static int pager_stmt_begin(
+static int SQLITE_TCLAPI pager_stmt_begin(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -188,7 +192,7 @@ static int pager_stmt_begin(
 **
 ** Rollback changes to a checkpoint
 */
-static int pager_stmt_rollback(
+static int SQLITE_TCLAPI pager_stmt_rollback(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -216,7 +220,7 @@ static int pager_stmt_rollback(
 **
 ** Commit changes to a checkpoint
 */
-static int pager_stmt_commit(
+static int SQLITE_TCLAPI pager_stmt_commit(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -243,7 +247,7 @@ static int pager_stmt_commit(
 **
 ** Return pager statistics.
 */
-static int pager_stats(
+static int SQLITE_TCLAPI pager_stats(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -276,7 +280,7 @@ static int pager_stats(
 **
 ** Return the size of the database file.
 */
-static int pager_pagecount(
+static int SQLITE_TCLAPI pager_pagecount(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -302,7 +306,7 @@ static int pager_pagecount(
 **
 ** Return a pointer to a page from the database.
 */
-static int page_get(
+static int SQLITE_TCLAPI page_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -339,7 +343,7 @@ static int page_get(
 ** Return a pointer to a page if the page is already in cache.
 ** If not in cache, return an empty string.
 */
-static int page_lookup(
+static int SQLITE_TCLAPI page_lookup(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -367,7 +371,7 @@ static int page_lookup(
 /*
 ** Usage:   pager_truncate ID PGNO
 */
-static int pager_truncate(
+static int SQLITE_TCLAPI pager_truncate(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -392,7 +396,7 @@ static int pager_truncate(
 **
 ** Drop a pointer to a page.
 */
-static int page_unref(
+static int SQLITE_TCLAPI page_unref(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -414,7 +418,7 @@ static int page_unref(
 **
 ** Return the content of a page
 */
-static int page_read(
+static int SQLITE_TCLAPI page_read(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -438,7 +442,7 @@ static int page_read(
 **
 ** Return the page number for a page.
 */
-static int page_number(
+static int SQLITE_TCLAPI page_number(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -462,7 +466,7 @@ static int page_number(
 **
 ** Write something into a page.
 */
-static int page_write(
+static int SQLITE_TCLAPI page_write(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -498,7 +502,7 @@ static int page_write(
 ** new pages after N.  If N is 2096 or bigger, this will test the
 ** ability of SQLite to write to large files.
 */
-static int fake_big_file(
+static int SQLITE_TCLAPI fake_big_file(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -551,7 +555,7 @@ static int fake_big_file(
 **
 ** Set the PENDING_BYTE using the sqlite3_test_control() interface.
 */
-static int testPendingByte(
+static int SQLITE_TCLAPI testPendingByte(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -616,7 +620,7 @@ static int faultSimCallback(int x){
 ** appended, whenever sqlite3FaultSim() is called.  Or, if SCRIPT is the
 ** empty string, cancel the sqlite3FaultSim() callback.
 */
-static int faultInstallCmd(
+static int SQLITE_TCLAPI faultInstallCmd(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -659,7 +663,7 @@ static int faultInstallCmd(
 ** Invoke the SQLITE_TESTCTRL_BITVEC_TEST operator on test_control.
 ** See comments on sqlite3BitvecBuiltinTest() for additional information.
 */
-static int testBitvecBuiltinTest(
+static int SQLITE_TCLAPI testBitvecBuiltinTest(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test3.c b/src/test3.c
index 2a41068e5f..6b4bfedbd9 100644
--- a/src/test3.c
+++ b/src/test3.c
@@ -15,7 +15,11 @@
 */
 #include "sqliteInt.h"
 #include "btreeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -33,7 +37,7 @@ static int nRefSqlite3 = 0;
 **
 ** Open a new database
 */
-static int btree_open(
+static int SQLITE_TCLAPI btree_open(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -79,7 +83,7 @@ static int btree_open(
 **
 ** Close the given database.
 */
-static int btree_close(
+static int SQLITE_TCLAPI btree_close(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -114,7 +118,7 @@ static int btree_close(
 **
 ** Start a new transaction
 */
-static int btree_begin_transaction(
+static int SQLITE_TCLAPI btree_begin_transaction(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -143,7 +147,7 @@ static int btree_begin_transaction(
 **
 ** Returns pager statistics
 */
-static int btree_pager_stats(
+static int SQLITE_TCLAPI btree_pager_stats(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -193,7 +197,7 @@ static int btree_pager_stats(
 **
 ** Create a new cursor.  Return the ID for the cursor.
 */
-static int btree_cursor(
+static int SQLITE_TCLAPI btree_cursor(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -242,14 +246,13 @@ static int btree_cursor(
 **
 ** Close a cursor opened using btree_cursor.
 */
-static int btree_close_cursor(
+static int SQLITE_TCLAPI btree_close_cursor(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  Btree *pBt;
   int rc;
 
   if( argc!=2 ){
@@ -258,12 +261,18 @@ static int btree_close_cursor(
     return TCL_ERROR;
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
-  pBt = pCur->pBtree;
-  sqlite3_mutex_enter(pBt->db->mutex);
-  sqlite3BtreeEnter(pBt);
+#if SQLITE_THREADSAFE>0
+  {
+    Btree *pBt = pCur->pBtree;
+    sqlite3_mutex_enter(pBt->db->mutex);
+    sqlite3BtreeEnter(pBt);
+    rc = sqlite3BtreeCloseCursor(pCur);
+    sqlite3BtreeLeave(pBt);
+    sqlite3_mutex_leave(pBt->db->mutex);
+  }
+#else
   rc = sqlite3BtreeCloseCursor(pCur);
-  sqlite3BtreeLeave(pBt);
-  sqlite3_mutex_leave(pBt->db->mutex);
+#endif
   ckfree((char *)pCur);
   if( rc ){
     Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
@@ -279,7 +288,7 @@ static int btree_close_cursor(
 ** or 1 if the cursor was already on the last entry in the table or if
 ** the table is empty.
 */
-static int btree_next(
+static int SQLITE_TCLAPI btree_next(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -314,7 +323,7 @@ static int btree_next(
 ** Move the cursor to the first entry in the table.  Return 0 if the
 ** cursor was left point to something and 1 if the table is empty.
 */
-static int btree_first(
+static int SQLITE_TCLAPI btree_first(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -349,7 +358,7 @@ static int btree_first(
 ** Return TRUE if the given cursor is not pointing at a valid entry.
 ** Return FALSE if the cursor does point to a valid entry.
 */
-static int btree_eof(
+static int SQLITE_TCLAPI btree_eof(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -378,15 +387,14 @@ static int btree_eof(
 **
 ** Return the number of bytes of payload
 */
-static int btree_payload_size(
+static int SQLITE_TCLAPI btree_payload_size(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
   const char **argv      /* Text of each argument */
 ){
   BtCursor *pCur;
-  int n2;
-  u64 n1;
+  u32 n;
   char zBuf[50];
 
   if( argc!=2 ){
@@ -396,17 +404,9 @@ static int btree_payload_size(
   }
   pCur = sqlite3TestTextToPtr(argv[1]);
   sqlite3BtreeEnter(pCur->pBtree);
-
-  /* The cursor may be in "require-seek" state. If this is the case, the
-  ** call to BtreeDataSize() will fix it. */
-  sqlite3BtreeDataSize(pCur, (u32*)&n2);
-  if( pCur->apPage[pCur->iPage]->intKey ){
-    n1 = 0;
-  }else{
-    sqlite3BtreeKeySize(pCur, (i64*)&n1);
-  }
+  n = sqlite3BtreePayloadSize(pCur);
   sqlite3BtreeLeave(pCur->pBtree);
-  sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", (int)(n1+n2));
+  sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n);
   Tcl_AppendResult(interp, zBuf, 0);
   return SQLITE_OK;
 }
@@ -425,7 +425,7 @@ static int btree_payload_size(
 ** This command returns nothing if it works.  It returns an error message
 ** if something goes wrong.
 */
-static int btree_varint_test(
+static int SQLITE_TCLAPI btree_varint_test(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -509,7 +509,7 @@ static int btree_varint_test(
 ** sqlite3 db test.db
 ** set bt [btree_from_db db]
 */
-static int btree_from_db(
+static int SQLITE_TCLAPI btree_from_db(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -547,9 +547,9 @@ static int btree_from_db(
 /*
 ** Usage:   btree_ismemdb ID
 **
-** Return true if the B-Tree is in-memory.
+** Return true if the B-Tree is currently stored entirely in memory.
 */
-static int btree_ismemdb(
+static int SQLITE_TCLAPI btree_ismemdb(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -557,6 +557,7 @@ static int btree_ismemdb(
 ){
   Btree *pBt;
   int res;
+  sqlite3_file *pFile;
 
   if( argc!=2 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0],
@@ -566,7 +567,8 @@ static int btree_ismemdb(
   pBt = sqlite3TestTextToPtr(argv[1]);
   sqlite3_mutex_enter(pBt->db->mutex);
   sqlite3BtreeEnter(pBt);
-  res = sqlite3PagerIsMemdb(sqlite3BtreePager(pBt));
+  pFile = sqlite3PagerFile(sqlite3BtreePager(pBt));
+  res = (pFile->pMethods==0);
   sqlite3BtreeLeave(pBt);
   sqlite3_mutex_leave(pBt->db->mutex);
   Tcl_SetObjResult(interp, Tcl_NewBooleanObj(res));
@@ -578,7 +580,7 @@ static int btree_ismemdb(
 **
 ** Set the size of the cache used by btree $ID.
 */
-static int btree_set_cache_size(
+static int SQLITE_TCLAPI btree_set_cache_size(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -608,7 +610,7 @@ static int btree_set_cache_size(
 **
 ** Set the size of the cache used by btree $ID.
 */
-static int btree_insert(
+static int SQLITE_TCLAPI btree_insert(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -616,27 +618,27 @@ static int btree_insert(
 ){
   BtCursor *pCur;
   int rc;
-  void *pKey = 0;
-  int nKey = 0;
-  void *pData = 0;
-  int nData = 0;
+  BtreePayload x;
 
   if( objc!=4 && objc!=3 ){
     Tcl_WrongNumArgs(interp, 1, objv, "?-intkey? CSR KEY VALUE");
     return TCL_ERROR;
   }
 
+  memset(&x, 0, sizeof(x));
   if( objc==4 ){
-    if( Tcl_GetIntFromObj(interp, objv[2], &nKey) ) return TCL_ERROR;
-    pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &nData);
+    if( Tcl_GetIntFromObj(interp, objv[2], &rc) ) return TCL_ERROR;
+    x.nKey = rc;
+    x.pData = (void*)Tcl_GetByteArrayFromObj(objv[3], &x.nData);
   }else{
-    pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &nKey);
+    x.pKey = (void*)Tcl_GetByteArrayFromObj(objv[2], &rc);
+    x.nKey = rc;
   }
   pCur = (BtCursor*)sqlite3TestTextToPtr(Tcl_GetString(objv[1]));
 
   sqlite3_mutex_enter(pCur->pBtree->db->mutex);
   sqlite3BtreeEnter(pCur->pBtree);
-  rc = sqlite3BtreeInsert(pCur, pKey, nKey, pData, nData, 0, 0, 0);
+  rc = sqlite3BtreeInsert(pCur, &x, 0, 0);
   sqlite3BtreeLeave(pCur->pBtree);
   sqlite3_mutex_leave(pCur->pBtree->db->mutex);
 
diff --git a/src/test4.c b/src/test4.c
index d689030303..c5b5ec12cc 100644
--- a/src/test4.c
+++ b/src/test4.c
@@ -12,7 +12,11 @@
 ** Code for testing the SQLite library in a multithreaded environment.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #if SQLITE_OS_UNIX && SQLITE_THREADSAFE
 #include 
 #include 
@@ -121,7 +125,7 @@ static int parse_thread_id(Tcl_Interp *interp, const char *zArg){
 ** NAME should be an upper case letter.  Start the thread running with
 ** an open connection to the given database.
 */
-static int tcl_thread_create(
+static int SQLITE_TCLAPI tcl_thread_create(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -170,7 +174,7 @@ static void thread_wait(Thread *p){
 **
 ** Wait on thread ID to reach its idle state.
 */
-static int tcl_thread_wait(
+static int SQLITE_TCLAPI tcl_thread_wait(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -214,7 +218,7 @@ static void stop_thread(Thread *p){
 ** Cause a thread to shut itself down.  Wait for the shutdown to be
 ** completed.  If ID is "*" then stop all threads.
 */
-static int tcl_thread_halt(
+static int SQLITE_TCLAPI tcl_thread_halt(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -249,7 +253,7 @@ static int tcl_thread_halt(
 ** Wait on the most recent thread_step to complete, then return the
 ** number of columns in the result set.
 */
-static int tcl_thread_argc(
+static int SQLITE_TCLAPI tcl_thread_argc(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -281,7 +285,7 @@ static int tcl_thread_argc(
 ** Wait on the most recent thread_step to complete, then return the
 ** value of the N-th columns in the result set.
 */
-static int tcl_thread_argv(
+static int SQLITE_TCLAPI tcl_thread_argv(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -317,7 +321,7 @@ static int tcl_thread_argv(
 ** Wait on the most recent thread_step to complete, then return the
 ** name of the N-th columns in the result set.
 */
-static int tcl_thread_colname(
+static int SQLITE_TCLAPI tcl_thread_colname(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -353,7 +357,7 @@ static int tcl_thread_colname(
 ** Wait on the most recent operation to complete, then return the
 ** result code from that operation.
 */
-static int tcl_thread_result(
+static int SQLITE_TCLAPI tcl_thread_result(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -385,7 +389,7 @@ static int tcl_thread_result(
 ** Wait on the most recent operation to complete, then return the
 ** error string.
 */
-static int tcl_thread_error(
+static int SQLITE_TCLAPI tcl_thread_error(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -430,7 +434,7 @@ static void do_compile(Thread *p){
 **
 ** Compile a new virtual machine.
 */
-static int tcl_thread_compile(
+static int SQLITE_TCLAPI tcl_thread_compile(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -483,7 +487,7 @@ static void do_step(Thread *p){
 **
 ** Advance the virtual machine by one step
 */
-static int tcl_thread_step(
+static int SQLITE_TCLAPI tcl_thread_step(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -525,7 +529,7 @@ static void do_finalize(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_thread_finalize(
+static int SQLITE_TCLAPI tcl_thread_finalize(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -556,7 +560,7 @@ static int tcl_thread_finalize(
 **
 ** Interchange the sqlite* pointer between two threads.
 */
-static int tcl_thread_swap(
+static int SQLITE_TCLAPI tcl_thread_swap(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -596,7 +600,7 @@ static int tcl_thread_swap(
 ** remove the pointer from the thread itself.  Afterwards, the thread
 ** can be stopped and the connection can be used by the main thread.
 */
-static int tcl_thread_db_get(
+static int SQLITE_TCLAPI tcl_thread_db_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -627,7 +631,7 @@ static int tcl_thread_db_get(
 ** Usage: thread_db_put ID DB
 **
 */
-static int tcl_thread_db_put(
+static int SQLITE_TCLAPI tcl_thread_db_put(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -659,7 +663,7 @@ static int tcl_thread_db_put(
 ** Return the database stmt pointer for the given thread.  Then
 ** remove the pointer from the thread itself. 
 */
-static int tcl_thread_stmt_get(
+static int SQLITE_TCLAPI tcl_thread_stmt_get(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test5.c b/src/test5.c
index 952e3325e5..0d9242862b 100644
--- a/src/test5.c
+++ b/src/test5.c
@@ -17,7 +17,11 @@
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -26,7 +30,7 @@
 ** object with the encoded representation of the string, including
 ** the NULL terminator.
 */
-static int binarize(
+static int SQLITE_TCLAPI binarize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -54,7 +58,7 @@ static int binarize(
 ** If  is 0, then the calls to sqlite3_value_text() are not
 ** actually made.
 */
-static int test_value_overhead(
+static int SQLITE_TCLAPI test_value_overhead(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -118,7 +122,7 @@ static u8 name_to_enc(Tcl_Interp *interp, Tcl_Obj *pObj){
 ** Usage:   test_translate    ??
 **
 */
-static int test_translate(
+static int SQLITE_TCLAPI test_translate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -182,7 +186,7 @@ static int test_translate(
 ** translation. If there is a problem an assert() will fail.
 **/
 void sqlite3UtfSelfTest(void);
-static int test_translate_selftest(
+static int SQLITE_TCLAPI test_translate_selftest(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test6.c b/src/test6.c
index 306482dcd3..849cdeb3b9 100644
--- a/src/test6.c
+++ b/src/test6.c
@@ -16,7 +16,11 @@
 */
 #if SQLITE_TEST          /* This file is used for testing only */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #ifndef SQLITE_OMIT_DISKIO  /* This file is a no-op if disk I/O is disabled */
 
@@ -157,13 +161,13 @@ static CrashGlobal g = {0, 0, SQLITE_DEFAULT_SECTOR_SIZE, 0, 0};
 static int sqlite3CrashTestEnable = 0;
 
 static void *crash_malloc(int nByte){
-  return (void *)Tcl_Alloc((size_t)nByte);
+  return (void *)Tcl_AttemptAlloc((size_t)nByte);
 }
 static void crash_free(void *p){
   Tcl_Free(p);
 }
 static void *crash_realloc(void *p, int n){
-  return (void *)Tcl_Realloc(p, (size_t)n);
+  return (void *)Tcl_AttemptRealloc(p, (size_t)n);
 }
 
 /*
@@ -215,7 +219,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){
   }
 
 #ifdef TRACE_CRASHTEST
-  printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  if( pFile ){
+    printf("Sync %s (is %s crash)\n", pFile->zName, (isCrash?"a":"not a"));
+  }
 #endif
 
   ppPtr = &g.pWriteList;
@@ -309,8 +315,9 @@ static int writeListSync(CrashFile *pFile, int isCrash){
         assert(pWrite->zBuf);
 
 #ifdef TRACE_CRASHTEST
-        printf("Trashing %d sectors @ %lld (sector %d) (%s)\n", 
-            1+iLast-iFirst, pWrite->iOffset, iFirst, pWrite->pFile->zName
+        printf("Trashing %d sectors (%d bytes) @ %lld (sector %d) (%s)\n", 
+            1+iLast-iFirst, (1+iLast-iFirst)*g.iSectorSize,
+            pWrite->iOffset, iFirst, pWrite->pFile->zName
         );
 #endif
 
@@ -701,6 +708,10 @@ static int cfCurrentTime(sqlite3_vfs *pCfVfs, double *pTimeOut){
   sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
   return pVfs->xCurrentTime(pVfs, pTimeOut);
 }
+static int cfGetLastError(sqlite3_vfs *pCfVfs, int n, char *z){
+  sqlite3_vfs *pVfs = (sqlite3_vfs *)pCfVfs->pAppData;
+  return pVfs->xGetLastError(pVfs, n, z);
+}
 
 static int processDevSymArgs(
   Tcl_Interp *interp,
@@ -796,18 +807,40 @@ static int processDevSymArgs(
 }
 
 /*
-** tclcmd:   sqlite_crash_enable ENABLE
+** tclcmd:   sqlite3_crash_now
+**
+** Simulate a crash immediately. This function does not return 
+** (writeListSync() calls exit(-1)).
+*/
+static int SQLITE_TCLAPI crashNowCmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+  writeListSync(0, 1);
+  assert( 0 );
+  return TCL_OK;
+}
+
+/*
+** tclcmd:   sqlite_crash_enable ENABLE ?DEFAULT?
 **
 ** Parameter ENABLE must be a boolean value. If true, then the "crash"
 ** vfs is added to the system. If false, it is removed.
 */
-static int crashEnableCmd(
+static int SQLITE_TCLAPI crashEnableCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
   int isEnable;
+  int isDefault = 0;
   static sqlite3_vfs crashVfs = {
     2,                  /* iVersion */
     0,                  /* szOsFile */
@@ -827,18 +860,21 @@ static int crashEnableCmd(
     cfRandomness,         /* xRandomness */
     cfSleep,              /* xSleep */
     cfCurrentTime,        /* xCurrentTime */
-    0,                    /* xGetlastError */
+    cfGetLastError,       /* xGetLastError */
     0,                    /* xCurrentTimeInt64 */
   };
 
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "ENABLE");
+  if( objc!=2 && objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "ENABLE ?DEFAULT?");
     return TCL_ERROR;
   }
 
   if( Tcl_GetBooleanFromObj(interp, objv[1], &isEnable) ){
     return TCL_ERROR;
   }
+  if( objc==3 && Tcl_GetBooleanFromObj(interp, objv[2], &isDefault) ){
+    return TCL_ERROR;
+  }
 
   if( (isEnable && crashVfs.pAppData) || (!isEnable && !crashVfs.pAppData) ){
     return TCL_OK;
@@ -849,7 +885,7 @@ static int crashEnableCmd(
     crashVfs.mxPathname = pOriginalVfs->mxPathname;
     crashVfs.pAppData = (void *)pOriginalVfs;
     crashVfs.szOsFile = sizeof(CrashFile) + pOriginalVfs->szOsFile;
-    sqlite3_vfs_register(&crashVfs, 0);
+    sqlite3_vfs_register(&crashVfs, isDefault);
   }else{
     crashVfs.pAppData = 0;
     sqlite3_vfs_unregister(&crashVfs);
@@ -876,7 +912,7 @@ static int crashEnableCmd(
 **   sqlite_crashparams -sect 1024 -char {atomic sequential} ./test.db 1
 **
 */
-static int crashParamsObjCmd(
+static int SQLITE_TCLAPI crashParamsObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -923,7 +959,7 @@ error:
   return TCL_ERROR;
 }
 
-static int devSymObjCmd(
+static int SQLITE_TCLAPI devSymObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -940,12 +976,33 @@ static int devSymObjCmd(
   devsym_register(iDc, iSectorSize);
 
   return TCL_OK;
+
+}
+
+/*
+** tclcmd: unregister_devsim
+*/
+static int SQLITE_TCLAPI dsUnregisterObjCmd(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  void devsym_unregister(void);
+
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+
+  devsym_unregister();
+  return TCL_OK;
 }
 
 /*
 ** tclcmd: register_jt_vfs ?-default? PARENT-VFS
 */
-static int jtObjCmd(
+static int SQLITE_TCLAPI jtObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -983,7 +1040,7 @@ static int jtObjCmd(
 /*
 ** tclcmd: unregister_jt_vfs
 */
-static int jtUnregisterObjCmd(
+static int SQLITE_TCLAPI jtUnregisterObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1009,7 +1066,9 @@ int Sqlitetest6_Init(Tcl_Interp *interp){
 #ifndef SQLITE_OMIT_DISKIO
   Tcl_CreateObjCommand(interp, "sqlite3_crash_enable", crashEnableCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_crashparams", crashParamsObjCmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "sqlite3_crash_now", crashNowCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "sqlite3_simulate_device", devSymObjCmd, 0, 0);
+  Tcl_CreateObjCommand(interp, "unregister_devsim", dsUnregisterObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "register_jt_vfs", jtObjCmd, 0, 0);
   Tcl_CreateObjCommand(interp, "unregister_jt_vfs", jtUnregisterObjCmd, 0, 0);
 #endif
diff --git a/src/test7.c b/src/test7.c
index 6ba3631b41..d57e4b826b 100644
--- a/src/test7.c
+++ b/src/test7.c
@@ -13,7 +13,11 @@
 ** Derived from test4.c.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /*
 ** This test only works on UNIX with a SQLITE_THREADSAFE build that includes
@@ -149,7 +153,7 @@ static int parse_client_id(Tcl_Interp *interp, const char *zArg){
 ** NAME should be an upper case letter.  Start the thread running with
 ** an open connection to the given database.
 */
-static int tcl_client_create(
+static int SQLITE_TCLAPI tcl_client_create(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -202,7 +206,7 @@ static void client_wait(Thread *p){
 **
 ** Wait on thread ID to reach its idle state.
 */
-static int tcl_client_wait(
+static int SQLITE_TCLAPI tcl_client_wait(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -246,7 +250,7 @@ static void stop_thread(Thread *p){
 ** Cause a client thread to shut itself down.  Wait for the shutdown to be
 ** completed.  If ID is "*" then stop all client threads.
 */
-static int tcl_client_halt(
+static int SQLITE_TCLAPI tcl_client_halt(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -294,7 +298,7 @@ static int tcl_client_halt(
 ** Wait on the most recent client_step to complete, then return the
 ** number of columns in the result set.
 */
-static int tcl_client_argc(
+static int SQLITE_TCLAPI tcl_client_argc(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -326,7 +330,7 @@ static int tcl_client_argc(
 ** Wait on the most recent client_step to complete, then return the
 ** value of the N-th columns in the result set.
 */
-static int tcl_client_argv(
+static int SQLITE_TCLAPI tcl_client_argv(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -362,7 +366,7 @@ static int tcl_client_argv(
 ** Wait on the most recent client_step to complete, then return the
 ** name of the N-th columns in the result set.
 */
-static int tcl_client_colname(
+static int SQLITE_TCLAPI tcl_client_colname(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -400,7 +404,7 @@ extern const char *sqlite3ErrName(int);
 ** Wait on the most recent operation to complete, then return the
 ** result code from that operation.
 */
-static int tcl_client_result(
+static int SQLITE_TCLAPI tcl_client_result(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -432,7 +436,7 @@ static int tcl_client_result(
 ** Wait on the most recent operation to complete, then return the
 ** error string.
 */
-static int tcl_client_error(
+static int SQLITE_TCLAPI tcl_client_error(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -477,7 +481,7 @@ static void do_compile(Thread *p){
 **
 ** Compile a new virtual machine.
 */
-static int tcl_client_compile(
+static int SQLITE_TCLAPI tcl_client_compile(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -530,7 +534,7 @@ static void do_step(Thread *p){
 **
 ** Advance the virtual machine by one step
 */
-static int tcl_client_step(
+static int SQLITE_TCLAPI tcl_client_step(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -572,7 +576,7 @@ static void do_finalize(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_client_finalize(
+static int SQLITE_TCLAPI tcl_client_finalize(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -616,7 +620,7 @@ static void do_reset(Thread *p){
 **
 ** Finalize the virtual machine.
 */
-static int tcl_client_reset(
+static int SQLITE_TCLAPI tcl_client_reset(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
@@ -647,7 +651,7 @@ static int tcl_client_reset(
 **
 ** Interchange the sqlite* pointer between two threads.
 */
-static int tcl_client_swap(
+static int SQLITE_TCLAPI tcl_client_swap(
   void *NotUsed,
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int argc,              /* Number of arguments */
diff --git a/src/test8.c b/src/test8.c
index fb781ac8fd..daab504e4e 100644
--- a/src/test8.c
+++ b/src/test8.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -1353,7 +1357,7 @@ static void moduleDestroy(void *p){
 /*
 ** Register the echo virtual table module.
 */
-static int register_echo_module(
+static int SQLITE_TCLAPI register_echo_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -1393,7 +1397,7 @@ static int register_echo_module(
 **
 ** sqlite3_declare_vtab DB SQL
 */
-static int declare_vtab(
+static int SQLITE_TCLAPI declare_vtab(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test9.c b/src/test9.c
index e5993e8ff7..5b139e8a56 100644
--- a/src/test9.c
+++ b/src/test9.c
@@ -15,14 +15,18 @@
 ** as there is not much point in binding to Tcl.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
 /*
 ** c_collation_test
 */
-static int c_collation_test(
+static int SQLITE_TCLAPI c_collation_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -63,7 +67,7 @@ error_out:
 /*
 ** c_realloc_test
 */
-static int c_realloc_test(
+static int SQLITE_TCLAPI c_realloc_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -104,7 +108,7 @@ error_out:
 /*
 ** c_misuse_test
 */
-static int c_misuse_test(
+static int SQLITE_TCLAPI c_misuse_test(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_async.c b/src/test_async.c
index b0b943185b..c32c74c660 100644
--- a/src/test_async.c
+++ b/src/test_async.c
@@ -15,7 +15,14 @@
 */
 
 #define TCL_THREADS 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifdef SQLITE_ENABLE_ASYNCIO
 
@@ -36,7 +43,7 @@ TCL_DECLARE_MUTEX(testasync_g_writerMutex);
 /*
 ** sqlite3async_initialize PARENT-VFS ISDEFAULT
 */
-static int testAsyncInit(
+static int SQLITE_TCLAPI testAsyncInit(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -69,7 +76,7 @@ static int testAsyncInit(
 /*
 ** sqlite3async_shutdown
 */
-static int testAsyncShutdown(
+static int SQLITE_TCLAPI testAsyncShutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -93,7 +100,7 @@ static Tcl_ThreadCreateType tclWriterThread(ClientData pIsStarted){
 **
 ** Start a new writer thread.
 */
-static int testAsyncStart(
+static int SQLITE_TCLAPI testAsyncStart(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -125,7 +132,7 @@ static int testAsyncStart(
 ** If the current writer thread is set to run forever then this
 ** command would block forever.  To prevent that, an error is returned. 
 */
-static int testAsyncWait(
+static int SQLITE_TCLAPI testAsyncWait(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -151,7 +158,7 @@ static int testAsyncWait(
 /*
 ** sqlite3async_control OPTION ?VALUE?
 */
-static int testAsyncControl(
+static int SQLITE_TCLAPI testAsyncControl(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_autoext.c b/src/test_autoext.c
index a5236d2390..e23e41a08a 100644
--- a/src/test_autoext.c
+++ b/src/test_autoext.c
@@ -11,7 +11,14 @@
 *************************************************************************
 ** Test extension for testing the sqlite3_auto_extension() function.
 */
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include "sqlite3ext.h"
 
 #ifndef SQLITE_OMIT_LOAD_EXTENSION
@@ -87,13 +94,13 @@ static int broken_init(
 **
 ** Register the "sqr" extension to be loaded automatically.
 */
-static int autoExtSqrObjCmd(
+static int SQLITE_TCLAPI autoExtSqrObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)sqr_init);
+  int rc = sqlite3_auto_extension((void(*)(void))sqr_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -103,13 +110,13 @@ static int autoExtSqrObjCmd(
 **
 ** Unregister the "sqr" extension.
 */
-static int cancelAutoExtSqrObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtSqrObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)sqr_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))sqr_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -119,13 +126,13 @@ static int cancelAutoExtSqrObjCmd(
 **
 ** Register the "cube" extension to be loaded automatically.
 */
-static int autoExtCubeObjCmd(
+static int SQLITE_TCLAPI autoExtCubeObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)cube_init);
+  int rc = sqlite3_auto_extension((void(*)(void))cube_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -135,13 +142,13 @@ static int autoExtCubeObjCmd(
 **
 ** Unregister the "cube" extension.
 */
-static int cancelAutoExtCubeObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtCubeObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)cube_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))cube_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -151,13 +158,13 @@ static int cancelAutoExtCubeObjCmd(
 **
 ** Register the broken extension to be loaded automatically.
 */
-static int autoExtBrokenObjCmd(
+static int SQLITE_TCLAPI autoExtBrokenObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_auto_extension((void*)broken_init);
+  int rc = sqlite3_auto_extension((void(*)(void))broken_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -167,13 +174,13 @@ static int autoExtBrokenObjCmd(
 **
 ** Unregister the broken extension.
 */
-static int cancelAutoExtBrokenObjCmd(
+static int SQLITE_TCLAPI cancelAutoExtBrokenObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  int rc = sqlite3_cancel_auto_extension((void*)broken_init);
+  int rc = sqlite3_cancel_auto_extension((void(*)(void))broken_init);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return SQLITE_OK;
 }
@@ -186,7 +193,7 @@ static int cancelAutoExtBrokenObjCmd(
 **
 ** Reset all auto-extensions
 */
-static int resetAutoExtObjCmd(
+static int SQLITE_TCLAPI resetAutoExtObjCmd(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_backup.c b/src/test_backup.c
index 6b4d6b9b1c..9b684a28f6 100644
--- a/src/test_backup.c
+++ b/src/test_backup.c
@@ -13,7 +13,14 @@
 **
 */
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include "sqlite3.h"
 #include 
 
@@ -23,7 +30,7 @@ extern const char *sqlite3ErrName(int);
 /* These functions are implemented in test1.c. */
 extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **);
 
-static int backupTestCmd(
+static int SQLITE_TCLAPI backupTestCmd(
   ClientData clientData, 
   Tcl_Interp *interp, 
   int objc,
@@ -98,7 +105,7 @@ static int backupTestCmd(
   return TCL_OK;
 }
 
-static void backupTestFinish(ClientData clientData){
+static void SQLITE_TCLAPI backupTestFinish(ClientData clientData){
   sqlite3_backup *pBackup = (sqlite3_backup *)clientData;
   sqlite3_backup_finish(pBackup);
 }
@@ -107,7 +114,7 @@ static void backupTestFinish(ClientData clientData){
 **     sqlite3_backup CMDNAME DESTHANDLE DESTNAME SRCHANDLE SRCNAME
 **
 */
-static int backupTestInit(
+static int SQLITE_TCLAPI backupTestInit(
   ClientData clientData, 
   Tcl_Interp *interp, 
   int objc,
diff --git a/src/test_bestindex.c b/src/test_bestindex.c
index a955c2782c..94017f0349 100644
--- a/src/test_bestindex.c
+++ b/src/test_bestindex.c
@@ -93,7 +93,11 @@
 
 
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
@@ -179,8 +183,8 @@ static int tclConnect(
     return SQLITE_ERROR;
   }
 
-  zCmd = sqlite3_malloc(strlen(argv[3])+1);
-  pTab = (tcl_vtab*)sqlite3_malloc(sizeof(tcl_vtab));
+  zCmd = sqlite3_malloc64(strlen(argv[3])+1);
+  pTab = (tcl_vtab*)sqlite3_malloc64(sizeof(tcl_vtab));
   if( zCmd && pTab ){
     memcpy(zCmd, argv[3], strlen(argv[3])+1);
     tclDequote(zCmd);
@@ -563,7 +567,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the echo virtual table module.
 */
-static int register_tcl_module(
+static int SQLITE_TCLAPI register_tcl_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_blob.c b/src/test_blob.c
index ec2227bb39..118f210738 100644
--- a/src/test_blob.c
+++ b/src/test_blob.c
@@ -12,7 +12,11 @@
 **
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -95,7 +99,7 @@ static char *blobStringFromObj(Tcl_Obj *pObj){
 **
 ** Tcl test harness for the sqlite3_blob_open() function.
 */
-static int test_blob_open(
+static int SQLITE_TCLAPI test_blob_open(
   ClientData clientData,          /* Not used */
   Tcl_Interp *interp,             /* Calling TCL interpreter */
   int objc,                       /* Number of arguments */
@@ -110,7 +114,7 @@ static int test_blob_open(
   const char *zVarname;
   int nVarname;
 
-  sqlite3_blob *pBlob = (sqlite3_blob*)0xFFFFFFFF;
+  sqlite3_blob *pBlob = (sqlite3_blob*)&flags;   /* Non-zero initialization */
   int rc;
 
   if( objc!=8 ){
@@ -146,7 +150,7 @@ static int test_blob_open(
 /*
 ** sqlite3_blob_close  HANDLE
 */
-static int test_blob_close(
+static int SQLITE_TCLAPI test_blob_close(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -174,7 +178,7 @@ static int test_blob_close(
 /*
 ** sqlite3_blob_bytes  HANDLE
 */
-static int test_blob_bytes(
+static int SQLITE_TCLAPI test_blob_bytes(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -210,7 +214,7 @@ static int test_blob_bytes(
 **   text representation of the returned error code (i.e. "SQLITE_NOMEM")
 **   and a Tcl exception is thrown.
 */
-static int test_blob_read(
+static int SQLITE_TCLAPI test_blob_read(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -235,7 +239,11 @@ static int test_blob_read(
   }
 
   if( nByte>0 ){
-    zBuf = (unsigned char *)Tcl_Alloc(nByte);
+    zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte);
+    if( zBuf==0 ){
+      Tcl_AppendResult(interp, "out of memory", 0);
+      return TCL_ERROR;
+    }
   }
   rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset);
   if( rc==SQLITE_OK ){
@@ -262,7 +270,7 @@ static int test_blob_read(
 **   result is set to the text representation of the returned error code 
 **   (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown.
 */
-static int test_blob_write(
+static int SQLITE_TCLAPI test_blob_write(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_btree.c b/src/test_btree.c
index dfe77051b6..03b8b207c9 100644
--- a/src/test_btree.c
+++ b/src/test_btree.c
@@ -14,7 +14,11 @@
 ** testing of the SQLite library.
 */
 #include "btreeInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /*
 ** Usage: sqlite3_shared_cache_report
@@ -22,7 +26,7 @@
 ** Return a list of file that are shared and the number of
 ** references to each file.
 */
-int sqlite3BtreeSharedCacheReport(
+int SQLITE_TCLAPI sqlite3BtreeSharedCacheReport(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_config.c b/src/test_config.c
index 30b421e00b..a25b0b3525 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -24,7 +24,11 @@
 #  include "os_win.h"
 #endif
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -79,6 +83,13 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "debug", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC
+  Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", 
+              SQLITE_DEFAULT_CKPTFULLFSYNC ? "1" : "0", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_DIRECT_OVERFLOW_READ
   Tcl_SetVar2(interp, "sqlite_options", "direct_read", "1", TCL_GLOBAL_ONLY);
 #else
@@ -97,6 +108,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS
+  Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","0",TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","1",TCL_GLOBAL_ONLY);
+#endif
+
 #if SQLITE_MAX_MMAP_SIZE>0
   Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY);
 #else
@@ -143,6 +160,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "mem5", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  Tcl_SetVar2(interp, "sqlite_options", "preupdate", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "preupdate", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_SNAPSHOT
   Tcl_SetVar2(interp, "sqlite_options", "snapshot", "1", TCL_GLOBAL_ONLY);
 #else
@@ -245,7 +268,7 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "between_opt", "1", TCL_GLOBAL_ONLY);
 #endif
 
-#ifdef SQLITE_OMIT_BUILTIN_TEST
+#ifdef SQLITE_UNTESTABLE
   Tcl_SetVar2(interp, "sqlite_options", "builtin_test", "0", TCL_GLOBAL_ONLY);
 #else
   Tcl_SetVar2(interp, "sqlite_options", "builtin_test", "1", TCL_GLOBAL_ONLY);
@@ -521,6 +544,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "schema_version", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_SESSION
+  Tcl_SetVar2(interp, "sqlite_options", "session", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "session", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_STAT4
   Tcl_SetVar2(interp, "sqlite_options", "stat4", "1", TCL_GLOBAL_ONLY);
 #else
@@ -576,7 +605,11 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
 #endif
 
   Tcl_SetVar2(interp, "sqlite_options", "threadsafe", 
-      STRINGVALUE(SQLITE_THREADSAFE), TCL_GLOBAL_ONLY);
+      SQLITE_THREADSAFE ? "1" : "0", TCL_GLOBAL_ONLY);
+  Tcl_SetVar2(interp, "sqlite_options", "threadsafe1", 
+      SQLITE_THREADSAFE==1 ? "1" : "0", TCL_GLOBAL_ONLY);
+  Tcl_SetVar2(interp, "sqlite_options", "threadsafe2", 
+      SQLITE_THREADSAFE==2 ? "1" : "0", TCL_GLOBAL_ONLY);
   assert( sqlite3_threadsafe()==SQLITE_THREADSAFE );
 
 #ifdef SQLITE_OMIT_TEMPDB
@@ -681,6 +714,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "sqllog", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_URI_00_ERROR
+  Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #define LINKVAR(x) { \
     static const int cv_ ## x = SQLITE_ ## x; \
     Tcl_LinkVar(interp, "SQLITE_" #x, (char *)&(cv_ ## x), \
@@ -701,6 +740,8 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   LINKVAR( DEFAULT_CACHE_SIZE );
   LINKVAR( DEFAULT_PAGE_SIZE );
   LINKVAR( DEFAULT_FILE_FORMAT );
+  LINKVAR( DEFAULT_SYNCHRONOUS );
+  LINKVAR( DEFAULT_WAL_SYNCHRONOUS );
   LINKVAR( MAX_ATTACHED );
   LINKVAR( MAX_DEFAULT_PAGE_SIZE );
   LINKVAR( MAX_WORKER_THREADS );
diff --git a/src/test_delete.c b/src/test_delete.c
new file mode 100644
index 0000000000..ca61965b27
--- /dev/null
+++ b/src/test_delete.c
@@ -0,0 +1,138 @@
+/*
+** 2016 September 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file contains test code to delete an SQLite database and all
+** of its associated files. Associated files include:
+**
+**   * The journal file.
+**   * The wal file.
+**   * The SQLITE_ENABLE_8_3_NAMES version of the db, journal or wal files.
+**   * Files created by the test_multiplex.c module to extend any of the 
+**     above.
+*/
+
+#if SQLITE_OS_WIN
+#  include 
+#  define F_OK 0
+#else
+#  include 
+#endif
+#include 
+#include 
+#include "sqlite3.h"
+
+/* The following #defines are copied from test_multiplex.c */
+#ifndef MX_CHUNK_NUMBER 
+# define MX_CHUNK_NUMBER 299
+#endif
+#ifndef SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET
+# define SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET 400
+#endif
+#ifndef SQLITE_MULTIPLEX_WAL_8_3_OFFSET
+# define SQLITE_MULTIPLEX_WAL_8_3_OFFSET 700
+#endif
+
+/*
+** This routine is a copy of (most of) the code from SQLite function
+** sqlite3FileSuffix3(). It modifies the filename in buffer z in the
+** same way as SQLite does when in 8.3 filenames mode.
+*/
+static void sqlite3Delete83Name(char *z){
+  int i, sz;
+  sz = (int)strlen(z);
+  for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
+  if( z[i]=='.' && (sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4);
+}
+
+/*
+** zFile is a filename. Assuming no error occurs, if this file exists, 
+** set *pbExists to true and unlink it. Or, if the file does not exist,
+** set *pbExists to false before returning.
+**
+** If an error occurs, the value of errno is returned. Or, if no error
+** occurs, zero is returned.
+*/
+static int sqlite3DeleteUnlinkIfExists(const char *zFile, int *pbExists){
+  int rc;
+  rc = access(zFile, F_OK);
+  if( rc ){
+    if( errno==ENOENT ){ 
+      if( pbExists ) *pbExists = 0;
+      return 0; 
+    }
+    return errno;
+  }
+  if( pbExists ) *pbExists = 1;
+  rc = unlink(zFile);
+  if( rc ) return errno;
+  return 0;
+}
+
+/*
+** Delete the database file identified by the string argument passed to this
+** function. The string must contain a filename, not an SQLite URI.
+*/
+SQLITE_API int sqlite3_delete_database(
+  const char *zFile               /* File to delete */
+){
+  char *zBuf;                     /* Buffer to sprintf() filenames to */
+  int nBuf;                       /* Size of buffer in bytes */
+  int rc = 0;                     /* System error code */
+  int i;                          /* Iterate through azFmt[] and aMFile[] */
+
+  const char *azFmt[] = { "%s", "%s-journal", "%s-wal", "%s-shm" };
+
+  struct MFile {
+    const char *zFmt;
+    int iOffset;
+    int b83;
+  } aMFile[] = {
+    { "%s%03d",         0,   0 },
+    { "%s-journal%03d", 0,   0 },
+    { "%s-wal%03d",     0,   0 },
+    { "%s%03d",         0,   1 },
+    { "%s-journal%03d", SQLITE_MULTIPLEX_JOURNAL_8_3_OFFSET, 1 },
+    { "%s-wal%03d",     SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 },
+  };
+
+  /* Allocate a buffer large enough for any of the files that need to be
+  ** deleted.  */
+  nBuf = (int)strlen(zFile) + 100;
+  zBuf = (char*)sqlite3_malloc(nBuf);
+  if( zBuf==0 ) return SQLITE_NOMEM;
+
+  /* Delete both the regular and 8.3 filenames versions of the database,
+  ** journal, wal and shm files.  */
+  for(i=0; rc==0 && izFmt, zFile, iChunk+p->iOffset);
+      if( p->b83 ) sqlite3Delete83Name(zBuf);
+      rc = sqlite3DeleteUnlinkIfExists(zBuf, &bExists);
+      if( bExists==0 || rc!=0 ) break;
+    }
+  }
+
+  sqlite3_free(zBuf);
+  return (rc ? SQLITE_ERROR : SQLITE_OK);
+}
diff --git a/src/test_demovfs.c b/src/test_demovfs.c
index 9410a309a6..ba7af863cd 100644
--- a/src/test_demovfs.c
+++ b/src/test_demovfs.c
@@ -641,10 +641,17 @@ sqlite3_vfs *sqlite3_demovfs(void){
 
 #ifdef SQLITE_TEST
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #if SQLITE_OS_UNIX
-static int register_demovfs(
+static int SQLITE_TCLAPI register_demovfs(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -653,7 +660,7 @@ static int register_demovfs(
   sqlite3_vfs_register(sqlite3_demovfs(), 1);
   return TCL_OK;
 }
-static int unregister_demovfs(
+static int SQLITE_TCLAPI unregister_demovfs(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_devsym.c b/src/test_devsym.c
index 21f0f684d8..9a1ba09d69 100644
--- a/src/test_devsym.c
+++ b/src/test_devsym.c
@@ -133,7 +133,8 @@ struct DevsymGlobal g = {0, 0, 512};
 */
 static int devsymClose(sqlite3_file *pFile){
   devsym_file *p = (devsym_file *)pFile;
-  return sqlite3OsClose(p->pReal);
+  sqlite3OsClose(p->pReal);
+  return SQLITE_OK;
 }
 
 /*
@@ -395,4 +396,11 @@ void devsym_register(int iDeviceChar, int iSectorSize){
   }
 }
 
+void devsym_unregister(){
+  sqlite3_vfs_unregister(&devsym_vfs);
+  g.pVfs = 0;
+  g.iDeviceChar = 0;
+  g.iSectorSize = 0;
+}
+
 #endif
diff --git a/src/test_fs.c b/src/test_fs.c
index 32026d0f38..dd590a6623 100644
--- a/src/test_fs.c
+++ b/src/test_fs.c
@@ -62,7 +62,11 @@
 **   SELECT * FROM fstree WHERE path LIKE '/home/dan/sqlite/%'
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #include 
 #include 
@@ -495,9 +499,13 @@ static int fstreeFilter(
   char aWild[2] = { '\0', '\0' };
 
 #if SQLITE_OS_WIN
-  zRoot = sqlite3_mprintf("%s%c", getenv("SystemDrive"), '/');
+  const char *zDrive = windirent_getenv("fstreeDrive");
+  if( zDrive==0 ){
+    zDrive = windirent_getenv("SystemDrive");
+  }
+  zRoot = sqlite3_mprintf("%s%c", zDrive, '/');
   nRoot = sqlite3Strlen30(zRoot);
-  zPrefix = sqlite3_mprintf("%s", getenv("SystemDrive"));
+  zPrefix = sqlite3_mprintf("%s", zDrive);
   nPrefix = sqlite3Strlen30(zPrefix);
 #else
   zRoot = "/";
@@ -871,7 +879,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the echo virtual table module.
 */
-static int register_fs_module(
+static int SQLITE_TCLAPI register_fs_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_func.c b/src/test_func.c
index 579177b70c..c7860fe887 100644
--- a/src/test_func.c
+++ b/src/test_func.c
@@ -13,7 +13,11 @@
 ** implements new SQL functions used by the test scripts.
 */
 #include "sqlite3.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -21,7 +25,6 @@
 #include "sqliteInt.h"
 #include "vdbeInt.h"
 
-
 /*
 ** Allocate nByte bytes of space using sqlite3_malloc(). If the
 ** allocation fails, call sqlite3_result_error_nomem() to notify
@@ -152,7 +155,7 @@ static void test_destructor_count(
 ** arguments. It returns the text value returned by the sqlite3_errmsg16()
 ** API function.
 */
-#ifndef SQLITE_OMIT_BUILTIN_TEST
+#ifndef SQLITE_UNTESTABLE
 void sqlite3BeginBenignMalloc(void);
 void sqlite3EndBenignMalloc(void);
 #else
@@ -640,7 +643,11 @@ static void test_setsubtype(
   sqlite3_result_subtype(context, (unsigned int)sqlite3_value_int(argv[1]));
 }
 
-static int registerTestFunctions(sqlite3 *db){
+static int registerTestFunctions(
+  sqlite3 *db,
+  char **pzErrMsg,
+  const sqlite3_api_routines *pThunk
+){
   static const struct {
      char *zName;
      signed char nArg;
@@ -689,16 +696,16 @@ static int registerTestFunctions(sqlite3 *db){
 ** the standard set of test functions to be loaded into each new
 ** database connection.
 */
-static int autoinstall_test_funcs(
+static int SQLITE_TCLAPI autoinstall_test_funcs(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
   Tcl_Obj *CONST objv[]
 ){
-  extern int Md5_Register(sqlite3*);
-  int rc = sqlite3_auto_extension((void*)registerTestFunctions);
+  extern int Md5_Register(sqlite3 *, char **, const sqlite3_api_routines *);
+  int rc = sqlite3_auto_extension((void(*)(void))registerTestFunctions);
   if( rc==SQLITE_OK ){
-    rc = sqlite3_auto_extension((void*)Md5_Register);
+    rc = sqlite3_auto_extension((void(*)(void))Md5_Register);
   }
   Tcl_SetObjResult(interp, Tcl_NewIntObj(rc));
   return TCL_OK;
@@ -717,7 +724,7 @@ static void tFinal(sqlite3_context *a){}
 ** Make various calls to sqlite3_create_function that do not have valid
 ** parameters.  Verify that the error condition is detected and reported.
 */
-static int abuse_create_function(
+static int SQLITE_TCLAPI abuse_create_function(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -783,6 +790,7 @@ abuse_err:
   return TCL_ERROR;
 }
 
+
 /*
 ** Register commands with the TCL interpreter.
 */
@@ -795,13 +803,13 @@ int Sqlitetest_func_Init(Tcl_Interp *interp){
      { "abuse_create_function",         abuse_create_function  },
   };
   int i;
-  extern int Md5_Register(sqlite3*);
+  extern int Md5_Register(sqlite3 *, char **, const sqlite3_api_routines *);
 
   for(i=0; i
 #include 
 #include 
@@ -94,7 +98,7 @@ int sqlite3TestHexToBin(const unsigned char *zIn, int N, unsigned char *aOut){
 ** beginning of the file.  Convert that information to hexadecimal
 ** and return the resulting HEX string.
 */
-static int hexio_read(
+static int SQLITE_TCLAPI hexio_read(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -144,7 +148,7 @@ static int hexio_read(
 ** Write DATA into file FILENAME beginning at OFFSET from the
 ** beginning of the file.  DATA is expressed in hexadecimal.
 */
-static int hexio_write(
+static int SQLITE_TCLAPI hexio_write(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -192,7 +196,7 @@ static int hexio_write(
 ** the value of that integer.  HEXDATA can contain between 2 and 8
 ** hexadecimal digits.
 */
-static int hexio_get_int(
+static int SQLITE_TCLAPI hexio_get_int(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -232,7 +236,7 @@ static int hexio_get_int(
 **
 ** Render INTEGER has a 16-bit big-endian integer in hexadecimal.
 */
-static int hexio_render_int16(
+static int SQLITE_TCLAPI hexio_render_int16(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -259,7 +263,7 @@ static int hexio_render_int16(
 **
 ** Render INTEGER has a 32-bit big-endian integer in hexadecimal.
 */
-static int hexio_render_int32(
+static int SQLITE_TCLAPI hexio_render_int32(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -289,7 +293,7 @@ static int hexio_render_int32(
 ** The UTF8 might not be well-formed.  Run this string through
 ** sqlite3Utf8to8() convert it back to hex and return the result.
 */
-static int utf8_to_utf8(
+static int SQLITE_TCLAPI utf8_to_utf8(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -340,7 +344,7 @@ static int getFts3Varint(const char *p, sqlite_int64 *v){
 ** Read a varint from the start of BLOB. Set variable VARNAME to contain
 ** the interpreted value. Return the number of bytes of BLOB consumed.
 */
-static int read_fts3varint(
+static int SQLITE_TCLAPI read_fts3varint(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_init.c b/src/test_init.c
index 502d95c052..58465785d8 100644
--- a/src/test_init.c
+++ b/src/test_init.c
@@ -27,7 +27,11 @@
 
 #include "sqliteInt.h"
 #include 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 static struct Wrapped {
   sqlite3_pcache_methods2 pcache;
@@ -184,7 +188,7 @@ static void installInitWrappers(void){
   sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcachemethods);
 }
 
-static int init_wrapper_install(
+static int SQLITE_TCLAPI init_wrapper_install(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -208,7 +212,7 @@ static int init_wrapper_install(
   return TCL_OK;
 }
 
-static int init_wrapper_uninstall(
+static int SQLITE_TCLAPI init_wrapper_uninstall(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -226,7 +230,7 @@ static int init_wrapper_uninstall(
   return TCL_OK;
 }
 
-static int init_wrapper_clear(
+static int SQLITE_TCLAPI init_wrapper_clear(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -243,7 +247,7 @@ static int init_wrapper_clear(
   return TCL_OK;
 }
 
-static int init_wrapper_query(
+static int SQLITE_TCLAPI init_wrapper_query(
   ClientData clientData, /* Unused */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_intarray.c b/src/test_intarray.c
index 70e34db3e1..ddfb13fe20 100644
--- a/src/test_intarray.c
+++ b/src/test_intarray.c
@@ -270,7 +270,14 @@ SQLITE_API int sqlite3_intarray_bind(
 ** Everything below is interface for testing this module.
 */
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 /*
 ** Routines to encode and decode pointers
@@ -286,7 +293,7 @@ extern const char *sqlite3ErrName(int);
 ** Invoke the sqlite3_intarray_create interface.  A string that becomes
 ** the first parameter to sqlite3_intarray_bind.
 */
-static int test_intarray_create(
+static int SQLITE_TCLAPI test_intarray_create(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
@@ -322,7 +329,7 @@ static int test_intarray_create(
 **
 ** Invoke the sqlite3_intarray_bind interface on the given array of integers.
 */
-static int test_intarray_bind(
+static int SQLITE_TCLAPI test_intarray_bind(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_intarray.h b/src/test_intarray.h
index 84b1f3fe66..cee55b5dae 100644
--- a/src/test_intarray.h
+++ b/src/test_intarray.h
@@ -75,8 +75,8 @@
 ** action to free the intarray objects.
 */
 #include "sqlite3.h"
-#ifndef _INTARRAY_H_
-#define _INTARRAY_H_
+#ifndef SQLITE_INTARRAY_H
+#define SQLITE_INTARRAY_H
 
 /*
 ** Make sure we can call this stuff from C++.
@@ -125,4 +125,4 @@ SQLITE_API int sqlite3_intarray_bind(
 #ifdef __cplusplus
 }  /* End of the 'extern "C"' block */
 #endif
-#endif /* _INTARRAY_H_ */
+#endif /* SQLITE_INTARRAY_H */
diff --git a/src/test_journal.c b/src/test_journal.c
index 6e320b7abb..8a449e888c 100644
--- a/src/test_journal.c
+++ b/src/test_journal.c
@@ -160,6 +160,7 @@ static int jtRandomness(sqlite3_vfs*, int nByte, char *zOut);
 static int jtSleep(sqlite3_vfs*, int microseconds);
 static int jtCurrentTime(sqlite3_vfs*, double*);
 static int jtCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int jtGetLastError(sqlite3_vfs*, int, char*);
 
 static sqlite3_vfs jt_vfs = {
   2,                             /* iVersion */
@@ -179,7 +180,7 @@ static sqlite3_vfs jt_vfs = {
   jtRandomness,                  /* xRandomness */
   jtSleep,                       /* xSleep */
   jtCurrentTime,                 /* xCurrentTime */
-  0,                             /* xGetLastError */
+  jtGetLastError,                /* xGetLastError */
   jtCurrentTimeInt64             /* xCurrentTimeInt64 */
 };
 
@@ -256,7 +257,8 @@ static int jtClose(sqlite3_file *pFile){
     *pp = p->pNext;
   }
   leaveJtMutex();
-  return sqlite3OsClose(p->pReal);
+  sqlite3OsClose(p->pReal);
+  return SQLITE_OK;
 }
 
 /*
@@ -284,9 +286,10 @@ static int jtRead(
 **   b) The file-name specified when the file was opened matches
 **      all but the final 8 characters of the journal file name.
 **
-**   c) There is currently a reserved lock on the file.
+**   c) There is currently a reserved lock on the file. This 
+**      condition is waived if the noLock argument is non-zero.
 **/
-static jt_file *locateDatabaseHandle(const char *zJournal){
+static jt_file *locateDatabaseHandle(const char *zJournal, int noLock){
   jt_file *pMain = 0;
   enterJtMutex();
   for(pMain=g.pList; pMain; pMain=pMain->pNext){
@@ -294,7 +297,7 @@ static jt_file *locateDatabaseHandle(const char *zJournal){
     if( (pMain->flags&SQLITE_OPEN_MAIN_DB)
      && ((int)strlen(pMain->zName)==nName)
      && 0==memcmp(pMain->zName, zJournal, nName)
-     && (pMain->eLock>=SQLITE_LOCK_RESERVED)
+     && ((pMain->eLock>=SQLITE_LOCK_RESERVED) || noLock)
     ){
       break;
     }
@@ -516,7 +519,7 @@ static int jtWrite(
   jt_file *p = (jt_file *)pFile;
   if( p->flags&SQLITE_OPEN_MAIN_JOURNAL ){
     if( iOfst==0 ){
-      jt_file *pMain = locateDatabaseHandle(p->zName);
+      jt_file *pMain = locateDatabaseHandle(p->zName, 0);
       assert( pMain );
   
       if( iAmt==28 ){
@@ -554,6 +557,9 @@ static int jtWrite(
       u32 pgno = (u32)(iOfst/p->nPagesize + 1);
       assert( (iAmt==1||iAmt==(int)p->nPagesize) &&
               ((iOfst+iAmt)%p->nPagesize)==0 );
+      /* The following assert() statements may fail if this layer is used
+      ** with a connection in "PRAGMA synchronous=off" mode. If they
+      ** fail with sync=normal or sync=full, this may indicate problem.  */
       assert( pgno<=p->nPage || p->nSync>0 );
       assert( pgno>p->nPage || sqlite3BitvecTest(p->pWritable, pgno) );
     }
@@ -561,7 +567,7 @@ static int jtWrite(
 
   rc = sqlite3OsWrite(p->pReal, zBuf, iAmt, iOfst);
   if( (p->flags&SQLITE_OPEN_MAIN_JOURNAL) && iAmt==12 ){
-    jt_file *pMain = locateDatabaseHandle(p->zName);
+    jt_file *pMain = locateDatabaseHandle(p->zName, 0);
     int rc2 = readJournalFile(p, pMain);
     if( rc==SQLITE_OK ) rc = rc2;
   }
@@ -575,7 +581,7 @@ static int jtTruncate(sqlite3_file *pFile, sqlite_int64 size){
   jt_file *p = (jt_file *)pFile;
   if( p->flags&SQLITE_OPEN_MAIN_JOURNAL && size==0 ){
     /* Truncating a journal file. This is the end of a transaction. */
-    jt_file *pMain = locateDatabaseHandle(p->zName);
+    jt_file *pMain = locateDatabaseHandle(p->zName, 0);
     closeTransaction(pMain);
   }
   if( p->flags&SQLITE_OPEN_MAIN_DB && p->pWritable ){
@@ -603,11 +609,10 @@ static int jtSync(sqlite3_file *pFile, int flags){
     ** jt_file.pWritable bitvec of the main database file associated with
     ** this journal file.
     */
-    pMain = locateDatabaseHandle(p->zName);
-    assert(pMain);
+    pMain = locateDatabaseHandle(p->zName, 0);
 
     /* Set the bitvec values */
-    if( pMain->pWritable ){
+    if( pMain && pMain->pWritable ){
       pMain->nSync++;
       rc = readJournalFile(p, pMain);
       if( rc!=SQLITE_OK ){
@@ -729,7 +734,7 @@ static int jtDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   int nPath = (int)strlen(zPath);
   if( nPath>8 && 0==strcmp("-journal", &zPath[nPath-8]) ){
     /* Deleting a journal file. The end of a transaction. */
-    jt_file *pMain = locateDatabaseHandle(zPath);
+    jt_file *pMain = locateDatabaseHandle(zPath, 0);
     if( pMain ){
       closeTransaction(pMain);
     }
@@ -824,6 +829,10 @@ static int jtCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
   return g.pVfs->xCurrentTimeInt64(g.pVfs, pTimeOut);
 }
 
+static int jtGetLastError(sqlite3_vfs *pVfs, int n, char *z){
+  return g.pVfs->xGetLastError(g.pVfs, n, z);
+}
+
 /**************************************************************************
 ** Start of public API.
 */
diff --git a/src/test_malloc.c b/src/test_malloc.c
index aaa640b03a..e8c248f958 100644
--- a/src/test_malloc.c
+++ b/src/test_malloc.c
@@ -14,7 +14,11 @@
 ** memory allocation subsystem.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -306,7 +310,7 @@ static int textToPointer(const char *z, void **pp){
 **
 ** Raw test interface for sqlite3_malloc().
 */
-static int test_malloc(
+static int SQLITE_TCLAPI test_malloc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -331,7 +335,7 @@ static int test_malloc(
 **
 ** Raw test interface for sqlite3_realloc().
 */
-static int test_realloc(
+static int SQLITE_TCLAPI test_realloc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -360,7 +364,7 @@ static int test_realloc(
 **
 ** Raw test interface for sqlite3_free().
 */
-static int test_free(
+static int SQLITE_TCLAPI test_free(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -391,7 +395,7 @@ int sqlite3TestBinToHex(char*,int);
 ** Set a chunk of memory (obtained from malloc, probably) to a
 ** specified hex pattern.
 */
-static int test_memset(
+static int SQLITE_TCLAPI test_memset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -437,7 +441,7 @@ static int test_memset(
 **
 ** Return memory as hexadecimal text.
 */
-static int test_memget(
+static int SQLITE_TCLAPI test_memget(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -484,7 +488,7 @@ static int test_memget(
 **
 ** Raw test interface for sqlite3_memory_used().
 */
-static int test_memory_used(
+static int SQLITE_TCLAPI test_memory_used(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -499,7 +503,7 @@ static int test_memory_used(
 **
 ** Raw test interface for sqlite3_memory_highwater().
 */
-static int test_memory_highwater(
+static int SQLITE_TCLAPI test_memory_highwater(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -524,7 +528,7 @@ static int test_memory_highwater(
 ** Set the depth of backtracing.  If SQLITE_MEMDEBUG is not defined
 ** then this routine is a no-op.
 */
-static int test_memdebug_backtrace(
+static int SQLITE_TCLAPI test_memdebug_backtrace(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -550,7 +554,7 @@ static int test_memdebug_backtrace(
 **
 ** Write a summary of unfreed memory to FILENAME.
 */
-static int test_memdebug_dump(
+static int SQLITE_TCLAPI test_memdebug_dump(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -575,7 +579,7 @@ static int test_memdebug_dump(
 **
 ** Return the total number of times malloc() has been called.
 */
-static int test_memdebug_malloc_count(
+static int SQLITE_TCLAPI test_memdebug_malloc_count(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -615,7 +619,7 @@ static int test_memdebug_malloc_count(
 **
 ** To disable simulated failures, use a COUNTER of -1.
 */
-static int test_memdebug_fail(
+static int SQLITE_TCLAPI test_memdebug_fail(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -681,7 +685,7 @@ static int test_memdebug_fail(
 ** simulated failure occurs. A negative return value indicates that
 ** no malloc() failure is scheduled.
 */
-static int test_memdebug_pending(
+static int SQLITE_TCLAPI test_memdebug_pending(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -714,7 +718,7 @@ static int sqlite3_memdebug_title_count = 0;
 **
 ** Each title overwrite the previous.
 */
-static int test_memdebug_settitle(
+static int SQLITE_TCLAPI test_memdebug_settitle(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -795,7 +799,7 @@ static void test_memdebug_log_clear(void){
   Tcl_InitHashTable(&aMallocLog, MALLOC_LOG_KEYINTS);
 }
 
-static int test_memdebug_log(
+static int SQLITE_TCLAPI test_memdebug_log(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -893,7 +897,7 @@ static int test_memdebug_log(
 **
 ** A negative SIZE causes the buffer pointer to be NULL.
 */
-static int test_config_scratch(
+static int SQLITE_TCLAPI test_config_scratch(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -933,7 +937,7 @@ static int test_config_scratch(
 **
 ** A negative SIZE causes the buffer pointer to be NULL.
 */
-static int test_config_pagecache(
+static int SQLITE_TCLAPI test_config_pagecache(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -976,7 +980,7 @@ static int test_config_pagecache(
 ** is certainty.  0 is never.  PRNG_SEED is the pseudo-random number generator
 ** seed.
 */
-static int test_alt_pcache(
+static int SQLITE_TCLAPI test_alt_pcache(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1017,7 +1021,7 @@ static int test_alt_pcache(
 **
 ** Enable or disable memory status reporting using SQLITE_CONFIG_MEMSTATUS.
 */
-static int test_config_memstatus(
+static int SQLITE_TCLAPI test_config_memstatus(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1038,7 +1042,7 @@ static int test_config_memstatus(
 ** Usage:    sqlite3_config_lookaside  SIZE  COUNT
 **
 */
-static int test_config_lookaside(
+static int SQLITE_TCLAPI test_config_lookaside(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1072,7 +1076,7 @@ static int test_config_lookaside(
 ** is 10KB in size.  A BUFID of 0 indicates that the buffer should be NULL
 ** which will cause sqlite3_db_config() to allocate space on its own.
 */
-static int test_db_config_lookaside(
+static int SQLITE_TCLAPI test_db_config_lookaside(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1093,7 +1097,7 @@ static int test_db_config_lookaside(
   if( Tcl_GetIntFromObj(interp, objv[3], &sz) ) return TCL_ERROR;
   if( Tcl_GetIntFromObj(interp, objv[4], &cnt) ) return TCL_ERROR;
   if( bufid==0 ){
-    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, 0, sz, cnt);
+    rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, (void*)0, sz, cnt);
   }else if( bufid>=1 && bufid<=2 && sz*cnt<=sizeof(azBuf[0]) ){
     rc = sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE, azBuf[bufid], sz,cnt);
   }else{
@@ -1107,7 +1111,7 @@ static int test_db_config_lookaside(
 /*
 ** Usage:    sqlite3_config_heap NBYTE NMINALLOC
 */
-static int test_config_heap(
+static int SQLITE_TCLAPI test_config_heap(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1144,7 +1148,7 @@ static int test_config_heap(
 /*
 ** Usage:    sqlite3_config_heap_size NBYTE
 */
-static int test_config_heap_size(
+static int SQLITE_TCLAPI test_config_heap_size(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1174,7 +1178,7 @@ static int test_config_heap_size(
 ** Invoke sqlite3_config() or sqlite3_db_config() with invalid
 ** opcodes and verify that they return errors.
 */
-static int test_config_error(
+static int SQLITE_TCLAPI test_config_error(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1212,7 +1216,7 @@ static int test_config_error(
 ** Enables or disables interpretation of URI parameters by default using
 ** SQLITE_CONFIG_URI.
 */
-static int test_config_uri(
+static int SQLITE_TCLAPI test_config_uri(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1241,7 +1245,7 @@ static int test_config_uri(
 ** Enables or disables the use of the covering-index scan optimization.
 ** SQLITE_CONFIG_COVERING_INDEX_SCAN.
 */
-static int test_config_cis(
+static int SQLITE_TCLAPI test_config_cis(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1269,7 +1273,7 @@ static int test_config_cis(
 **
 ** Set the minimum PMA size.
 */
-static int test_config_pmasz(
+static int SQLITE_TCLAPI test_config_pmasz(
   void * clientData, 
   Tcl_Interp *interp,
   int objc,
@@ -1299,7 +1303,7 @@ static int test_config_pmasz(
 **
 ** Write a summary of unfreed memsys3 allocations to FILENAME.
 */
-static int test_dump_memsys3(
+static int SQLITE_TCLAPI test_dump_memsys3(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1335,7 +1339,7 @@ static int test_dump_memsys3(
 ** Return a list of three elements which are the sqlite3_status() return
 ** code, the current value, and the high-water mark value.
 */
-static int test_status(
+static int SQLITE_TCLAPI test_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1392,7 +1396,7 @@ static int test_status(
 ** Return a list of three elements which are the sqlite3_db_status() return
 ** code, the current value, and the high-water mark value.
 */
-static int test_db_status(
+static int SQLITE_TCLAPI test_db_status(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1417,7 +1421,8 @@ static int test_db_status(
     { "CACHE_HIT",           SQLITE_DBSTATUS_CACHE_HIT           },
     { "CACHE_MISS",          SQLITE_DBSTATUS_CACHE_MISS          },
     { "CACHE_WRITE",         SQLITE_DBSTATUS_CACHE_WRITE         },
-    { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        }
+    { "DEFERRED_FKS",        SQLITE_DBSTATUS_DEFERRED_FKS        },
+    { "CACHE_USED_SHARED",   SQLITE_DBSTATUS_CACHE_USED_SHARED   },
   };
   Tcl_Obj *pResult;
   if( objc!=4 ){
@@ -1452,7 +1457,7 @@ static int test_db_status(
 /*
 ** install_malloc_faultsim BOOLEAN
 */
-static int test_install_malloc_faultsim(
+static int SQLITE_TCLAPI test_install_malloc_faultsim(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1476,7 +1481,7 @@ static int test_install_malloc_faultsim(
 /*
 ** sqlite3_install_memsys3
 */
-static int test_install_memsys3(
+static int SQLITE_TCLAPI test_install_memsys3(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1491,7 +1496,7 @@ static int test_install_memsys3(
   return TCL_OK;
 }
 
-static int test_vfs_oom_test(
+static int SQLITE_TCLAPI test_vfs_oom_test(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_multiplex.c b/src/test_multiplex.c
index 82845ea7e1..ed8c9f7fa7 100644
--- a/src/test_multiplex.c
+++ b/src/test_multiplex.c
@@ -68,19 +68,6 @@
 #define MAX_PAGE_SIZE       0x10000
 #define DEFAULT_SECTOR_SIZE 0x1000
 
-/*
-** For a build without mutexes, no-op the mutex calls.
-*/
-#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0
-#define sqlite3_mutex_alloc(X)    ((sqlite3_mutex*)8)
-#define sqlite3_mutex_free(X)
-#define sqlite3_mutex_enter(X)
-#define sqlite3_mutex_try(X)      SQLITE_OK
-#define sqlite3_mutex_leave(X)
-#define sqlite3_mutex_held(X)     ((void)(X),1)
-#define sqlite3_mutex_notheld(X)  ((void)(X),1)
-#endif /* SQLITE_THREADSAFE==0 */
-
 /* Maximum chunk number */
 #define MX_CHUNK_NUMBER 299
 
@@ -139,7 +126,6 @@ struct multiplexGroup {
   unsigned int szChunk;            /* Chunk size used for this group */
   unsigned char bEnabled;          /* TRUE to use Multiplex VFS for this file */
   unsigned char bTruncate;         /* TRUE to enable truncation of databases */
-  multiplexGroup *pNext, *pPrev;   /* Doubly linked list of all group objects */
 };
 
 /*
@@ -187,28 +173,9 @@ static struct {
   /* True when this shim has been initialized.
   */
   int isInitialized;
-
-  /* For run-time access any of the other global data structures in this
-  ** shim, the following mutex must be held. In practice, all this mutex
-  ** protects is add/remove operations to/from the linked list of group objects
-  ** starting at pGroups below. More specifically, it protects the value of
-  ** pGroups itself, and the pNext/pPrev fields of each multiplexGroup
-  ** structure.  */
-  sqlite3_mutex *pMutex;
-
-  /* List of multiplexGroup objects.
-  */
-  multiplexGroup *pGroups;
 } gMultiplex;
 
 /************************* Utility Routines *********************************/
-/*
-** Acquire and release the mutex used to serialize access to the
-** list of multiplexGroups.
-*/
-static void multiplexEnter(void){ sqlite3_mutex_enter(gMultiplex.pMutex); }
-static void multiplexLeave(void){ sqlite3_mutex_leave(gMultiplex.pMutex); }
-
 /*
 ** Compute a string length that is limited to what can be stored in
 ** lower 30 bits of a 32-bit signed integer.
@@ -519,7 +486,6 @@ static int multiplexOpen(
   /* We need to create a group structure and manage
   ** access to this group of files.
   */
-  multiplexEnter();
   pMultiplexOpen = (multiplexConn*)pConn;
 
   if( rc==SQLITE_OK ){
@@ -539,7 +505,7 @@ static int multiplexOpen(
     memset(pGroup, 0, sz);
     pMultiplexOpen->pGroup = pGroup;
     pGroup->bEnabled = (unsigned char)-1;
-    pGroup->bTruncate = sqlite3_uri_boolean(zUri, "truncate", 
+    pGroup->bTruncate = (unsigned char)sqlite3_uri_boolean(zUri, "truncate", 
                                    (flags & SQLITE_OPEN_MAIN_DB)==0);
     pGroup->szChunk = (int)sqlite3_uri_int64(zUri, "chunksize",
                                         SQLITE_MULTIPLEX_CHUNK_SIZE);
@@ -626,16 +592,11 @@ static int multiplexOpen(
       }else{
         pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV2;
       }
-      /* place this group at the head of our list */
-      pGroup->pNext = gMultiplex.pGroups;
-      if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup;
-      gMultiplex.pGroups = pGroup;
     }else{
       multiplexFreeComponents(pGroup);
       sqlite3_free(pGroup);
     }
   }
-  multiplexLeave();
   sqlite3_free(zToFree);
   return rc;
 }
@@ -717,7 +678,11 @@ static int multiplexCurrentTime(sqlite3_vfs *a, double *b){
   return gMultiplex.pOrigVfs->xCurrentTime(gMultiplex.pOrigVfs, b);
 }
 static int multiplexGetLastError(sqlite3_vfs *a, int b, char *c){
-  return gMultiplex.pOrigVfs->xGetLastError(gMultiplex.pOrigVfs, b, c);
+  if( gMultiplex.pOrigVfs->xGetLastError ){
+    return gMultiplex.pOrigVfs->xGetLastError(gMultiplex.pOrigVfs, b, c);
+  }else{
+    return 0;
+  }
 }
 static int multiplexCurrentTimeInt64(sqlite3_vfs *a, sqlite3_int64 *b){
   return gMultiplex.pOrigVfs->xCurrentTimeInt64(gMultiplex.pOrigVfs, b);
@@ -734,17 +699,8 @@ static int multiplexClose(sqlite3_file *pConn){
   multiplexConn *p = (multiplexConn*)pConn;
   multiplexGroup *pGroup = p->pGroup;
   int rc = SQLITE_OK;
-  multiplexEnter();
   multiplexFreeComponents(pGroup);
-  /* remove from linked list */
-  if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev;
-  if( pGroup->pPrev ){
-    pGroup->pPrev->pNext = pGroup->pNext;
-  }else{
-    gMultiplex.pGroups = pGroup->pNext;
-  }
   sqlite3_free(pGroup);
-  multiplexLeave();
   return rc;
 }
 
@@ -841,7 +797,6 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){
   multiplexConn *p = (multiplexConn*)pConn;
   multiplexGroup *pGroup = p->pGroup;
   int rc = SQLITE_OK;
-  multiplexEnter();
   if( !pGroup->bEnabled ){
     sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
@@ -873,7 +828,6 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){
     }
     if( rc ) rc = SQLITE_IOERR_TRUNCATE;
   }
-  multiplexLeave();
   return rc;
 }
 
@@ -884,7 +838,6 @@ static int multiplexSync(sqlite3_file *pConn, int flags){
   multiplexGroup *pGroup = p->pGroup;
   int rc = SQLITE_OK;
   int i;
-  multiplexEnter();
   for(i=0; inReal; i++){
     sqlite3_file *pSubOpen = pGroup->aReal[i].p;
     if( pSubOpen ){
@@ -892,7 +845,6 @@ static int multiplexSync(sqlite3_file *pConn, int flags){
       if( rc2!=SQLITE_OK ) rc = rc2;
     }
   }
-  multiplexLeave();
   return rc;
 }
 
@@ -904,7 +856,6 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){
   multiplexGroup *pGroup = p->pGroup;
   int rc = SQLITE_OK;
   int i;
-  multiplexEnter();
   if( !pGroup->bEnabled ){
     sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
@@ -920,7 +871,6 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){
       *pSize = i*(sqlite3_int64)pGroup->szChunk + sz;
     }
   }
-  multiplexLeave();
   return rc;
 }
 
@@ -974,7 +924,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
     case MULTIPLEX_CTRL_ENABLE:
       if( pArg ) {
         int bEnabled = *(int *)pArg;
-        pGroup->bEnabled = bEnabled;
+        pGroup->bEnabled = (unsigned char)bEnabled;
         rc = SQLITE_OK;
       }
       break;
@@ -1147,11 +1097,6 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){
   pOrigVfs = sqlite3_vfs_find(zOrigVfsName);
   if( pOrigVfs==0 ) return SQLITE_ERROR;
   assert( pOrigVfs!=&gMultiplex.sThisVfs );
-  gMultiplex.pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
-  if( !gMultiplex.pMutex ){
-    return SQLITE_NOMEM;
-  }
-  gMultiplex.pGroups = NULL;
   gMultiplex.isInitialized = 1;
   gMultiplex.pOrigVfs = pOrigVfs;
   gMultiplex.sThisVfs = *pOrigVfs;
@@ -1193,7 +1138,7 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){
   gMultiplex.sIoMethodsV2.xShmUnmap = multiplexShmUnmap;
   sqlite3_vfs_register(&gMultiplex.sThisVfs, makeDefault);
 
-  sqlite3_auto_extension((void*)multiplexFuncInit);
+  sqlite3_auto_extension((void(*)(void))multiplexFuncInit);
 
   return SQLITE_OK;
 }
@@ -1210,14 +1155,7 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){
 int sqlite3_multiplex_shutdown(int eForce){
   int rc = SQLITE_OK;
   if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE;
-  if( gMultiplex.pGroups ){
-    sqlite3_log(SQLITE_MISUSE, "sqlite3_multiplex_shutdown() called "
-                "while database connections are still open");
-    if( !eForce ) return SQLITE_MISUSE;
-    rc = SQLITE_MISUSE;
-  }
   gMultiplex.isInitialized = 0;
-  sqlite3_mutex_free(gMultiplex.pMutex);
   sqlite3_vfs_unregister(&gMultiplex.sThisVfs);
   memset(&gMultiplex, 0, sizeof(gMultiplex));
   return rc;
@@ -1225,14 +1163,21 @@ int sqlite3_multiplex_shutdown(int eForce){
 
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 extern const char *sqlite3ErrName(int);
 
 
 /*
 ** tclcmd: sqlite3_multiplex_initialize NAME MAKEDEFAULT
 */
-static int test_multiplex_initialize(
+static int SQLITE_TCLAPI test_multiplex_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1263,7 +1208,7 @@ static int test_multiplex_initialize(
 /*
 ** tclcmd: sqlite3_multiplex_shutdown
 */
-static int test_multiplex_shutdown(
+static int SQLITE_TCLAPI test_multiplex_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1288,65 +1233,10 @@ static int test_multiplex_shutdown(
   return TCL_OK;
 }
 
-/*
-** tclcmd:  sqlite3_multiplex_dump
-*/
-static int test_multiplex_dump(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  Tcl_Obj *pResult;
-  Tcl_Obj *pGroupTerm;
-  multiplexGroup *pGroup;
-  int i;
-  int nChunks = 0;
-
-  UNUSED_PARAMETER(clientData);
-  UNUSED_PARAMETER(objc);
-  UNUSED_PARAMETER(objv);
-
-  pResult = Tcl_NewObj();
-  multiplexEnter();
-  for(pGroup=gMultiplex.pGroups; pGroup; pGroup=pGroup->pNext){
-    pGroupTerm = Tcl_NewObj();
-
-    if( pGroup->zName ){
-      pGroup->zName[pGroup->nName] = '\0';
-      Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewStringObj(pGroup->zName, -1));
-    }else{
-      Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewObj());
-    }
-    Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewIntObj(pGroup->nName));
-    Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewIntObj(pGroup->flags));
-
-    /* count number of chunks with open handles */
-    for(i=0; inReal; i++){
-      if( pGroup->aReal[i].p!=0 ) nChunks++;
-    }
-    Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewIntObj(nChunks));
-
-    Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewIntObj(pGroup->szChunk));
-    Tcl_ListObjAppendElement(interp, pGroupTerm,
-          Tcl_NewIntObj(pGroup->nReal));
-
-    Tcl_ListObjAppendElement(interp, pResult, pGroupTerm);
-  }
-  multiplexLeave();
-  Tcl_SetObjResult(interp, pResult);
-  return TCL_OK;
-}
-
 /*
 ** Tclcmd: test_multiplex_control HANDLE DBNAME SUB-COMMAND ?INT-VALUE?
 */
-static int test_multiplex_control(
+static int SQLITE_TCLAPI test_multiplex_control(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -1417,7 +1307,6 @@ int Sqlitemultiplex_Init(Tcl_Interp *interp){
   } aCmd[] = {
     { "sqlite3_multiplex_initialize", test_multiplex_initialize },
     { "sqlite3_multiplex_shutdown", test_multiplex_shutdown },
-    { "sqlite3_multiplex_dump", test_multiplex_dump },
     { "sqlite3_multiplex_control", test_multiplex_control },
   };
   int i;
diff --git a/src/test_multiplex.h b/src/test_multiplex.h
index d973e4af27..790c778a35 100644
--- a/src/test_multiplex.h
+++ b/src/test_multiplex.h
@@ -20,8 +20,8 @@
 **
 */
 
-#ifndef _TEST_MULTIPLEX_H
-#define _TEST_MULTIPLEX_H
+#ifndef SQLITE_TEST_MULTIPLEX_H
+#define SQLITE_TEST_MULTIPLEX_H
 
 /*
 ** CAPI: File-control Operations Supported by Multiplex VFS
@@ -96,4 +96,4 @@ extern int sqlite3_multiplex_shutdown(int eForce);
 }  /* End of the 'extern "C"' block */
 #endif
 
-#endif /* _TEST_MULTIPLEX_H */
+#endif /* SQLITE_TEST_MULTIPLEX_H */
diff --git a/src/test_mutex.c b/src/test_mutex.c
index 995b89a4c6..8f43e5ad30 100644
--- a/src/test_mutex.c
+++ b/src/test_mutex.c
@@ -12,7 +12,11 @@
 ** This file contains test logic for the sqlite3_mutex interfaces.
 */
 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include "sqlite3.h"
 #include "sqliteInt.h"
 #include 
@@ -152,7 +156,7 @@ static void counterMutexLeave(sqlite3_mutex *p){
 /*
 ** sqlite3_shutdown
 */
-static int test_shutdown(
+static int SQLITE_TCLAPI test_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -173,7 +177,7 @@ static int test_shutdown(
 /*
 ** sqlite3_initialize
 */
-static int test_initialize(
+static int SQLITE_TCLAPI test_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -194,7 +198,7 @@ static int test_initialize(
 /*
 ** install_mutex_counters BOOLEAN
 */
-static int test_install_mutex_counters(
+static int SQLITE_TCLAPI test_install_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -255,7 +259,7 @@ static int test_install_mutex_counters(
 /*
 ** read_mutex_counters
 */
-static int test_read_mutex_counters(
+static int SQLITE_TCLAPI test_read_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -284,7 +288,7 @@ static int test_read_mutex_counters(
 /*
 ** clear_mutex_counters
 */
-static int test_clear_mutex_counters(
+static int SQLITE_TCLAPI test_clear_mutex_counters(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -308,7 +312,7 @@ static int test_clear_mutex_counters(
 ** will be invalid since the mutex has already been freed.  The
 ** return pointer just checks to see if the mutex really was allocated.
 */
-static int test_alloc_mutex(
+static int SQLITE_TCLAPI test_alloc_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -335,7 +339,7 @@ static int test_alloc_mutex(
 **
 ** Or OPTION can be an raw integer.
 */
-static int test_config(
+static int SQLITE_TCLAPI test_config(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -397,7 +401,7 @@ static sqlite3_mutex *getStaticMutexPointer(
   return counterMutexAlloc(iMutex);
 }
 
-static int test_enter_static_mutex(
+static int SQLITE_TCLAPI test_enter_static_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -416,7 +420,7 @@ static int test_enter_static_mutex(
   return TCL_OK;
 }
 
-static int test_leave_static_mutex(
+static int SQLITE_TCLAPI test_leave_static_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -435,7 +439,7 @@ static int test_leave_static_mutex(
   return TCL_OK;
 }
 
-static int test_enter_db_mutex(
+static int SQLITE_TCLAPI test_enter_db_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -454,7 +458,7 @@ static int test_enter_db_mutex(
   return TCL_OK;
 }
 
-static int test_leave_db_mutex(
+static int SQLITE_TCLAPI test_leave_db_mutex(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_onefile.c b/src/test_onefile.c
index 122be700e6..46c8bc9c83 100644
--- a/src/test_onefile.c
+++ b/src/test_onefile.c
@@ -508,7 +508,7 @@ static int fsSync(sqlite3_file *pFile, int flags){
   if( p->eType==DATABASE_FILE ){
     unsigned char zSize[4];
     zSize[0] = (pReal->nDatabase&0xFF000000)>>24;
-    zSize[1] = (pReal->nDatabase&0x00FF0000)>>16;
+    zSize[1] = (unsigned char)((pReal->nDatabase&0x00FF0000)>>16);
     zSize[2] = (pReal->nDatabase&0x0000FF00)>>8;
     zSize[3] = (pReal->nDatabase&0x000000FF);
     rc = pRealFile->pMethods->xWrite(pRealFile, zSize, 4, 0);
diff --git a/src/test_osinst.c b/src/test_osinst.c
index 4ae23a87c6..a008baba45 100644
--- a/src/test_osinst.c
+++ b/src/test_osinst.c
@@ -644,9 +644,9 @@ static void vfslog_flush(VfslogVfs *p){
 
 static void put32bits(unsigned char *p, unsigned int v){
   p[0] = v>>24;
-  p[1] = v>>16;
-  p[2] = v>>8;
-  p[3] = v;
+  p[1] = (unsigned char)(v>>16);
+  p[2] = (unsigned char)(v>>8);
+  p[3] = (unsigned char)v;
 }
 
 static void vfslog_call(
@@ -1104,9 +1104,16 @@ int sqlite3_vfslog_register(sqlite3 *db){
 
 #if defined(SQLITE_TEST) || defined(TCLSH)
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
-static int test_vfslog(
+static int SQLITE_TCLAPI test_vfslog(
   void *clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_quota.c b/src/test_quota.c
index e8e0b34072..e87e9772f6 100644
--- a/src/test_quota.c
+++ b/src/test_quota.c
@@ -111,7 +111,7 @@ struct quotaFile {
 
 /*
 ** An instance of the following object represents each open connection
-** to a file that participates in quota tracking.  This object is a 
+** to a file that participates in quota tracking.  This object is a
 ** subclass of sqlite3_file.  The sqlite3_file object for the underlying
 ** VFS is appended to this structure.
 */
@@ -154,11 +154,11 @@ static struct {
   */
   sqlite3_vfs sThisVfs;
 
-  /* The sIoMethods defines the methods used by sqlite3_file objects 
+  /* The sIoMethods defines the methods used by sqlite3_file objects
   ** associated with this shim.  It is initialized at start-time and does
   ** not require a mutex.
   **
-  ** When the underlying VFS is called to open a file, it might return 
+  ** When the underlying VFS is called to open a file, it might return
   ** either a version 1 or a version 2 sqlite3_file object.  This shim
   ** has to create a wrapper sqlite3_file of the same version.  Hence
   ** there are two I/O method structures, one for version 1 and the other
@@ -190,7 +190,7 @@ static struct {
 static void quotaEnter(void){ sqlite3_mutex_enter(gQuota.pMutex); }
 static void quotaLeave(void){ sqlite3_mutex_leave(gQuota.pMutex); }
 
-/* Count the number of open files in a quotaGroup 
+/* Count the number of open files in a quotaGroup
 */
 static int quotaGroupOpenFileCount(quotaGroup *pGroup){
   int N = 0;
@@ -399,7 +399,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){
   return zMbcs;
 #else
   return (char*)zUtf8;  /* No-op on unix */
-#endif  
+#endif
 }
 
 /*
@@ -410,7 +410,7 @@ static void quota_mbcs_free(char *zOld){
   sqlite3_free(zOld);
 #else
   /* No-op on unix */
-#endif  
+#endif
 }
 
 /************************* VFS Method Wrappers *****************************/
@@ -428,7 +428,7 @@ static int quotaOpen(
   int flags,                  /* Flags to control the opening */
   int *pOutFlags              /* Flags showing results of opening */
 ){
-  int rc;                                    /* Result code */         
+  int rc;                                    /* Result code */
   quotaConn *pQuotaOpen;                     /* The new quota file descriptor */
   quotaFile *pFile;                          /* Corresponding quotaFile obj */
   quotaGroup *pGroup;                        /* The group file belongs to */
@@ -488,7 +488,7 @@ static int quotaDelete(
   const char *zName,          /* Name of file to be deleted */
   int syncDir                 /* Do a directory sync after deleting */
 ){
-  int rc;                                    /* Result code */         
+  int rc;                                    /* Result code */
   quotaFile *pFile;                          /* Files in the quota */
   quotaGroup *pGroup;                        /* The group file belongs to */
   sqlite3_vfs *pOrigVfs = gQuota.pOrigVfs;   /* Real VFS */
@@ -581,7 +581,7 @@ static int quotaWrite(
     szNew = pGroup->iSize - pFile->iSize + iEnd;
     if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
       if( pGroup->xCallback ){
-        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, 
+        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew,
                           pGroup->pArg);
       }
       if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
@@ -738,7 +738,7 @@ static int quotaShmUnmap(sqlite3_file *pConn, int deleteFlag){
 /*
 ** Initialize the quota VFS shim.  Use the VFS named zOrigVfsName
 ** as the VFS that does the actual work.  Use the default if
-** zOrigVfsName==NULL.  
+** zOrigVfsName==NULL.
 **
 ** The quota VFS shim is named "quota".  It will become the default
 ** VFS if makeDefault is non-zero.
@@ -908,7 +908,7 @@ int sqlite3_quota_file(const char *zFilename){
 
   if( rc==SQLITE_OK ){
     zFull[strlen(zFull)+1] = '\0';
-    rc = quotaOpen(&gQuota.sThisVfs, zFull, fd, 
+    rc = quotaOpen(&gQuota.sThisVfs, zFull, fd,
                    SQLITE_OPEN_READONLY | SQLITE_OPEN_MAIN_DB, &outFlags);
     if( rc==SQLITE_OK ){
       fd->pMethods->xFileSize(fd, &iSize);
@@ -1016,7 +1016,7 @@ size_t sqlite3_quota_fwrite(
     szNew = pGroup->iSize - pFile->iSize + iEnd;
     if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
       if( pGroup->xCallback ){
-        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew, 
+        pGroup->xCallback(pFile->zFilename, &pGroup->iLimit, szNew,
                           pGroup->pArg);
       }
       if( szNew>pGroup->iLimit && pGroup->iLimit>0 ){
@@ -1203,7 +1203,7 @@ sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){
 sqlite3_int64 sqlite3_quota_file_size(quota_FILE *p){
   return p->pFile ? p->pFile->iSize : -1;
 }
- 
+
 /*
 ** Determine the amount of data in bytes available for reading
 ** in the given file.
@@ -1275,10 +1275,17 @@ int sqlite3_quota_remove(const char *zFilename){
   sqlite3_free(zFull);
   return rc;
 }
-  
+
 /***************************** Test Code ***********************************/
 #ifdef SQLITE_TEST
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 /*
 ** Argument passed to a TCL quota-over-limit callback.
@@ -1350,7 +1357,7 @@ static void tclCallbackDestructor(void *pObj){
 /*
 ** tclcmd: sqlite3_quota_initialize NAME MAKEDEFAULT
 */
-static int test_quota_initialize(
+static int SQLITE_TCLAPI test_quota_initialize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1379,7 +1386,7 @@ static int test_quota_initialize(
 /*
 ** tclcmd: sqlite3_quota_shutdown
 */
-static int test_quota_shutdown(
+static int SQLITE_TCLAPI test_quota_shutdown(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1402,7 +1409,7 @@ static int test_quota_shutdown(
 /*
 ** tclcmd: sqlite3_quota_set PATTERN LIMIT SCRIPT
 */
-static int test_quota_set(
+static int SQLITE_TCLAPI test_quota_set(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1456,7 +1463,7 @@ static int test_quota_set(
 /*
 ** tclcmd: sqlite3_quota_file FILENAME
 */
-static int test_quota_file(
+static int SQLITE_TCLAPI test_quota_file(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1482,7 +1489,7 @@ static int test_quota_file(
 /*
 ** tclcmd:  sqlite3_quota_dump
 */
-static int test_quota_dump(
+static int SQLITE_TCLAPI test_quota_dump(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1530,7 +1537,7 @@ static int test_quota_dump(
 /*
 ** tclcmd: sqlite3_quota_fopen FILENAME MODE
 */
-static int test_quota_fopen(
+static int SQLITE_TCLAPI test_quota_fopen(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1560,7 +1567,7 @@ extern void *sqlite3TestTextToPtr(const char*);
 /*
 ** tclcmd: sqlite3_quota_fread HANDLE SIZE NELEM
 */
-static int test_quota_fread(
+static int SQLITE_TCLAPI test_quota_fread(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1594,7 +1601,7 @@ static int test_quota_fread(
 /*
 ** tclcmd: sqlite3_quota_fwrite HANDLE SIZE NELEM CONTENT
 */
-static int test_quota_fwrite(
+static int SQLITE_TCLAPI test_quota_fwrite(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1622,7 +1629,7 @@ static int test_quota_fwrite(
 /*
 ** tclcmd: sqlite3_quota_fclose HANDLE
 */
-static int test_quota_fclose(
+static int SQLITE_TCLAPI test_quota_fclose(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1644,7 +1651,7 @@ static int test_quota_fclose(
 /*
 ** tclcmd: sqlite3_quota_fflush HANDLE ?HARDSYNC?
 */
-static int test_quota_fflush(
+static int SQLITE_TCLAPI test_quota_fflush(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1670,7 +1677,7 @@ static int test_quota_fflush(
 /*
 ** tclcmd: sqlite3_quota_fseek HANDLE OFFSET WHENCE
 */
-static int test_quota_fseek(
+static int SQLITE_TCLAPI test_quota_fseek(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1708,7 +1715,7 @@ static int test_quota_fseek(
 /*
 ** tclcmd: sqlite3_quota_rewind HANDLE
 */
-static int test_quota_rewind(
+static int SQLITE_TCLAPI test_quota_rewind(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1727,7 +1734,7 @@ static int test_quota_rewind(
 /*
 ** tclcmd: sqlite3_quota_ftell HANDLE
 */
-static int test_quota_ftell(
+static int SQLITE_TCLAPI test_quota_ftell(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1748,7 +1755,7 @@ static int test_quota_ftell(
 /*
 ** tclcmd: sqlite3_quota_ftruncate HANDLE SIZE
 */
-static int test_quota_ftruncate(
+static int SQLITE_TCLAPI test_quota_ftruncate(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1773,7 +1780,7 @@ static int test_quota_ftruncate(
 /*
 ** tclcmd: sqlite3_quota_file_size HANDLE
 */
-static int test_quota_file_size(
+static int SQLITE_TCLAPI test_quota_file_size(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1794,7 +1801,7 @@ static int test_quota_file_size(
 /*
 ** tclcmd: sqlite3_quota_file_truesize HANDLE
 */
-static int test_quota_file_truesize(
+static int SQLITE_TCLAPI test_quota_file_truesize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1815,7 +1822,7 @@ static int test_quota_file_truesize(
 /*
 ** tclcmd: sqlite3_quota_file_mtime HANDLE
 */
-static int test_quota_file_mtime(
+static int SQLITE_TCLAPI test_quota_file_mtime(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1838,7 +1845,7 @@ static int test_quota_file_mtime(
 /*
 ** tclcmd: sqlite3_quota_remove FILENAME
 */
-static int test_quota_remove(
+static int SQLITE_TCLAPI test_quota_remove(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1862,7 +1869,7 @@ static int test_quota_remove(
 ** Test the glob pattern matching.  Return 1 if TEXT matches PATTERN
 ** and return 0 if it does not.
 */
-static int test_quota_glob(
+static int SQLITE_TCLAPI test_quota_glob(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1888,7 +1895,7 @@ static int test_quota_glob(
 ** Return the number of bytes from the current file point to the end of
 ** the file.
 */
-static int test_quota_file_available(
+static int SQLITE_TCLAPI test_quota_file_available(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -1911,7 +1918,7 @@ static int test_quota_file_available(
 **
 ** Return true if the file handle is in the error state.
 */
-static int test_quota_ferror(
+static int SQLITE_TCLAPI test_quota_ferror(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_rtree.c b/src/test_rtree.c
index 797ec0026c..0c6dbf3cd7 100644
--- a/src/test_rtree.c
+++ b/src/test_rtree.c
@@ -14,7 +14,11 @@
 */
 
 #include "sqlite3.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 /* Solely for the UNUSED_PARAMETER() macro. */
 #include "sqliteInt.h"
@@ -353,7 +357,11 @@ static int bfs_query_func(sqlite3_rtree_query_info *p){
 *************************************************************************/
 
 #include 
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 typedef struct Cube Cube;
 struct Cube {
@@ -432,7 +440,7 @@ static int cube_geom(
 }
 #endif /* SQLITE_ENABLE_RTREE */
 
-static int register_cube_geom(
+static int SQLITE_TCLAPI register_cube_geom(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -460,7 +468,7 @@ static int register_cube_geom(
   return TCL_OK;
 }
 
-static int register_circle_geom(
+static int SQLITE_TCLAPI register_circle_geom(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_schema.c b/src/test_schema.c
index 4ee18193b0..cdf085797f 100644
--- a/src/test_schema.c
+++ b/src/test_schema.c
@@ -35,10 +35,14 @@
 ** to be compiled into an sqlite dynamic extension.
 */
 #ifdef SQLITE_TEST
-  #include "sqliteInt.h"
-  #include "tcl.h"
+#  include "sqliteInt.h"
+#  if defined(INCLUDE_SQLITE_TCL_H)
+#    include "sqlite_tcl.h"
+#  else
+#    include "tcl.h"
+#  endif
 #else
-  #include "sqlite3ext.h"
+#  include "sqlite3ext.h"
   SQLITE_EXTENSION_INIT1
 #endif
 
@@ -302,7 +306,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the schema virtual table module.
 */
-static int register_schema_module(
+static int SQLITE_TCLAPI register_schema_module(
   ClientData clientData, /* Not used */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_sqllog.c b/src/test_sqllog.c
index 31d5ad2f5b..9b207cf07b 100644
--- a/src/test_sqllog.c
+++ b/src/test_sqllog.c
@@ -313,7 +313,7 @@ static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){
 
       /* Generate a file-name to use for the copy of this database */
       iDb = sqllogglobal.iNextDb++;
-      zInit = sqlite3_mprintf("%s_%d.db", sqllogglobal.zPrefix, iDb);
+      zInit = sqlite3_mprintf("%s_%02d.db", sqllogglobal.zPrefix, iDb);
 
       /* Create the backup */
       assert( sqllogglobal.bRec==0 );
@@ -376,7 +376,7 @@ static void sqllogOpenlog(struct SLConn *p){
       char *zVar = getenv(ENVIRONMENT_VARIABLE1_NAME);
       if( zVar==0 || strlen(zVar)+10>=(sizeof(sqllogglobal.zPrefix)) ) return;
       sqlite3_snprintf(sizeof(sqllogglobal.zPrefix), sqllogglobal.zPrefix,
-                        "%s/sqllog_%d", zVar, getProcessId());
+                        "%s/sqllog_%05d", zVar, getProcessId());
       sqlite3_snprintf(sizeof(sqllogglobal.zIdx), sqllogglobal.zIdx,
                         "%s.idx", sqllogglobal.zPrefix);
       if( getenv(ENVIRONMENT_VARIABLE2_NAME) ){
@@ -387,7 +387,7 @@ static void sqllogOpenlog(struct SLConn *p){
     }
 
     /* Open the log file */
-    zLog = sqlite3_mprintf("%s_%d.sql", sqllogglobal.zPrefix, p->iLog);
+    zLog = sqlite3_mprintf("%s_%05d.sql", sqllogglobal.zPrefix, p->iLog);
     p->fd = fopen(zLog, "w");
     sqlite3_free(zLog);
     if( p->fd==0 ){
diff --git a/src/test_superlock.c b/src/test_superlock.c
index cac789842d..45d0d623a0 100644
--- a/src/test_superlock.c
+++ b/src/test_superlock.c
@@ -256,7 +256,14 @@ int sqlite3demo_superlock(
 
 #ifdef SQLITE_TEST
 
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 struct InterpAndScript {
   Tcl_Interp *interp;
@@ -264,11 +271,11 @@ struct InterpAndScript {
 };
 typedef struct InterpAndScript InterpAndScript;
 
-static void superunlock_del(ClientData cd){
+static void SQLITE_TCLAPI superunlock_del(ClientData cd){
   sqlite3demo_superunlock((void *)cd);
 }
 
-static int superunlock_cmd(
+static int SQLITE_TCLAPI superunlock_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -300,7 +307,7 @@ static int superlock_busy(void *pCtx, int nBusy){
 /*
 ** Tclcmd: sqlite3demo_superlock CMDNAME PATH VFS BUSY-HANDLER-SCRIPT
 */
-static int superlock_cmd(
+static int SQLITE_TCLAPI superlock_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_syscall.c b/src/test_syscall.c
index 0dac2e897e..947f9a9d9d 100644
--- a/src/test_syscall.c
+++ b/src/test_syscall.c
@@ -76,7 +76,11 @@
 
 #include "sqliteInt.h"
 #include "sqlite3.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 #include 
@@ -108,10 +112,13 @@ static int ts_ftruncate(int fd, off_t n);
 static int ts_fcntl(int fd, int cmd, ... );
 static int ts_read(int fd, void *aBuf, size_t nBuf);
 static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off);
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off);
+/* Note:  pread64() and pwrite64() actually use off64_t as the type on their
+** last parameter.  But that datatype is not defined on many systems 
+** (ex: Mac, OpenBSD).  So substitute a likely equivalent: sqlite3_uint64 */
+static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off);
 static int ts_write(int fd, const void *aBuf, size_t nBuf);
 static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off);
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off);
+static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off);
 static int ts_fchmod(int fd, mode_t mode);
 static int ts_fallocate(int fd, off_t off, off_t len);
 static void *ts_mmap(void *, size_t, int, int, int, off_t);
@@ -155,11 +162,11 @@ struct TestSyscallArray {
 #define orig_fcntl     ((int(*)(int,int,...))aSyscall[7].xOrig)
 #define orig_read      ((ssize_t(*)(int,void*,size_t))aSyscall[8].xOrig)
 #define orig_pread     ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].xOrig)
-#define orig_pread64   ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].xOrig)
+#define orig_pread64   ((ssize_t(*)(int,void*,size_t,sqlite3_uint64))aSyscall[10].xOrig)
 #define orig_write     ((ssize_t(*)(int,const void*,size_t))aSyscall[11].xOrig)
 #define orig_pwrite    ((ssize_t(*)(int,const void*,size_t,off_t))\
                        aSyscall[12].xOrig)
-#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,off_t))\
+#define orig_pwrite64  ((ssize_t(*)(int,const void*,size_t,sqlite3_uint64))\
                        aSyscall[13].xOrig)
 #define orig_fchmod    ((int(*)(int,mode_t))aSyscall[14].xOrig)
 #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig)
@@ -326,7 +333,7 @@ static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pread64().
 */
-static int ts_pread64(int fd, void *aBuf, size_t nBuf, off_t off){
+static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pread64") ){
     return -1;
   }
@@ -357,7 +364,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){
 /*
 ** A wrapper around pwrite64().
 */
-static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off){
+static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){
   if( tsIsFailErrno("pwrite64") ){
     return -1;
   }
@@ -415,7 +422,7 @@ static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){
   return orig_mremap(a, b, c, d, pArg);
 }
 
-static int test_syscall_install(
+static int SQLITE_TCLAPI test_syscall_install(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -451,7 +458,7 @@ static int test_syscall_install(
   return TCL_OK;
 }
 
-static int test_syscall_uninstall(
+static int SQLITE_TCLAPI test_syscall_uninstall(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -475,7 +482,7 @@ static int test_syscall_uninstall(
   return TCL_OK;
 }
 
-static int test_syscall_reset(
+static int SQLITE_TCLAPI test_syscall_reset(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -513,7 +520,7 @@ static int test_syscall_reset(
   return TCL_OK;
 }
 
-static int test_syscall_exists(
+static int SQLITE_TCLAPI test_syscall_exists(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -534,7 +541,7 @@ static int test_syscall_exists(
   return TCL_OK;
 }
 
-static int test_syscall_fault(
+static int SQLITE_TCLAPI test_syscall_fault(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -563,7 +570,7 @@ static int test_syscall_fault(
   return TCL_OK;
 }
 
-static int test_syscall_errno(
+static int SQLITE_TCLAPI test_syscall_errno(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -609,7 +616,7 @@ static int test_syscall_errno(
   return TCL_OK;
 }
 
-static int test_syscall_list(
+static int SQLITE_TCLAPI test_syscall_list(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -639,7 +646,7 @@ static int test_syscall_list(
   return TCL_OK;
 }
 
-static int test_syscall_defaultvfs(
+static int SQLITE_TCLAPI test_syscall_defaultvfs(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -661,7 +668,7 @@ static int ts_getpagesize(void){
   return gSyscall.pgsz;
 }
 
-static int test_syscall_pagesize(
+static int SQLITE_TCLAPI test_syscall_pagesize(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -696,7 +703,7 @@ static int test_syscall_pagesize(
   return TCL_OK;
 }
 
-static int test_syscall(
+static int SQLITE_TCLAPI test_syscall(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -719,14 +726,20 @@ static int test_syscall(
   };
   int iCmd;
   int rc;
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
 
   if( objc<2 ){
     Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ...");
     return TCL_ERROR;
   }
-  rc = Tcl_GetIndexFromObjStruct(interp, 
-      objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd
-  );
+  if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){
+    Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0);
+    rc = TCL_ERROR;
+  }else{
+    rc = Tcl_GetIndexFromObjStruct(interp, 
+        objv[1], aCmd, sizeof(aCmd[0]), "sub-command", 0, &iCmd
+    );
+  }
   if( rc!=TCL_OK ) return rc;
   return aCmd[iCmd].xCmd(clientData, interp, objc, objv);
 }
diff --git a/src/test_tclvar.c b/src/test_tclvar.c
index 63ed394734..8f7352112d 100644
--- a/src/test_tclvar.c
+++ b/src/test_tclvar.c
@@ -17,7 +17,11 @@
 ** access to TCL variables.
 */
 #include "sqliteInt.h"
-#include "tcl.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include 
 #include 
 
@@ -407,7 +411,7 @@ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb);
 /*
 ** Register the echo virtual table module.
 */
-static int register_tclvar_module(
+static int SQLITE_TCLAPI register_tclvar_module(
   ClientData clientData, /* Pointer to sqlite3_enable_XXX function */
   Tcl_Interp *interp,    /* The TCL interpreter that invoked this command */
   int objc,              /* Number of arguments */
diff --git a/src/test_thread.c b/src/test_thread.c
index a4d96e1942..20b4cf148b 100644
--- a/src/test_thread.c
+++ b/src/test_thread.c
@@ -16,7 +16,11 @@
 */
 
 #include "sqliteInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 #if SQLITE_THREADSAFE
 
@@ -72,7 +76,7 @@ extern int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int);
 /*
 ** Handler for events of type EvalEvent.
 */
-static int tclScriptEvent(Tcl_Event *evPtr, int flags){
+static int SQLITE_TCLAPI tclScriptEvent(Tcl_Event *evPtr, int flags){
   int rc;
   EvalEvent *p = (EvalEvent *)evPtr;
   rc = Tcl_Eval(p->interp, p->zScript);
@@ -167,7 +171,7 @@ static Tcl_ThreadCreateType tclScriptThread(ClientData pSqlThread){
 **
 **     The caller can wait for the script to terminate using [vwait VARNAME].
 */
-static int sqlthread_spawn(
+static int SQLITE_TCLAPI sqlthread_spawn(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -220,7 +224,7 @@ static int sqlthread_spawn(
 **
 **     NOTE: At the moment, this doesn't work. FIXME.
 */
-static int sqlthread_parent(
+static int SQLITE_TCLAPI sqlthread_parent(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -265,7 +269,7 @@ static int xBusy(void *pArg, int nBusy){
 **     Open a database handle and return the string representation of
 **     the pointer value.
 */
-static int sqlthread_open(
+static int SQLITE_TCLAPI sqlthread_open(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -276,7 +280,7 @@ static int sqlthread_open(
   const char *zFilename;
   sqlite3 *db;
   char zBuf[100];
-  extern void Md5_Register(sqlite3*);
+  extern int Md5_Register(sqlite3*,char**,const sqlite3_api_routines*);
 
   UNUSED_PARAMETER(clientData);
   UNUSED_PARAMETER(objc);
@@ -299,7 +303,7 @@ static int sqlthread_open(
     }
   }
 #endif
-  Md5_Register(db);
+  Md5_Register(db, 0, 0);
   sqlite3_busy_handler(db, xBusy, 0);
   
   if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR;
@@ -315,7 +319,7 @@ static int sqlthread_open(
 **     Return the current thread-id (Tcl_GetCurrentThread()) cast to
 **     an integer.
 */
-static int sqlthread_id(
+static int SQLITE_TCLAPI sqlthread_id(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -333,7 +337,7 @@ static int sqlthread_id(
 /*
 ** Dispatch routine for the sub-commands of [sqlthread].
 */
-static int sqlthread_proc(
+static int SQLITE_TCLAPI sqlthread_proc(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -381,7 +385,7 @@ static int sqlthread_proc(
 ** implemented as a script in Tcl 8.5, it is not usually available to
 ** testfixture.
 */ 
-static int clock_seconds_proc(
+static int SQLITE_TCLAPI clock_seconds_proc(
   ClientData clientData,
   Tcl_Interp *interp,
   int objc,
@@ -543,7 +547,7 @@ int sqlite3_blocking_prepare_v2(
 **
 ** Advance the statement to the next row.
 */
-static int blocking_step_proc(
+static int SQLITE_TCLAPI blocking_step_proc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
@@ -569,7 +573,7 @@ static int blocking_step_proc(
 ** Usage: sqlite3_blocking_prepare_v2 DB sql bytes ?tailvar?
 ** Usage: sqlite3_nonblocking_prepare_v2 DB sql bytes ?tailvar?
 */
-static int blocking_prepare_v2_proc(
+static int SQLITE_TCLAPI blocking_prepare_v2_proc(
   void * clientData,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_vfs.c b/src/test_vfs.c
index a8c6ae7bf5..fb987a6163 100644
--- a/src/test_vfs.c
+++ b/src/test_vfs.c
@@ -28,7 +28,11 @@
 
 #include "sqlite3.h"
 #include "sqliteInt.h"
-#include 
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 
 typedef struct Testvfs Testvfs;
 typedef struct TestvfsShm TestvfsShm;
@@ -306,7 +310,6 @@ static void tvfsExecTcl(
 ** Close an tvfs-file.
 */
 static int tvfsClose(sqlite3_file *pFile){
-  int rc;
   TestvfsFile *pTestfile = (TestvfsFile *)pFile;
   TestvfsFd *pFd = pTestfile->pFd;
   Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
@@ -324,10 +327,10 @@ static int tvfsClose(sqlite3_file *pFile){
   if( pFile->pMethods ){
     ckfree((char *)pFile->pMethods);
   }
-  rc = sqlite3OsClose(pFd->pReal);
+  sqlite3OsClose(pFd->pReal);
   ckfree((char *)pFd);
   pTestfile->pFd = 0;
-  return rc;
+  return SQLITE_OK;
 }
 
 /*
@@ -1037,7 +1040,7 @@ static int tvfsUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *p){
   return sqlite3OsUnfetch(pFd->pReal, iOfst, p);
 }
 
-static int testvfs_obj_cmd(
+static int SQLITE_TCLAPI testvfs_obj_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
@@ -1349,7 +1352,7 @@ static int testvfs_obj_cmd(
   return TCL_OK;
 }
 
-static void testvfs_obj_del(ClientData cd){
+static void SQLITE_TCLAPI testvfs_obj_del(ClientData cd){
   Testvfs *p = (Testvfs *)cd;
   if( p->pScript ) Tcl_DecrRefCount(p->pScript);
   sqlite3_vfs_unregister(p->pVfs);
@@ -1392,7 +1395,7 @@ static void testvfs_obj_del(ClientData cd){
 **
 ** where LOCK is of the form "OFFSET NBYTE lock/unlock shared/exclusive"
 */
-static int testvfs_cmd(
+static int SQLITE_TCLAPI testvfs_cmd(
   ClientData cd,
   Tcl_Interp *interp,
   int objc,
diff --git a/src/test_windirent.c b/src/test_windirent.c
index 11d7dc07d0..ca78d345d9 100644
--- a/src/test_windirent.c
+++ b/src/test_windirent.c
@@ -17,6 +17,34 @@
 
 #include "test_windirent.h"
 
+/*
+** Implementation of the POSIX getenv() function using the Win32 API.
+** This function is not thread-safe.
+*/
+const char *windirent_getenv(
+  const char *name
+){
+  static char value[32768]; /* Maximum length, per MSDN */
+  DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */
+  DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */
+
+  memset(value, 0, sizeof(value));
+  dwRet = GetEnvironmentVariableA(name, value, dwSize);
+  if( dwRet==0 || dwRet>dwSize ){
+    /*
+    ** The function call to GetEnvironmentVariableA() failed -OR-
+    ** the buffer is not large enough.  Either way, return NULL.
+    */
+    return 0;
+  }else{
+    /*
+    ** The function call to GetEnvironmentVariableA() succeeded
+    ** -AND- the buffer contains the entire value.
+    */
+    return value;
+  }
+}
+
 /*
 ** Implementation of the POSIX opendir() function using the MSVCRT.
 */
@@ -32,9 +60,10 @@ LPDIR opendir(
 
   /* TODO: Remove this if Unix-style root paths are not used. */
   if( sqlite3_stricmp(dirname, "/")==0 ){
-    dirname = getenv("SystemDrive");
+    dirname = windirent_getenv("SystemDrive");
   }
 
+  memset(&data, 0, sizeof(struct _finddata_t));
   _snprintf(data.name, namesize, "%s\\*", dirname);
   dirp->d_handle = _findfirst(data.name, &data);
 
@@ -43,12 +72,18 @@ LPDIR opendir(
     return NULL;
   }
 
-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN || data.attrib&_A_SYSTEM ){
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ){
+next:
+
+    memset(&data, 0, sizeof(struct _finddata_t));
     if( _findnext(dirp->d_handle, &data)==-1 ){
       closedir(dirp);
       return NULL;
     }
+
+    /* TODO: Remove this block to allow hidden and/or system files. */
+    if( is_filtered(data) ) goto next;
   }
 
   dirp->d_first.d_attributes = data.attrib;
@@ -77,11 +112,11 @@ LPDIRENT readdir(
 
 next:
 
+  memset(&data, 0, sizeof(struct _finddata_t));
   if( _findnext(dirp->d_handle, &data)==-1 ) return NULL;
 
-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN ) goto next;
-  if( data.attrib&_A_SYSTEM ) goto next;
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ) goto next;
 
   dirp->d_next.d_ino++;
   dirp->d_next.d_attributes = data.attrib;
@@ -118,14 +153,14 @@ INT readdir_r(
 
 next:
 
+  memset(&data, 0, sizeof(struct _finddata_t));
   if( _findnext(dirp->d_handle, &data)==-1 ){
     *result = NULL;
     return ENOENT;
   }
 
-  /* TODO: Remove this block to allow hidden and system files. */
-  if( data.attrib&_A_HIDDEN ) goto next;
-  if( data.attrib&_A_SYSTEM ) goto next;
+  /* TODO: Remove this block to allow hidden and/or system files. */
+  if( is_filtered(data) ) goto next;
 
   entry->d_ino = (ino_t)-1; /* not available */
   entry->d_attributes = data.attrib;
diff --git a/src/test_windirent.h b/src/test_windirent.h
index 0b8d1a7b51..578e2a7c22 100644
--- a/src/test_windirent.h
+++ b/src/test_windirent.h
@@ -92,6 +92,24 @@ struct DIR {
   DIRENT d_next;     /* DIRENT constructed based on "_findnext". */
 };
 
+/*
+** Provide a macro, for use by the implementation, to determine if a
+** particular directory entry should be skipped over when searching for
+** the next directory entry that should be returned by the readdir() or
+** readdir_r() functions.
+*/
+
+#ifndef is_filtered
+#  define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM))
+#endif
+
+/*
+** Provide the function prototype for the POSIX compatiable getenv()
+** function.  This function is not thread-safe.
+*/
+
+extern const char *windirent_getenv(const char *name);
+
 /*
 ** Finally, we can provide the function prototypes for the opendir(),
 ** readdir(), readdir_r(), and closedir() POSIX functions.
diff --git a/src/tokenize.c b/src/tokenize.c
index 19a5ddf04d..7f5f298987 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -81,13 +81,13 @@ static const unsigned char aiClass[] = {
 /* 1x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
 /* 2x */   27, 27, 27, 27, 27,  7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
 /* 3x */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
-/* 4x */    7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10,
+/* 4x */    7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10,
 /* 5x */   24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15,  4, 21, 18, 19, 27,
-/* 6x */   11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22,  1, 13,  7,
+/* 6x */   11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22,  1, 13,  6,
 /* 7x */   27, 27, 27, 27, 27, 27, 27, 27, 27,  8,  5,  5,  5,  8, 14,  8,
 /* 8x */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
 /* 9x */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
-/* 9x */   25,  1,  1,  1,  1,  1,  1,  0,  1,  1, 27, 27, 27, 27, 27, 27,
+/* Ax */   27, 25,  1,  1,  1,  1,  1,  0,  1,  1, 27, 27, 27, 27, 27, 27,
 /* Bx */   27, 27, 27, 27, 27, 27, 27, 27, 27, 27,  9, 27, 27, 27, 27, 27,
 /* Cx */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
 /* Dx */   27,  1,  1,  1,  1,  1,  1,  1,  1,  1, 27, 27, 27, 27, 27, 27,
@@ -475,12 +475,15 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
 */
 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   int nErr = 0;                   /* Number of errors encountered */
-  int i;                          /* Loop counter */
   void *pEngine;                  /* The LEMON-generated LALR(1) parser */
+  int n = 0;                      /* Length of the next token token */
   int tokenType;                  /* type of the next token */
   int lastTokenParsed = -1;       /* type of the previous token */
   sqlite3 *db = pParse->db;       /* The database connection */
   int mxSqlLen;                   /* Max length of an SQL string */
+#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
+  yyParser sEngine;    /* Space to hold the Lemon-generated Parser object */
+#endif
 
   assert( zSql!=0 );
   mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH];
@@ -489,27 +492,41 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   }
   pParse->rc = SQLITE_OK;
   pParse->zTail = zSql;
-  i = 0;
   assert( pzErrMsg!=0 );
   /* sqlite3ParserTrace(stdout, "parser: "); */
+#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
+  pEngine = &sEngine;
+  sqlite3ParserInit(pEngine);
+#else
   pEngine = sqlite3ParserAlloc(sqlite3Malloc);
   if( pEngine==0 ){
     sqlite3OomFault(db);
     return SQLITE_NOMEM_BKPT;
   }
+#endif
   assert( pParse->pNewTable==0 );
   assert( pParse->pNewTrigger==0 );
   assert( pParse->nVar==0 );
-  assert( pParse->nzVar==0 );
-  assert( pParse->azVar==0 );
-  while( zSql[i]!=0 ){
-    assert( i>=0 );
-    pParse->sLastToken.z = &zSql[i];
-    pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
-    i += pParse->sLastToken.n;
-    if( i>mxSqlLen ){
-      pParse->rc = SQLITE_TOOBIG;
-      break;
+  assert( pParse->pVList==0 );
+  while( 1 ){
+    if( zSql[0]!=0 ){
+      n = sqlite3GetToken((u8*)zSql, &tokenType);
+      mxSqlLen -= n;
+      if( mxSqlLen<0 ){
+        pParse->rc = SQLITE_TOOBIG;
+        break;
+      }
+    }else{
+      /* Upon reaching the end of input, call the parser two more times
+      ** with tokens TK_SEMI and 0, in that order. */
+      if( lastTokenParsed==TK_SEMI ){
+        tokenType = 0;
+      }else if( lastTokenParsed==0 ){
+        break;
+      }else{
+        tokenType = TK_SEMI;
+      }
+      zSql -= n;
     }
     if( tokenType>=TK_SPACE ){
       assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
@@ -518,27 +535,21 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
         break;
       }
       if( tokenType==TK_ILLEGAL ){
-        sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"",
-                        &pParse->sLastToken);
+        sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
         break;
       }
+      zSql += n;
     }else{
+      pParse->sLastToken.z = zSql;
+      pParse->sLastToken.n = n;
       sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
       lastTokenParsed = tokenType;
+      zSql += n;
       if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
     }
   }
   assert( nErr==0 );
-  pParse->zTail = &zSql[i];
-  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
-    assert( zSql[i]==0 );
-    if( lastTokenParsed!=TK_SEMI ){
-      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
-    }
-    if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
-      sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
-    }
-  }
+  pParse->zTail = zSql;
 #ifdef YYTRACKMAXSTACKDEPTH
   sqlite3_mutex_enter(sqlite3MallocMutex());
   sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK,
@@ -546,7 +557,11 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   );
   sqlite3_mutex_leave(sqlite3MallocMutex());
 #endif /* YYDEBUG */
+#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
+  sqlite3ParserFinalize(pEngine);
+#else
   sqlite3ParserFree(pEngine, sqlite3_free);
+#endif
   if( db->mallocFailed ){
     pParse->rc = SQLITE_NOMEM_BKPT;
   }
@@ -583,10 +598,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
     sqlite3DeleteTable(db, pParse->pNewTable);
   }
 
-  sqlite3WithDelete(db, pParse->pWithToFree);
+  if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree);
   sqlite3DeleteTrigger(db, pParse->pNewTrigger);
-  for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
-  sqlite3DbFree(db, pParse->azVar);
+  sqlite3DbFree(db, pParse->pVList);
   while( pParse->pAinc ){
     AutoincInfo *p = pParse->pAinc;
     pParse->pAinc = p->pNext;
diff --git a/src/treeview.c b/src/treeview.c
index 907159c06d..61d4626a22 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -65,6 +65,7 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){
   va_start(ap, zFormat);
   sqlite3VXPrintf(&acc, zFormat, ap);
   va_end(ap);
+  assert( acc.nChar>0 );
   if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1);
   sqlite3StrAccumFinish(&acc);
   fprintf(stdout,"%s", zBuf);
@@ -120,11 +121,15 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){
 
 
 /*
-** Generate a human-readable description of a the Select object.
+** Generate a human-readable description of a Select object.
 */
 void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
   int n = 0;
   int cnt = 0;
+  if( p==0 ){
+    sqlite3TreeViewLine(pView, "nil-SELECT");
+    return;
+  } 
   pView = sqlite3TreeViewPush(pView, moreToFollow);
   if( p->pWith ){
     sqlite3TreeViewWith(pView, p->pWith, 1);
@@ -445,6 +450,21 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
       break;
     }
 #endif
+    case TK_MATCH: {
+      sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s",
+                          pExpr->iTable, pExpr->iColumn, zFlgs);
+      sqlite3TreeViewExpr(pView, pExpr->pRight, 0);
+      break;
+    }
+    case TK_VECTOR: {
+      sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR");
+      break;
+    }
+    case TK_SELECT_COLUMN: {
+      sqlite3TreeViewLine(pView, "SELECT-COLUMN %d", pExpr->iColumn);
+      sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0);
+      break;
+    }
     default: {
       sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
       break;
@@ -461,21 +481,20 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
   sqlite3TreeViewPop(pView);
 }
 
+
 /*
 ** Generate a human-readable explanation of an expression list.
 */
-void sqlite3TreeViewExprList(
+void sqlite3TreeViewBareExprList(
   TreeView *pView,
   const ExprList *pList,
-  u8 moreToFollow,
   const char *zLabel
 ){
-  int i;
-  pView = sqlite3TreeViewPush(pView, moreToFollow);
   if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST";
   if( pList==0 ){
     sqlite3TreeViewLine(pView, "%s (empty)", zLabel);
   }else{
+    int i;
     sqlite3TreeViewLine(pView, "%s", zLabel);
     for(i=0; inExpr; i++){
       int j = pList->a[i].u.x.iOrderByCol;
@@ -487,6 +506,15 @@ void sqlite3TreeViewExprList(
       if( j ) sqlite3TreeViewPop(pView);
     }
   }
+}
+void sqlite3TreeViewExprList(
+  TreeView *pView,
+  const ExprList *pList,
+  u8 moreToFollow,
+  const char *zLabel
+){
+  pView = sqlite3TreeViewPush(pView, moreToFollow);
+  sqlite3TreeViewBareExprList(pView, pList, zLabel);
   sqlite3TreeViewPop(pView);
 }
 
diff --git a/src/trigger.c b/src/trigger.c
index 72c31eb2a3..bdf964084b 100644
--- a/src/trigger.c
+++ b/src/trigger.c
@@ -96,7 +96,6 @@ void sqlite3BeginTrigger(
   int iDb;                /* The database to store the trigger in */
   Token *pName;           /* The unqualified db name */
   DbFixer sFix;           /* State vector for the DB fixer */
-  int iTabDb;             /* Index of the database holding pTab */
 
   assert( pName1!=0 );   /* pName1->z might be NULL, but not pName1 itself */
   assert( pName2!=0 );
@@ -209,13 +208,13 @@ void sqlite3BeginTrigger(
         " trigger on table: %S", pTableName, 0);
     goto trigger_cleanup;
   }
-  iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema);
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
   {
+    int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema);
     int code = SQLITE_CREATE_TRIGGER;
-    const char *zDb = db->aDb[iTabDb].zName;
-    const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb;
+    const char *zDb = db->aDb[iTabDb].zDbSName;
+    const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb;
     if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER;
     if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){
       goto trigger_cleanup;
@@ -309,7 +308,7 @@ void sqlite3FinishTrigger(
     z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n);
     sqlite3NestedParse(pParse,
        "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
-       db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
+       db->aDb[iDb].zDbSName, MASTER_NAME, zName,
        pTrig->table, z);
     sqlite3DbFree(db, z);
     sqlite3ChangeCookie(pParse, iDb);
@@ -498,7 +497,7 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){
   assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) );
   for(i=OMIT_TEMPDB; inDb; i++){
     int j = (i<2) ? i^1 : i;  /* Search TEMP before MAIN */
-    if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue;
+    if( zDb && sqlite3StrICmp(db->aDb[j].zDbSName, zDb) ) continue;
     assert( sqlite3SchemaMutexHeld(db, j, 0) );
     pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName);
     if( pTrigger ) break;
@@ -544,7 +543,7 @@ void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){
 #ifndef SQLITE_OMIT_AUTHORIZATION
   {
     int code = SQLITE_DROP_TRIGGER;
-    const char *zDb = db->aDb[iDb].zName;
+    const char *zDb = db->aDb[iDb].zDbSName;
     const char *zTab = SCHEMA_TABLE(iDb);
     if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER;
     if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) ||
@@ -560,7 +559,7 @@ void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){
   if( (v = sqlite3GetVdbe(pParse))!=0 ){
     sqlite3NestedParse(pParse,
        "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'",
-       db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pTrigger->zName
+       db->aDb[iDb].zDbSName, MASTER_NAME, pTrigger->zName
     );
     sqlite3ChangeCookie(pParse, iDb);
     sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0);
@@ -663,8 +662,10 @@ static SrcList *targetSrcList(
     pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget);
     iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema);
     if( iDb==0 || iDb>=2 ){
+      const char *zDb;
       assert( iDbnDb );
-      pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName);
+      zDb = db->aDb[iDb].zDbSName;
+      pSrc->a[pSrc->nSrc-1].zDatabase =  sqlite3DbStrDup(db, zDb);
     }
   }
   return pSrc;
@@ -878,7 +879,6 @@ static TriggerPrg *codeRowTrigger(
     }
     pProgram->nMem = pSubParse->nMem;
     pProgram->nCsr = pSubParse->nTab;
-    pProgram->nOnce = pSubParse->nOnce;
     pProgram->token = (void *)pTrigger;
     pPrg->aColmask[0] = pSubParse->oldmask;
     pPrg->aColmask[1] = pSubParse->newmask;
diff --git a/src/update.c b/src/update.c
index 9b351662e2..8fca7eb988 100644
--- a/src/update.c
+++ b/src/update.c
@@ -69,14 +69,14 @@ void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){
     sqlite3ValueFromExpr(sqlite3VdbeDb(v), pCol->pDflt, enc, 
                          pCol->affinity, &pValue);
     if( pValue ){
-      sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM);
+      sqlite3VdbeAppendP4(v, pValue, P4_MEM);
     }
-#ifndef SQLITE_OMIT_FLOATING_POINT
-    if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){
-      sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg);
-    }
-#endif
   }
+#ifndef SQLITE_OMIT_FLOATING_POINT
+  if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){
+    sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg);
+  }
+#endif
 }
 
 /*
@@ -105,7 +105,7 @@ void sqlite3Update(
   int iDataCur;          /* Cursor for the canonical data btree */
   int iIdxCur;           /* Cursor for the first index */
   sqlite3 *db;           /* The database structure */
-  int *aRegIdx = 0;      /* One register assigned to each index to be updated */
+  int *aRegIdx = 0;      /* First register in array assigned to each index */
   int *aXRef = 0;        /* aXRef[i] is the index in pChanges->a[] of the
                          ** an expression for the i-th column of the table.
                          ** aXRef[i]==-1 if the i-th column is not changed. */
@@ -117,10 +117,11 @@ void sqlite3Update(
   AuthContext sContext;  /* The authorization context */
   NameContext sNC;       /* The name-context to resolve expressions in */
   int iDb;               /* Database containing the table being updated */
-  int okOnePass;         /* True for one-pass algorithm without the FIFO */
+  int eOnePass;          /* ONEPASS_XXX value from where.c */
   int hasFK;             /* True if foreign key processing is required */
   int labelBreak;        /* Jump here to break out of UPDATE loop */
   int labelContinue;     /* Jump here to continue next step of UPDATE loop */
+  int flags;             /* Flags for sqlite3WhereBegin() */
 
 #ifndef SQLITE_OMIT_TRIGGER
   int isView;            /* True when updating a view (INSTEAD OF trigger) */
@@ -131,6 +132,10 @@ void sqlite3Update(
   int iEph = 0;          /* Ephemeral table holding all primary key values */
   int nKey = 0;          /* Number of elements in regKey for WITHOUT ROWID */
   int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
+  int addrOpen = 0;      /* Address of OP_OpenEphemeral */
+  int iPk = 0;           /* First of nPk cells holding PRIMARY KEY value */
+  i16 nPk = 0;           /* Number of components of the PRIMARY KEY */
+  int bReplace = 0;      /* True if REPLACE conflict resolution might happen */
 
   /* Register Allocations */
   int regRowCount = 0;   /* A count of rows changed */
@@ -249,7 +254,7 @@ void sqlite3Update(
       int rc;
       rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName,
                             j<0 ? "ROWID" : pTab->aCol[j].zName,
-                            db->aDb[iDb].zName);
+                            db->aDb[iDb].zDbSName);
       if( rc==SQLITE_DENY ){
         goto update_cleanup;
       }else if( rc==SQLITE_IGNORE ){
@@ -282,12 +287,19 @@ void sqlite3Update(
     int reg;
     if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){
       reg = ++pParse->nMem;
+      pParse->nMem += pIdx->nColumn;
     }else{
       reg = 0;
       for(i=0; inKeyCol; i++){
         i16 iIdxCol = pIdx->aiColumn[i];
         if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){
           reg = ++pParse->nMem;
+          pParse->nMem += pIdx->nColumn;
+          if( (onError==OE_Replace)
+           || (onError==OE_Default && pIdx->onError==OE_Replace) 
+          ){
+            bReplace = 1;
+          }
           break;
         }
       }
@@ -295,6 +307,11 @@ void sqlite3Update(
     if( reg==0 ) aToOpen[j+1] = 0;
     aRegIdx[j] = reg;
   }
+  if( bReplace ){
+    /* If REPLACE conflict resolution might be invoked, open cursors on all 
+    ** indexes in case they are needed to delete records.  */
+    memset(aToOpen, 1, nIdx+1);
+  }
 
   /* Begin generating code. */
   v = sqlite3GetVdbe(pParse);
@@ -347,109 +364,130 @@ void sqlite3Update(
   }
 #endif
 
-  /* Begin the database scan
-  */
+  /* Initialize the count of updated rows */
+  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){
+    regRowCount = ++pParse->nMem;
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
+  }
+
   if( HasRowid(pTab) ){
     sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid);
-    pWInfo = sqlite3WhereBegin(
-        pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur
-    );
-    if( pWInfo==0 ) goto update_cleanup;
-    okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
-  
-    /* Remember the rowid of every item to be updated.
-    */
-    sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
-    if( !okOnePass ){
-      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
-    }
-  
-    /* End the database scan loop.
-    */
-    sqlite3WhereEnd(pWInfo);
   }else{
-    int iPk;         /* First of nPk memory cells holding PRIMARY KEY value */
-    i16 nPk;         /* Number of components of the PRIMARY KEY */
-    int addrOpen;    /* Address of the OpenEphemeral instruction */
-
     assert( pPk!=0 );
     nPk = pPk->nKeyCol;
     iPk = pParse->nMem+1;
     pParse->nMem += nPk;
     regKey = ++pParse->nMem;
     iEph = pParse->nTab++;
+
     sqlite3VdbeAddOp2(v, OP_Null, 0, iPk);
     addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk);
     sqlite3VdbeSetP4KeyInfo(pParse, pPk);
-    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, 
-                               WHERE_ONEPASS_DESIRED, iIdxCur);
-    if( pWInfo==0 ) goto update_cleanup;
-    okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
+  }
+
+  /* Begin the database scan. 
+  **
+  ** Do not consider a single-pass strategy for a multi-row update if
+  ** there are any triggers or foreign keys to process, or rows may
+  ** be deleted as a result of REPLACE conflict handling. Any of these
+  ** things might disturb a cursor being used to scan through the table
+  ** or index, causing a single-pass approach to malfunction.  */
+  flags = WHERE_ONEPASS_DESIRED|WHERE_SEEK_UNIQ_TABLE;
+  if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){
+    flags |= WHERE_ONEPASS_MULTIROW;
+  }
+  pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags, iIdxCur);
+  if( pWInfo==0 ) goto update_cleanup;
+
+  /* A one-pass strategy that might update more than one row may not
+  ** be used if any column of the index used for the scan is being
+  ** updated. Otherwise, if there is an index on "b", statements like
+  ** the following could create an infinite loop:
+  **
+  **   UPDATE t1 SET b=b+1 WHERE b>?
+  **
+  ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI
+  ** strategy that uses an index for which one or more columns are being
+  ** updated.  */
+  eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
+  if( eOnePass==ONEPASS_MULTI ){
+    int iCur = aiCurOnePass[1];
+    if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){
+      eOnePass = ONEPASS_OFF;
+    }
+    assert( iCur!=iDataCur || !HasRowid(pTab) );
+  }
+  
+  if( HasRowid(pTab) ){
+    /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF
+    ** mode, write the rowid into the FIFO. In either of the one-pass modes,
+    ** leave it in register regOldRowid.  */
+    sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid);
+    if( eOnePass==ONEPASS_OFF ){
+      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid);
+    }
+  }else{
+    /* Read the PK of the current row into an array of registers. In
+    ** ONEPASS_OFF mode, serialize the array into a record and store it in
+    ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change
+    ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table 
+    ** is not required) and leave the PK fields in the array of registers.  */
     for(i=0; iaiColumn[i]>=0 );
-      sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i],
-                                      iPk+i);
+      sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i);
     }
-    if( okOnePass ){
+    if( eOnePass ){
       sqlite3VdbeChangeToNoop(v, addrOpen);
       nKey = nPk;
       regKey = iPk;
     }else{
       sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey,
                         sqlite3IndexAffinityStr(db, pPk), nPk);
-      sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey);
+      sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk);
     }
-    sqlite3WhereEnd(pWInfo);
   }
 
-  /* Initialize the count of updated rows
-  */
-  if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){
-    regRowCount = ++pParse->nMem;
-    sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount);
+  if( eOnePass!=ONEPASS_MULTI ){
+    sqlite3WhereEnd(pWInfo);
   }
 
   labelBreak = sqlite3VdbeMakeLabel(v);
   if( !isView ){
-    /* 
-    ** Open every index that needs updating.  Note that if any
-    ** index could potentially invoke a REPLACE conflict resolution 
-    ** action, then we need to open all indices because we might need
-    ** to be deleting some records.
-    */
-    if( onError==OE_Replace ){
-      memset(aToOpen, 1, nIdx+1);
-    }else{
-      for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
-        if( pIdx->onError==OE_Replace ){
-          memset(aToOpen, 1, nIdx+1);
-          break;
-        }
-      }
-    }
-    if( okOnePass ){
+    int addrOnce = 0;
+
+    /* Open every index that needs updating. */
+    if( eOnePass!=ONEPASS_OFF ){
       if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0;
       if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0;
     }
+
+    if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){
+      addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
+    }
     sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen,
                                0, 0);
+    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
   }
 
   /* Top of the update loop */
-  if( okOnePass ){
-    if( aToOpen[iDataCur-iBaseCur] && !isView ){
+  if( eOnePass!=ONEPASS_OFF ){
+    if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){
       assert( pPk );
       sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey);
       VdbeCoverageNeverTaken(v);
     }
-    labelContinue = labelBreak;
+    if( eOnePass==ONEPASS_SINGLE ){
+      labelContinue = labelBreak;
+    }else{
+      labelContinue = sqlite3VdbeMakeLabel(v);
+    }
     sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak);
     VdbeCoverageIf(v, pPk==0);
     VdbeCoverageIf(v, pPk!=0);
   }else if( pPk ){
     labelContinue = sqlite3VdbeMakeLabel(v);
     sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v);
-    addrTop = sqlite3VdbeAddOp2(v, OP_RowKey, iEph, regKey);
+    addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey);
     sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0);
     VdbeCoverage(v);
   }else{
@@ -567,7 +605,6 @@ void sqlite3Update(
 
   if( !isView ){
     int addr1 = 0;        /* Address of jump instruction */
-    int bReplace = 0;     /* True if REPLACE conflict resolution might happen */
 
     /* Do constraint checks. */
     assert( regOldRowid>0 );
@@ -590,11 +627,34 @@ void sqlite3Update(
       VdbeCoverageNeverTaken(v);
     }
     sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1);
-  
-    /* If changing the record number, delete the old record.  */
-    if( hasFK || chngKey || pPk!=0 ){
+
+    /* If changing the rowid value, or if there are foreign key constraints
+    ** to process, delete the old record. Otherwise, add a noop OP_Delete
+    ** to invoke the pre-update hook.
+    **
+    ** That (regNew==regnewRowid+1) is true is also important for the 
+    ** pre-update hook. If the caller invokes preupdate_new(), the returned
+    ** value is copied from memory cell (regNewRowid+1+iCol), where iCol
+    ** is the column index supplied by the user.
+    */
+    assert( regNew==regNewRowid+1 );
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    sqlite3VdbeAddOp3(v, OP_Delete, iDataCur,
+        OPFLAG_ISUPDATE | ((hasFK || chngKey) ? 0 : OPFLAG_ISNOOP),
+        regNewRowid
+    );
+    if( eOnePass==ONEPASS_MULTI ){
+      assert( hasFK==0 && chngKey==0 );
+      sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION);
+    }
+    if( !pParse->nested ){
+      sqlite3VdbeAppendP4(v, pTab, P4_TABLE);
+    }
+#else
+    if( hasFK || chngKey ){
       sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0);
     }
+#endif
     if( bReplace || chngKey ){
       sqlite3VdbeJumpHere(v, addr1);
     }
@@ -604,8 +664,11 @@ void sqlite3Update(
     }
   
     /* Insert the new index entries and the new record. */
-    sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur,
-                             regNewRowid, aRegIdx, 1, 0, 0);
+    sqlite3CompleteInsertion(
+        pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, 
+        OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), 
+        0, 0
+    );
 
     /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to
     ** handle rows (possibly in other tables) that refer via a foreign key
@@ -627,8 +690,11 @@ void sqlite3Update(
   /* Repeat the above with the next record to be updated, until
   ** all record selected by the WHERE clause have been updated.
   */
-  if( okOnePass ){
+  if( eOnePass==ONEPASS_SINGLE ){
     /* Nothing to do at end-of-loop for a single-pass */
+  }else if( eOnePass==ONEPASS_MULTI ){
+    sqlite3VdbeResolveLabel(v, labelContinue);
+    sqlite3WhereEnd(pWInfo);
   }else if( pPk ){
     sqlite3VdbeResolveLabel(v, labelContinue);
     sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v);
@@ -637,15 +703,6 @@ void sqlite3Update(
   }
   sqlite3VdbeResolveLabel(v, labelBreak);
 
-  /* Close all tables */
-  for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){
-    assert( aRegIdx );
-    if( aToOpen[i+1] ){
-      sqlite3VdbeAddOp2(v, OP_Close, iIdxCur+i, 0);
-    }
-  }
-  if( iDataCurcolFlags & COLFLAG_HASTYPE)==0 ) return zDflt;
+  return pCol->zName + strlen(pCol->zName) + 1;
+}
+
+/*
+** Helper function for sqlite3Error() - called rarely.  Broken out into
+** a separate routine to avoid unnecessary register saves on entry to
+** sqlite3Error().
+*/
+static SQLITE_NOINLINE void  sqlite3ErrorFinish(sqlite3 *db, int err_code){
+  if( db->pErr ) sqlite3ValueSetNull(db->pErr);
+  sqlite3SystemError(db, err_code);
 }
 
 /*
 ** Set the current error code to err_code and clear any prior error message.
+** Also set iSysErrno (by calling sqlite3System) if the err_code indicates
+** that would be appropriate.
 */
 void sqlite3Error(sqlite3 *db, int err_code){
   assert( db!=0 );
   db->errCode = err_code;
-  if( db->pErr ) sqlite3ValueSetNull(db->pErr);
+  if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code);
+}
+
+/*
+** Load the sqlite3.iSysErrno field if that is an appropriate thing
+** to do based on the SQLite error code in rc.
+*/
+void sqlite3SystemError(sqlite3 *db, int rc){
+  if( rc==SQLITE_IOERR_NOMEM ) return;
+  rc &= 0xff;
+  if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){
+    db->iSysErrno = sqlite3OsGetLastError(db->pVfs);
+  }
 }
 
 /*
@@ -150,6 +178,7 @@ void sqlite3Error(sqlite3 *db, int err_code){
 void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){
   assert( db!=0 );
   db->errCode = err_code;
+  sqlite3SystemError(db, err_code);
   if( zFormat==0 ){
     sqlite3Error(db, err_code);
   }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){
@@ -213,18 +242,13 @@ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){
 ** brackets from around identifiers.  For example:  "[a-b-c]" becomes
 ** "a-b-c".
 */
-int sqlite3Dequote(char *z){
+void sqlite3Dequote(char *z){
   char quote;
   int i, j;
-  if( z==0 ) return -1;
+  if( z==0 ) return;
   quote = z[0];
-  switch( quote ){
-    case '\'':  break;
-    case '"':   break;
-    case '`':   break;                /* For MySQL compatibility */
-    case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
-    default:    return -1;
-  }
+  if( !sqlite3Isquote(quote) ) return;
+  if( quote=='[' ) quote = ']';
   for(i=1, j=0;; i++){
     assert( z[i] );
     if( z[i]==quote ){
@@ -239,7 +263,6 @@ int sqlite3Dequote(char *z){
     }
   }
   z[j] = 0;
-  return j;
 }
 
 /*
@@ -332,7 +355,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   int eValid = 1;  /* True exponent is either not used or is well-formed */
   double result;
   int nDigits = 0;
-  int nonNum = 0;
+  int nonNum = 0;  /* True if input contains UTF16 with high byte non-zero */
 
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
   *pResult = 0.0;   /* Default return value, in case of an error */
@@ -345,7 +368,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
     assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
     for(i=3-enc; i=zEnd ) goto do_atof_calc;
 
@@ -393,7 +414,12 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   if( *z=='e' || *z=='E' ){
     z+=incr;
     eValid = 0;
-    if( z>=zEnd ) goto do_atof_calc;
+
+    /* This branch is needed to avoid a (harmless) buffer overread.  The 
+    ** special comment alerts the mutation tester that the correct answer
+    ** is obtained even if the branch is omitted */
+    if( z>=zEnd ) goto do_atof_calc;              /*PREVENTS-HARMLESS-OVERREAD*/
+
     /* get sign of exponent */
     if( *z=='-' ){
       esign = -1;
@@ -410,9 +436,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){
   }
 
   /* skip trailing spaces */
-  if( nDigits && eValid ){
-    while( z0 ){
-      while( s<(LARGEST_INT64/10) && e>0 ) e--,s*=10;
-    }else{
-      while( !(s%10) && e>0 ) e--,s/=10;
+    /* Attempt to reduce exponent.
+    **
+    ** Branches that are not required for the correct answer but which only
+    ** help to obtain the correct answer faster are marked with special
+    ** comments, as a hint to the mutation tester.
+    */
+    while( e>0 ){                                       /*OPTIMIZATION-IF-TRUE*/
+      if( esign>0 ){
+        if( s>=(LARGEST_INT64/10) ) break;             /*OPTIMIZATION-IF-FALSE*/
+        s *= 10;
+      }else{
+        if( s%10!=0 ) break;                           /*OPTIMIZATION-IF-FALSE*/
+        s /= 10;
+      }
+      e--;
     }
 
     /* adjust the sign of significand */
     s = sign<0 ? -s : s;
 
-    /* if exponent, scale significand as appropriate
-    ** and store in result. */
-    if( e ){
+    if( e==0 ){                                         /*OPTIMIZATION-IF-TRUE*/
+      result = (double)s;
+    }else{
       LONGDOUBLE_TYPE scale = 1.0;
       /* attempt to handle extremely small/large numbers better */
-      if( e>307 && e<342 ){
-        while( e%308 ) { scale *= 1.0e+1; e -= 1; }
-        if( esign<0 ){
-          result = s / scale;
-          result /= 1.0e+308;
-        }else{
-          result = s * scale;
-          result *= 1.0e+308;
-        }
-      }else if( e>=342 ){
-        if( esign<0 ){
-          result = 0.0*s;
-        }else{
-          result = 1e308*1e308*s;  /* Infinity */
+      if( e>307 ){                                      /*OPTIMIZATION-IF-TRUE*/
+        if( e<342 ){                                    /*OPTIMIZATION-IF-TRUE*/
+          while( e%308 ) { scale *= 1.0e+1; e -= 1; }
+          if( esign<0 ){
+            result = s / scale;
+            result /= 1.0e+308;
+          }else{
+            result = s * scale;
+            result *= 1.0e+308;
+          }
+        }else{ assert( e>=342 );
+          if( esign<0 ){
+            result = 0.0*s;
+          }else{
+            result = 1e308*1e308*s;  /* Infinity */
+          }
         }
       }else{
         /* 1.0e+22 is the largest power of 10 than can be 
@@ -471,8 +505,6 @@ do_atof_calc:
           result = s * scale;
         }
       }
-    } else {
-      result = (double)s;
     }
   }
 
@@ -480,7 +512,7 @@ do_atof_calc:
   *pResult = result;
 
   /* return true if number and no extra non-whitespace chracters after */
-  return z>=zEnd && nDigits>0 && eValid && nonNum==0;
+  return z==zEnd && nDigits>0 && eValid && nonNum==0;
 #else
   return !sqlite3Atoi64(z, pResult, length, enc);
 #endif /* SQLITE_OMIT_FLOATING_POINT */
@@ -542,7 +574,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
   int neg = 0; /* assume positive */
   int i;
   int c = 0;
-  int nonNum = 0;
+  int nonNum = 0;  /* True if input contains UTF16 with high byte non-zero */
   const char *zStart;
   const char *zEnd = zNum + length;
   assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE );
@@ -553,7 +585,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
     assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 );
     for(i=3-enc; i19*incr || nonNum ){
+  if( &zNum[i]19*incr                  /* Too many digits */
+   || nonNum                     /* UTF16 with high-order bytes non-zero */
+  ){
     /* zNum is empty or contains non-numeric text or is longer
     ** than 19 digits (thus guaranteeing that it is too large) */
     return 1;
@@ -623,7 +658,6 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
 #ifndef SQLITE_OMIT_HEX_INTEGER
   if( z[0]=='0'
    && (z[1]=='x' || z[1]=='X')
-   && sqlite3Isxdigit(z[2])
   ){
     u64 u = 0;
     int i, k;
@@ -1106,13 +1140,11 @@ u32 sqlite3Get4byte(const u8 *p){
   u32 x;
   memcpy(&x,p,4);
   return x;
-#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
-    && defined(__GNUC__) && GCC_VERSION>=4003000
+#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
   u32 x;
   memcpy(&x,p,4);
   return __builtin_bswap32(x);
-#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
-    && defined(_MSC_VER) && _MSC_VER>=1300
+#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
   u32 x;
   memcpy(&x,p,4);
   return _byteswap_ulong(x);
@@ -1124,12 +1156,10 @@ u32 sqlite3Get4byte(const u8 *p){
 void sqlite3Put4byte(unsigned char *p, u32 v){
 #if SQLITE_BYTEORDER==4321
   memcpy(p,&v,4);
-#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
-    && defined(__GNUC__) && GCC_VERSION>=4003000
+#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000
   u32 x = __builtin_bswap32(v);
   memcpy(p,&x,4);
-#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \
-    && defined(_MSC_VER) && _MSC_VER>=1300
+#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
   u32 x = _byteswap_ulong(v);
   memcpy(p,&x,4);
 #else
@@ -1245,6 +1275,9 @@ int sqlite3SafetyCheckSickOrOk(sqlite3 *db){
 ** overflow, leave *pA unchanged and return 1.
 */
 int sqlite3AddInt64(i64 *pA, i64 iB){
+#if GCC_VERSION>=5004000
+  return __builtin_add_overflow(*pA, iB, pA);
+#else
   i64 iA = *pA;
   testcase( iA==0 ); testcase( iA==1 );
   testcase( iB==-1 ); testcase( iB==0 );
@@ -1259,8 +1292,12 @@ int sqlite3AddInt64(i64 *pA, i64 iB){
   }
   *pA += iB;
   return 0; 
+#endif
 }
 int sqlite3SubInt64(i64 *pA, i64 iB){
+#if GCC_VERSION>=5004000
+  return __builtin_sub_overflow(*pA, iB, pA);
+#else
   testcase( iB==SMALLEST_INT64+1 );
   if( iB==SMALLEST_INT64 ){
     testcase( (*pA)==(-1) ); testcase( (*pA)==0 );
@@ -1270,38 +1307,28 @@ int sqlite3SubInt64(i64 *pA, i64 iB){
   }else{
     return sqlite3AddInt64(pA, -iB);
   }
+#endif
 }
-#define TWOPOWER32 (((i64)1)<<32)
-#define TWOPOWER31 (((i64)1)<<31)
 int sqlite3MulInt64(i64 *pA, i64 iB){
+#if GCC_VERSION>=5004000
+  return __builtin_mul_overflow(*pA, iB, pA);
+#else
   i64 iA = *pA;
-  i64 iA1, iA0, iB1, iB0, r;
-
-  iA1 = iA/TWOPOWER32;
-  iA0 = iA % TWOPOWER32;
-  iB1 = iB/TWOPOWER32;
-  iB0 = iB % TWOPOWER32;
-  if( iA1==0 ){
-    if( iB1==0 ){
-      *pA *= iB;
-      return 0;
+  if( iB>0 ){
+    if( iA>LARGEST_INT64/iB ) return 1;
+    if( iA0 ){
+      if( iBLARGEST_INT64/-iB ) return 1;
     }
-    r = iA0*iB1;
-  }else if( iB1==0 ){
-    r = iA1*iB0;
-  }else{
-    /* If both iA1 and iB1 are non-zero, overflow will result */
-    return 1;
   }
-  testcase( r==(-TWOPOWER31)-1 );
-  testcase( r==(-TWOPOWER31) );
-  testcase( r==TWOPOWER31 );
-  testcase( r==TWOPOWER31-1 );
-  if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1;
-  r *= TWOPOWER32;
-  if( sqlite3AddInt64(&r, iA0*iB0) ) return 1;
-  *pA = r;
+  *pA = iA*iB;
   return 0;
+#endif
 }
 
 /*
@@ -1385,7 +1412,7 @@ LogEst sqlite3LogEst(u64 x){
     if( x<2 ) return 0;
     while( x<8 ){  y -= 10; x <<= 1; }
   }else{
-    while( x>255 ){ y += 40; x >>= 4; }
+    while( x>255 ){ y += 40; x >>= 4; }  /*OPTIMIZATION-IF-TRUE*/
     while( x>15 ){  y += 10; x >>= 1; }
   }
   return a[x&7] + y - 10;
@@ -1419,7 +1446,6 @@ LogEst sqlite3LogEstFromDouble(double x){
 */
 u64 sqlite3LogEstToInt(LogEst x){
   u64 n;
-  if( x<10 ) return 1;
   n = x%10;
   x /= 10;
   if( n>=5 ) n -= 2;
@@ -1435,3 +1461,106 @@ u64 sqlite3LogEstToInt(LogEst x){
   return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x);
 }
 #endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */
+
+/*
+** Add a new name/number pair to a VList.  This might require that the
+** VList object be reallocated, so return the new VList.  If an OOM
+** error occurs, the original VList returned and the
+** db->mallocFailed flag is set.
+**
+** A VList is really just an array of integers.  To destroy a VList,
+** simply pass it to sqlite3DbFree().
+**
+** The first integer is the number of integers allocated for the whole
+** VList.  The second integer is the number of integers actually used.
+** Each name/number pair is encoded by subsequent groups of 3 or more
+** integers.
+**
+** Each name/number pair starts with two integers which are the numeric
+** value for the pair and the size of the name/number pair, respectively.
+** The text name overlays one or more following integers.  The text name
+** is always zero-terminated.
+**
+** Conceptually:
+**
+**    struct VList {
+**      int nAlloc;   // Number of allocated slots 
+**      int nUsed;    // Number of used slots 
+**      struct VListEntry {
+**        int iValue;    // Value for this entry
+**        int nSlot;     // Slots used by this entry
+**        // ... variable name goes here
+**      } a[0];
+**    }
+**
+** During code generation, pointers to the variable names within the
+** VList are taken.  When that happens, nAlloc is set to zero as an 
+** indication that the VList may never again be enlarged, since the
+** accompanying realloc() would invalidate the pointers.
+*/
+VList *sqlite3VListAdd(
+  sqlite3 *db,           /* The database connection used for malloc() */
+  VList *pIn,            /* The input VList.  Might be NULL */
+  const char *zName,     /* Name of symbol to add */
+  int nName,             /* Bytes of text in zName */
+  int iVal               /* Value to associate with zName */
+){
+  int nInt;              /* number of sizeof(int) objects needed for zName */
+  char *z;               /* Pointer to where zName will be stored */
+  int i;                 /* Index in pIn[] where zName is stored */
+
+  nInt = nName/4 + 3;
+  assert( pIn==0 || pIn[0]>=3 );  /* Verify ok to add new elements */
+  if( pIn==0 || pIn[1]+nInt > pIn[0] ){
+    /* Enlarge the allocation */
+    int nAlloc = (pIn ? pIn[0]*2 : 10) + nInt;
+    VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int));
+    if( pOut==0 ) return pIn;
+    if( pIn==0 ) pOut[1] = 2;
+    pIn = pOut;
+    pIn[0] = nAlloc;
+  }
+  i = pIn[1];
+  pIn[i] = iVal;
+  pIn[i+1] = nInt;
+  z = (char*)&pIn[i+2];
+  pIn[1] = i+nInt;
+  assert( pIn[1]<=pIn[0] );
+  memcpy(z, zName, nName);
+  z[nName] = 0;
+  return pIn;
+}
+
+/*
+** Return a pointer to the name of a variable in the given VList that
+** has the value iVal.  Or return a NULL if there is no such variable in
+** the list
+*/
+const char *sqlite3VListNumToName(VList *pIn, int iVal){
+  int i, mx;
+  if( pIn==0 ) return 0;
+  mx = pIn[1];
+  i = 2;
+  do{
+    if( pIn[i]==iVal ) return (char*)&pIn[i+2];
+    i += pIn[i+1];
+  }while( iflags&SQLITE_CountRows) );
-  return vacuumFinalize(db, pStmt, pzErrMsg);
-}
-
-/*
-** Execute zSql on database db. The statement returns exactly
-** one column. Execute this as SQL on the same database.
-*/
-static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
   sqlite3_stmt *pStmt;
   int rc;
 
-  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  /* printf("SQL: [%s]\n", zSql); fflush(stdout); */
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
   if( rc!=SQLITE_OK ) return rc;
-
-  while( SQLITE_ROW==sqlite3_step(pStmt) ){
-    rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0));
-    if( rc!=SQLITE_OK ){
-      vacuumFinalize(db, pStmt, pzErrMsg);
-      return rc;
+  while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+    const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0);
+    assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 );
+    if( zSubSql ){
+      assert( zSubSql[0]!='S' );
+      rc = execSql(db, pzErrMsg, zSubSql);
+      if( rc!=SQLITE_OK ) break;
     }
   }
-
-  return vacuumFinalize(db, pStmt, pzErrMsg);
+  assert( rc!=SQLITE_ROW );
+  if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+  if( rc ){
+    sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db));
+  }
+  (void)sqlite3_finalize(pStmt);
+  return rc;
+}
+static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){
+  char *z;
+  va_list ap;
+  int rc;
+  va_start(ap, zSql);
+  z = sqlite3VMPrintf(db, zSql, ap);
+  va_end(ap);
+  if( z==0 ) return SQLITE_NOMEM;
+  rc = execSql(db, pzErrMsg, z);
+  sqlite3DbFree(db, z);
+  return rc;
 }
 
 /*
@@ -101,11 +96,29 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){
 ** transient would cause the database file to appear to be deleted
 ** following reboot.
 */
-void sqlite3Vacuum(Parse *pParse){
+void sqlite3Vacuum(Parse *pParse, Token *pNm){
   Vdbe *v = sqlite3GetVdbe(pParse);
-  if( v ){
-    sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0);
-    sqlite3VdbeUsesBtree(v, 0);
+  int iDb = 0;
+  if( v==0 ) return;
+  if( pNm ){
+#ifndef SQLITE_BUG_COMPATIBLE_20160819
+    /* Default behavior:  Report an error if the argument to VACUUM is
+    ** not recognized */
+    iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm);
+    if( iDb<0 ) return;
+#else
+    /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments
+    ** to VACUUM are silently ignored.  This is a back-out of a bug fix that
+    ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270).
+    ** The buggy behavior is required for binary compatibility with some
+    ** legacy applications. */
+    iDb = sqlite3FindDb(pParse->db, pNm);
+    if( iDb<0 ) iDb = 0;
+#endif
+  }
+  if( iDb!=1 ){
+    sqlite3VdbeAddOp1(v, OP_Vacuum, iDb);
+    sqlite3VdbeUsesBtree(v, iDb);
   }
   return;
 }
@@ -113,19 +126,19 @@ void sqlite3Vacuum(Parse *pParse){
 /*
 ** This routine implements the OP_Vacuum opcode of the VDBE.
 */
-int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
+int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){
   int rc = SQLITE_OK;     /* Return code from service routines */
   Btree *pMain;           /* The database being vacuumed */
   Btree *pTemp;           /* The temporary database we vacuum into */
-  char *zSql = 0;         /* SQL statements */
   int saved_flags;        /* Saved value of the db->flags */
   int saved_nChange;      /* Saved value of db->nChange */
   int saved_nTotalChange; /* Saved value of db->nTotalChange */
-  void (*saved_xTrace)(void*,const char*);  /* Saved db->xTrace */
+  u8 saved_mTrace;        /* Saved trace settings */
   Db *pDb = 0;            /* Database to detach at end of vacuum */
   int isMemDb;            /* True if vacuuming a :memory: database */
   int nRes;               /* Bytes of reserved space at the end of each page */
   int nDb;                /* Number of attached databases */
+  const char *zDbMain;    /* Schema name of database to vacuum */
 
   if( !db->autoCommit ){
     sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction");
@@ -142,12 +155,14 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   saved_flags = db->flags;
   saved_nChange = db->nChange;
   saved_nTotalChange = db->nTotalChange;
-  saved_xTrace = db->xTrace;
-  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin;
-  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder);
-  db->xTrace = 0;
+  saved_mTrace = db->mTrace;
+  db->flags |= (SQLITE_WriteSchema | SQLITE_IgnoreChecks
+                 | SQLITE_PreferBuiltin | SQLITE_Vacuum);
+  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows);
+  db->mTrace = 0;
 
-  pMain = db->aDb[0].pBt;
+  zDbMain = db->aDb[iDb].zDbSName;
+  pMain = db->aDb[iDb].pBt;
   isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));
 
   /* Attach the temporary database as 'vacuum_db'. The synchronous pragma
@@ -165,18 +180,12 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   ** to write the journal header file.
   */
   nDb = db->nDb;
-  if( sqlite3TempInMemory(db) ){
-    zSql = "ATTACH ':memory:' AS vacuum_db;";
-  }else{
-    zSql = "ATTACH '' AS vacuum_db;";
-  }
-  rc = execSql(db, pzErrMsg, zSql);
-  if( db->nDb>nDb ){
-    pDb = &db->aDb[db->nDb-1];
-    assert( strcmp(pDb->zName,"vacuum_db")==0 );
-  }
+  rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db");
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  pTemp = db->aDb[db->nDb-1].pBt;
+  assert( (db->nDb-1)==nDb );
+  pDb = &db->aDb[nDb];
+  assert( strcmp(pDb->zDbSName,"vacuum_db")==0 );
+  pTemp = pDb->pBt;
 
   /* The call to execSql() to attach the temp database has left the file
   ** locked (as there was more than one active statement when the transaction
@@ -197,14 +206,15 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   }
 #endif
 
-  rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF");
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
+  sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size);
+  sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0));
+  sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL);
 
   /* Begin a transaction and take an exclusive lock on the main database
   ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below,
   ** to ensure that we do not try to change the page-size on a WAL database.
   */
-  rc = execSql(db, pzErrMsg, "BEGIN;");
+  rc = execSql(db, pzErrMsg, "BEGIN");
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
   rc = sqlite3BtreeBeginTrans(pMain, 2);
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
@@ -231,64 +241,48 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
   /* Query the schema of the main database. Create a mirror schema
   ** in the temporary database.
   */
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) "
-      "  FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'"
-      "   AND coalesce(rootpage,1)>0"
+  db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT sql FROM \"%w\".sqlite_master"
+      " WHERE type='table'AND name<>'sqlite_sequence'"
+      " AND coalesce(rootpage,1)>0",
+      zDbMain
   );
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)"
-      "  FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' ");
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) "
-      "  FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'");
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT sql FROM \"%w\".sqlite_master"
+      " WHERE type='index' AND length(sql)>10",
+      zDbMain
+  );
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
+  db->init.iDb = 0;
 
   /* Loop through the tables in the main database. For each, do
   ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy
   ** the contents to the temporary database.
   */
-  assert( (db->flags & SQLITE_Vacuum)==0 );
-  db->flags |= SQLITE_Vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'INSERT INTO vacuum_db.' || quote(name) "
-      "|| ' SELECT * FROM main.' || quote(name) || ';'"
-      "FROM main.sqlite_master "
-      "WHERE type = 'table' AND name!='sqlite_sequence' "
-      "  AND coalesce(rootpage,1)>0"
+  rc = execSqlF(db, pzErrMsg,
+      "SELECT'INSERT INTO vacuum_db.'||quote(name)"
+      "||' SELECT*FROM\"%w\".'||quote(name)"
+      "FROM vacuum_db.sqlite_master "
+      "WHERE type='table'AND coalesce(rootpage,1)>0",
+      zDbMain
   );
   assert( (db->flags & SQLITE_Vacuum)!=0 );
   db->flags &= ~SQLITE_Vacuum;
   if( rc!=SQLITE_OK ) goto end_of_vacuum;
 
-  /* Copy over the sequence table
-  */
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' "
-      "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' "
-  );
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-  rc = execExecSql(db, pzErrMsg,
-      "SELECT 'INSERT INTO vacuum_db.' || quote(name) "
-      "|| ' SELECT * FROM main.' || quote(name) || ';' "
-      "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';"
-  );
-  if( rc!=SQLITE_OK ) goto end_of_vacuum;
-
-
   /* Copy the triggers, views, and virtual tables from the main database
   ** over to the temporary database.  None of these objects has any
   ** associated storage, so all we have to do is copy their entries
   ** from the SQLITE_MASTER table.
   */
-  rc = execSql(db, pzErrMsg,
-      "INSERT INTO vacuum_db.sqlite_master "
-      "  SELECT type, name, tbl_name, rootpage, sql"
-      "    FROM main.sqlite_master"
-      "   WHERE type='view' OR type='trigger'"
-      "      OR (type='table' AND rootpage=0)"
+  rc = execSqlF(db, pzErrMsg,
+      "INSERT INTO vacuum_db.sqlite_master"
+      " SELECT*FROM \"%w\".sqlite_master"
+      " WHERE type IN('view','trigger')"
+      " OR(type='table'AND rootpage=0)",
+      zDbMain
   );
   if( rc ) goto end_of_vacuum;
 
@@ -342,10 +336,11 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){
 
 end_of_vacuum:
   /* Restore the original value of db->flags */
+  db->init.iDb = 0;
   db->flags = saved_flags;
   db->nChange = saved_nChange;
   db->nTotalChange = saved_nTotalChange;
-  db->xTrace = saved_xTrace;
+  db->mTrace = saved_mTrace;
   sqlite3BtreeSetPageSize(pMain, -1, -1, 1);
 
   /* Currently there is an SQL level transaction open on the vacuum
diff --git a/src/vdbe.c b/src/vdbe.c
index 3351a44551..406394c5be 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -86,6 +86,16 @@ static void updateMaxBlobsize(Mem *p){
 }
 #endif
 
+/*
+** This macro evaluates to true if either the update hook or the preupdate
+** hook are enabled for database connect DB.
+*/
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback)
+#else
+# define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback)
+#endif
+
 /*
 ** The next global variable is incremented each time the OP_Found opcode
 ** is executed. This is used to test whether or not the foreign key
@@ -101,7 +111,7 @@ int sqlite3_found_count = 0;
 ** Test a register to see if it exceeds the current maximum blob size.
 ** If it does, record the new maximum blob size.
 */
-#if defined(SQLITE_TEST) && !defined(SQLITE_OMIT_BUILTIN_TEST)
+#if defined(SQLITE_TEST) && !defined(SQLITE_UNTESTABLE)
 # define UPDATE_MAX_BLOBSIZE(P)  updateMaxBlobsize(P)
 #else
 # define UPDATE_MAX_BLOBSIZE(P)
@@ -192,11 +202,11 @@ static VdbeCursor *allocateCursor(
   **     be freed lazily via the sqlite3_release_memory() API. This
   **     minimizes the number of malloc calls made by the system.
   **
-  ** Memory cells for cursors are allocated at the top of the address
-  ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for
-  ** cursor 1 is managed by memory cell (p->nMem-1), etc.
+  ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from
+  ** the top of the register space.  Cursor 1 is at Mem[p->nMem-1].
+  ** Cursor 2 is at Mem[p->nMem-2]. And so forth.
   */
-  Mem *pMem = &p->aMem[p->nMem-iCur];
+  Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem;
 
   int nByte;
   VdbeCursor *pCx = 0;
@@ -204,14 +214,14 @@ static VdbeCursor *allocateCursor(
       ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + 
       (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0);
 
-  assert( iCurnCursor );
-  if( p->apCsr[iCur] ){
+  assert( iCur>=0 && iCurnCursor );
+  if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/
     sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
     p->apCsr[iCur] = 0;
   }
   if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){
     p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
-    memset(pCx, 0, sizeof(VdbeCursor));
+    memset(pCx, 0, offsetof(VdbeCursor,pAltCursor));
     pCx->eCurType = eCurType;
     pCx->iDb = iDb;
     pCx->nField = nField;
@@ -282,7 +292,7 @@ static void applyAffinity(
   if( affinity>=SQLITE_AFF_NUMERIC ){
     assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL
              || affinity==SQLITE_AFF_NUMERIC );
-    if( (pRec->flags & MEM_Int)==0 ){
+    if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/
       if( (pRec->flags & MEM_Real)==0 ){
         if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1);
       }else{
@@ -292,10 +302,13 @@ static void applyAffinity(
   }else if( affinity==SQLITE_AFF_TEXT ){
     /* Only attempt the conversion to TEXT if there is an integer or real
     ** representation (blob and NULL do not get converted) but no string
-    ** representation.
-    */
-    if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){
-      sqlite3VdbeMemStringify(pRec, enc, 1);
+    ** representation.  It would be harmless to repeat the conversion if 
+    ** there is already a string rep, but it is pointless to waste those
+    ** CPU cycles. */
+    if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/
+      if( (pRec->flags&(MEM_Real|MEM_Int)) ){
+        sqlite3VdbeMemStringify(pRec, enc, 1);
+      }
     }
     pRec->flags &= ~(MEM_Real|MEM_Int);
   }
@@ -390,9 +403,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){
     }else{
       c = 's';
     }
-
-    sqlite3_snprintf(100, zCsr, "%c", c);
-    zCsr += sqlite3Strlen30(zCsr);
+    *(zCsr++) = c;
     sqlite3_snprintf(100, zCsr, "%d[", pMem->n);
     zCsr += sqlite3Strlen30(zCsr);
     for(i=0; i<16 && in; i++){
@@ -404,9 +415,7 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){
       if( z<32 || z>126 ) *zCsr++ = '.';
       else *zCsr++ = z;
     }
-
-    sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]);
-    zCsr += sqlite3Strlen30(zCsr);
+    *(zCsr++) = ']';
     if( f & MEM_Zero ){
       sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero);
       zCsr += sqlite3Strlen30(zCsr);
@@ -529,10 +538,10 @@ static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){
 static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){
   Mem *pOut;
   assert( pOp->p2>0 );
-  assert( pOp->p2<=(p->nMem-p->nCursor) );
+  assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
   pOut = &p->aMem[pOp->p2];
   memAboutToChange(p, pOut);
-  if( VdbeMemDynamic(pOut) ){
+  if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/
     return out2PrereleaseWithClear(pOut);
   }else{
     pOut->flags = MEM_Int;
@@ -560,7 +569,7 @@ int sqlite3VdbeExec(
   sqlite3 *db = p->db;       /* The database */
   u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */
   u8 encoding = ENC(db);     /* The database encoding */
-  int iCompare = 0;          /* Result of last OP_Compare operation */
+  int iCompare = 0;          /* Result of last comparison */
   unsigned nVmStep = 0;      /* Number of virtual machine steps */
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */
@@ -570,8 +579,6 @@ int sqlite3VdbeExec(
   Mem *pIn2 = 0;             /* 2nd input operand */
   Mem *pIn3 = 0;             /* 3rd input operand */
   Mem *pOut = 0;             /* Output operand */
-  int *aPermute = 0;         /* Permutation of columns for OP_Compare */
-  i64 lastRowid = db->lastRowid;  /* Saved value of the last insert ROWID */
 #ifdef VDBE_PROFILE
   u64 start;                 /* CPU clock count at start of opcode */
 #endif
@@ -586,7 +593,6 @@ int sqlite3VdbeExec(
   }
   assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY );
   assert( p->bIsReader || p->readOnly!=0 );
-  p->rc = SQLITE_OK;
   p->iCurrentTime = 0;
   assert( p->explain==0 );
   p->pResultSet = 0;
@@ -664,37 +670,39 @@ int sqlite3VdbeExec(
 
     /* Sanity checking on other operands */
 #ifdef SQLITE_DEBUG
-    assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
-    if( (pOp->opflags & OPFLG_IN1)!=0 ){
-      assert( pOp->p1>0 );
-      assert( pOp->p1<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p1]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) );
-      REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
-    }
-    if( (pOp->opflags & OPFLG_IN2)!=0 ){
-      assert( pOp->p2>0 );
-      assert( pOp->p2<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p2]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) );
-      REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
-    }
-    if( (pOp->opflags & OPFLG_IN3)!=0 ){
-      assert( pOp->p3>0 );
-      assert( pOp->p3<=(p->nMem-p->nCursor) );
-      assert( memIsValid(&aMem[pOp->p3]) );
-      assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) );
-      REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
-    }
-    if( (pOp->opflags & OPFLG_OUT2)!=0 ){
-      assert( pOp->p2>0 );
-      assert( pOp->p2<=(p->nMem-p->nCursor) );
-      memAboutToChange(p, &aMem[pOp->p2]);
-    }
-    if( (pOp->opflags & OPFLG_OUT3)!=0 ){
-      assert( pOp->p3>0 );
-      assert( pOp->p3<=(p->nMem-p->nCursor) );
-      memAboutToChange(p, &aMem[pOp->p3]);
+    {
+      u8 opProperty = sqlite3OpcodeProperty[pOp->opcode];
+      if( (opProperty & OPFLG_IN1)!=0 ){
+        assert( pOp->p1>0 );
+        assert( pOp->p1<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p1]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) );
+        REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
+      }
+      if( (opProperty & OPFLG_IN2)!=0 ){
+        assert( pOp->p2>0 );
+        assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p2]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) );
+        REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
+      }
+      if( (opProperty & OPFLG_IN3)!=0 ){
+        assert( pOp->p3>0 );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
+        assert( memIsValid(&aMem[pOp->p3]) );
+        assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) );
+        REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
+      }
+      if( (opProperty & OPFLG_OUT2)!=0 ){
+        assert( pOp->p2>0 );
+        assert( pOp->p2<=(p->nMem+1 - p->nCursor) );
+        memAboutToChange(p, &aMem[pOp->p2]);
+      }
+      if( (opProperty & OPFLG_OUT3)!=0 ){
+        assert( pOp->p3>0 );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
+        memAboutToChange(p, &aMem[pOp->p3]);
+      }
     }
 #endif
 #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
@@ -792,7 +800,7 @@ check_for_interrupt:
 ** and then jump to address P2.
 */
 case OP_Gosub: {            /* jump */
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pIn1 = &aMem[pOp->p1];
   assert( VdbeMemDynamic(pIn1)==0 );
   memAboutToChange(p, pIn1);
@@ -832,7 +840,7 @@ case OP_Return: {           /* in1 */
 ** See also: EndCoroutine
 */
 case OP_InitCoroutine: {     /* jump */
-  assert( pOp->p1>0 &&  pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 &&  pOp->p1<=(p->nMem+1 - p->nCursor) );
   assert( pOp->p2>=0 && pOp->p2nOp );
   assert( pOp->p3>=0 && pOp->p3nOp );
   pOut = &aMem[pOp->p1];
@@ -890,7 +898,7 @@ case OP_Yield: {            /* in1, jump */
 }
 
 /* Opcode:  HaltIfNull  P1 P2 P3 P4 P5
-** Synopsis:  if r[P3]=null halt
+** Synopsis: if r[P3]=null halt
 **
 ** Check the value in register P3.  If it is NULL then Halt using
 ** parameter P1, P2, and P4 as if this were a Halt instruction.  If the
@@ -934,8 +942,6 @@ case OP_HaltIfNull: {      /* in3 */
 ** is the same as executing Halt.
 */
 case OP_Halt: {
-  const char *zType;
-  const char *zLogFmt;
   VdbeFrame *pFrame;
   int pcx;
 
@@ -947,7 +953,6 @@ case OP_Halt: {
     p->nFrame--;
     sqlite3VdbeSetChanges(db, p->nChange);
     pcx = sqlite3VdbeFrameRestore(pFrame);
-    lastRowid = db->lastRowid;
     if( pOp->p2==OE_Ignore ){
       /* Instruction pcx is the OP_Program that invoked the sub-program 
       ** currently being halted. If the p2 instruction of this OP_Halt
@@ -964,34 +969,28 @@ case OP_Halt: {
   p->rc = pOp->p1;
   p->errorAction = (u8)pOp->p2;
   p->pc = pcx;
+  assert( pOp->p5<=4 );
   if( p->rc ){
     if( pOp->p5 ){
       static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK",
                                              "FOREIGN KEY" };
-      assert( pOp->p5>=1 && pOp->p5<=4 );
       testcase( pOp->p5==1 );
       testcase( pOp->p5==2 );
       testcase( pOp->p5==3 );
       testcase( pOp->p5==4 );
-      zType = azType[pOp->p5-1];
+      sqlite3VdbeError(p, "%s constraint failed", azType[pOp->p5-1]);
+      if( pOp->p4.z ){
+        p->zErrMsg = sqlite3MPrintf(db, "%z: %s", p->zErrMsg, pOp->p4.z);
+      }
     }else{
-      zType = 0;
-    }
-    assert( zType!=0 || pOp->p4.z!=0 );
-    zLogFmt = "abort at %d in [%s]: %s";
-    if( zType && pOp->p4.z ){
-      sqlite3VdbeError(p, "%s constraint failed: %s", zType, pOp->p4.z);
-    }else if( pOp->p4.z ){
       sqlite3VdbeError(p, "%s", pOp->p4.z);
-    }else{
-      sqlite3VdbeError(p, "%s constraint failed", zType);
     }
-    sqlite3_log(pOp->p1, zLogFmt, pcx, p->zSql, p->zErrMsg);
+    sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg);
   }
   rc = sqlite3VdbeHalt(p);
   assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR );
   if( rc==SQLITE_BUSY ){
-    p->rc = rc = SQLITE_BUSY;
+    p->rc = SQLITE_BUSY;
   }else{
     assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT );
     assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 );
@@ -1057,10 +1056,7 @@ case OP_String8: {         /* same as TK_STRING, out2 */
 #ifndef SQLITE_OMIT_UTF16
   if( encoding!=SQLITE_UTF8 ){
     rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC);
-    if( rc ){
-      assert( rc==SQLITE_TOOBIG ); /* This is the only possible error here */
-      goto too_big;
-    }
+    assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG );
     if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem;
     assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z );
     assert( VdbeMemDynamic(pOut)==0 );
@@ -1073,10 +1069,12 @@ case OP_String8: {         /* same as TK_STRING, out2 */
     pOp->p4.z = pOut->z;
     pOp->p1 = pOut->n;
   }
+  testcase( rc==SQLITE_TOOBIG );
 #endif
   if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){
     goto too_big;
   }
+  assert( rc==SQLITE_OK );
   /* Fall through to the next case, OP_String */
 }
   
@@ -1085,10 +1083,12 @@ case OP_String8: {         /* same as TK_STRING, out2 */
 **
 ** The string value P4 of length P1 (bytes) is stored in register P2.
 **
-** If P5!=0 and the content of register P3 is greater than zero, then
+** If P3 is not zero and the content of register P3 is equal to P5, then
 ** the datatype of the register P2 is converted to BLOB.  The content is
 ** the same sequence of bytes, it is merely interpreted as a BLOB instead
-** of a string, as if it had been CAST.
+** of a string, as if it had been CAST.  In other words:
+**
+** if( P3!=0 and reg[P3]==P5 ) reg[P2] := CAST(reg[P2] as BLOB)
 */
 case OP_String: {          /* out2 */
   assert( pOp->p4.z!=0 );
@@ -1099,19 +1099,18 @@ case OP_String: {          /* out2 */
   pOut->enc = encoding;
   UPDATE_MAX_BLOBSIZE(pOut);
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
-  if( pOp->p5 ){
-    assert( pOp->p3>0 );
-    assert( pOp->p3<=(p->nMem-p->nCursor) );
+  if( pOp->p3>0 ){
+    assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
     pIn3 = &aMem[pOp->p3];
     assert( pIn3->flags & MEM_Int );
-    if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term;
+    if( pIn3->u.i==pOp->p5 ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term;
   }
 #endif
   break;
 }
 
 /* Opcode: Null P1 P2 P3 * *
-** Synopsis:  r[P2..P3]=NULL
+** Synopsis: r[P2..P3]=NULL
 **
 ** Write a NULL into registers P2.  If P3 greater than P2, then also write
 ** NULL into register P3 and every register in between P2 and P3.  If P3
@@ -1127,20 +1126,22 @@ case OP_Null: {           /* out2 */
   u16 nullFlag;
   pOut = out2Prerelease(p, pOp);
   cnt = pOp->p3-pOp->p2;
-  assert( pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
   pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null;
+  pOut->n = 0;
   while( cnt>0 ){
     pOut++;
     memAboutToChange(p, pOut);
     sqlite3VdbeMemSetNull(pOut);
     pOut->flags = nullFlag;
+    pOut->n = 0;
     cnt--;
   }
   break;
 }
 
 /* Opcode: SoftNull P1 * * * *
-** Synopsis:  r[P1]=NULL
+** Synopsis: r[P1]=NULL
 **
 ** Set register P1 to have the value NULL as seen by the OP_MakeRecord
 ** instruction, but do not free any string or blob memory associated with
@@ -1148,7 +1149,7 @@ case OP_Null: {           /* out2 */
 ** previously copied using OP_SCopy, the copies will continue to be valid.
 */
 case OP_SoftNull: {
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pOut = &aMem[pOp->p1];
   pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined;
   break;
@@ -1181,19 +1182,19 @@ case OP_Variable: {            /* out2 */
   Mem *pVar;       /* Value being transferred */
 
   assert( pOp->p1>0 && pOp->p1<=p->nVar );
-  assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] );
+  assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) );
   pVar = &p->aVar[pOp->p1 - 1];
   if( sqlite3VdbeMemTooBig(pVar) ){
     goto too_big;
   }
-  pOut = out2Prerelease(p, pOp);
+  pOut = &aMem[pOp->p2];
   sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static);
   UPDATE_MAX_BLOBSIZE(pOut);
   break;
 }
 
 /* Opcode: Move P1 P2 P3 * *
-** Synopsis:  r[P2@P3]=r[P1@P3]
+** Synopsis: r[P2@P3]=r[P1@P3]
 **
 ** Move the P3 values in register P1..P1+P3-1 over into
 ** registers P2..P2+P3-1.  Registers P1..P1+P3-1 are
@@ -1215,8 +1216,8 @@ case OP_Move: {
   pIn1 = &aMem[p1];
   pOut = &aMem[p2];
   do{
-    assert( pOut<=&aMem[(p->nMem-p->nCursor)] );
-    assert( pIn1<=&aMem[(p->nMem-p->nCursor)] );
+    assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] );
+    assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] );
     assert( memIsValid(pIn1) );
     memAboutToChange(p, pOut);
     sqlite3VdbeMemMove(pOut, pIn1);
@@ -1303,7 +1304,7 @@ case OP_IntCopy: {            /* out2 */
 }
 
 /* Opcode: ResultRow P1 P2 * * *
-** Synopsis:  output=r[P1@P2]
+** Synopsis: output=r[P1@P2]
 **
 ** The registers P1 through P1+P2-1 contain a single row of
 ** results. This opcode causes the sqlite3_step() call to terminate
@@ -1316,7 +1317,7 @@ case OP_ResultRow: {
   int i;
   assert( p->nResColumn==pOp->p2 );
   assert( pOp->p1>0 );
-  assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 );
+  assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 );
 
 #ifndef SQLITE_OMIT_PROGRESS_CALLBACK
   /* Run the progress counter just before returning.
@@ -1376,6 +1377,10 @@ case OP_ResultRow: {
   }
   if( db->mallocFailed ) goto no_mem;
 
+  if( db->mTrace & SQLITE_TRACE_ROW ){
+    db->xTrace(SQLITE_TRACE_ROW, db->pTraceArg, p, 0);
+  }
+
   /* Return SQLITE_ROW
   */
   p->pc = (int)(pOp - aOp) + 1;
@@ -1432,14 +1437,14 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 }
 
 /* Opcode: Add P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]+r[P2]
+** Synopsis: r[P3]=r[P1]+r[P2]
 **
 ** Add the value in register P1 to the value in register P2
 ** and store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Multiply P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]*r[P2]
+** Synopsis: r[P3]=r[P1]*r[P2]
 **
 **
 ** Multiply the value in register P1 by the value in register P2
@@ -1447,14 +1452,14 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Subtract P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]-r[P1]
+** Synopsis: r[P3]=r[P2]-r[P1]
 **
 ** Subtract the value in register P1 from the value in register P2
 ** and store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: Divide P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]/r[P1]
+** Synopsis: r[P3]=r[P2]/r[P1]
 **
 ** Divide the value in register P1 by the value in register P2
 ** and store the result in register P3 (P3=P2/P1). If the value in 
@@ -1462,7 +1467,7 @@ case OP_Concat: {           /* same as TK_CONCAT, in1, in2, out3 */
 ** NULL, the result is NULL.
 */
 /* Opcode: Remainder P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]%r[P1]
+** Synopsis: r[P3]=r[P2]%r[P1]
 **
 ** Compute the remainder after integer register P2 is divided by 
 ** register P1 and store the result in register P3. 
@@ -1628,8 +1633,8 @@ case OP_Function0: {
 
   assert( pOp->p4type==P4_FUNCDEF );
   n = pOp->p5;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
-  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
+  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3p2 || pOp->p3>=pOp->p2+n );
   pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*));
   if( pCtx==0 ) goto no_mem;
@@ -1660,23 +1665,21 @@ case OP_Function: {
     for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i];
   }
 
-  memAboutToChange(p, pCtx->pOut);
+  memAboutToChange(p, pOut);
 #ifdef SQLITE_DEBUG
   for(i=0; iargc; i++){
     assert( memIsValid(pCtx->argv[i]) );
     REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]);
   }
 #endif
-  MemSetTypeFlag(pCtx->pOut, MEM_Null);
+  MemSetTypeFlag(pOut, MEM_Null);
   pCtx->fErrorOrAux = 0;
-  db->lastRowid = lastRowid;
   (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */
-  lastRowid = db->lastRowid;  /* Remember rowid changes made by xSFunc */
 
   /* If the function returned an error, throw an exception */
   if( pCtx->fErrorOrAux ){
     if( pCtx->isError ){
-      sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut));
+      sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut));
       rc = pCtx->isError;
     }
     sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1);
@@ -1685,31 +1688,31 @@ case OP_Function: {
 
   /* Copy the result of the function into register P3 */
   if( pOut->flags & (MEM_Str|MEM_Blob) ){
-    sqlite3VdbeChangeEncoding(pCtx->pOut, encoding);
-    if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big;
+    sqlite3VdbeChangeEncoding(pOut, encoding);
+    if( sqlite3VdbeMemTooBig(pOut) ) goto too_big;
   }
 
-  REGISTER_TRACE(pOp->p3, pCtx->pOut);
-  UPDATE_MAX_BLOBSIZE(pCtx->pOut);
+  REGISTER_TRACE(pOp->p3, pOut);
+  UPDATE_MAX_BLOBSIZE(pOut);
   break;
 }
 
 /* Opcode: BitAnd P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]&r[P2]
+** Synopsis: r[P3]=r[P1]&r[P2]
 **
 ** Take the bit-wise AND of the values in register P1 and P2 and
 ** store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: BitOr P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P1]|r[P2]
+** Synopsis: r[P3]=r[P1]|r[P2]
 **
 ** Take the bit-wise OR of the values in register P1 and P2 and
 ** store the result in register P3.
 ** If either input is NULL, the result is NULL.
 */
 /* Opcode: ShiftLeft P1 P2 P3 * *
-** Synopsis:  r[P3]=r[P2]<>r[P1]
+** Synopsis: r[P3]=r[P2]>>r[P1]
 **
 ** Shift the integer value in register P2 to the right by the
 ** number of bits specified by the integer in register P1.
@@ -1777,7 +1780,7 @@ case OP_ShiftRight: {           /* same as TK_RSHIFT, in1, in2, out3 */
 }
 
 /* Opcode: AddImm  P1 P2 * * *
-** Synopsis:  r[P1]=r[P1]+P2
+** Synopsis: r[P1]=r[P1]+P2
 ** 
 ** Add the constant P2 to the value in register P1.
 ** The result is always an integer.
@@ -1869,14 +1872,61 @@ case OP_Cast: {                  /* in1 */
 }
 #endif /* SQLITE_OMIT_CAST */
 
+/* Opcode: Eq P1 P2 P3 P4 P5
+** Synopsis: IF r[P3]==r[P1]
+**
+** Compare the values in register P1 and P3.  If reg(P3)==reg(P1) then
+** jump to address P2.  Or if the SQLITE_STOREP2 flag is set in P5, then
+** store the result of comparison in register P2.
+**
+** The SQLITE_AFF_MASK portion of P5 must be an affinity character -
+** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made 
+** to coerce both inputs according to this affinity before the
+** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric
+** affinity is used. Note that the affinity conversions are stored
+** back into the input registers P1 and P3.  So this opcode can cause
+** persistent changes to registers P1 and P3.
+**
+** Once any conversions have taken place, and neither value is NULL, 
+** the values are compared. If both values are blobs then memcmp() is
+** used to determine the results of the comparison.  If both values
+** are text, then the appropriate collating function specified in
+** P4 is used to do the comparison.  If P4 is not specified then
+** memcmp() is used to compare text string.  If both values are
+** numeric, then a numeric comparison is used. If the two values
+** are of different types, then numbers are considered less than
+** strings and strings are considered less than blobs.
+**
+** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either
+** true or false and is never NULL.  If both operands are NULL then the result
+** of comparison is true.  If either operand is NULL then the result is false.
+** If neither operand is NULL the result is the same as it would be if
+** the SQLITE_NULLEQ flag were omitted from P5.
+**
+** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the
+** content of r[P2] is only changed if the new value is NULL or 0 (false).
+** In other words, a prior r[P2] value will not be overwritten by 1 (true).
+*/
+/* Opcode: Ne P1 P2 P3 P4 P5
+** Synopsis: IF r[P3]!=r[P1]
+**
+** This works just like the Eq opcode except that the jump is taken if
+** the operands in registers P1 and P3 are not equal.  See the Eq opcode for
+** additional information.
+**
+** If both SQLITE_STOREP2 and SQLITE_KEEPNULL flags are set then the
+** content of r[P2] is only changed if the new value is NULL or 1 (true).
+** In other words, a prior r[P2] value will not be overwritten by 0 (false).
+*/
 /* Opcode: Lt P1 P2 P3 P4 P5
-** Synopsis: if r[P1]r[P3] goto P2
+** Synopsis: IF r[P3]>r[P1]
 **
 ** This works just like the Lt opcode except that the jump is taken if
 ** the content of register P3 is greater than the content of
 ** register P1.  See the Lt opcode for additional information.
 */
 /* Opcode: Ge P1 P2 P3 P4 P5
-** Synopsis: if r[P1]>=r[P3] goto P2
+** Synopsis: IF r[P3]>=r[P1]
 **
 ** This works just like the Lt opcode except that the jump is taken if
 ** the content of register P3 is greater than or equal to the content of
@@ -1957,7 +1974,7 @@ case OP_Lt:               /* same as TK_LT, jump, in1, in3 */
 case OP_Le:               /* same as TK_LE, jump, in1, in3 */
 case OP_Gt:               /* same as TK_GT, jump, in1, in3 */
 case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
-  int res;            /* Result of the comparison of pIn1 against pIn3 */
+  int res, res2;      /* Result of the comparison of pIn1 against pIn3 */
   char affinity;      /* Affinity to use for comparison */
   u16 flags1;         /* Copy of initial value of pIn1->flags */
   u16 flags3;         /* Copy of initial value of pIn3->flags */
@@ -1976,13 +1993,12 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne );
       assert( (flags1 & MEM_Cleared)==0 );
       assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 );
-      if( (flags1&MEM_Null)!=0
-       && (flags3&MEM_Null)!=0
+      if( (flags1&flags3&MEM_Null)!=0
        && (flags3&MEM_Cleared)==0
       ){
-        res = 0;  /* Results are equal */
+        res = 0;  /* Operands are equal */
       }else{
-        res = 1;  /* Results are not equal */
+        res = 1;  /* Operands are not equal */
       }
     }else{
       /* SQLITE_NULLEQ is clear and at least one operand is NULL,
@@ -1991,6 +2007,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       */
       if( pOp->p5 & SQLITE_STOREP2 ){
         pOut = &aMem[pOp->p2];
+        iCompare = 1;    /* Operands are not equal */
         memAboutToChange(p, pOut);
         MemSetTypeFlag(pOut, MEM_Null);
         REGISTER_TRACE(pOp->p2, pOut);
@@ -2006,11 +2023,23 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
     /* Neither operand is NULL.  Do a comparison. */
     affinity = pOp->p5 & SQLITE_AFF_MASK;
     if( affinity>=SQLITE_AFF_NUMERIC ){
-      if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
-        applyNumericAffinity(pIn1,0);
+      if( (flags1 | flags3)&MEM_Str ){
+        if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
+          applyNumericAffinity(pIn1,0);
+          testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */
+          flags3 = pIn3->flags;
+        }
+        if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
+          applyNumericAffinity(pIn3,0);
+        }
       }
-      if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){
-        applyNumericAffinity(pIn3,0);
+      /* Handle the common case of integer comparison here, as an
+      ** optimization, to avoid a call to sqlite3MemCompare() */
+      if( (pIn1->flags & pIn3->flags & MEM_Int)!=0 ){
+        if( pIn3->u.i > pIn1->u.i ){ res = +1; goto compare_op; }
+        if( pIn3->u.i < pIn1->u.i ){ res = -1; goto compare_op; }
+        res = 0;
+        goto compare_op;
       }
     }else if( affinity==SQLITE_AFF_TEXT ){
       if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){
@@ -2019,6 +2048,7 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
         sqlite3VdbeMemStringify(pIn1, encoding, 1);
         testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) );
         flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
+        assert( pIn1!=pIn3 );
       }
       if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){
         testcase( pIn3->flags & MEM_Int );
@@ -2029,23 +2059,16 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
       }
     }
     assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 );
-    if( flags1 & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pIn1);
-      flags1 &= ~MEM_Zero;
-    }
-    if( flags3 & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pIn3);
-      flags3 &= ~MEM_Zero;
-    }
     res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl);
   }
+compare_op:
   switch( pOp->opcode ){
-    case OP_Eq:    res = res==0;     break;
-    case OP_Ne:    res = res!=0;     break;
-    case OP_Lt:    res = res<0;      break;
-    case OP_Le:    res = res<=0;     break;
-    case OP_Gt:    res = res>0;      break;
-    default:       res = res>=0;     break;
+    case OP_Eq:    res2 = res==0;     break;
+    case OP_Ne:    res2 = res;        break;
+    case OP_Lt:    res2 = res<0;      break;
+    case OP_Le:    res2 = res<=0;     break;
+    case OP_Gt:    res2 = res>0;      break;
+    default:       res2 = res>=0;     break;
   }
 
   /* Undo any changes made by applyAffinity() to the input registers. */
@@ -2056,23 +2079,59 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 
   if( pOp->p5 & SQLITE_STOREP2 ){
     pOut = &aMem[pOp->p2];
+    iCompare = res;
+    res2 = res2!=0;  /* For this path res2 must be exactly 0 or 1 */
+    if( (pOp->p5 & SQLITE_KEEPNULL)!=0 ){
+      /* The KEEPNULL flag prevents OP_Eq from overwriting a NULL with 1
+      ** and prevents OP_Ne from overwriting NULL with 0.  This flag
+      ** is only used in contexts where either:
+      **   (1) op==OP_Eq && (r[P2]==NULL || r[P2]==0)
+      **   (2) op==OP_Ne && (r[P2]==NULL || r[P2]==1)
+      ** Therefore it is not necessary to check the content of r[P2] for
+      ** NULL. */
+      assert( pOp->opcode==OP_Ne || pOp->opcode==OP_Eq );
+      assert( res2==0 || res2==1 );
+      testcase( res2==0 && pOp->opcode==OP_Eq );
+      testcase( res2==1 && pOp->opcode==OP_Eq );
+      testcase( res2==0 && pOp->opcode==OP_Ne );
+      testcase( res2==1 && pOp->opcode==OP_Ne );
+      if( (pOp->opcode==OP_Eq)==res2 ) break;
+    }
     memAboutToChange(p, pOut);
     MemSetTypeFlag(pOut, MEM_Int);
-    pOut->u.i = res;
+    pOut->u.i = res2;
     REGISTER_TRACE(pOp->p2, pOut);
   }else{
     VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3);
-    if( res ){
+    if( res2 ){
       goto jump_to_p2;
     }
   }
   break;
 }
 
+/* Opcode: ElseNotEq * P2 * * *
+**
+** This opcode must immediately follow an OP_Lt or OP_Gt comparison operator.
+** If result of an OP_Eq comparison on the same two operands
+** would have be NULL or false (0), then then jump to P2. 
+** If the result of an OP_Eq comparison on the two previous operands
+** would have been true (1), then fall through.
+*/
+case OP_ElseNotEq: {       /* same as TK_ESCAPE, jump */
+  assert( pOp>aOp );
+  assert( pOp[-1].opcode==OP_Lt || pOp[-1].opcode==OP_Gt );
+  assert( pOp[-1].p5 & SQLITE_STOREP2 );
+  VdbeBranchTaken(iCompare!=0, 2);
+  if( iCompare!=0 ) goto jump_to_p2;
+  break;
+}
+
+
 /* Opcode: Permutation * * * P4 *
 **
-** Set the permutation used by the OP_Compare operator to be the array
-** of integers in P4.
+** Set the permutation used by the OP_Compare operator in the next
+** instruction.  The permutation is stored in the P4 operand.
 **
 ** The permutation is only valid until the next OP_Compare that has
 ** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should 
@@ -2084,7 +2143,8 @@ case OP_Ge: {             /* same as TK_GE, jump, in1, in3 */
 case OP_Permutation: {
   assert( pOp->p4type==P4_INTARRAY );
   assert( pOp->p4.ai );
-  aPermute = pOp->p4.ai + 1;
+  assert( pOp[1].opcode==OP_Compare );
+  assert( pOp[1].p5 & OPFLAG_PERMUTE );
   break;
 }
 
@@ -2117,23 +2177,32 @@ case OP_Compare: {
   int idx;
   CollSeq *pColl;    /* Collating sequence to use on this term */
   int bRev;          /* True for DESCENDING sort order */
+  int *aPermute;     /* The permutation */
 
-  if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0;
+  if( (pOp->p5 & OPFLAG_PERMUTE)==0 ){
+    aPermute = 0;
+  }else{
+    assert( pOp>aOp );
+    assert( pOp[-1].opcode==OP_Permutation );
+    assert( pOp[-1].p4type==P4_INTARRAY );
+    aPermute = pOp[-1].p4.ai + 1;
+    assert( aPermute!=0 );
+  }
   n = pOp->p3;
   pKeyInfo = pOp->p4.pKeyInfo;
   assert( n>0 );
   assert( pKeyInfo!=0 );
   p1 = pOp->p1;
   p2 = pOp->p2;
-#if SQLITE_DEBUG
+#ifdef SQLITE_DEBUG
   if( aPermute ){
     int k, mx = 0;
     for(k=0; kmx ) mx = aPermute[k];
-    assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 );
-    assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 );
+    assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 );
+    assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 );
   }else{
-    assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 );
-    assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 );
+    assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 );
+    assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 );
   }
 #endif /* SQLITE_DEBUG */
   for(i=0; ip1nOnceFlag );
-  VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2);
-  if( p->aOnceFlag[pOp->p1] ){
-    goto jump_to_p2;
+  u32 iAddr;                /* Address of this instruction */
+  assert( p->aOp[0].opcode==OP_Init );
+  if( p->pFrame ){
+    iAddr = (int)(pOp - p->aOp);
+    if( (p->pFrame->aOnce[iAddr/8] & (1<<(iAddr & 7)))!=0 ){
+      VdbeBranchTaken(1, 2);
+      goto jump_to_p2;
+    }
+    p->pFrame->aOnce[iAddr/8] |= 1<<(iAddr & 7);
   }else{
-    p->aOnceFlag[pOp->p1] = 1;
+    if( p->aOp[0].p1==pOp->p1 ){
+      VdbeBranchTaken(1, 2);
+      goto jump_to_p2;
+    }
   }
+  VdbeBranchTaken(0, 2);
+  pOp->p1 = p->aOp[0].p1;
   break;
 }
 
@@ -2318,7 +2402,7 @@ case OP_IfNot: {            /* jump, in1 */
 }
 
 /* Opcode: IsNull P1 P2 * * *
-** Synopsis:  if r[P1]==NULL goto P2
+** Synopsis: if r[P1]==NULL goto P2
 **
 ** Jump to P2 if the value in register P1 is NULL.
 */
@@ -2346,7 +2430,7 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 }
 
 /* Opcode: Column P1 P2 P3 P4 P5
-** Synopsis:  r[P3]=PX
+** Synopsis: r[P3]=PX
 **
 ** Interpret the data that cursor P1 points to as a structure built using
 ** the MakeRecord instruction.  (See the MakeRecord opcode for additional
@@ -2371,7 +2455,6 @@ case OP_NotNull: {            /* same as TK_NOTNULL, jump, in1 */
 ** skipped for length() and all content loading can be skipped for typeof().
 */
 case OP_Column: {
-  i64 payloadSize64; /* Number of bytes in the record */
   int p2;            /* column number to retrieve */
   VdbeCursor *pC;    /* The VDBE cursor */
   BtCursor *pCrsr;   /* The BTree cursor */
@@ -2394,8 +2477,9 @@ case OP_Column: {
 
   /* If the cursor cache is stale, bring it up-to-date */
   rc = sqlite3VdbeCursorMoveto(&pC, &p2);
+  if( rc ) goto abort_due_to_error;
 
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pDest = &aMem[pOp->p3];
   memAboutToChange(p, pDest);
   assert( pOp->p1>=0 && pOp->p1nCursor );
@@ -2405,10 +2489,8 @@ case OP_Column: {
   assert( pC->eCurType!=CURTYPE_VTAB );
   assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow );
   assert( pC->eCurType!=CURTYPE_SORTER );
-  pCrsr = pC->uc.pCursor;
 
-  if( rc ) goto abort_due_to_error;
-  if( pC->cacheStatus!=p->cacheCtr ){
+  if( pC->cacheStatus!=p->cacheCtr ){                /*OPTIMIZATION-IF-FALSE*/
     if( pC->nullRow ){
       if( pC->eCurType==CURTYPE_PSEUDO ){
         assert( pC->uc.pseudoTableReg>0 );
@@ -2422,24 +2504,12 @@ case OP_Column: {
         goto op_column_out;
       }
     }else{
+      pCrsr = pC->uc.pCursor;
       assert( pC->eCurType==CURTYPE_BTREE );
       assert( pCrsr );
-      if( pC->isTable==0 ){
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64);
-        assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */
-        /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the
-        ** payload size, so it is impossible for payloadSize64 to be
-        ** larger than 32 bits. */
-        assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 );
-        pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail);
-        pC->payloadSize = (u32)payloadSize64;
-      }else{
-        assert( sqlite3BtreeCursorIsValid(pCrsr) );
-        VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize);
-        assert( rc==SQLITE_OK );   /* DataSize() cannot fail */
-        pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail);
-      }
+      assert( sqlite3BtreeCursorIsValid(pCrsr) );
+      pC->payloadSize = sqlite3BtreePayloadSize(pCrsr);
+      pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &avail);
       assert( avail<=65536 );  /* Maximum page size is 64KiB */
       if( pC->payloadSize <= (u32)avail ){
         pC->szRow = pC->payloadSize;
@@ -2455,7 +2525,7 @@ case OP_Column: {
     aOffset[0] = offset;
 
 
-    if( availaRow does not have to hold the entire row, but it does at least
       ** need to cover the header of the record.  If pC->aRow does not contain
       ** the complete header, then set it to zero, forcing the header to be
@@ -2476,14 +2546,15 @@ case OP_Column: {
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
+    }else if( offset>0 ){ /*OPTIMIZATION-IF-TRUE*/
+      /* The following goto is an optimization.  It can be omitted and
+      ** everything will still work.  But OP_Column is measurably faster
+      ** by skipping the subsequent conditional, which is always true.
+      */
+      zData = pC->aRow;
+      assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
+      goto op_column_read_header;
     }
-
-    /* The following goto is an optimization.  It can be omitted and
-    ** everything will still work.  But OP_Column is measurably faster
-    ** by skipping the subsequent conditional, which is always true.
-    */
-    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
-    goto op_column_read_header;
   }
 
   /* Make sure at least the first p2+1 entries of the header have been
@@ -2493,12 +2564,11 @@ case OP_Column: {
     /* If there is more header available for parsing in the record, try
     ** to extract additional fields up through the p2+1-th field 
     */
-    op_column_read_header:
     if( pC->iHdrOffsetaRow==0 ){
         memset(&sMem, 0, sizeof(sMem));
-        rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem);
+        rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, 0, aOffset[0], &sMem);
         if( rc!=SQLITE_OK ) goto abort_due_to_error;
         zData = (u8*)sMem.z;
       }else{
@@ -2506,11 +2576,11 @@ case OP_Column: {
       }
   
       /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */
+    op_column_read_header:
       i = pC->nHdrParsed;
       offset64 = aOffset[i];
       zHdr = zData + pC->iHdrOffset;
       zEndHdr = zData + aOffset[0];
-      assert( i<=p2 && zHdraType[i++] = t;
         aOffset[i] = (u32)(offset64 & 0xffffffff);
       }while( i<=p2 && zHdrnHdrParsed = i;
-      pC->iHdrOffset = (u32)(zHdr - zData);
-      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
-  
+
       /* The record is corrupt if any of the following are true:
       ** (1) the bytes of the header extend past the declared header size
       ** (2) the entire header was used but not all data was used
@@ -2534,9 +2601,14 @@ case OP_Column: {
       if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize))
        || (offset64 > pC->payloadSize)
       ){
+        if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
         rc = SQLITE_CORRUPT_BKPT;
         goto abort_due_to_error;
       }
+
+      pC->nHdrParsed = i;
+      pC->iHdrOffset = (u32)(zHdr - zData);
+      if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem);
     }else{
       t = 0;
     }
@@ -2564,9 +2636,10 @@ case OP_Column: {
   assert( p2nHdrParsed );
   assert( rc==SQLITE_OK );
   assert( sqlite3VdbeCheckMemInvariants(pDest) );
-  if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest);
+  if( VdbeMemDynamic(pDest) ){
+    sqlite3VdbeMemSetNull(pDest);
+  }
   assert( t==pC->aType[p2] );
-  pDest->enc = encoding;
   if( pC->szRow>=aOffset[p2+1] ){
     /* This is the common case where the desired content fits on the original
     ** page - where the content is not on an overflow page */
@@ -2580,6 +2653,7 @@ case OP_Column: {
       */
       static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term };
       pDest->n = len = (t-12)/2;
+      pDest->enc = encoding;
       if( pDest->szMalloc < len+2 ){
         pDest->flags = MEM_Null;
         if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem;
@@ -2592,6 +2666,7 @@ case OP_Column: {
       pDest->flags = aFlag[t&1];
     }
   }else{
+    pDest->enc = encoding;
     /* This branch happens only when content is on overflow pages */
     if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
           && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
@@ -2602,12 +2677,16 @@ case OP_Column: {
       **    2. the length(X) function if X is a blob, and
       **    3. if the content length is zero.
       ** So we might as well use bogus content rather than reading
-      ** content from disk. */
-      static u8 aZero[8];  /* This is the bogus content */
+      ** content from disk. 
+      **
+      ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the
+      ** buffer passed to it, debugging function VdbeMemPrettyPrint() may
+      ** read up to 16. So 16 bytes of bogus content is supplied.
+      */
+      static u8 aZero[16];  /* This is the bogus content */
       sqlite3VdbeSerialGet(aZero, t, pDest);
     }else{
-      rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable,
-                                   pDest);
+      rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest);
       if( rc!=SQLITE_OK ) goto abort_due_to_error;
       sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest);
       pDest->flags &= ~MEM_Ephem;
@@ -2638,7 +2717,7 @@ case OP_Affinity: {
   assert( zAffinity[pOp->p2]==0 );
   pIn1 = &aMem[pOp->p1];
   while( (cAff = *(zAffinity++))!=0 ){
-    assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] );
+    assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] );
     assert( memIsValid(pIn1) );
     applyAffinity(pIn1, cAff, encoding);
     pIn1++;
@@ -2700,7 +2779,7 @@ case OP_MakeRecord: {
   nZero = 0;         /* Number of zero bytes at the end of the record */
   nField = pOp->p1;
   zAffinity = pOp->p4.z;
-  assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 );
+  assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 );
   pData0 = &aMem[nField];
   nField = pOp->p2;
   pLast = &pData0[nField-1];
@@ -2722,6 +2801,20 @@ case OP_MakeRecord: {
     }while( zAffinity[0] );
   }
 
+#ifdef SQLITE_ENABLE_NULL_TRIM
+  /* NULLs can be safely trimmed from the end of the record, as long as
+  ** as the schema format is 2 or more and none of the omitted columns
+  ** have a non-NULL default value.  Also, the record must be left with
+  ** at least one field.  If P5>0 then it will be one more than the
+  ** index of the right-most column with a non-NULL default value */
+  if( pOp->p5 ){
+    while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){
+      pLast--;
+      nField--;
+    }
+  }
+#endif
+
   /* Loop through the elements that will make up the record to figure
   ** out how much space is required for the new record.
   */
@@ -2741,7 +2834,9 @@ case OP_MakeRecord: {
     testcase( serial_type==127 );
     testcase( serial_type==128 );
     nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type);
-  }while( (--pRec)>=pData0 );
+    if( pRec==pData0 ) break;
+    pRec--;
+  }while(1);
 
   /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint
   ** which determines the total number of bytes in the header. The varint
@@ -2790,7 +2885,7 @@ case OP_MakeRecord: {
   assert( i==nHdr );
   assert( j==nByte );
 
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pOut->n = (int)nByte;
   pOut->flags = MEM_Blob;
   if( nZero ){
@@ -2889,7 +2984,7 @@ case OP_Savepoint: {
         }else{
           db->nSavepoint++;
         }
-    
+
         /* Link the new savepoint into the database handle's list. */
         pNew->pNext = db->pSavepoint;
         db->pSavepoint = pNew;
@@ -3119,12 +3214,12 @@ case OP_Transaction: {
     rc = sqlite3BtreeBeginTrans(pBt, pOp->p2);
     testcase( rc==SQLITE_BUSY_SNAPSHOT );
     testcase( rc==SQLITE_BUSY_RECOVERY );
-    if( (rc&0xff)==SQLITE_BUSY ){
-      p->pc = (int)(pOp - aOp);
-      p->rc = rc;
-      goto vdbe_return;
-    }
     if( rc!=SQLITE_OK ){
+      if( (rc&0xff)==SQLITE_BUSY ){
+        p->pc = (int)(pOp - aOp);
+        p->rc = rc;
+        goto vdbe_return;
+      }
       goto abort_due_to_error;
     }
 
@@ -3151,10 +3246,9 @@ case OP_Transaction: {
     }
 
     /* Gather the schema version number for checking:
-    ** IMPLEMENTATION-OF: R-32195-19465 The schema version is used by SQLite
-    ** each time a query is executed to ensure that the internal cache of the
-    ** schema used when compiling the SQL query matches the schema of the
-    ** database against which the compiled query is actually executed.
+    ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema
+    ** version is checked to ensure that the schema has not changed since the
+    ** SQL statement was prepared.
     */
     sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta);
     iGen = db->aDb[pOp->p1].pSchema->iGeneration;
@@ -3376,7 +3470,7 @@ case OP_OpenWrite:
   }
   if( pOp->p5 & OPFLAG_P2ISREG ){
     assert( p2>0 );
-    assert( p2<=(p->nMem-p->nCursor) );
+    assert( p2<=(p->nMem+1 - p->nCursor) );
     pIn2 = &aMem[p2];
     assert( memIsValid(pIn2) );
     assert( (pIn2->flags & MEM_Int)!=0 );
@@ -3471,10 +3565,10 @@ case OP_OpenEphemeral: {
   if( pCx==0 ) goto no_mem;
   pCx->nullRow = 1;
   pCx->isEphemeral = 1;
-  rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, 
+  rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, 
                         BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags);
   if( rc==SQLITE_OK ){
-    rc = sqlite3BtreeBeginTrans(pCx->pBt, 1);
+    rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1);
   }
   if( rc==SQLITE_OK ){
     /* If a transient index is required, create it by calling
@@ -3482,21 +3576,20 @@ case OP_OpenEphemeral: {
     ** opening it. If a transient table is required, just use the
     ** automatically created table with root-page 1 (an BLOB_INTKEY table).
     */
-    if( (pKeyInfo = pOp->p4.pKeyInfo)!=0 ){
+    if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){
       int pgno;
       assert( pOp->p4type==P4_KEYINFO );
-      rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); 
+      rc = sqlite3BtreeCreateTable(pCx->pBtx, &pgno, BTREE_BLOBKEY | pOp->p5); 
       if( rc==SQLITE_OK ){
         assert( pgno==MASTER_ROOT+1 );
         assert( pKeyInfo->db==db );
         assert( pKeyInfo->enc==ENC(db) );
-        pCx->pKeyInfo = pKeyInfo;
-        rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR,
+        rc = sqlite3BtreeCursor(pCx->pBtx, pgno, BTREE_WRCSR,
                                 pKeyInfo, pCx->uc.pCursor);
       }
       pCx->isTable = 0;
     }else{
-      rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR,
+      rc = sqlite3BtreeCursor(pCx->pBtx, MASTER_ROOT, BTREE_WRCSR,
                               0, pCx->uc.pCursor);
       pCx->isTable = 1;
     }
@@ -3728,7 +3821,8 @@ case OP_SeekGT: {       /* jump, in3 */
 
   if( pC->isTable ){
     /* The BTREE_SEEK_EQ flag is only set on index cursors */
-    assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 );
+    assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0
+              || CORRUPT_DB );
 
     /* The input value in P3 might be of any type: integer, real, string,
     ** blob, or NULL.  But it needs to be an integer before we can do
@@ -3815,7 +3909,6 @@ case OP_SeekGT: {       /* jump, in3 */
 #ifdef SQLITE_DEBUG
     { int i; for(i=0; iuc.pCursor, &r, 0, 0, &res);
     if( rc!=SQLITE_OK ){
@@ -3863,7 +3956,6 @@ seek_not_found:
   }
   break;
 }
-  
 
 /* Opcode: Found P1 P2 P3 P4 *
 ** Synopsis: key=r[P3@P4]
@@ -3932,10 +4024,9 @@ case OP_Found: {        /* jump, in3 */
   int ii;
   VdbeCursor *pC;
   int res;
-  char *pFree;
+  UnpackedRecord *pFree;
   UnpackedRecord *pIdxKey;
   UnpackedRecord r;
-  char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*4 + 7];
 
 #ifdef SQLITE_TEST
   if( pOp->opcode!=OP_NoConflict ) sqlite3_found_count++;
@@ -3952,26 +4043,24 @@ case OP_Found: {        /* jump, in3 */
   assert( pC->eCurType==CURTYPE_BTREE );
   assert( pC->uc.pCursor!=0 );
   assert( pC->isTable==0 );
-  pFree = 0;
   if( pOp->p4.i>0 ){
     r.pKeyInfo = pC->pKeyInfo;
     r.nField = (u16)pOp->p4.i;
     r.aMem = pIn3;
+#ifdef SQLITE_DEBUG
     for(ii=0; iip3+ii, &r.aMem[ii]);
-#endif
     }
+#endif
     pIdxKey = &r;
+    pFree = 0;
   }else{
-    pIdxKey = sqlite3VdbeAllocUnpackedRecord(
-        pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree
-    );
+    pFree = pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo);
     if( pIdxKey==0 ) goto no_mem;
     assert( pIn3->flags & MEM_Blob );
-    ExpandBlob(pIn3);
+    (void)ExpandBlob(pIn3);
     sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey);
   }
   pIdxKey->default_rc = 0;
@@ -3988,7 +4077,7 @@ case OP_Found: {        /* jump, in3 */
     }
   }
   rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res);
-  sqlite3DbFree(db, pFree);
+  if( pFree ) sqlite3DbFree(db, pFree);
   if( rc!=SQLITE_OK ){
     goto abort_due_to_error;
   }
@@ -4007,6 +4096,30 @@ case OP_Found: {        /* jump, in3 */
   break;
 }
 
+/* Opcode: SeekRowid P1 P2 P3 * *
+** Synopsis: intkey=r[P3]
+**
+** P1 is the index of a cursor open on an SQL table btree (with integer
+** keys).  If register P3 does not contain an integer or if P1 does not
+** contain a record with rowid P3 then jump immediately to P2.  
+** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain
+** a record with rowid P3 then 
+** leave the cursor pointing at that record and fall through to the next
+** instruction.
+**
+** The OP_NotExists opcode performs the same operation, but with OP_NotExists
+** the P3 register must be guaranteed to contain an integer value.  With this
+** opcode, register P3 might not contain an integer.
+**
+** The OP_NotFound opcode performs the same operation on index btrees
+** (with arbitrary multi-value keys).
+**
+** This opcode leaves the cursor in a state where it cannot be advanced
+** in either direction.  In other words, the Next and Prev opcodes will
+** not work following this opcode.
+**
+** See also: Found, NotFound, NoConflict, SeekRowid
+*/
 /* Opcode: NotExists P1 P2 P3 * *
 ** Synopsis: intkey=r[P3]
 **
@@ -4017,6 +4130,10 @@ case OP_Found: {        /* jump, in3 */
 ** leave the cursor pointing at that record and fall through to the next
 ** instruction.
 **
+** The OP_SeekRowid opcode performs the same operation but also allows the
+** P3 register to contain a non-integer value, in which case the jump is
+** always taken.  This opcode requires that P3 always contain an integer.
+**
 ** The OP_NotFound opcode performs the same operation on index btrees
 ** (with arbitrary multi-value keys).
 **
@@ -4024,14 +4141,21 @@ case OP_Found: {        /* jump, in3 */
 ** in either direction.  In other words, the Next and Prev opcodes will
 ** not work following this opcode.
 **
-** See also: Found, NotFound, NoConflict
+** See also: Found, NotFound, NoConflict, SeekRowid
 */
-case OP_NotExists: {        /* jump, in3 */
+case OP_SeekRowid: {        /* jump, in3 */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
   u64 iKey;
 
+  pIn3 = &aMem[pOp->p3];
+  if( (pIn3->flags & MEM_Int)==0 ){
+    applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding);
+    if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2;
+  }
+  /* Fall through into OP_NotExists */
+case OP_NotExists:          /* jump, in3 */
   pIn3 = &aMem[pOp->p3];
   assert( pIn3->flags & MEM_Int );
   assert( pOp->p1>=0 && pOp->p1nCursor );
@@ -4150,8 +4274,7 @@ case OP_NewRowid: {           /* out2 */
         v = 1;   /* IMP: R-61914-48074 */
       }else{
         assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
-        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
+        v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
         if( v>=MAX_ROWID ){
           pC->useRandomRowid = 1;
         }else{
@@ -4171,7 +4294,7 @@ case OP_NewRowid: {           /* out2 */
         pMem = &pFrame->aMem[pOp->p3];
       }else{
         /* Assert that P3 is a valid memory cell. */
-        assert( pOp->p3<=(p->nMem-p->nCursor) );
+        assert( pOp->p3<=(p->nMem+1 - p->nCursor) );
         pMem = &aMem[pOp->p3];
         memAboutToChange(p, pMem);
       }
@@ -4181,7 +4304,7 @@ case OP_NewRowid: {           /* out2 */
       sqlite3VdbeMemIntegerify(pMem);
       assert( (pMem->flags & MEM_Int)!=0 );  /* mem(P3) holds an integer */
       if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){
-        rc = SQLITE_FULL;   /* IMP: R-12275-61338 */
+        rc = SQLITE_FULL;   /* IMP: R-17817-00630 */
         goto abort_due_to_error;
       }
       if( vu.i+1 ){
@@ -4233,22 +4356,19 @@ case OP_NewRowid: {           /* out2 */
 ** then rowid is stored for subsequent return by the
 ** sqlite3_last_insert_rowid() function (otherwise it is unmodified).
 **
-** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of
-** the last seek operation (OP_NotExists) was a success, then this
-** operation will not attempt to find the appropriate row before doing
-** the insert but will instead overwrite the row that the cursor is
-** currently pointing to.  Presumably, the prior OP_NotExists opcode
-** has already positioned the cursor correctly.  This is an optimization
-** that boosts performance by avoiding redundant seeks.
+** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might
+** run faster by avoiding an unnecessary seek on cursor P1.  However,
+** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior
+** seeks on the cursor or if the most recent seek used a key equal to P3.
 **
 ** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an
 ** UPDATE operation.  Otherwise (if the flag is clear) then this opcode
 ** is part of an INSERT operation.  The difference is only important to
 ** the update hook.
 **
-** Parameter P4 may point to a string containing the table-name, or
-** may be NULL. If it is not NULL, then the update-hook 
-** (sqlite3.xUpdateCallback) is invoked following a successful insert.
+** Parameter P4 may point to a Table structure, or may be NULL. If it is 
+** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked 
+** following a successful insert.
 **
 ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically
 ** allocated, then ownership of P2 is transferred to the pseudo-cursor
@@ -4260,7 +4380,7 @@ case OP_NewRowid: {           /* out2 */
 ** for indices is OP_IdxInsert.
 */
 /* Opcode: InsertInt P1 P2 P3 P4 P5
-** Synopsis:  intkey=P3 data=r[P2]
+** Synopsis: intkey=P3 data=r[P2]
 **
 ** This works exactly like OP_Insert except that the key is the
 ** integer value P3, not the value of the integer stored in register P3.
@@ -4269,14 +4389,14 @@ case OP_Insert:
 case OP_InsertInt: {
   Mem *pData;       /* MEM cell holding data for the record to be inserted */
   Mem *pKey;        /* MEM cell holding key  for the record */
-  i64 iKey;         /* The integer ROWID or key for the record to be inserted */
   VdbeCursor *pC;   /* Cursor to table into which insert is written */
-  int nZero;        /* Number of zero-bytes to append */
   int seekResult;   /* Result of prior seek or 0 if no USESEEKRESULT flag */
   const char *zDb;  /* database name - used by the update hook */
-  const char *zTbl; /* Table name - used by the opdate hook */
+  Table *pTab;      /* Table structure - used by update and pre-update hooks */
   int op;           /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
+  BtreePayload x;   /* Payload to be inserted */
 
+  op = 0;
   pData = &aMem[pOp->p2];
   assert( pOp->p1>=0 && pOp->p1nCursor );
   assert( memIsValid(pData) );
@@ -4284,7 +4404,8 @@ case OP_InsertInt: {
   assert( pC!=0 );
   assert( pC->eCurType==CURTYPE_BTREE );
   assert( pC->uc.pCursor!=0 );
-  assert( pC->isTable );
+  assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable );
+  assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC );
   REGISTER_TRACE(pOp->p2, pData);
 
   if( pOp->opcode==OP_Insert ){
@@ -4292,47 +4413,66 @@ case OP_InsertInt: {
     assert( pKey->flags & MEM_Int );
     assert( memIsValid(pKey) );
     REGISTER_TRACE(pOp->p3, pKey);
-    iKey = pKey->u.i;
+    x.nKey = pKey->u.i;
   }else{
     assert( pOp->opcode==OP_InsertInt );
-    iKey = pOp->p3;
+    x.nKey = pOp->p3;
   }
 
+  if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
+    assert( pC->iDb>=0 );
+    zDb = db->aDb[pC->iDb].zDbSName;
+    pTab = pOp->p4.pTab;
+    assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) );
+    op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT);
+  }else{
+    pTab = 0; /* Not needed.  Silence a compiler warning. */
+    zDb = 0;  /* Not needed.  Silence a compiler warning. */
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  /* Invoke the pre-update hook, if any */
+  if( db->xPreUpdateCallback 
+   && pOp->p4type==P4_TABLE
+   && !(pOp->p5 & OPFLAG_ISUPDATE)
+  ){
+    sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, x.nKey, pOp->p2);
+  }
+  if( pOp->p5 & OPFLAG_ISNOOP ) break;
+#endif
+
   if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
-  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey;
+  if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey;
   if( pData->flags & MEM_Null ){
-    pData->z = 0;
-    pData->n = 0;
+    x.pData = 0;
+    x.nData = 0;
   }else{
     assert( pData->flags & (MEM_Blob|MEM_Str) );
+    x.pData = pData->z;
+    x.nData = pData->n;
   }
   seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0);
   if( pData->flags & MEM_Zero ){
-    nZero = pData->u.nZero;
+    x.nZero = pData->u.nZero;
   }else{
-    nZero = 0;
+    x.nZero = 0;
   }
-  rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey,
-                          pData->z, pData->n, nZero,
-                          (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
+  x.pKey = 0;
+  rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
+      (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), seekResult
   );
   pC->deferredMoveto = 0;
   pC->cacheStatus = CACHE_STALE;
 
   /* Invoke the update-hook if required. */
   if( rc ) goto abort_due_to_error;
-  if( db->xUpdateCallback && pOp->p4.z ){
-    zDb = db->aDb[pC->iDb].zName;
-    zTbl = pOp->p4.z;
-    op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT);
-    assert( pC->isTable );
-    db->xUpdateCallback(db->pUpdateArg, op, zDb, zTbl, iKey);
-    assert( pC->iDb>=0 );
+  if( db->xUpdateCallback && op ){
+    db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, x.nKey);
   }
   break;
 }
 
-/* Opcode: Delete P1 P2 * P4 P5
+/* Opcode: Delete P1 P2 P3 P4 P5
 **
 ** Delete the record at which the P1 cursor is currently pointing.
 **
@@ -4356,15 +4496,24 @@ case OP_InsertInt: {
 ** P1 must not be pseudo-table.  It has to be a real table with
 ** multiple rows.
 **
-** If P4 is not NULL, then it is the name of the table that P1 is
-** pointing to.  The update hook will be invoked, if it exists.
-** If P4 is not NULL then the P1 cursor must have been positioned
-** using OP_NotFound prior to invoking this opcode.
+** If P4 is not NULL then it points to a Table object. In this case either 
+** the update or pre-update hook, or both, may be invoked. The P1 cursor must
+** have been positioned using OP_NotFound prior to invoking this opcode in 
+** this case. Specifically, if one is configured, the pre-update hook is 
+** invoked if P4 is not NULL. The update-hook is invoked if one is configured, 
+** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2.
+**
+** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address
+** of the memory cell that contains the value that the rowid of the row will
+** be set to by the update.
 */
 case OP_Delete: {
   VdbeCursor *pC;
-  u8 hasUpdateCallback;
+  const char *zDb;
+  Table *pTab;
+  int opflags;
 
+  opflags = pOp->p2;
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
@@ -4372,22 +4521,50 @@ case OP_Delete: {
   assert( pC->uc.pCursor!=0 );
   assert( pC->deferredMoveto==0 );
 
-  hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable;
-  if( pOp->p5 && hasUpdateCallback ){
-    sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
-  }
-
 #ifdef SQLITE_DEBUG
-  /* The seek operation that positioned the cursor prior to OP_Delete will
-  ** have also set the pC->movetoTarget field to the rowid of the row that
-  ** is being deleted */
-  if( pOp->p4.z && pC->isTable && pOp->p5==0 ){
-    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
-    assert( pC->movetoTarget==iKey ); 
+  if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){
+    /* If p5 is zero, the seek operation that positioned the cursor prior to
+    ** OP_Delete will have also set the pC->movetoTarget field to the rowid of
+    ** the row that is being deleted */
+    i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor);
+    assert( pC->movetoTarget==iKey );
   }
 #endif
 
+  /* If the update-hook or pre-update-hook will be invoked, set zDb to
+  ** the name of the db to pass as to it. Also set local pTab to a copy
+  ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was
+  ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set 
+  ** VdbeCursor.movetoTarget to the current rowid.  */
+  if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){
+    assert( pC->iDb>=0 );
+    assert( pOp->p4.pTab!=0 );
+    zDb = db->aDb[pC->iDb].zDbSName;
+    pTab = pOp->p4.pTab;
+    if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){
+      pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor);
+    }
+  }else{
+    zDb = 0;   /* Not needed.  Silence a compiler warning. */
+    pTab = 0;  /* Not needed.  Silence a compiler warning. */
+  }
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+  /* Invoke the pre-update-hook if required. */
+  if( db->xPreUpdateCallback && pOp->p4.pTab ){
+    assert( !(opflags & OPFLAG_ISUPDATE) 
+         || HasRowid(pTab)==0 
+         || (aMem[pOp->p3].flags & MEM_Int) 
+    );
+    sqlite3VdbePreUpdateHook(p, pC,
+        (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, 
+        zDb, pTab, pC->movetoTarget,
+        pOp->p3
+    );
+  }
+  if( opflags & OPFLAG_ISNOOP ) break;
+#endif
+ 
   /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ 
   assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 );
   assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION );
@@ -4409,15 +4586,19 @@ case OP_Delete: {
 
   rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5);
   pC->cacheStatus = CACHE_STALE;
+  pC->seekResult = 0;
+  if( rc ) goto abort_due_to_error;
 
   /* Invoke the update-hook if required. */
-  if( rc ) goto abort_due_to_error;
-  if( hasUpdateCallback ){
-    db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
-                        db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
-    assert( pC->iDb>=0 );
+  if( opflags & OPFLAG_NCHANGE ){
+    p->nChange++;
+    if( db->xUpdateCallback && HasRowid(pTab) ){
+      db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName,
+          pC->movetoTarget);
+      assert( pC->iDb>=0 );
+    }
   }
-  if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++;
+
   break;
 }
 /* Opcode: ResetCount * * * * *
@@ -4434,7 +4615,7 @@ case OP_ResetCount: {
 }
 
 /* Opcode: SorterCompare P1 P2 P3 P4
-** Synopsis:  if key(P1)!=trim(r[P3],P4) goto P2
+** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2
 **
 ** P1 is a sorter cursor. This instruction compares a prefix of the
 ** record blob in register P3 against a prefix of the entry that 
@@ -4492,53 +4673,54 @@ case OP_SorterData: {
   break;
 }
 
-/* Opcode: RowData P1 P2 * * *
+/* Opcode: RowData P1 P2 P3 * *
 ** Synopsis: r[P2]=data
 **
-** Write into register P2 the complete row data for cursor P1.
+** Write into register P2 the complete row content for the row at 
+** which cursor P1 is currently pointing.
 ** There is no interpretation of the data.  
 ** It is just copied onto the P2 register exactly as 
 ** it is found in the database file.
 **
-** If the P1 cursor must be pointing to a valid row (not a NULL row)
-** of a real table, not a pseudo-table.
-*/
-/* Opcode: RowKey P1 P2 * * *
-** Synopsis: r[P2]=key
-**
-** Write into register P2 the complete row key for cursor P1.
-** There is no interpretation of the data.  
-** The key is copied onto the P2 register exactly as 
-** it is found in the database file.
+** If cursor P1 is an index, then the content is the key of the row.
+** If cursor P2 is a table, then the content extracted is the data.
 **
 ** If the P1 cursor must be pointing to a valid row (not a NULL row)
 ** of a real table, not a pseudo-table.
+**
+** If P3!=0 then this opcode is allowed to make an ephermeral pointer
+** into the database page.  That means that the content of the output
+** register will be invalidated as soon as the cursor moves - including
+** moves caused by other cursors that "save" the the current cursors
+** position in order that they can write to the same table.  If P3==0
+** then a copy of the data is made into memory.  P3!=0 is faster, but
+** P3==0 is safer.
+**
+** If P3!=0 then the content of the P2 register is unsuitable for use
+** in OP_Result and any OP_Result will invalidate the P2 register content.
+** The P2 register content is invalidated by opcodes like OP_Function or
+** by any use of another cursor pointing to the same table.
 */
-case OP_RowKey:
 case OP_RowData: {
   VdbeCursor *pC;
   BtCursor *pCrsr;
   u32 n;
-  i64 n64;
 
-  pOut = &aMem[pOp->p2];
-  memAboutToChange(p, pOut);
+  pOut = out2Prerelease(p, pOp);
 
-  /* Note that RowKey and RowData are really exactly the same instruction */
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
   assert( pC->eCurType==CURTYPE_BTREE );
   assert( isSorter(pC)==0 );
-  assert( pC->isTable || pOp->opcode!=OP_RowData );
-  assert( pC->isTable==0 || pOp->opcode==OP_RowData );
   assert( pC->nullRow==0 );
   assert( pC->uc.pCursor!=0 );
   pCrsr = pC->uc.pCursor;
 
-  /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
-  ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
-  ** the cursor.  If this where not the case, on of the following assert()s
+  /* The OP_RowData opcodes always follow OP_NotExists or
+  ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions
+  ** that might invalidate the cursor.
+  ** If this where not the case, on of the following assert()s
   ** would fail.  Should this ever change (because of changes in the code
   ** generator) then the fix would be to insert a call to
   ** sqlite3VdbeCursorMoveto().
@@ -4550,34 +4732,14 @@ case OP_RowData: {
   if( rc!=SQLITE_OK ) goto abort_due_to_error;
 #endif
 
-  if( pC->isTable==0 ){
-    assert( !pC->isTable );
-    VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64);
-    assert( rc==SQLITE_OK );    /* True because of CursorMoveto() call above */
-    if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
-    n = (u32)n64;
-  }else{
-    VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n);
-    assert( rc==SQLITE_OK );    /* DataSize() cannot fail */
-    if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-      goto too_big;
-    }
+  n = sqlite3BtreePayloadSize(pCrsr);
+  if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+    goto too_big;
   }
   testcase( n==0 );
-  if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
-    goto no_mem;
-  }
-  pOut->n = n;
-  MemSetTypeFlag(pOut, MEM_Blob);
-  if( pC->isTable==0 ){
-    rc = sqlite3BtreeKey(pCrsr, 0, n, pOut->z);
-  }else{
-    rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z);
-  }
+  rc = sqlite3VdbeMemFromBtree(pCrsr, 0, n, pOut);
   if( rc ) goto abort_due_to_error;
-  pOut->enc = SQLITE_UTF8;  /* In case the blob is ever cast to text */
+  if( !pOp->p3 ) Deephemeralize(pOut);
   UPDATE_MAX_BLOBSIZE(pOut);
   REGISTER_TRACE(pOp->p2, pOut);
   break;
@@ -4628,8 +4790,7 @@ case OP_Rowid: {                 /* out2 */
       pOut->flags = MEM_Null;
       break;
     }
-    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
-    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
+    v = sqlite3BtreeIntegerKey(pC->uc.pCursor);
   }
   pOut->u.i = v;
   break;
@@ -4667,6 +4828,13 @@ case OP_NullRow: {
 ** This opcode leaves the cursor configured to move in reverse order,
 ** from the end toward the beginning.  In other words, the cursor is
 ** configured to use Prev, not Next.
+**
+** If P3 is -1, then the cursor is positioned at the end of the btree
+** for the purpose of appending a new entry onto the btree.  In that
+** case P2 must be 0.  It is assumed that the cursor is used only for
+** appending and so if the cursor is valid, then the cursor must already
+** be pointing at the end of the btree and so no changes are made to
+** the cursor.
 */
 case OP_Last: {        /* jump */
   VdbeCursor *pC;
@@ -4680,23 +4848,63 @@ case OP_Last: {        /* jump */
   pCrsr = pC->uc.pCursor;
   res = 0;
   assert( pCrsr!=0 );
-  rc = sqlite3BtreeLast(pCrsr, &res);
-  pC->nullRow = (u8)res;
-  pC->deferredMoveto = 0;
-  pC->cacheStatus = CACHE_STALE;
   pC->seekResult = pOp->p3;
 #ifdef SQLITE_DEBUG
   pC->seekOp = OP_Last;
 #endif
-  if( rc ) goto abort_due_to_error;
-  if( pOp->p2>0 ){
-    VdbeBranchTaken(res!=0,2);
-    if( res ) goto jump_to_p2;
+  if( pOp->p3==0 || !sqlite3BtreeCursorIsValidNN(pCrsr) ){
+    rc = sqlite3BtreeLast(pCrsr, &res);
+    pC->nullRow = (u8)res;
+    pC->deferredMoveto = 0;
+    pC->cacheStatus = CACHE_STALE;
+    if( rc ) goto abort_due_to_error;
+    if( pOp->p2>0 ){
+      VdbeBranchTaken(res!=0,2);
+      if( res ) goto jump_to_p2;
+    }
+  }else{
+    assert( pOp->p2==0 );
   }
   break;
 }
 
+/* Opcode: IfSmaller P1 P2 P3 * *
+**
+** Estimate the number of rows in the table P1.  Jump to P2 if that
+** estimate is less than approximately 2**(0.1*P3).
+*/
+case OP_IfSmaller: {        /* jump */
+  VdbeCursor *pC;
+  BtCursor *pCrsr;
+  int res;
+  i64 sz;
 
+  assert( pOp->p1>=0 && pOp->p1nCursor );
+  pC = p->apCsr[pOp->p1];
+  assert( pC!=0 );
+  pCrsr = pC->uc.pCursor;
+  assert( pCrsr );
+  rc = sqlite3BtreeFirst(pCrsr, &res);
+  if( rc ) goto abort_due_to_error;
+  if( res==0 ){
+    sz = sqlite3BtreeRowCountEst(pCrsr);
+    if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1;
+  }
+  VdbeBranchTaken(res!=0,2);
+  if( res ) goto jump_to_p2;
+  break;
+}
+
+
+/* Opcode: SorterSort P1 P2 * * *
+**
+** After all records have been inserted into the Sorter object
+** identified by P1, invoke this opcode to actually do the sorting.
+** Jump to P2 if there are no records to be sorted.
+**
+** This opcode is an alias for OP_Sort and OP_Rewind that is used
+** for Sorter objects.
+*/
 /* Opcode: Sort P1 P2 * * *
 **
 ** This opcode does exactly the same thing as OP_Rewind except that
@@ -4824,6 +5032,13 @@ case OP_Rewind: {        /* jump */
 ** This opcode works just like Prev except that if cursor P1 is not
 ** open it behaves a no-op.
 */
+/* Opcode: SorterNext P1 P2 * * P5
+**
+** This opcode works just like OP_Next except that P1 must be a
+** sorter object for which the OP_SorterSort opcode has been
+** invoked.  This opcode advances the cursor to the next sorted
+** record, or jumps to P2 if there are no more sorted records.
+*/
 case OP_SorterNext: {  /* jump */
   VdbeCursor *pC;
   int res;
@@ -4880,32 +5095,45 @@ next_tail:
   goto check_for_interrupt;
 }
 
-/* Opcode: IdxInsert P1 P2 P3 * P5
+/* Opcode: IdxInsert P1 P2 P3 P4 P5
 ** Synopsis: key=r[P2]
 **
 ** Register P2 holds an SQL index key made using the
 ** MakeRecord instructions.  This opcode writes that key
 ** into the index P1.  Data for the entry is nil.
 **
-** P3 is a flag that provides a hint to the b-tree layer that this
-** insert is likely to be an append.
+** If P4 is not zero, then it is the number of values in the unpacked
+** key of reg(P2).  In that case, P3 is the index of the first register
+** for the unpacked key.  The availability of the unpacked key can sometimes
+** be an optimization.
+**
+** If P5 has the OPFLAG_APPEND bit set, that is a hint to the b-tree layer
+** that this insert is likely to be an append.
 **
 ** If P5 has the OPFLAG_NCHANGE bit set, then the change counter is
 ** incremented by this instruction.  If the OPFLAG_NCHANGE bit is clear,
 ** then the change counter is unchanged.
 **
-** If P5 has the OPFLAG_USESEEKRESULT bit set, then the cursor must have
-** just done a seek to the spot where the new entry is to be inserted.
-** This flag avoids doing an extra seek.
+** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might
+** run faster by avoiding an unnecessary seek on cursor P1.  However,
+** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior
+** seeks on the cursor or if the most recent seek used a key equivalent
+** to P2. 
 **
 ** This instruction only works for indices.  The equivalent instruction
 ** for tables is OP_Insert.
 */
+/* Opcode: SorterInsert P1 P2 * * *
+** Synopsis: key=r[P2]
+**
+** Register P2 holds an SQL index key made using the
+** MakeRecord instructions.  This opcode writes that key
+** into the sorter P1.  Data for the entry is nil.
+*/
 case OP_SorterInsert:       /* in2 */
 case OP_IdxInsert: {        /* in2 */
   VdbeCursor *pC;
-  int nKey;
-  const char *zKey;
+  BtreePayload x;
 
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
@@ -4921,9 +5149,12 @@ case OP_IdxInsert: {        /* in2 */
   if( pOp->opcode==OP_SorterInsert ){
     rc = sqlite3VdbeSorterWrite(pC, pIn2);
   }else{
-    nKey = pIn2->n;
-    zKey = pIn2->z;
-    rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
+    x.nKey = pIn2->n;
+    x.pKey = pIn2->z;
+    x.aMem = aMem + pOp->p3;
+    x.nMem = (u16)pOp->p4.i;
+    rc = sqlite3BtreeInsert(pC->uc.pCursor, &x,
+         (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), 
         ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
         );
     assert( pC->deferredMoveto==0 );
@@ -4947,7 +5178,7 @@ case OP_IdxDelete: {
   UnpackedRecord r;
 
   assert( pOp->p3>0 );
-  assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 );
+  assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 );
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
@@ -4967,11 +5198,12 @@ case OP_IdxDelete: {
   }
   assert( pC->deferredMoveto==0 );
   pC->cacheStatus = CACHE_STALE;
+  pC->seekResult = 0;
   break;
 }
 
 /* Opcode: Seek P1 * P3 P4 *
-** Synopsis:  Move P3 to P1.rowid
+** Synopsis: Move P3 to P1.rowid
 **
 ** P1 is an open index cursor and P3 is a cursor on the corresponding
 ** table.  This opcode does a deferred seek of the P3 table cursor
@@ -5044,7 +5276,6 @@ case OP_IdxRowid: {              /* out2 */
     }else{
       pOut = out2Prerelease(p, pOp);
       pOut->u.i = rowid;
-      pOut->flags = MEM_Int;
     }
   }else{
     assert( pOp->opcode==OP_IdxRowid );
@@ -5308,6 +5539,18 @@ case OP_CreateTable: {          /* out2 */
   break;
 }
 
+/* Opcode: SqlExec * * * P4 *
+**
+** Run the SQL statement or statements specified in the P4 string.
+*/
+case OP_SqlExec: {
+  db->nSqlExec++;
+  rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0);
+  db->nSqlExec--;
+  if( rc ) goto abort_due_to_error;
+  break;
+}
+
 /* Opcode: ParseSchema P1 * * P4 *
 **
 ** Read and parse all entries from the SQLITE_MASTER table of database P1
@@ -5336,13 +5579,13 @@ case OP_ParseSchema: {
   assert( iDb>=0 && iDbnDb );
   assert( DbHasProperty(db, iDb, DB_SchemaLoaded) );
   /* Used to be a conditional */ {
-    zMaster = SCHEMA_TABLE(iDb);
+    zMaster = MASTER_NAME;
     initData.db = db;
     initData.iDb = pOp->p1;
     initData.pzErrMsg = &p->zErrMsg;
     zSql = sqlite3MPrintf(db,
        "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid",
-       db->aDb[iDb].zName, zMaster, pOp->p4.z);
+       db->aDb[iDb].zDbSName, zMaster, pOp->p4.z);
     if( zSql==0 ){
       rc = SQLITE_NOMEM_BKPT;
     }else{
@@ -5422,20 +5665,19 @@ case OP_DropTrigger: {
 
 
 #ifndef SQLITE_OMIT_INTEGRITY_CHECK
-/* Opcode: IntegrityCk P1 P2 P3 * P5
+/* Opcode: IntegrityCk P1 P2 P3 P4 P5
 **
 ** Do an analysis of the currently open database.  Store in
 ** register P1 the text of an error message describing any problems.
 ** If no problems are found, store a NULL in register P1.
 **
-** The register P3 contains the maximum number of allowed errors.
+** The register P3 contains one less than the maximum number of allowed errors.
 ** At most reg(P3) errors will be reported.
 ** In other words, the analysis stops as soon as reg(P1) errors are 
 ** seen.  Reg(P1) is updated with the number of errors remaining.
 **
-** The root page numbers of all tables in the database are integer
-** stored in reg(P1), reg(P1+1), reg(P1+2), ....  There are P2 tables
-** total.
+** The root page numbers of all tables in the database are integers
+** stored in P4_INTARRAY argument.
 **
 ** If P5 is not zero, the check is done on the auxiliary database
 ** file, not the main database file.
@@ -5445,37 +5687,31 @@ case OP_DropTrigger: {
 case OP_IntegrityCk: {
   int nRoot;      /* Number of tables to check.  (Number of root pages.) */
   int *aRoot;     /* Array of rootpage numbers for tables to be checked */
-  int j;          /* Loop counter */
   int nErr;       /* Number of errors reported */
   char *z;        /* Text of the error report */
   Mem *pnErr;     /* Register keeping track of errors remaining */
 
   assert( p->bIsReader );
   nRoot = pOp->p2;
+  aRoot = pOp->p4.ai;
   assert( nRoot>0 );
-  aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(nRoot+1) );
-  if( aRoot==0 ) goto no_mem;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( aRoot[nRoot]==0 );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pnErr = &aMem[pOp->p3];
   assert( (pnErr->flags & MEM_Int)!=0 );
   assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 );
   pIn1 = &aMem[pOp->p1];
-  for(j=0; jp5nDb );
   assert( DbMaskTest(p->btreeMask, pOp->p5) );
   z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot,
-                                 (int)pnErr->u.i, &nErr);
-  sqlite3DbFree(db, aRoot);
-  pnErr->u.i -= nErr;
+                                 (int)pnErr->u.i+1, &nErr);
   sqlite3VdbeMemSetNull(pIn1);
   if( nErr==0 ){
     assert( z==0 );
   }else if( z==0 ){
     goto no_mem;
   }else{
+    pnErr->u.i -= nErr-1;
     sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free);
   }
   UPDATE_MAX_BLOBSIZE(pIn1);
@@ -5485,7 +5721,7 @@ case OP_IntegrityCk: {
 #endif /* SQLITE_OMIT_INTEGRITY_CHECK */
 
 /* Opcode: RowSetAdd P1 P2 * * *
-** Synopsis:  rowset(P1)=r[P2]
+** Synopsis: rowset(P1)=r[P2]
 **
 ** Insert the integer value held by register P2 into a boolean index
 ** held in register P1.
@@ -5505,7 +5741,7 @@ case OP_RowSetAdd: {       /* in1, in2 */
 }
 
 /* Opcode: RowSetRead P1 P2 P3 * *
-** Synopsis:  r[P3]=rowset(P1)
+** Synopsis: r[P3]=rowset(P1)
 **
 ** Extract the smallest value from boolean index P1 and put that value into
 ** register P3.  Or, if boolean index P1 is initially empty, leave P3
@@ -5650,10 +5886,12 @@ case OP_Program: {        /* jump */
     ** variable nMem (and later, VdbeFrame.nChildMem) to this value.
     */
     nMem = pProgram->nMem + pProgram->nCsr;
+    assert( nMem>0 );
+    if( pProgram->nCsr==0 ) nMem++;
     nByte = ROUND8(sizeof(VdbeFrame))
               + nMem * sizeof(Mem)
-              + pProgram->nCsr * sizeof(VdbeCursor *)
-              + pProgram->nOnce * sizeof(u8);
+              + pProgram->nCsr * sizeof(VdbeCursor*)
+              + (pProgram->nOp + 7)/8;
     pFrame = sqlite3DbMallocZero(db, nByte);
     if( !pFrame ){
       goto no_mem;
@@ -5673,8 +5911,6 @@ case OP_Program: {        /* jump */
     pFrame->aOp = p->aOp;
     pFrame->nOp = p->nOp;
     pFrame->token = pProgram->token;
-    pFrame->aOnceFlag = p->aOnceFlag;
-    pFrame->nOnceFlag = p->nOnceFlag;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     pFrame->anExec = p->anExec;
 #endif
@@ -5686,14 +5922,15 @@ case OP_Program: {        /* jump */
     }
   }else{
     pFrame = pRt->u.pFrame;
-    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem );
+    assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem 
+        || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) );
     assert( pProgram->nCsr==pFrame->nChildCsr );
     assert( (int)(pOp - aOp)==pFrame->pc );
   }
 
   p->nFrame++;
   pFrame->pParent = p->pFrame;
-  pFrame->lastRowid = lastRowid;
+  pFrame->lastRowid = db->lastRowid;
   pFrame->nChange = p->nChange;
   pFrame->nDbChange = p->db->nChange;
   assert( pFrame->pAuxData==0 );
@@ -5701,19 +5938,18 @@ case OP_Program: {        /* jump */
   p->pAuxData = 0;
   p->nChange = 0;
   p->pFrame = pFrame;
-  p->aMem = aMem = &VdbeFrameMem(pFrame)[-1];
+  p->aMem = aMem = VdbeFrameMem(pFrame);
   p->nMem = pFrame->nChildMem;
   p->nCursor = (u16)pFrame->nChildCsr;
-  p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
+  p->apCsr = (VdbeCursor **)&aMem[p->nMem];
+  pFrame->aOnce = (u8*)&p->apCsr[pProgram->nCsr];
+  memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8);
   p->aOp = aOp = pProgram->aOp;
   p->nOp = pProgram->nOp;
-  p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor];
-  p->nOnceFlag = pProgram->nOnce;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   p->anExec = 0;
 #endif
   pOp = &aOp[-1];
-  memset(p->aOnceFlag, 0, p->nOnceFlag);
 
   break;
 }
@@ -5857,29 +6093,42 @@ case OP_IfPos: {        /* jump, in1 */
 ** Otherwise, r[P2] is set to the sum of r[P1] and r[P3].
 */
 case OP_OffsetLimit: {    /* in1, out2, in3 */
+  i64 x;
   pIn1 = &aMem[pOp->p1];
   pIn3 = &aMem[pOp->p3];
   pOut = out2Prerelease(p, pOp);
   assert( pIn1->flags & MEM_Int );
   assert( pIn3->flags & MEM_Int );
-  pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0);
+  x = pIn1->u.i;
+  if( x<=0 || sqlite3AddInt64(&x, pIn3->u.i>0?pIn3->u.i:0) ){
+    /* If the LIMIT is less than or equal to zero, loop forever.  This
+    ** is documented.  But also, if the LIMIT+OFFSET exceeds 2^63 then
+    ** also loop forever.  This is undocumented.  In fact, one could argue
+    ** that the loop should terminate.  But assuming 1 billion iterations
+    ** per second (far exceeding the capabilities of any current hardware)
+    ** it would take nearly 300 years to actually reach the limit.  So
+    ** looping forever is a reasonable approximation. */
+    pOut->u.i = -1;
+  }else{
+    pOut->u.i = x;
+  }
   break;
 }
 
-/* Opcode: IfNotZero P1 P2 P3 * *
-** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2
+/* Opcode: IfNotZero P1 P2 * * *
+** Synopsis: if r[P1]!=0 then r[P1]--, goto P2
 **
 ** Register P1 must contain an integer.  If the content of register P1 is
-** initially nonzero, then subtract P3 from the value in register P1 and
-** jump to P2.  If register P1 is initially zero, leave it unchanged
-** and fall through.
+** initially greater than zero, then decrement the value in register P1.
+** If it is non-zero (negative or positive) and then also jump to P2.  
+** If register P1 is initially zero, leave it unchanged and fall through.
 */
 case OP_IfNotZero: {        /* jump, in1 */
   pIn1 = &aMem[pOp->p1];
   assert( pIn1->flags&MEM_Int );
   VdbeBranchTaken(pIn1->u.i<0, 2);
   if( pIn1->u.i ){
-     pIn1->u.i -= pOp->p3;
+     if( pIn1->u.i>0 ) pIn1->u.i--;
      goto jump_to_p2;
   }
   break;
@@ -5888,34 +6137,19 @@ case OP_IfNotZero: {        /* jump, in1 */
 /* Opcode: DecrJumpZero P1 P2 * * *
 ** Synopsis: if (--r[P1])==0 goto P2
 **
-** Register P1 must hold an integer.  Decrement the value in register P1
-** then jump to P2 if the new value is exactly zero.
+** Register P1 must hold an integer.  Decrement the value in P1
+** and jump to P2 if the new value is exactly zero.
 */
 case OP_DecrJumpZero: {      /* jump, in1 */
   pIn1 = &aMem[pOp->p1];
   assert( pIn1->flags&MEM_Int );
-  pIn1->u.i--;
+  if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--;
   VdbeBranchTaken(pIn1->u.i==0, 2);
   if( pIn1->u.i==0 ) goto jump_to_p2;
   break;
 }
 
 
-/* Opcode: JumpZeroIncr P1 P2 * * *
-** Synopsis: if (r[P1]++)==0 ) goto P2
-**
-** The register P1 must contain an integer.  If register P1 is initially
-** zero, then jump to P2.  Increment register P1 regardless of whether or
-** not the jump is taken.
-*/
-case OP_JumpZeroIncr: {        /* jump, in1 */
-  pIn1 = &aMem[pOp->p1];
-  assert( pIn1->flags&MEM_Int );
-  VdbeBranchTaken(pIn1->u.i==0, 2);
-  if( (pIn1->u.i++)==0 ) goto jump_to_p2;
-  break;
-}
-
 /* Opcode: AggStep0 * P2 P3 P4 P5
 ** Synopsis: accum=r[P3] step(r[P2@P5])
 **
@@ -5950,8 +6184,8 @@ case OP_AggStep0: {
 
   assert( pOp->p4type==P4_FUNCDEF );
   n = pOp->p5;
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
-  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
+  assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) );
   assert( pOp->p3p2 || pOp->p3>=pOp->p2+n );
   pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*));
   if( pCtx==0 ) goto no_mem;
@@ -6030,7 +6264,7 @@ case OP_AggStep: {
 */
 case OP_AggFinal: {
   Mem *pMem;
-  assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) );
+  assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) );
   pMem = &aMem[pOp->p1];
   assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
   rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc);
@@ -6155,7 +6389,7 @@ case OP_JournalMode: {    /* out2 */
         ** file. An EXCLUSIVE lock may still be held on the database file 
         ** after a successful return. 
         */
-        rc = sqlite3PagerCloseWal(pPager);
+        rc = sqlite3PagerCloseWal(pPager, db);
         if( rc==SQLITE_OK ){
           sqlite3PagerSetJournalMode(pPager, eNew);
         }
@@ -6190,15 +6424,14 @@ case OP_JournalMode: {    /* out2 */
 #endif /* SQLITE_OMIT_PRAGMA */
 
 #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
-/* Opcode: Vacuum * * * * *
+/* Opcode: Vacuum P1 * * * *
 **
-** Vacuum the entire database.  This opcode will cause other virtual
-** machines to be created and run.  It may not be called from within
-** a transaction.
+** Vacuum the entire database P1.  P1 is 0 for "main", and 2 or more
+** for an attached database.  The "temp" database may not be vacuumed.
 */
 case OP_Vacuum: {
   assert( p->readOnly==0 );
-  rc = sqlite3RunVacuum(&p->zErrMsg, db);
+  rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1);
   if( rc ) goto abort_due_to_error;
   break;
 }
@@ -6472,7 +6705,7 @@ case OP_VColumn: {
 
   VdbeCursor *pCur = p->apCsr[pOp->p1];
   assert( pCur->eCurType==CURTYPE_VTAB );
-  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
+  assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) );
   pDest = &aMem[pOp->p3];
   memAboutToChange(p, pDest);
   if( pCur->nullRow ){
@@ -6640,7 +6873,7 @@ case OP_VUpdate: {
     sqlite3VtabImportErrmsg(p, pVtab);
     if( rc==SQLITE_OK && pOp->p1 ){
       assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) );
-      db->lastRowid = lastRowid = rowid;
+      db->lastRowid = rowid;
     }
     if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){
       if( pOp->p5==OE_Ignore ){
@@ -6696,8 +6929,8 @@ case OP_MaxPgcnt: {            /* out2 */
 #endif
 
 
-/* Opcode: Init * P2 * P4 *
-** Synopsis:  Start at P2
+/* Opcode: Init P1 P2 * P4 *
+** Synopsis: Start at P2
 **
 ** Programs contain a single instance of this opcode as the very first
 ** opcode.
@@ -6707,27 +6940,54 @@ case OP_MaxPgcnt: {            /* out2 */
 ** Or if P4 is blank, use the string returned by sqlite3_sql().
 **
 ** If P2 is not zero, jump to instruction P2.
+**
+** Increment the value of P1 so that OP_Once opcodes will jump the
+** first time they are evaluated for this run.
 */
 case OP_Init: {          /* jump */
   char *zTrace;
-  char *z;
+  int i;
+
+  /* If the P4 argument is not NULL, then it must be an SQL comment string.
+  ** The "--" string is broken up to prevent false-positives with srcck1.c.
+  **
+  ** This assert() provides evidence for:
+  ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that
+  ** would have been returned by the legacy sqlite3_trace() interface by
+  ** using the X argument when X begins with "--" and invoking
+  ** sqlite3_expanded_sql(P) otherwise.
+  */
+  assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 );
+  assert( pOp==p->aOp );  /* Always instruction 0 */
 
 #ifndef SQLITE_OMIT_TRACE
-  if( db->xTrace
+  if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0
    && !p->doingRerun
    && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0
   ){
-    z = sqlite3VdbeExpandSql(p, zTrace);
-    db->xTrace(db->pTraceArg, z);
-    sqlite3DbFree(db, z);
+#ifndef SQLITE_OMIT_DEPRECATED
+    if( db->mTrace & SQLITE_TRACE_LEGACY ){
+      void (*x)(void*,const char*) = (void(*)(void*,const char*))db->xTrace;
+      char *z = sqlite3VdbeExpandSql(p, zTrace);
+      x(db->pTraceArg, z);
+      sqlite3_free(z);
+    }else
+#endif
+    if( db->nVdbeExec>1 ){
+      char *z = sqlite3MPrintf(db, "-- %s", zTrace);
+      (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, z);
+      sqlite3DbFree(db, z);
+    }else{
+      (void)db->xTrace(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace);
+    }
   }
 #ifdef SQLITE_USE_FCNTL_TRACE
   zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
   if( zTrace ){
-    int i;
-    for(i=0; inDb; i++){
-      if( DbMaskTest(p->btreeMask, i)==0 ) continue;
-      sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace);
+    int j;
+    for(j=0; jnDb; j++){
+      if( DbMaskTest(p->btreeMask, j)==0 ) continue;
+      sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace);
     }
   }
 #endif /* SQLITE_USE_FCNTL_TRACE */
@@ -6739,8 +6999,15 @@ case OP_Init: {          /* jump */
   }
 #endif /* SQLITE_DEBUG */
 #endif /* SQLITE_OMIT_TRACE */
-  if( pOp->p2 ) goto jump_to_p2;
-  break;
+  assert( pOp->p2>0 );
+  if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){
+    for(i=1; inOp; i++){
+      if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0;
+    }
+    pOp->p1 = 0;
+  }
+  pOp->p1++;
+  goto jump_to_p2;
 }
 
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
@@ -6808,11 +7075,12 @@ default: {          /* This is really OP_Noop and OP_Explain */
 
 #ifdef SQLITE_DEBUG
     if( db->flags & SQLITE_VdbeTrace ){
+      u8 opProperty = sqlite3OpcodeProperty[pOrigOp->opcode];
       if( rc!=0 ) printf("rc=%d\n",rc);
-      if( pOrigOp->opflags & (OPFLG_OUT2) ){
+      if( opProperty & (OPFLG_OUT2) ){
         registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]);
       }
-      if( pOrigOp->opflags & OPFLG_OUT3 ){
+      if( opProperty & OPFLG_OUT3 ){
         registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]);
       }
     }
@@ -6830,6 +7098,7 @@ abort_due_to_error:
     sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc));
   }
   p->rc = rc;
+  sqlite3SystemError(db, rc);
   testcase( sqlite3GlobalConfig.xLog!=0 );
   sqlite3_log(rc, "statement aborts at %d: [%s] %s", 
                    (int)(pOp - aOp), p->zSql, p->zErrMsg);
@@ -6844,7 +7113,6 @@ abort_due_to_error:
   ** release the mutexes on btrees that were acquired at the
   ** top. */
 vdbe_return:
-  db->lastRowid = lastRowid;
   testcase( nVmStep>0 );
   p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep;
   sqlite3VdbeLeave(p);
diff --git a/src/vdbe.h b/src/vdbe.h
index 4c02f5844d..be19bc5316 100644
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -15,8 +15,8 @@
 ** or VDBE.  The VDBE implements an abstract machine that runs a
 ** simple program to access and modify the underlying database.
 */
-#ifndef _SQLITE_VDBE_H_
-#define _SQLITE_VDBE_H_
+#ifndef SQLITE_VDBE_H
+#define SQLITE_VDBE_H
 #include 
 
 /*
@@ -41,8 +41,7 @@ typedef struct SubProgram SubProgram;
 struct VdbeOp {
   u8 opcode;          /* What operation to perform */
   signed char p4type; /* One of the P4_xxx constants for p4 */
-  u8 opflags;         /* Mask of the OPFLG_* flags in opcodes.h */
-  u8 p5;              /* Fifth parameter is an unsigned character */
+  u16 p5;             /* Fifth parameter is an unsigned 16-bit integer */
   int p1;             /* First operand */
   int p2;             /* Second parameter (often the jump destination) */
   int p3;             /* The third parameter */
@@ -60,6 +59,7 @@ struct VdbeOp {
     KeyInfo *pKeyInfo;     /* Used when p4type is P4_KEYINFO */
     int *ai;               /* Used when p4type is P4_INTARRAY */
     SubProgram *pProgram;  /* Used when p4type is P4_SUBPROGRAM */
+    Table *pTab;           /* Used when p4type is P4_TABLE */
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
     Expr *pExpr;           /* Used when p4type is P4_EXPR */
 #endif
@@ -87,7 +87,7 @@ struct SubProgram {
   int nOp;                      /* Elements in aOp[] */
   int nMem;                     /* Number of memory cells required */
   int nCsr;                     /* Number of cursors required */
-  int nOnce;                    /* Number of OP_Once instructions */
+  u8 *aOnce;                    /* Array of OP_Once flags */
   void *token;                  /* id that may be used to recursive triggers */
   SubProgram *pNext;            /* Next sub-program already visited */
 };
@@ -110,21 +110,21 @@ typedef struct VdbeOpList VdbeOpList;
 #define P4_NOTUSED    0   /* The P4 parameter is not used */
 #define P4_DYNAMIC  (-1)  /* Pointer to a string obtained from sqliteMalloc() */
 #define P4_STATIC   (-2)  /* Pointer to a static string */
-#define P4_COLLSEQ  (-4)  /* P4 is a pointer to a CollSeq structure */
-#define P4_FUNCDEF  (-5)  /* P4 is a pointer to a FuncDef structure */
-#define P4_KEYINFO  (-6)  /* P4 is a pointer to a KeyInfo structure */
-#define P4_EXPR     (-7)  /* P4 is a pointer to an Expr tree */
-#define P4_MEM      (-8)  /* P4 is a pointer to a Mem*    structure */
+#define P4_COLLSEQ  (-3)  /* P4 is a pointer to a CollSeq structure */
+#define P4_FUNCDEF  (-4)  /* P4 is a pointer to a FuncDef structure */
+#define P4_KEYINFO  (-5)  /* P4 is a pointer to a KeyInfo structure */
+#define P4_EXPR     (-6)  /* P4 is a pointer to an Expr tree */
+#define P4_MEM      (-7)  /* P4 is a pointer to a Mem*    structure */
 #define P4_TRANSIENT  0   /* P4 is a pointer to a transient string */
-#define P4_VTAB     (-10) /* P4 is a pointer to an sqlite3_vtab structure */
-#define P4_MPRINTF  (-11) /* P4 is a string obtained from sqlite3_mprintf() */
-#define P4_REAL     (-12) /* P4 is a 64-bit floating point value */
-#define P4_INT64    (-13) /* P4 is a 64-bit signed integer */
-#define P4_INT32    (-14) /* P4 is a 32-bit signed integer */
-#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */
-#define P4_SUBPROGRAM  (-18) /* P4 is a pointer to a SubProgram structure */
-#define P4_ADVANCE  (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */
-#define P4_FUNCCTX  (-20) /* P4 is a pointer to an sqlite3_context object */
+#define P4_VTAB     (-8) /* P4 is a pointer to an sqlite3_vtab structure */
+#define P4_REAL     (-9) /* P4 is a 64-bit floating point value */
+#define P4_INT64    (-10) /* P4 is a 64-bit signed integer */
+#define P4_INT32    (-11) /* P4 is a 32-bit signed integer */
+#define P4_INTARRAY (-12) /* P4 is a vector of 32-bit integers */
+#define P4_SUBPROGRAM  (-13) /* P4 is a pointer to a SubProgram structure */
+#define P4_ADVANCE  (-14) /* P4 is a pointer to BtreeNext() or BtreePrev() */
+#define P4_TABLE    (-15) /* P4 is a pointer to a Table structure */
+#define P4_FUNCCTX  (-16) /* P4 is a pointer to an sqlite3_context object */
 
 /* Error message codes for OP_Halt */
 #define P5_ConstraintNotNull 1
@@ -183,8 +183,10 @@ int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
 void sqlite3VdbeEndCoroutine(Vdbe*,int);
 #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
   void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N);
+  void sqlite3VdbeVerifyNoResultRow(Vdbe *p);
 #else
 # define sqlite3VdbeVerifyNoMallocRequired(A,B)
+# define sqlite3VdbeVerifyNoResultRow(A)
 #endif
 VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
 void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
@@ -192,16 +194,18 @@ void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
 void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
 void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
 void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
-void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
+void sqlite3VdbeChangeP5(Vdbe*, u16 P5);
 void sqlite3VdbeJumpHere(Vdbe*, int addr);
 int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
 int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);
 void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);
+void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type);
 void sqlite3VdbeSetP4KeyInfo(Parse*, Index*);
 void sqlite3VdbeUsesBtree(Vdbe*, int);
 VdbeOp *sqlite3VdbeGetOp(Vdbe*, int);
 int sqlite3VdbeMakeLabel(Vdbe*);
 void sqlite3VdbeRunOnlyOnce(Vdbe*);
+void sqlite3VdbeReusable(Vdbe*);
 void sqlite3VdbeDelete(Vdbe*);
 void sqlite3VdbeClearObject(sqlite3*,Vdbe*);
 void sqlite3VdbeMakeReady(Vdbe*,Parse*);
@@ -231,7 +235,7 @@ int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*);
 void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*);
 int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*);
 int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int);
-UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **);
+UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*);
 
 typedef int (*RecordCompare)(int,const void*,UnpackedRecord*);
 RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*);
@@ -306,4 +310,4 @@ void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*);
 # define sqlite3VdbeScanStatus(a,b,c,d,e)
 #endif
 
-#endif
+#endif /* SQLITE_VDBE_H */
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index b53952fd44..c0435a5a79 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -15,8 +15,8 @@
 ** 6000 lines long) it was split up into several smaller files and
 ** this header information was factored out.
 */
-#ifndef _VDBEINT_H_
-#define _VDBEINT_H_
+#ifndef SQLITE_VDBEINT_H
+#define SQLITE_VDBEINT_H
 
 /*
 ** The maximum number of times that a statement will try to reparse
@@ -52,9 +52,6 @@ typedef unsigned Bool;
 /* Opaque type used by code in vdbesort.c */
 typedef struct VdbeSorter VdbeSorter;
 
-/* Opaque type used by the explainer */
-typedef struct Explain Explain;
-
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;
 
@@ -76,59 +73,68 @@ typedef struct AuxData AuxData;
 */
 typedef struct VdbeCursor VdbeCursor;
 struct VdbeCursor {
-  u8 eCurType;          /* One of the CURTYPE_* values above */
-  i8 iDb;               /* Index of cursor database in db->aDb[] (or -1) */
-  u8 nullRow;           /* True if pointing to a row with no data */
-  u8 deferredMoveto;    /* A call to sqlite3BtreeMoveto() is needed */
-  u8 isTable;           /* True for rowid tables.  False for indexes */
+  u8 eCurType;            /* One of the CURTYPE_* values above */
+  i8 iDb;                 /* Index of cursor database in db->aDb[] (or -1) */
+  u8 nullRow;             /* True if pointing to a row with no data */
+  u8 deferredMoveto;      /* A call to sqlite3BtreeMoveto() is needed */
+  u8 isTable;             /* True for rowid tables.  False for indexes */
 #ifdef SQLITE_DEBUG
-  u8 seekOp;            /* Most recent seek operation on this cursor */
-  u8 wrFlag;            /* The wrFlag argument to sqlite3BtreeCursor() */
+  u8 seekOp;              /* Most recent seek operation on this cursor */
+  u8 wrFlag;              /* The wrFlag argument to sqlite3BtreeCursor() */
 #endif
-  Bool isEphemeral:1;   /* True for an ephemeral table */
-  Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */
-  Bool isOrdered:1;     /* True if the table is not BTREE_UNORDERED */
-  Pgno pgnoRoot;        /* Root page of the open btree cursor */
-  i16 nField;           /* Number of fields in the header */
-  u16 nHdrParsed;       /* Number of header fields parsed so far */
+  Bool isEphemeral:1;     /* True for an ephemeral table */
+  Bool useRandomRowid:1;  /* Generate new record numbers semi-randomly */
+  Bool isOrdered:1;       /* True if the table is not BTREE_UNORDERED */
+  Btree *pBtx;            /* Separate file holding temporary table */
+  i64 seqCount;           /* Sequence counter */
+  int *aAltMap;           /* Mapping from table to index column numbers */
+
+  /* Cached OP_Column parse information is only valid if cacheStatus matches
+  ** Vdbe.cacheCtr.  Vdbe.cacheCtr will never take on the value of
+  ** CACHE_STALE (0) and so setting cacheStatus=CACHE_STALE guarantees that
+  ** the cache is out of date. */
+  u32 cacheStatus;        /* Cache is valid if this matches Vdbe.cacheCtr */
+  int seekResult;         /* Result of previous sqlite3BtreeMoveto() or 0
+                          ** if there have been no prior seeks on the cursor. */
+  /* NB: seekResult does not distinguish between "no seeks have ever occurred
+  ** on this cursor" and "the most recent seek was an exact match". */
+
+  /* When a new VdbeCursor is allocated, only the fields above are zeroed.
+  ** The fields that follow are uninitialized, and must be individually
+  ** initialized prior to first use. */
+  VdbeCursor *pAltCursor; /* Associated index cursor from which to read */
   union {
     BtCursor *pCursor;          /* CURTYPE_BTREE.  Btree cursor */
     sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB.   Vtab cursor */
     int pseudoTableReg;         /* CURTYPE_PSEUDO. Reg holding content. */
     VdbeSorter *pSorter;        /* CURTYPE_SORTER. Sorter object */
   } uc;
-  Btree *pBt;           /* Separate file holding temporary table */
-  KeyInfo *pKeyInfo;    /* Info about index keys needed by index cursors */
-  int seekResult;       /* Result of previous sqlite3BtreeMoveto() */
-  i64 seqCount;         /* Sequence counter */
-  i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
-  VdbeCursor *pAltCursor; /* Associated index cursor from which to read */
-  int *aAltMap;           /* Mapping from table to index column numbers */
+  KeyInfo *pKeyInfo;      /* Info about index keys needed by index cursors */
+  u32 iHdrOffset;         /* Offset to next unparsed byte of the header */
+  Pgno pgnoRoot;          /* Root page of the open btree cursor */
+  i16 nField;             /* Number of fields in the header */
+  u16 nHdrParsed;         /* Number of header fields parsed so far */
+  i64 movetoTarget;       /* Argument to the deferred sqlite3BtreeMoveto() */
+  u32 *aOffset;           /* Pointer to aType[nField] */
+  const u8 *aRow;         /* Data for the current row, if all on one page */
+  u32 payloadSize;        /* Total number of bytes in the record */
+  u32 szRow;              /* Byte available in aRow */
 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
-  u64 maskUsed;         /* Mask of columns used by this cursor */
+  u64 maskUsed;           /* Mask of columns used by this cursor */
 #endif
 
-  /* Cached information about the header for the data record that the
-  ** cursor is currently pointing to.  Only valid if cacheStatus matches
-  ** Vdbe.cacheCtr.  Vdbe.cacheCtr will never take on the value of
-  ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that
-  ** the cache is out of date.
-  **
-  ** aRow might point to (ephemeral) data for the current row, or it might
-  ** be NULL.
-  */
-  u32 cacheStatus;      /* Cache is valid if this matches Vdbe.cacheCtr */
-  u32 payloadSize;      /* Total number of bytes in the record */
-  u32 szRow;            /* Byte available in aRow */
-  u32 iHdrOffset;       /* Offset to next unparsed byte of the header */
-  const u8 *aRow;       /* Data for the current row, if all on one page */
-  u32 *aOffset;         /* Pointer to aType[nField] */
-  u32 aType[1];         /* Type values for all entries in the record */
   /* 2*nField extra array elements allocated for aType[], beyond the one
   ** static element declared in the structure.  nField total array slots for
   ** aType[] and nField+1 array slots for aOffset[] */
+  u32 aType[1];           /* Type values record decode.  MUST BE LAST */
 };
 
+
+/*
+** A value for VdbeCursor.cacheStatus that means the cache is always invalid.
+*/
+#define CACHE_STALE 0
+
 /*
 ** When a sub-program is executed (OP_Program), a structure of this type
 ** is allocated to store the current value of the program counter, as
@@ -157,8 +163,8 @@ struct VdbeFrame {
   Op *aOp;                /* Program instructions for parent frame */
   i64 *anExec;            /* Event counters from parent frame */
   Mem *aMem;              /* Array of memory cells for parent frame */
-  u8 *aOnceFlag;          /* Array of OP_Once flags for parent frame */
   VdbeCursor **apCsr;     /* Array of Vdbe cursors for parent frame */
+  u8 *aOnce;              /* Bitmask used by OP_Once */
   void *token;            /* Copy of SubProgram.token */
   i64 lastRowid;          /* Last insert rowid (sqlite3.lastRowid) */
   AuxData *pAuxData;      /* Linked list of auxdata allocations */
@@ -166,7 +172,6 @@ struct VdbeFrame {
   int pc;                 /* Program Counter in parent (calling) frame */
   int nOp;                /* Size of aOp array */
   int nMem;               /* Number of entries in aMem */
-  int nOnceFlag;          /* Number of entries in aOnceFlag */
   int nChildMem;          /* Number of memory cells for child frame */
   int nChildCsr;          /* Number of cursors for child frame */
   int nChange;            /* Statement changes (Vdbe.nChange)     */
@@ -175,11 +180,6 @@ struct VdbeFrame {
 
 #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))])
 
-/*
-** A value for VdbeCursor.cacheValid that means the cache is always invalid.
-*/
-#define CACHE_STALE 0
-
 /*
 ** Internally, the vdbe manipulates nearly all SQL values as Mem
 ** structures. Each Mem struct may cache multiple representations (string,
@@ -320,18 +320,6 @@ struct sqlite3_context {
   sqlite3_value *argv[1]; /* Argument set */
 };
 
-/*
-** An Explain object accumulates indented output which is helpful
-** in describing recursive data structures.
-*/
-struct Explain {
-  Vdbe *pVdbe;       /* Attach the explanation to this Vdbe */
-  StrAccum str;      /* The string being accumulated */
-  int nIndent;       /* Number of elements in aIndent */
-  u16 aIndent[100];  /* Levels of indentation */
-  char zBase[100];   /* Initial space */
-};
-
 /* A bitfield type for use inside of structures.  Always follow with :N where
 ** N is the number of bits.
 */
@@ -356,53 +344,56 @@ struct ScanStatus {
 */
 struct Vdbe {
   sqlite3 *db;            /* The database connection that owns this statement */
+  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
+  Parse *pParse;          /* Parsing context used to create this Vdbe */
+  ynVar nVar;             /* Number of entries in aVar[] */
+  u32 magic;              /* Magic number for sanity checking */
+  int nMem;               /* Number of memory locations currently allocated */
+  int nCursor;            /* Number of slots in apCsr[] */
+  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
+  int pc;                 /* The program counter */
+  int rc;                 /* Value to return */
+  int nChange;            /* Number of db changes made since last reset */
+  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
+  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
+  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
+  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
+  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
+
+  /* When allocating a new Vdbe object, all of the fields below should be
+  ** initialized to zero or NULL */
+
   Op *aOp;                /* Space to hold the virtual machine's program */
   Mem *aMem;              /* The memory locations */
   Mem **apArg;            /* Arguments to currently executing user function */
   Mem *aColName;          /* Column names to return */
   Mem *pResultSet;        /* Pointer to an array of results */
-  Parse *pParse;          /* Parsing context used to create this Vdbe */
-  int nMem;               /* Number of memory locations currently allocated */
-  int nOp;                /* Number of instructions in the program */
-  int nCursor;            /* Number of slots in apCsr[] */
-  u32 magic;              /* Magic number for sanity checking */
   char *zErrMsg;          /* Error message written here */
-  Vdbe *pPrev,*pNext;     /* Linked list of VDBEs with the same Vdbe.db */
   VdbeCursor **apCsr;     /* One element of this array for each open cursor */
   Mem *aVar;              /* Values for the OP_Variable opcode. */
-  char **azVar;           /* Name of variables */
-  ynVar nVar;             /* Number of entries in aVar[] */
-  ynVar nzVar;            /* Number of entries in azVar[] */
-  u32 cacheCtr;           /* VdbeCursor row cache generation counter */
-  int pc;                 /* The program counter */
-  int rc;                 /* Value to return */
+  VList *pVList;          /* Name of variables */
+#ifndef SQLITE_OMIT_TRACE
+  i64 startTime;          /* Time when query started - used for profiling */
+#endif
+  int nOp;                /* Number of instructions in the program */
 #ifdef SQLITE_DEBUG
   int rcApp;              /* errcode set by sqlite3_result_error_code() */
 #endif
   u16 nResColumn;         /* Number of columns in one row of the result set */
   u8 errorAction;         /* Recovery action to do in case of an error */
   u8 minWriteFileFormat;  /* Minimum file format for writable database files */
+  bft expired:1;          /* True if the VM needs to be recompiled */
+  bft doingRerun:1;       /* True if rerunning after an auto-reprepare */
   bft explain:2;          /* True if EXPLAIN present on SQL command */
   bft changeCntOn:1;      /* True to update the change-counter */
-  bft expired:1;          /* True if the VM needs to be recompiled */
   bft runOnlyOnce:1;      /* Automatically expire on reset */
   bft usesStmtJournal:1;  /* True if uses a statement journal */
   bft readOnly:1;         /* True for statements that do not write */
   bft bIsReader:1;        /* True for statements that read */
   bft isPrepareV2:1;      /* True if prepared with prepare_v2() */
-  bft doingRerun:1;       /* True if rerunning after an auto-reprepare */
-  int nChange;            /* Number of db changes made since last reset */
   yDbMask btreeMask;      /* Bitmask of db->aDb[] entries referenced */
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
-  int iStatement;         /* Statement number (or 0 if has not opened stmt) */
   u32 aCounter[5];        /* Counters used by sqlite3_stmt_status() */
-#ifndef SQLITE_OMIT_TRACE
-  i64 startTime;          /* Time when query started - used for profiling */
-#endif
-  i64 iCurrentTime;       /* Value of julianday('now') for this statement */
-  i64 nFkConstraint;      /* Number of imm. FK constraints this VM */
-  i64 nStmtDefCons;       /* Number of def. constraints when stmt started */
-  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
   char *zSql;             /* Text of the SQL statement that generated this */
   void *pFree;            /* Free this when deleting the vdbe */
   VdbeFrame *pFrame;      /* Parent frame */
@@ -410,8 +401,6 @@ struct Vdbe {
   int nFrame;             /* Number of frames in pFrame list */
   u32 expmask;            /* Binding to these vars invalidates VM */
   SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */
-  int nOnceFlag;          /* Size of array aOnceFlag[] */
-  u8 *aOnceFlag;          /* Flags for OP_Once */
   AuxData *pAuxData;      /* Linked list of auxdata allocations */
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   i64 *anExec;            /* Number of times each op has been executed */
@@ -423,10 +412,31 @@ struct Vdbe {
 /*
 ** The following are allowed values for Vdbe.magic
 */
-#define VDBE_MAGIC_INIT     0x26bceaa5    /* Building a VDBE program */
-#define VDBE_MAGIC_RUN      0xbdf20da3    /* VDBE is ready to execute */
-#define VDBE_MAGIC_HALT     0x519c2973    /* VDBE has completed execution */
-#define VDBE_MAGIC_DEAD     0xb606c3c8    /* The VDBE has been deallocated */
+#define VDBE_MAGIC_INIT     0x16bceaa5    /* Building a VDBE program */
+#define VDBE_MAGIC_RUN      0x2df20da3    /* VDBE is ready to execute */
+#define VDBE_MAGIC_HALT     0x319c2973    /* VDBE has completed execution */
+#define VDBE_MAGIC_RESET    0x48fa9f76    /* Reset and ready to run again */
+#define VDBE_MAGIC_DEAD     0x5606c3c8    /* The VDBE has been deallocated */
+
+/*
+** Structure used to store the context required by the 
+** sqlite3_preupdate_*() API functions.
+*/
+struct PreUpdate {
+  Vdbe *v;
+  VdbeCursor *pCsr;               /* Cursor to read old values from */
+  int op;                         /* One of SQLITE_INSERT, UPDATE, DELETE */
+  u8 *aRecord;                    /* old.* database record */
+  KeyInfo keyinfo;
+  UnpackedRecord *pUnpacked;      /* Unpacked version of aRecord[] */
+  UnpackedRecord *pNewUnpacked;   /* Unpacked version of new.* record */
+  int iNewReg;                    /* Register for new.* values */
+  i64 iKey1;                      /* First key value passed to hook */
+  i64 iKey2;                      /* Second key value passed to hook */
+  Mem *aNew;                      /* Array of new.* values */
+  Table *pTab;                    /* Schema object being upated */          
+  Index *pPk;                     /* PK index if pTab is WITHOUT ROWID */
+};
 
 /*
 ** Function prototypes
@@ -478,7 +488,7 @@ void sqlite3VdbeIntegerAffinity(Mem*);
 int sqlite3VdbeMemRealify(Mem*);
 int sqlite3VdbeMemNumerify(Mem*);
 void sqlite3VdbeMemCast(Mem*,u8,u8);
-int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*);
+int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*);
 void sqlite3VdbeMemRelease(Mem *p);
 int sqlite3VdbeMemFinalize(Mem*, FuncDef*);
 const char *sqlite3OpcodeName(int);
@@ -487,6 +497,9 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int n);
 int sqlite3VdbeCloseStatement(Vdbe *, int);
 void sqlite3VdbeFrameDelete(VdbeFrame*);
 int sqlite3VdbeFrameRestore(VdbeFrame *);
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int);
+#endif
 int sqlite3VdbeTransferError(Vdbe *p);
 
 int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *);
@@ -536,4 +549,4 @@ int sqlite3VdbeMemHandleBom(Mem *pMem);
   #define ExpandBlob(P) SQLITE_OK
 #endif
 
-#endif /* !defined(_VDBEINT_H_) */
+#endif /* !defined(SQLITE_VDBEINT_H) */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index c2038137d1..1f2699dc4d 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -60,12 +60,19 @@ static int vdbeSafetyNotNull(Vdbe *p){
 */
 static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){
   sqlite3_int64 iNow;
+  sqlite3_int64 iElapse;
   assert( p->startTime>0 );
-  assert( db->xProfile!=0 );
+  assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 );
   assert( db->init.busy==0 );
   assert( p->zSql!=0 );
   sqlite3OsCurrentTimeInt64(db->pVfs, &iNow);
-  db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000);
+  iElapse = (iNow - p->startTime)*1000000;
+  if( db->xProfile ){
+    db->xProfile(db->pProfileArg, p->zSql, iElapse);
+  }
+  if( db->mTrace & SQLITE_TRACE_PROFILE ){
+    db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse);
+  }
   p->startTime = 0;
 }
 /*
@@ -147,7 +154,8 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){
     sqlite3VdbeMemRelease(&p->aVar[i]);
     p->aVar[i].flags = MEM_Null;
   }
-  if( p->isPrepareV2 && p->expmask ){
+  assert( p->isPrepareV2 || p->expmask==0 );
+  if( p->expmask ){
     p->expired = 1;
   }
   sqlite3_mutex_leave(mutex);
@@ -162,7 +170,7 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){
 const void *sqlite3_value_blob(sqlite3_value *pVal){
   Mem *p = (Mem*)pVal;
   if( p->flags & (MEM_Blob|MEM_Str) ){
-    if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ){
+    if( ExpandBlob(p)!=SQLITE_OK ){
       assert( p->flags==MEM_Null && p->z==0 );
       return 0;
     }
@@ -492,7 +500,7 @@ static int doWalCallbacks(sqlite3 *db){
       nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt));
       sqlite3BtreeLeave(pBt);
       if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){
-        rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry);
+        rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry);
       }
     }
   }
@@ -569,7 +577,8 @@ static int sqlite3Step(Vdbe *p){
     );
 
 #ifndef SQLITE_OMIT_TRACE
-    if( db->xProfile && !db->init.busy && p->zSql ){
+    if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0)
+        && !db->init.busy && p->zSql ){
       sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
     }else{
       assert( p->startTime==0 );
@@ -944,14 +953,13 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){
   Mem *pOut;
 
   pVm = (Vdbe *)pStmt;
-  if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){
-    sqlite3_mutex_enter(pVm->db->mutex);
+  if( pVm==0 ) return (Mem*)columnNullValue();
+  assert( pVm->db );
+  sqlite3_mutex_enter(pVm->db->mutex);
+  if( pVm->pResultSet!=0 && inResColumn && i>=0 ){
     pOut = &pVm->pResultSet[i];
   }else{
-    if( pVm && ALWAYS(pVm->db) ){
-      sqlite3_mutex_enter(pVm->db->mutex);
-      sqlite3Error(pVm->db, SQLITE_RANGE);
-    }
+    sqlite3Error(pVm->db, SQLITE_RANGE);
     pOut = (Mem*)columnNullValue();
   }
   return pOut;
@@ -984,6 +992,8 @@ static void columnMallocFailure(sqlite3_stmt *pStmt)
   */
   Vdbe *p = (Vdbe *)pStmt;
   if( p ){
+    assert( p->db!=0 );
+    assert( sqlite3_mutex_held(p->db->mutex) );
     p->rc = sqlite3ApiExit(p->db, p->rc);
     sqlite3_mutex_leave(p->db->mutex);
   }
@@ -1249,9 +1259,8 @@ static int vdbeUnbind(Vdbe *p, int i){
   ** as if there had been a schema change, on the first sqlite3_step() call
   ** following any change to the bindings of that parameter.
   */
-  if( p->isPrepareV2 &&
-     ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff)
-  ){
+  assert( p->isPrepareV2 || p->expmask==0 );
+  if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1;
   }
   return SQLITE_OK;
@@ -1461,10 +1470,8 @@ int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){
 */
 const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){
   Vdbe *p = (Vdbe*)pStmt;
-  if( p==0 || i<1 || i>p->nzVar ){
-    return 0;
-  }
-  return p->azVar[i-1];
+  if( p==0 ) return 0;
+  return sqlite3VListNumToName(p->pVList, i);
 }
 
 /*
@@ -1473,19 +1480,8 @@ const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){
 ** return 0.
 */
 int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){
-  int i;
-  if( p==0 ){
-    return 0;
-  }
-  if( zName ){
-    for(i=0; inzVar; i++){
-      const char *z = p->azVar[i];
-      if( z && strncmp(z,zName,nName)==0 && z[nName]==0 ){
-        return i+1;
-      }
-    }
-  }
-  return 0;
+  if( p==0 || zName==0 ) return 0;
+  return sqlite3VListNameToNum(p->pVList, zName, nName);
 }
 int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){
   return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName));
@@ -1527,10 +1523,12 @@ int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){
   if( pFrom->nVar!=pTo->nVar ){
     return SQLITE_ERROR;
   }
-  if( pTo->isPrepareV2 && pTo->expmask ){
+  assert( pTo->isPrepareV2 || pTo->expmask==0 );
+  if( pTo->expmask ){
     pTo->expired = 1;
   }
-  if( pFrom->isPrepareV2 && pFrom->expmask ){
+  assert( pFrom->isPrepareV2 || pFrom->expmask==0 );
+  if( pFrom->expmask ){
     pFrom->expired = 1;
   }
   return sqlite3TransferBindings(pFromStmt, pToStmt);
@@ -1560,7 +1558,7 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){
 */
 int sqlite3_stmt_busy(sqlite3_stmt *pStmt){
   Vdbe *v = (Vdbe*)pStmt;
-  return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN;
+  return v!=0 && v->magic==VDBE_MAGIC_RUN && v->pc>=0;
 }
 
 /*
@@ -1604,6 +1602,225 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
   return (int)v;
 }
 
+/*
+** Return the SQL associated with a prepared statement
+*/
+const char *sqlite3_sql(sqlite3_stmt *pStmt){
+  Vdbe *p = (Vdbe *)pStmt;
+  return p ? p->zSql : 0;
+}
+
+/*
+** Return the SQL associated with a prepared statement with
+** bound parameters expanded.  Space to hold the returned string is
+** obtained from sqlite3_malloc().  The caller is responsible for
+** freeing the returned string by passing it to sqlite3_free().
+**
+** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of
+** expanded bound parameters.
+*/
+char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
+#ifdef SQLITE_OMIT_TRACE
+  return 0;
+#else
+  char *z = 0;
+  const char *zSql = sqlite3_sql(pStmt);
+  if( zSql ){
+    Vdbe *p = (Vdbe *)pStmt;
+    sqlite3_mutex_enter(p->db->mutex);
+    z = sqlite3VdbeExpandSql(p, zSql);
+    sqlite3_mutex_leave(p->db->mutex);
+  }
+  return z;
+#endif
+}
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Allocate and populate an UnpackedRecord structure based on the serialized
+** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure
+** if successful, or a NULL pointer if an OOM error is encountered.
+*/
+static UnpackedRecord *vdbeUnpackRecord(
+  KeyInfo *pKeyInfo, 
+  int nKey, 
+  const void *pKey
+){
+  UnpackedRecord *pRet;           /* Return value */
+
+  pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo);
+  if( pRet ){
+    memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1));
+    sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet);
+  }
+  return pRet;
+}
+
+/*
+** This function is called from within a pre-update callback to retrieve
+** a field of the row currently being updated or deleted.
+*/
+int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
+  PreUpdate *p = db->pPreUpdate;
+  Mem *pMem;
+  int rc = SQLITE_OK;
+
+  /* Test that this call is being made from within an SQLITE_DELETE or
+  ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */
+  if( !p || p->op==SQLITE_INSERT ){
+    rc = SQLITE_MISUSE_BKPT;
+    goto preupdate_old_out;
+  }
+  if( p->pPk ){
+    iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx);
+  }
+  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+    rc = SQLITE_RANGE;
+    goto preupdate_old_out;
+  }
+
+  /* If the old.* record has not yet been loaded into memory, do so now. */
+  if( p->pUnpacked==0 ){
+    u32 nRec;
+    u8 *aRec;
+
+    nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor);
+    aRec = sqlite3DbMallocRaw(db, nRec);
+    if( !aRec ) goto preupdate_old_out;
+    rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec);
+    if( rc==SQLITE_OK ){
+      p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec);
+      if( !p->pUnpacked ) rc = SQLITE_NOMEM;
+    }
+    if( rc!=SQLITE_OK ){
+      sqlite3DbFree(db, aRec);
+      goto preupdate_old_out;
+    }
+    p->aRecord = aRec;
+  }
+
+  pMem = *ppValue = &p->pUnpacked->aMem[iIdx];
+  if( iIdx==p->pTab->iPKey ){
+    sqlite3VdbeMemSetInt64(pMem, p->iKey1);
+  }else if( iIdx>=p->pUnpacked->nField ){
+    *ppValue = (sqlite3_value *)columnNullValue();
+  }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
+    if( pMem->flags & MEM_Int ){
+      sqlite3VdbeMemRealify(pMem);
+    }
+  }
+
+ preupdate_old_out:
+  sqlite3Error(db, rc);
+  return sqlite3ApiExit(db, rc);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is called from within a pre-update callback to retrieve
+** the number of columns in the row being updated, deleted or inserted.
+*/
+int sqlite3_preupdate_count(sqlite3 *db){
+  PreUpdate *p = db->pPreUpdate;
+  return (p ? p->keyinfo.nField : 0);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is designed to be called from within a pre-update callback
+** only. It returns zero if the change that caused the callback was made
+** immediately by a user SQL statement. Or, if the change was made by a
+** trigger program, it returns the number of trigger programs currently
+** on the stack (1 for a top-level trigger, 2 for a trigger fired by a 
+** top-level trigger etc.).
+**
+** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL
+** or SET DEFAULT action is considered a trigger.
+*/
+int sqlite3_preupdate_depth(sqlite3 *db){
+  PreUpdate *p = db->pPreUpdate;
+  return (p ? p->v->nFrame : 0);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** This function is called from within a pre-update callback to retrieve
+** a field of the row currently being updated or inserted.
+*/
+int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
+  PreUpdate *p = db->pPreUpdate;
+  int rc = SQLITE_OK;
+  Mem *pMem;
+
+  if( !p || p->op==SQLITE_DELETE ){
+    rc = SQLITE_MISUSE_BKPT;
+    goto preupdate_new_out;
+  }
+  if( p->pPk && p->op!=SQLITE_UPDATE ){
+    iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx);
+  }
+  if( iIdx>=p->pCsr->nField || iIdx<0 ){
+    rc = SQLITE_RANGE;
+    goto preupdate_new_out;
+  }
+
+  if( p->op==SQLITE_INSERT ){
+    /* For an INSERT, memory cell p->iNewReg contains the serialized record
+    ** that is being inserted. Deserialize it. */
+    UnpackedRecord *pUnpack = p->pNewUnpacked;
+    if( !pUnpack ){
+      Mem *pData = &p->v->aMem[p->iNewReg];
+      rc = ExpandBlob(pData);
+      if( rc!=SQLITE_OK ) goto preupdate_new_out;
+      pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z);
+      if( !pUnpack ){
+        rc = SQLITE_NOMEM;
+        goto preupdate_new_out;
+      }
+      p->pNewUnpacked = pUnpack;
+    }
+    pMem = &pUnpack->aMem[iIdx];
+    if( iIdx==p->pTab->iPKey ){
+      sqlite3VdbeMemSetInt64(pMem, p->iKey2);
+    }else if( iIdx>=pUnpack->nField ){
+      pMem = (sqlite3_value *)columnNullValue();
+    }
+  }else{
+    /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required
+    ** value. Make a copy of the cell contents and return a pointer to it.
+    ** It is not safe to return a pointer to the memory cell itself as the
+    ** caller may modify the value text encoding.
+    */
+    assert( p->op==SQLITE_UPDATE );
+    if( !p->aNew ){
+      p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField);
+      if( !p->aNew ){
+        rc = SQLITE_NOMEM;
+        goto preupdate_new_out;
+      }
+    }
+    assert( iIdx>=0 && iIdxpCsr->nField );
+    pMem = &p->aNew[iIdx];
+    if( pMem->flags==0 ){
+      if( iIdx==p->pTab->iPKey ){
+        sqlite3VdbeMemSetInt64(pMem, p->iKey2);
+      }else{
+        rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]);
+        if( rc!=SQLITE_OK ) goto preupdate_new_out;
+      }
+    }
+  }
+  *ppValue = pMem;
+
+ preupdate_new_out:
+  sqlite3Error(db, rc);
+  return sqlite3ApiExit(db, rc);
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
 /*
 ** Return status data for a single loop within query pStmt.
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 9c75d1e078..a8f215420a 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -21,8 +21,9 @@
 Vdbe *sqlite3VdbeCreate(Parse *pParse){
   sqlite3 *db = pParse->db;
   Vdbe *p;
-  p = sqlite3DbMallocZero(db, sizeof(Vdbe) );
+  p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) );
   if( p==0 ) return 0;
+  memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp));
   p->db = db;
   if( db->pVdbe ){
     db->pVdbe->pPrev = p;
@@ -56,6 +57,7 @@ void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){
 void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){
   assert( isPrepareV2==1 || isPrepareV2==0 );
   if( p==0 ) return;
+  if( !isPrepareV2 ) p->expmask = 0;
 #if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG)
   if( !isPrepareV2 ) return;
 #endif
@@ -64,20 +66,13 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){
   p->isPrepareV2 = (u8)isPrepareV2;
 }
 
-/*
-** Return the SQL associated with a prepared statement
-*/
-const char *sqlite3_sql(sqlite3_stmt *pStmt){
-  Vdbe *p = (Vdbe *)pStmt;
-  return p ? p->zSql : 0;
-}
-
 /*
 ** Swap all content between two VDBE structures.
 */
 void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
   Vdbe tmp, *pTmp;
   char *zTmp;
+  assert( pA->db==pB->db );
   tmp = *pA;
   *pA = *pB;
   *pB = tmp;
@@ -91,6 +86,7 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
   pA->zSql = pB->zSql;
   pB->zSql = zTmp;
   pB->isPrepareV2 = pA->isPrepareV2;
+  pB->expmask = pA->expmask;
 }
 
 /*
@@ -121,6 +117,12 @@ static int growOpArray(Vdbe *v, int nOp){
   UNUSED_PARAMETER(nOp);
 #endif
 
+  /* Ensure that the size of a VDBE does not grow too large */
+  if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){
+    sqlite3OomFault(p->db);
+    return SQLITE_NOMEM;
+  }
+
   assert( nOp<=(1024/sizeof(Op)) );
   assert( nNew>=(p->nOpAlloc+nOp) );
   pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op));
@@ -191,9 +193,8 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
   if( p->db->flags & SQLITE_VdbeAddopTrace ){
     int jj, kk;
     Parse *pParse = p->pParse;
-    for(jj=kk=0; jjnColCache; jj++){
       struct yColCache *x = pParse->aColCache + jj;
-      if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue;
       printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn);
       kk++;
     }
@@ -320,7 +321,11 @@ int sqlite3VdbeAddOp4Int(
   int p4              /* The P4 operand as an integer */
 ){
   int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
-  sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
+  if( p->db->mallocFailed==0 ){
+    VdbeOp *pOp = &p->aOp[addr];
+    pOp->p4type = P4_INT32;
+    pOp->p4.i = p4;
+  }
   return addr;
 }
 
@@ -381,7 +386,6 @@ void sqlite3VdbeResolveLabel(Vdbe *v, int x){
   if( p->aLabel ){
     p->aLabel[j] = v->nOp;
   }
-  p->iFixedOp = v->nOp - 1;
 }
 
 /*
@@ -391,6 +395,13 @@ void sqlite3VdbeRunOnlyOnce(Vdbe *p){
   p->runOnlyOnce = 1;
 }
 
+/*
+** Mark the VDBE as one that can only be run multiple times.
+*/
+void sqlite3VdbeReusable(Vdbe *p){
+  p->runOnlyOnce = 0;
+}
+
 #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */
 
 /*
@@ -537,73 +548,84 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){
 ** (4) Initialize the p4.xAdvance pointer on opcodes that use it.
 **
 ** (5) Reclaim the memory allocated for storing labels.
+**
+** This routine will only function correctly if the mkopcodeh.tcl generator
+** script numbers the opcodes correctly.  Changes to this routine must be
+** coordinated with changes to mkopcodeh.tcl.
 */
 static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
-  int i;
   int nMaxArgs = *pMaxFuncArgs;
   Op *pOp;
   Parse *pParse = p->pParse;
   int *aLabel = pParse->aLabel;
   p->readOnly = 1;
   p->bIsReader = 0;
-  for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
-    u8 opcode = pOp->opcode;
+  pOp = &p->aOp[p->nOp-1];
+  while(1){
 
-    /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing
-    ** cases from this switch! */
-    switch( opcode ){
-      case OP_Transaction: {
-        if( pOp->p2!=0 ) p->readOnly = 0;
-        /* fall thru */
-      }
-      case OP_AutoCommit:
-      case OP_Savepoint: {
-        p->bIsReader = 1;
-        break;
-      }
+    /* Only JUMP opcodes and the short list of special opcodes in the switch
+    ** below need to be considered.  The mkopcodeh.tcl generator script groups
+    ** all these opcodes together near the front of the opcode list.  Skip
+    ** any opcode that does not need processing by virtual of the fact that
+    ** it is larger than SQLITE_MX_JUMP_OPCODE, as a performance optimization.
+    */
+    if( pOp->opcode<=SQLITE_MX_JUMP_OPCODE ){
+      /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing
+      ** cases from this switch! */
+      switch( pOp->opcode ){
+        case OP_Transaction: {
+          if( pOp->p2!=0 ) p->readOnly = 0;
+          /* fall thru */
+        }
+        case OP_AutoCommit:
+        case OP_Savepoint: {
+          p->bIsReader = 1;
+          break;
+        }
 #ifndef SQLITE_OMIT_WAL
-      case OP_Checkpoint:
+        case OP_Checkpoint:
 #endif
-      case OP_Vacuum:
-      case OP_JournalMode: {
-        p->readOnly = 0;
-        p->bIsReader = 1;
-        break;
-      }
+        case OP_Vacuum:
+        case OP_JournalMode: {
+          p->readOnly = 0;
+          p->bIsReader = 1;
+          break;
+        }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-      case OP_VUpdate: {
-        if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
-        break;
-      }
-      case OP_VFilter: {
-        int n;
-        assert( p->nOp - i >= 3 );
-        assert( pOp[-1].opcode==OP_Integer );
-        n = pOp[-1].p1;
-        if( n>nMaxArgs ) nMaxArgs = n;
-        break;
-      }
+        case OP_VUpdate: {
+          if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
+          break;
+        }
+        case OP_VFilter: {
+          int n;
+          assert( (pOp - p->aOp) >= 3 );
+          assert( pOp[-1].opcode==OP_Integer );
+          n = pOp[-1].p1;
+          if( n>nMaxArgs ) nMaxArgs = n;
+          break;
+        }
 #endif
-      case OP_Next:
-      case OP_NextIfOpen:
-      case OP_SorterNext: {
-        pOp->p4.xAdvance = sqlite3BtreeNext;
-        pOp->p4type = P4_ADVANCE;
-        break;
+        case OP_Next:
+        case OP_NextIfOpen:
+        case OP_SorterNext: {
+          pOp->p4.xAdvance = sqlite3BtreeNext;
+          pOp->p4type = P4_ADVANCE;
+          break;
+        }
+        case OP_Prev:
+        case OP_PrevIfOpen: {
+          pOp->p4.xAdvance = sqlite3BtreePrevious;
+          pOp->p4type = P4_ADVANCE;
+          break;
+        }
       }
-      case OP_Prev:
-      case OP_PrevIfOpen: {
-        pOp->p4.xAdvance = sqlite3BtreePrevious;
-        pOp->p4type = P4_ADVANCE;
-        break;
+      if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){
+        assert( ADDR(pOp->p2)nLabel );
+        pOp->p2 = aLabel[ADDR(pOp->p2)];
       }
     }
-
-    pOp->opflags = sqlite3OpcodeProperty[opcode];
-    if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
-      assert( ADDR(pOp->p2)nLabel );
-      pOp->p2 = aLabel[ADDR(pOp->p2)];
-    }
+    if( pOp==p->aOp ) break;
+    pOp--;
   }
   sqlite3DbFree(p->db, pParse->aLabel);
   pParse->aLabel = 0;
@@ -634,6 +656,22 @@ void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){
 }
 #endif
 
+/*
+** Verify that the VM passed as the only argument does not contain
+** an OP_ResultRow opcode. Fail an assert() if it does. This is used
+** by code in pragma.c to ensure that the implementation of certain
+** pragmas comports with the flags specified in the mkpragmatab.tcl
+** script.
+*/
+#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS)
+void sqlite3VdbeVerifyNoResultRow(Vdbe *p){
+  int i;
+  for(i=0; inOp; i++){
+    assert( p->aOp[i].opcode!=OP_ResultRow );
+  }
+}
+#endif
+
 /*
 ** This function returns a pointer to the array of opcodes associated with
 ** the Vdbe passed as the first argument. It is the callers responsibility
@@ -753,8 +791,9 @@ void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){
 void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
   sqlite3VdbeGetOp(p,addr)->p3 = val;
 }
-void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
-  if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5;
+void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
+  assert( p->nOp>0 || p->db->mallocFailed );
+  if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5;
 }
 
 /*
@@ -762,7 +801,6 @@ void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
 ** the address of the next instruction to be coded.
 */
 void sqlite3VdbeJumpHere(Vdbe *p, int addr){
-  p->pParse->iFixedOp = p->nOp - 1;
   sqlite3VdbeChangeP2(p, addr, p->nOp);
 }
 
@@ -772,7 +810,7 @@ void sqlite3VdbeJumpHere(Vdbe *p, int addr){
 ** the FuncDef is not ephermal, then do nothing.
 */
 static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){
-  if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
+  if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){
     sqlite3DbFree(db, pDef);
   }
 }
@@ -782,53 +820,53 @@ static void vdbeFreeOpArray(sqlite3 *, Op *, int);
 /*
 ** Delete a P4 value if necessary.
 */
+static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){
+  if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
+  sqlite3DbFree(db, p);
+}
+static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){
+  freeEphemeralFunction(db, p->pFunc);
+  sqlite3DbFree(db, p);
+}
 static void freeP4(sqlite3 *db, int p4type, void *p4){
-  if( p4 ){
-    assert( db );
-    switch( p4type ){
-      case P4_FUNCCTX: {
-        freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc);
-        /* Fall through into the next case */
-      }
-      case P4_REAL:
-      case P4_INT64:
-      case P4_DYNAMIC:
-      case P4_INTARRAY: {
-        sqlite3DbFree(db, p4);
-        break;
-      }
-      case P4_KEYINFO: {
-        if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4);
-        break;
-      }
+  assert( db );
+  switch( p4type ){
+    case P4_FUNCCTX: {
+      freeP4FuncCtx(db, (sqlite3_context*)p4);
+      break;
+    }
+    case P4_REAL:
+    case P4_INT64:
+    case P4_DYNAMIC:
+    case P4_INTARRAY: {
+      sqlite3DbFree(db, p4);
+      break;
+    }
+    case P4_KEYINFO: {
+      if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4);
+      break;
+    }
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
-      case P4_EXPR: {
-        sqlite3ExprDelete(db, (Expr*)p4);
-        break;
-      }
+    case P4_EXPR: {
+      sqlite3ExprDelete(db, (Expr*)p4);
+      break;
+    }
 #endif
-      case P4_MPRINTF: {
-        if( db->pnBytesFreed==0 ) sqlite3_free(p4);
-        break;
-      }
-      case P4_FUNCDEF: {
-        freeEphemeralFunction(db, (FuncDef*)p4);
-        break;
-      }
-      case P4_MEM: {
-        if( db->pnBytesFreed==0 ){
-          sqlite3ValueFree((sqlite3_value*)p4);
-        }else{
-          Mem *p = (Mem*)p4;
-          if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc);
-          sqlite3DbFree(db, p);
-        }
-        break;
-      }
-      case P4_VTAB : {
-        if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
-        break;
+    case P4_FUNCDEF: {
+      freeEphemeralFunction(db, (FuncDef*)p4);
+      break;
+    }
+    case P4_MEM: {
+      if( db->pnBytesFreed==0 ){
+        sqlite3ValueFree((sqlite3_value*)p4);
+      }else{
+        freeP4Mem(db, (Mem*)p4);
       }
+      break;
+    }
+    case P4_VTAB : {
+      if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4);
+      break;
     }
   }
 }
@@ -881,7 +919,7 @@ int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
 ** then remove it.  Return true if and only if an opcode was removed.
 */
 int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
-  if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){
+  if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){
     return sqlite3VdbeChangeToNoop(p, p->nOp-1);
   }else{
     return 0;
@@ -958,16 +996,42 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){
   }
 }
 
+/*
+** Change the P4 operand of the most recently coded instruction 
+** to the value defined by the arguments.  This is a high-speed
+** version of sqlite3VdbeChangeP4().
+**
+** The P4 operand must not have been previously defined.  And the new
+** P4 must not be P4_INT32.  Use sqlite3VdbeChangeP4() in either of
+** those cases.
+*/
+void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){
+  VdbeOp *pOp;
+  assert( n!=P4_INT32 && n!=P4_VTAB );
+  assert( n<=0 );
+  if( p->db->mallocFailed ){
+    freeP4(p->db, n, pP4);
+  }else{
+    assert( pP4!=0 );
+    assert( p->nOp>0 );
+    pOp = &p->aOp[p->nOp-1];
+    assert( pOp->p4type==P4_NOTUSED );
+    pOp->p4type = n;
+    pOp->p4.p = pP4;
+  }
+}
+
 /*
 ** Set the P4 on the most recently added opcode to the KeyInfo for the
 ** index given.
 */
 void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){
   Vdbe *v = pParse->pVdbe;
+  KeyInfo *pKeyInfo;
   assert( v!=0 );
   assert( pIdx!=0 );
-  sqlite3VdbeChangeP4(v, -1, (char*)sqlite3KeyInfoOfIndex(pParse, pIdx),
-                      P4_KEYINFO);
+  pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pIdx);
+  if( pKeyInfo ) sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO);
 }
 
 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
@@ -1079,12 +1143,21 @@ static int displayComment(
   const char *zSynopsis;
   int nOpName;
   int ii, jj;
+  char zAlt[50];
   zOpName = sqlite3OpcodeName(pOp->opcode);
   nOpName = sqlite3Strlen30(zOpName);
   if( zOpName[nOpName+1] ){
     int seenCom = 0;
     char c;
     zSynopsis = zOpName += nOpName + 1;
+    if( strncmp(zSynopsis,"IF ",3)==0 ){
+      if( pOp->p5 & SQLITE_STOREP2 ){
+        sqlite3_snprintf(sizeof(zAlt), zAlt, "r[P2] = (%s)", zSynopsis+3);
+      }else{
+        sqlite3_snprintf(sizeof(zAlt), zAlt, "if %s goto P2", zSynopsis+3);
+      }
+      zSynopsis = zAlt;
+    }
     for(ii=jj=0; jjzName, pDef->nArg);
       break;
     }
-#ifdef SQLITE_DEBUG
+#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
     case P4_FUNCCTX: {
       FuncDef *pDef = pOp->p4.pCtx->pFunc;
       sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg);
@@ -1310,6 +1383,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){
       zTemp[0] = 0;
       break;
     }
+    case P4_TABLE: {
+      sqlite3XPrintf(&x, "%s", pOp->p4.pTab->zName);
+      break;
+    }
     default: {
       zP4 = pOp->p4.z;
       if( zP4==0 ){
@@ -1431,6 +1508,21 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){
 }
 #endif
 
+/*
+** Initialize an array of N Mem element.
+*/
+static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){
+  while( (N--)>0 ){
+    p->db = db;
+    p->flags = flags;
+    p->szMalloc = 0;
+#ifdef SQLITE_DEBUG
+    p->pScopyFrom = 0;
+#endif
+    p++;
+  }
+}
+
 /*
 ** Release an array of N Mem elements
 */
@@ -1642,6 +1734,7 @@ int sqlite3VdbeList(
     pMem->flags = MEM_Str|MEM_Term;
     zP4 = displayP4(pOp, pMem->z, pMem->szMalloc);
     if( zP4!=pMem->z ){
+      pMem->n = 0;
       sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0);
     }else{
       assert( pMem->z!=0 );
@@ -1784,7 +1877,7 @@ void sqlite3VdbeRewind(Vdbe *p){
   int i;
 #endif
   assert( p!=0 );
-  assert( p->magic==VDBE_MAGIC_INIT );
+  assert( p->magic==VDBE_MAGIC_INIT || p->magic==VDBE_MAGIC_RESET );
 
   /* There should be at least one opcode.
   */
@@ -1794,7 +1887,7 @@ void sqlite3VdbeRewind(Vdbe *p){
   p->magic = VDBE_MAGIC_RUN;
 
 #ifdef SQLITE_DEBUG
-  for(i=1; inMem; i++){
+  for(i=0; inMem; i++){
     assert( p->aMem[i].db==p->db );
   }
 #endif
@@ -1841,7 +1934,6 @@ void sqlite3VdbeMakeReady(
   int nMem;                      /* Number of VM memory registers */
   int nCursor;                   /* Number of cursors required */
   int nArg;                      /* Number of arguments in subprograms */
-  int nOnce;                     /* Number of OP_Once instructions */
   int n;                         /* Loop counter */
   struct ReusableSpace x;        /* Reusable bulk memory */
 
@@ -1856,19 +1948,14 @@ void sqlite3VdbeMakeReady(
   nMem = pParse->nMem;
   nCursor = pParse->nTab;
   nArg = pParse->nMaxArg;
-  nOnce = pParse->nOnce;
-  if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */
   
-  /* For each cursor required, also allocate a memory cell. Memory
-  ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by
-  ** the vdbe program. Instead they are used to allocate memory for
-  ** VdbeCursor/BtCursor structures. The blob of memory associated with 
-  ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1)
-  ** stores the blob of memory associated with cursor 1, etc.
-  **
+  /* Each cursor uses a memory cell.  The first cursor (cursor 0) can
+  ** use aMem[0] which is not otherwise used by the VDBE program.  Allocate
+  ** space at the end of aMem[] for cursors 1 and greater.
   ** See also: allocateCursor().
   */
   nMem += nCursor;
+  if( nCursor==0 && nMem>0 ) nMem++;  /* Space for aMem[0] even if not used */
 
   /* Figure out how much reusable memory is available at the end of the
   ** opcode array.  This extra memory will be reallocated for other elements
@@ -1879,10 +1966,7 @@ void sqlite3VdbeMakeReady(
   assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) );
   x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n);  /* Bytes of unused memory */
   assert( x.nFree>=0 );
-  if( x.nFree>0 ){
-    memset(x.pSpace, 0, x.nFree);
-    assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) );
-  }
+  assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) );
 
   resolveP2Values(p, &nArg);
   p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort);
@@ -1907,37 +1991,32 @@ void sqlite3VdbeMakeReady(
     p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem));
     p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*));
     p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*));
-    p->aOnceFlag = allocSpace(&x, p->aOnceFlag, nOnce);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
     p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64));
 #endif
     if( x.nNeeded==0 ) break;
-    x.pSpace = p->pFree = sqlite3DbMallocZero(db, x.nNeeded);
+    x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded);
     x.nFree = x.nNeeded;
   }while( !db->mallocFailed );
 
-  p->nCursor = nCursor;
-  p->nOnceFlag = nOnce;
-  if( p->aVar ){
-    p->nVar = (ynVar)nVar;
-    for(n=0; naVar[n].flags = MEM_Null;
-      p->aVar[n].db = db;
-    }
-  }
-  p->nzVar = pParse->nzVar;
-  p->azVar = pParse->azVar;
-  pParse->nzVar =  0;
-  pParse->azVar = 0;
-  if( p->aMem ){
-    p->aMem--;                      /* aMem[] goes from 1..nMem */
-    p->nMem = nMem;                 /*       not from 0..nMem-1 */
-    for(n=1; n<=nMem; n++){
-      p->aMem[n].flags = MEM_Undefined;
-      p->aMem[n].db = db;
-    }
-  }
+  p->pVList = pParse->pVList;
+  pParse->pVList =  0;
   p->explain = pParse->explain;
+  if( db->mallocFailed ){
+    p->nVar = 0;
+    p->nCursor = 0;
+    p->nMem = 0;
+  }else{
+    p->nCursor = nCursor;
+    p->nVar = (ynVar)nVar;
+    initMemArray(p->aVar, nVar, db, MEM_Null);
+    p->nMem = nMem;
+    initMemArray(p->aMem, nMem, db, MEM_Undefined);
+    memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*));
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+    memset(p->anExec, 0, p->nOp*sizeof(i64));
+#endif
+  }
   sqlite3VdbeRewind(p);
 }
 
@@ -1949,15 +2028,15 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
   if( pCx==0 ){
     return;
   }
-  assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE );
+  assert( pCx->pBtx==0 || pCx->eCurType==CURTYPE_BTREE );
   switch( pCx->eCurType ){
     case CURTYPE_SORTER: {
       sqlite3VdbeSorterClose(p->db, pCx);
       break;
     }
     case CURTYPE_BTREE: {
-      if( pCx->pBt ){
-        sqlite3BtreeClose(pCx->pBt);
+      if( pCx->pBtx ){
+        sqlite3BtreeClose(pCx->pBtx);
         /* The pCx->pCursor will be close automatically, if it exists, by
         ** the call above. */
       }else{
@@ -2006,8 +2085,6 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   v->anExec = pFrame->anExec;
 #endif
-  v->aOnceFlag = pFrame->aOnceFlag;
-  v->nOnceFlag = pFrame->nOnceFlag;
   v->aOp = pFrame->aOp;
   v->nOp = pFrame->nOp;
   v->aMem = pFrame->aMem;
@@ -2042,7 +2119,7 @@ static void closeAllCursors(Vdbe *p){
   assert( p->nFrame==0 );
   closeCursorsInFrame(p);
   if( p->aMem ){
-    releaseMemArray(&p->aMem[1], p->nMem);
+    releaseMemArray(p->aMem, p->nMem);
   }
   while( p->pDelFrame ){
     VdbeFrame *pDel = p->pDelFrame;
@@ -2067,7 +2144,7 @@ static void Cleanup(Vdbe *p){
   int i;
   if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 );
   if( p->aMem ){
-    for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
+    for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined );
   }
 #endif
 
@@ -2091,13 +2168,9 @@ void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){
   sqlite3DbFree(db, p->aColName);
   n = nResColumn*COLNAME_N;
   p->nResColumn = (u16)nResColumn;
-  p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n );
+  p->aColName = pColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n );
   if( p->aColName==0 ) return;
-  while( n-- > 0 ){
-    pColName->flags = MEM_Null;
-    pColName->db = p->db;
-    pColName++;
-  }
+  initMemArray(p->aColName, n, p->db, MEM_Null);
 }
 
 /*
@@ -2432,60 +2505,59 @@ static void checkActiveVdbeCnt(sqlite3 *db){
 ** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. 
 ** Otherwise SQLITE_OK.
 */
-int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){
+static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){
   sqlite3 *const db = p->db;
   int rc = SQLITE_OK;
+  int i;
+  const int iSavepoint = p->iStatement-1;
 
-  /* If p->iStatement is greater than zero, then this Vdbe opened a 
-  ** statement transaction that should be closed here. The only exception
-  ** is that an IO error may have occurred, causing an emergency rollback.
-  ** In this case (db->nStatement==0), and there is nothing to do.
-  */
-  if( db->nStatement && p->iStatement ){
-    int i;
-    const int iSavepoint = p->iStatement-1;
+  assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE);
+  assert( db->nStatement>0 );
+  assert( p->iStatement==(db->nStatement+db->nSavepoint) );
 
-    assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE);
-    assert( db->nStatement>0 );
-    assert( p->iStatement==(db->nStatement+db->nSavepoint) );
-
-    for(i=0; inDb; i++){ 
-      int rc2 = SQLITE_OK;
-      Btree *pBt = db->aDb[i].pBt;
-      if( pBt ){
-        if( eOp==SAVEPOINT_ROLLBACK ){
-          rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint);
-        }
-        if( rc2==SQLITE_OK ){
-          rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint);
-        }
-        if( rc==SQLITE_OK ){
-          rc = rc2;
-        }
-      }
-    }
-    db->nStatement--;
-    p->iStatement = 0;
-
-    if( rc==SQLITE_OK ){
+  for(i=0; inDb; i++){ 
+    int rc2 = SQLITE_OK;
+    Btree *pBt = db->aDb[i].pBt;
+    if( pBt ){
       if( eOp==SAVEPOINT_ROLLBACK ){
-        rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint);
+        rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint);
+      }
+      if( rc2==SQLITE_OK ){
+        rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint);
       }
       if( rc==SQLITE_OK ){
-        rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint);
+        rc = rc2;
       }
     }
+  }
+  db->nStatement--;
+  p->iStatement = 0;
 
-    /* If the statement transaction is being rolled back, also restore the 
-    ** database handles deferred constraint counter to the value it had when 
-    ** the statement transaction was opened.  */
+  if( rc==SQLITE_OK ){
     if( eOp==SAVEPOINT_ROLLBACK ){
-      db->nDeferredCons = p->nStmtDefCons;
-      db->nDeferredImmCons = p->nStmtDefImmCons;
+      rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint);
     }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint);
+    }
+  }
+
+  /* If the statement transaction is being rolled back, also restore the 
+  ** database handles deferred constraint counter to the value it had when 
+  ** the statement transaction was opened.  */
+  if( eOp==SAVEPOINT_ROLLBACK ){
+    db->nDeferredCons = p->nStmtDefCons;
+    db->nDeferredImmCons = p->nStmtDefImmCons;
   }
   return rc;
 }
+int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){
+  if( p->db->nStatement && p->iStatement ){
+    return vdbeCloseStatement(p, eOp);
+  }
+  return SQLITE_OK;
+}
+
 
 /*
 ** This function is called when a transaction opened by the database 
@@ -2545,14 +2617,13 @@ int sqlite3VdbeHalt(Vdbe *p){
   ** one, or the complete transaction if there is no statement transaction.
   */
 
-  if( db->mallocFailed ){
-    p->rc = SQLITE_NOMEM_BKPT;
-  }
-  if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag);
-  closeAllCursors(p);
   if( p->magic!=VDBE_MAGIC_RUN ){
     return SQLITE_OK;
   }
+  if( db->mallocFailed ){
+    p->rc = SQLITE_NOMEM_BKPT;
+  }
+  closeAllCursors(p);
   checkActiveVdbeCnt(db);
 
   /* No commit or rollback needed if the program never started or if the
@@ -2860,7 +2931,7 @@ int sqlite3VdbeReset(Vdbe *p){
   }
 #endif
   p->iCurrentTime = 0;
-  p->magic = VDBE_MAGIC_INIT;
+  p->magic = VDBE_MAGIC_RESET;
   return p->rc & db->errMask;
 }
  
@@ -2922,26 +2993,29 @@ void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){
 */
 void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   SubProgram *pSub, *pNext;
-  int i;
   assert( p->db==0 || p->db==db );
-  releaseMemArray(p->aVar, p->nVar);
   releaseMemArray(p->aColName, p->nResColumn*COLNAME_N);
   for(pSub=p->pProgram; pSub; pSub=pNext){
     pNext = pSub->pNext;
     vdbeFreeOpArray(db, pSub->aOp, pSub->nOp);
     sqlite3DbFree(db, pSub);
   }
-  for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]);
-  sqlite3DbFree(db, p->azVar);
+  if( p->magic!=VDBE_MAGIC_INIT ){
+    releaseMemArray(p->aVar, p->nVar);
+    sqlite3DbFree(db, p->pVList);
+    sqlite3DbFree(db, p->pFree);
+  }
   vdbeFreeOpArray(db, p->aOp, p->nOp);
   sqlite3DbFree(db, p->aColName);
   sqlite3DbFree(db, p->zSql);
-  sqlite3DbFree(db, p->pFree);
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
-  for(i=0; inScan; i++){
-    sqlite3DbFree(db, p->aScan[i].zName);
+  {
+    int i;
+    for(i=0; inScan; i++){
+      sqlite3DbFree(db, p->aScan[i].zName);
+    }
+    sqlite3DbFree(db, p->aScan);
   }
-  sqlite3DbFree(db, p->aScan);
 #endif
 }
 
@@ -3442,30 +3516,13 @@ u32 sqlite3VdbeSerialGet(
 ** If an OOM error occurs, NULL is returned.
 */
 UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(
-  KeyInfo *pKeyInfo,              /* Description of the record */
-  char *pSpace,                   /* Unaligned space available */
-  int szSpace,                    /* Size of pSpace[] in bytes */
-  char **ppFree                   /* OUT: Caller should free this pointer */
+  KeyInfo *pKeyInfo               /* Description of the record */
 ){
   UnpackedRecord *p;              /* Unpacked record to return */
-  int nOff;                       /* Increment pSpace by nOff to align it */
   int nByte;                      /* Number of bytes required for *p */
-
-  /* We want to shift the pointer pSpace up such that it is 8-byte aligned.
-  ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift 
-  ** it by.  If pSpace is already 8-byte aligned, nOff should be zero.
-  */
-  nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7;
   nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1);
-  if( nByte>szSpace+nOff ){
-    p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte);
-    *ppFree = (char *)p;
-    if( !p ) return 0;
-  }else{
-    p = (UnpackedRecord*)&pSpace[nOff];
-    *ppFree = 0;
-  }
-
+  p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte);
+  if( !p ) return 0;
   p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))];
   assert( pKeyInfo->aSortOrder!=0 );
   p->pKeyInfo = pKeyInfo;
@@ -3504,6 +3561,7 @@ void sqlite3VdbeRecordUnpack(
     pMem->db = pKeyInfo->db;
     /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */
     pMem->szMalloc = 0;
+    pMem->z = 0;
     d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem);
     pMem++;
     if( (++u)>=p->nField ) break;
@@ -3512,7 +3570,7 @@ void sqlite3VdbeRecordUnpack(
   p->nField = u;
 }
 
-#if SQLITE_DEBUG
+#ifdef SQLITE_DEBUG
 /*
 ** This function compares two index or table record keys in the same way
 ** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(),
@@ -3617,7 +3675,7 @@ debugCompareEnd:
 }
 #endif
 
-#if SQLITE_DEBUG
+#ifdef SQLITE_DEBUG
 /*
 ** Count the number of fields (a.k.a. columns) in the record given by
 ** pKey,nKey.  The verify that this count is less than or equal to the
@@ -3691,15 +3749,49 @@ static int vdbeCompareMemString(
   }
 }
 
+/*
+** The input pBlob is guaranteed to be a Blob that is not marked
+** with MEM_Zero.  Return true if it could be a zero-blob.
+*/
+static int isAllZero(const char *z, int n){
+  int i;
+  for(i=0; iz, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n);
+  int c;
+  int n1 = pB1->n;
+  int n2 = pB2->n;
+
+  /* It is possible to have a Blob value that has some non-zero content
+  ** followed by zero content.  But that only comes up for Blobs formed
+  ** by the OP_MakeRecord opcode, and such Blobs never get passed into
+  ** sqlite3MemCompare(). */
+  assert( (pB1->flags & MEM_Zero)==0 || n1==0 );
+  assert( (pB2->flags & MEM_Zero)==0 || n2==0 );
+
+  if( (pB1->flags|pB2->flags) & MEM_Zero ){
+    if( pB1->flags & pB2->flags & MEM_Zero ){
+      return pB1->u.nZero - pB2->u.nZero;
+    }else if( pB1->flags & MEM_Zero ){
+      if( !isAllZero(pB2->z, pB2->n) ) return -1;
+      return pB1->u.nZero - n2;
+    }else{
+      if( !isAllZero(pB1->z, pB1->n) ) return +1;
+      return n1 - pB2->u.nZero;
+    }
+  }
+  c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1);
   if( c ) return c;
-  return pB1->n - pB2->n;
+  return n1 - n2;
 }
 
 /*
@@ -4005,6 +4097,7 @@ int sqlite3VdbeRecordCompareWithSkip(
 
     /* RHS is a blob */
     else if( pRhs->flags & MEM_Blob ){
+      assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 );
       getVarint32(&aKey1[idx1], serial_type);
       testcase( serial_type==12 );
       if( serial_type<12 || (serial_type & 0x01) ){
@@ -4016,6 +4109,12 @@ int sqlite3VdbeRecordCompareWithSkip(
         if( (d1+nStr) > (unsigned)nKey1 ){
           pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT;
           return 0;                /* Corruption */
+        }else if( pRhs->flags & MEM_Zero ){
+          if( !isAllZero((const char*)&aKey1[d1],nStr) ){
+            rc = 1;
+          }else{
+            rc = nStr - pRhs->u.nZero;
+          }
         }else{
           int nCmp = MIN(nStr, pRhs->n);
           rc = memcmp(&aKey1[d1], pRhs->z, nCmp);
@@ -4086,7 +4185,7 @@ static int vdbeRecordCompareInt(
   int res;
   u32 y;
   u64 x;
-  i64 v = pPKey2->aMem[0].u.i;
+  i64 v;
   i64 lhs;
 
   vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo);
@@ -4145,6 +4244,7 @@ static int vdbeRecordCompareInt(
       return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2);
   }
 
+  v = pPKey2->aMem[0].u.i;
   if( v>lhs ){
     res = pPKey2->r1;
   }else if( veCurType==CURTYPE_BTREE );
   pCur = pC->uc.pCursor;
   assert( sqlite3BtreeCursorIsValid(pCur) );
-  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
-  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
+  nCellKey = sqlite3BtreePayloadSize(pCur);
   /* nCellKey will always be between 0 and 0xffffffff because of the way
   ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */
   if( nCellKey<=0 || nCellKey>0x7fffffff ){
@@ -4378,7 +4476,7 @@ int sqlite3VdbeIdxKeyCompare(
     return SQLITE_CORRUPT_BKPT;
   }
   sqlite3VdbeMemInit(&m, db, 0);
-  rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m);
+  rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, &m);
   if( rc ){
     return rc;
   }
@@ -4460,8 +4558,8 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){
 */
 void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){
   assert( iVar>0 );
-  if( iVar>32 ){
-    v->expmask = 0xffffffff;
+  if( iVar>=32 ){
+    v->expmask |= 0x80000000;
   }else{
     v->expmask |= ((u32)1 << (iVar-1));
   }
@@ -4483,3 +4581,92 @@ void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){
   }
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+
+/*
+** If the second argument is not NULL, release any allocations associated 
+** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord
+** structure itself, using sqlite3DbFree().
+**
+** This function is used to free UnpackedRecord structures allocated by
+** the vdbeUnpackRecord() function found in vdbeapi.c.
+*/
+static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){
+  if( p ){
+    int i;
+    for(i=0; iaMem[i];
+      if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem);
+    }
+    sqlite3DbFree(db, p);
+  }
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+/*
+** Invoke the pre-update hook. If this is an UPDATE or DELETE pre-update call,
+** then cursor passed as the second argument should point to the row about
+** to be update or deleted. If the application calls sqlite3_preupdate_old(),
+** the required value will be read from the row the cursor points to.
+*/
+void sqlite3VdbePreUpdateHook(
+  Vdbe *v,                        /* Vdbe pre-update hook is invoked by */
+  VdbeCursor *pCsr,               /* Cursor to grab old.* values from */
+  int op,                         /* SQLITE_INSERT, UPDATE or DELETE */
+  const char *zDb,                /* Database name */
+  Table *pTab,                    /* Modified table */
+  i64 iKey1,                      /* Initial key value */
+  int iReg                        /* Register for new.* record */
+){
+  sqlite3 *db = v->db;
+  i64 iKey2;
+  PreUpdate preupdate;
+  const char *zTbl = pTab->zName;
+  static const u8 fakeSortOrder = 0;
+
+  assert( db->pPreUpdate==0 );
+  memset(&preupdate, 0, sizeof(PreUpdate));
+  if( HasRowid(pTab)==0 ){
+    iKey1 = iKey2 = 0;
+    preupdate.pPk = sqlite3PrimaryKeyIndex(pTab);
+  }else{
+    if( op==SQLITE_UPDATE ){
+      iKey2 = v->aMem[iReg].u.i;
+    }else{
+      iKey2 = iKey1;
+    }
+  }
+
+  assert( pCsr->nField==pTab->nCol 
+       || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1)
+  );
+
+  preupdate.v = v;
+  preupdate.pCsr = pCsr;
+  preupdate.op = op;
+  preupdate.iNewReg = iReg;
+  preupdate.keyinfo.db = db;
+  preupdate.keyinfo.enc = ENC(db);
+  preupdate.keyinfo.nField = pTab->nCol;
+  preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder;
+  preupdate.iKey1 = iKey1;
+  preupdate.iKey2 = iKey2;
+  preupdate.pTab = pTab;
+
+  db->pPreUpdate = &preupdate;
+  db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2);
+  db->pPreUpdate = 0;
+  sqlite3DbFree(db, preupdate.aRecord);
+  vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pUnpacked);
+  vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pNewUnpacked);
+  if( preupdate.aNew ){
+    int i;
+    for(i=0; inField; i++){
+      sqlite3VdbeMemRelease(&preupdate.aNew[i]);
+    }
+    sqlite3DbFree(db, preupdate.aNew);
+  }
+}
+#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
diff --git a/src/vdbeblob.c b/src/vdbeblob.c
index 2ba5301e7a..5692c1c606 100644
--- a/src/vdbeblob.c
+++ b/src/vdbeblob.c
@@ -23,13 +23,14 @@
 */
 typedef struct Incrblob Incrblob;
 struct Incrblob {
-  int flags;              /* Copy of "flags" passed to sqlite3_blob_open() */
   int nByte;              /* Size of open blob, in bytes */
   int iOffset;            /* Byte offset of blob in cursor data */
-  int iCol;               /* Table column this handle is open on */
+  u16 iCol;               /* Table column this handle is open on */
   BtCursor *pCsr;         /* Cursor pointing at blob row */
   sqlite3_stmt *pStmt;    /* Statement holding cursor open */
   sqlite3 *db;            /* The associated database */
+  char *zDb;              /* Database name */
+  Table *pTab;            /* Table object */
 };
 
 
@@ -55,17 +56,27 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
   char *zErr = 0;                 /* Error message */
   Vdbe *v = (Vdbe *)p->pStmt;
 
-  /* Set the value of the SQL statements only variable to integer iRow. 
-  ** This is done directly instead of using sqlite3_bind_int64() to avoid 
-  ** triggering asserts related to mutexes.
+  /* Set the value of register r[1] in the SQL statement to integer iRow. 
+  ** This is done directly as a performance optimization
   */
-  assert( v->aVar[0].flags&MEM_Int );
-  v->aVar[0].u.i = iRow;
+  v->aMem[1].flags = MEM_Int;
+  v->aMem[1].u.i = iRow;
 
-  rc = sqlite3_step(p->pStmt);
+  /* If the statement has been run before (and is paused at the OP_ResultRow)
+  ** then back it up to the point where it does the OP_SeekRowid.  This could
+  ** have been down with an extra OP_Goto, but simply setting the program
+  ** counter is faster. */
+  if( v->pc>3 ){
+    v->pc = 3;
+    rc = sqlite3VdbeExec(v);
+  }else{
+    rc = sqlite3_step(p->pStmt);
+  }
   if( rc==SQLITE_ROW ){
     VdbeCursor *pC = v->apCsr[0];
-    u32 type = pC->aType[p->iCol];
+    u32 type = pC->nHdrParsed>p->iCol ? pC->aType[p->iCol] : 0;
+    testcase( pC->nHdrParsed==p->iCol );
+    testcase( pC->nHdrParsed==p->iCol+1 );
     if( type<12 ){
       zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
           type==0?"null": type==7?"real": "integer"
@@ -110,7 +121,7 @@ int sqlite3_blob_open(
   const char *zTable,     /* The table containing the blob */
   const char *zColumn,    /* The column containing the blob */
   sqlite_int64 iRow,      /* The row containing the glob */
-  int flags,              /* True -> read/write access, false -> read-only */
+  int wrFlag,             /* True -> read/write access, false -> read-only */
   sqlite3_blob **ppBlob   /* Handle for accessing the blob returned here */
 ){
   int nAttempt = 0;
@@ -132,7 +143,7 @@ int sqlite3_blob_open(
     return SQLITE_MISUSE_BKPT;
   }
 #endif
-  flags = !!flags;                /* flags = (flags ? 1 : 0); */
+  wrFlag = !!wrFlag;                /* wrFlag = (wrFlag ? 1 : 0); */
 
   sqlite3_mutex_enter(db->mutex);
 
@@ -173,6 +184,8 @@ int sqlite3_blob_open(
       sqlite3BtreeLeaveAll(db);
       goto blob_open_out;
     }
+    pBlob->pTab = pTab;
+    pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName;
 
     /* Now search pTab for the exact column. */
     for(iCol=0; iColnCol; iCol++) {
@@ -190,9 +203,8 @@ int sqlite3_blob_open(
 
     /* If the value is being opened for writing, check that the
     ** column is not indexed, and that it is not part of a foreign key. 
-    ** It is against the rules to open a column to which either of these
-    ** descriptions applies for writing.  */
-    if( flags ){
+    */
+    if( wrFlag ){
       const char *zFault = 0;
       Index *pIdx;
 #ifndef SQLITE_OMIT_FOREIGN_KEY
@@ -253,19 +265,17 @@ int sqlite3_blob_open(
       static const VdbeOpList openBlob[] = {
         {OP_TableLock,      0, 0, 0},  /* 0: Acquire a read or write lock */
         {OP_OpenRead,       0, 0, 0},  /* 1: Open a cursor */
-        {OP_Variable,       1, 1, 0},  /* 2: Move ?1 into reg[1] */
-        {OP_NotExists,      0, 7, 1},  /* 3: Seek the cursor */
-        {OP_Column,         0, 0, 1},  /* 4  */
-        {OP_ResultRow,      1, 0, 0},  /* 5  */
-        {OP_Goto,           0, 2, 0},  /* 6  */
-        {OP_Close,          0, 0, 0},  /* 7  */
-        {OP_Halt,           0, 0, 0},  /* 8  */
+        /* blobSeekToRow() will initialize r[1] to the desired rowid */
+        {OP_NotExists,      0, 5, 1},  /* 2: Seek the cursor to rowid=r[1] */
+        {OP_Column,         0, 0, 1},  /* 3  */
+        {OP_ResultRow,      1, 0, 0},  /* 4  */
+        {OP_Halt,           0, 0, 0},  /* 5  */
       };
       Vdbe *v = (Vdbe *)pBlob->pStmt;
       int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
       VdbeOp *aOp;
 
-      sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, 
+      sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, 
                            pTab->pSchema->schema_cookie,
                            pTab->pSchema->iGeneration);
       sqlite3VdbeChangeP5(v, 1);     
@@ -282,7 +292,7 @@ int sqlite3_blob_open(
 #else
         aOp[0].p1 = iDb;
         aOp[0].p2 = pTab->tnum;
-        aOp[0].p3 = flags;
+        aOp[0].p3 = wrFlag;
         sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT);
       }
       if( db->mallocFailed==0 ){
@@ -290,7 +300,7 @@ int sqlite3_blob_open(
 
         /* Remove either the OP_OpenWrite or OpenRead. Set the P2 
         ** parameter of the other to pTab->tnum.  */
-        if( flags ) aOp[1].opcode = OP_OpenWrite;
+        if( wrFlag ) aOp[1].opcode = OP_OpenWrite;
         aOp[1].p2 = pTab->tnum;
         aOp[1].p3 = iDb;   
 
@@ -303,23 +313,21 @@ int sqlite3_blob_open(
         */
         aOp[1].p4type = P4_INT32;
         aOp[1].p4.i = pTab->nCol+1;
-        aOp[4].p2 = pTab->nCol;
+        aOp[3].p2 = pTab->nCol;
 
-        pParse->nVar = 1;
+        pParse->nVar = 0;
         pParse->nMem = 1;
         pParse->nTab = 1;
         sqlite3VdbeMakeReady(v, pParse);
       }
     }
    
-    pBlob->flags = flags;
     pBlob->iCol = iCol;
     pBlob->db = db;
     sqlite3BtreeLeaveAll(db);
     if( db->mallocFailed ){
       goto blob_open_out;
     }
-    sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
     rc = blobSeekToRow(pBlob, iRow, &zErr);
   } while( (++nAttempt)db );
     sqlite3BtreeEnterCursor(p->pCsr);
+
+#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
+    if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){
+      /* If a pre-update hook is registered and this is a write cursor, 
+      ** invoke it here. 
+      ** 
+      ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this
+      ** operation should really be an SQLITE_UPDATE. This is probably
+      ** incorrect, but is convenient because at this point the new.* values 
+      ** are not easily obtainable. And for the sessions module, an 
+      ** SQLITE_UPDATE where the PK columns do not change is handled in the 
+      ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually
+      ** slightly more efficient). Since you cannot write to a PK column
+      ** using the incremental-blob API, this works. For the sessions module
+      ** anyhow.
+      */
+      sqlite3_int64 iKey;
+      iKey = sqlite3BtreeIntegerKey(p->pCsr);
+      sqlite3VdbePreUpdateHook(
+          v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1
+      );
+    }
+#endif
+
     rc = xCall(p->pCsr, iOffset+p->iOffset, n, z);
     sqlite3BtreeLeaveCursor(p->pCsr);
     if( rc==SQLITE_ABORT ){
@@ -413,7 +445,7 @@ static int blobReadWrite(
 ** Read data from a blob handle.
 */
 int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){
-  return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData);
+  return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreePayloadChecked);
 }
 
 /*
diff --git a/src/vdbemem.c b/src/vdbemem.c
index b5139fe32e..656e19bfa8 100644
--- a/src/vdbemem.c
+++ b/src/vdbemem.c
@@ -189,18 +189,18 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
 ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails.
 */
 int sqlite3VdbeMemMakeWriteable(Mem *pMem){
-  int f;
   assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
   assert( (pMem->flags&MEM_RowSet)==0 );
-  ExpandBlob(pMem);
-  f = pMem->flags;
-  if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){
-    if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){
-      return SQLITE_NOMEM_BKPT;
+  if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){
+    if( ExpandBlob(pMem) ) return SQLITE_NOMEM;
+    if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){
+      if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){
+        return SQLITE_NOMEM_BKPT;
+      }
+      pMem->z[pMem->n] = 0;
+      pMem->z[pMem->n+1] = 0;
+      pMem->flags |= MEM_Term;
     }
-    pMem->z[pMem->n] = 0;
-    pMem->z[pMem->n+1] = 0;
-    pMem->flags |= MEM_Term;
   }
   pMem->flags &= ~MEM_Ephem;
 #ifdef SQLITE_DEBUG
@@ -216,25 +216,24 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){
 */
 #ifndef SQLITE_OMIT_INCRBLOB
 int sqlite3VdbeMemExpandBlob(Mem *pMem){
-  if( pMem->flags & MEM_Zero ){
-    int nByte;
-    assert( pMem->flags&MEM_Blob );
-    assert( (pMem->flags&MEM_RowSet)==0 );
-    assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
+  int nByte;
+  assert( pMem->flags & MEM_Zero );
+  assert( pMem->flags&MEM_Blob );
+  assert( (pMem->flags&MEM_RowSet)==0 );
+  assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
 
-    /* Set nByte to the number of bytes required to store the expanded blob. */
-    nByte = pMem->n + pMem->u.nZero;
-    if( nByte<=0 ){
-      nByte = 1;
-    }
-    if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){
-      return SQLITE_NOMEM_BKPT;
-    }
-
-    memset(&pMem->z[pMem->n], 0, pMem->u.nZero);
-    pMem->n += pMem->u.nZero;
-    pMem->flags &= ~(MEM_Zero|MEM_Term);
+  /* Set nByte to the number of bytes required to store the expanded blob. */
+  nByte = pMem->n + pMem->u.nZero;
+  if( nByte<=0 ){
+    nByte = 1;
   }
+  if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){
+    return SQLITE_NOMEM_BKPT;
+  }
+
+  memset(&pMem->z[pMem->n], 0, pMem->u.nZero);
+  pMem->n += pMem->u.nZero;
+  pMem->flags &= ~(MEM_Zero|MEM_Term);
   return SQLITE_OK;
 }
 #endif
@@ -294,6 +293,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){
 
 
   if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){
+    pMem->enc = 0;
     return SQLITE_NOMEM_BKPT;
   }
 
@@ -575,7 +575,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){
     }
   }
   assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 );
-  pMem->flags &= ~(MEM_Str|MEM_Blob);
+  pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero);
   return SQLITE_OK;
 }
 
@@ -593,7 +593,7 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){
       if( (pMem->flags & MEM_Blob)==0 ){
         sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding);
         assert( pMem->flags & MEM_Str || pMem->db->mallocFailed );
-        MemSetTypeFlag(pMem, MEM_Blob);
+        if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob);
       }else{
         pMem->flags &= ~(MEM_TypeMask&~MEM_Blob);
       }
@@ -761,7 +761,7 @@ int sqlite3VdbeMemTooBig(Mem *p){
 void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){
   int i;
   Mem *pX;
-  for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){
+  for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){
     if( pX->pScopyFrom==pMem ){
       pX->flags |= MEM_Undefined;
       pX->pScopyFrom = 0;
@@ -802,10 +802,6 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){
 int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){
   int rc = SQLITE_OK;
 
-  /* The pFrom==0 case in the following assert() is when an sqlite3_value
-  ** from sqlite3_value_dup() is used as the argument
-  ** to sqlite3_result_value(). */
-  assert( pTo->db==pFrom->db || pFrom->db==0 );
   assert( (pFrom->flags & MEM_RowSet)==0 );
   if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo);
   memcpy(pTo, pFrom, MEMCELLSIZE);
@@ -938,10 +934,9 @@ int sqlite3VdbeMemSetStr(
 
 /*
 ** Move data out of a btree key or data field and into a Mem structure.
-** The data or key is taken from the entry that pCur is currently pointing
+** The data is payload from the entry that pCur is currently pointing
 ** to.  offset and amt determine what portion of the data or key to retrieve.
-** key is true to get the key or false to get data.  The result is written
-** into the pMem element.
+** The result is written into the pMem element.
 **
 ** The pMem object must have been initialized.  This routine will use
 ** pMem->zMalloc to hold the content from the btree, if possible.  New
@@ -956,17 +951,12 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize(
   BtCursor *pCur,   /* Cursor pointing at record to retrieve. */
   u32 offset,       /* Offset from the start of data to return bytes from. */
   u32 amt,          /* Number of bytes to return. */
-  int key,          /* If true, retrieve from the btree key, not data. */
   Mem *pMem         /* OUT: Return data in this Mem structure. */
 ){
   int rc;
   pMem->flags = MEM_Null;
   if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){
-    if( key ){
-      rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z);
-    }else{
-      rc = sqlite3BtreeData(pCur, offset, amt, pMem->z);
-    }
+    rc = sqlite3BtreePayload(pCur, offset, amt, pMem->z);
     if( rc==SQLITE_OK ){
       pMem->z[amt] = 0;
       pMem->z[amt+1] = 0;
@@ -982,7 +972,6 @@ int sqlite3VdbeMemFromBtree(
   BtCursor *pCur,   /* Cursor pointing at record to retrieve. */
   u32 offset,       /* Offset from the start of data to return bytes from. */
   u32 amt,          /* Number of bytes to return. */
-  int key,          /* If true, retrieve from the btree key, not data. */
   Mem *pMem         /* OUT: Return data in this Mem structure. */
 ){
   char *zData;        /* Data from the btree layer */
@@ -995,11 +984,7 @@ int sqlite3VdbeMemFromBtree(
   /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() 
   ** that both the BtShared and database handle mutexes are held. */
   assert( (pMem->flags & MEM_RowSet)==0 );
-  if( key ){
-    zData = (char *)sqlite3BtreeKeyFetch(pCur, &available);
-  }else{
-    zData = (char *)sqlite3BtreeDataFetch(pCur, &available);
-  }
+  zData = (char *)sqlite3BtreePayloadFetch(pCur, &available);
   assert( zData!=0 );
 
   if( offset+amt<=available ){
@@ -1007,7 +992,7 @@ int sqlite3VdbeMemFromBtree(
     pMem->flags = MEM_Blob|MEM_Ephem;
     pMem->n = (int)amt;
   }else{
-    rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem);
+    rc = vdbeMemFromBtreeResize(pCur, offset, amt, pMem);
   }
 
   return rc;
@@ -1025,10 +1010,8 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){
   assert( (pVal->flags & MEM_RowSet)==0 );
   assert( (pVal->flags & (MEM_Null))==0 );
   if( pVal->flags & (MEM_Blob|MEM_Str) ){
+    if( ExpandBlob(pVal) ) return 0;
     pVal->flags |= MEM_Str;
-    if( pVal->flags & MEM_Zero ){
-      sqlite3VdbeMemExpandBlob(pVal);
-    }
     if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){
       sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED);
     }
@@ -1281,10 +1264,7 @@ static int valueFromExpr(
   const char *zNeg = "";
   int rc = SQLITE_OK;
 
-  if( !pExpr ){
-    *ppVal = 0;
-    return SQLITE_OK;
-  }
+  assert( pExpr!=0 );
   while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft;
   if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;
 
@@ -1354,6 +1334,7 @@ static int valueFromExpr(
   }else if( op==TK_NULL ){
     pVal = valueNew(db, pCtx);
     if( pVal==0 ) goto no_mem;
+    sqlite3VdbeMemNumerify(pVal);
   }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
   else if( op==TK_BLOB ){
@@ -1408,7 +1389,7 @@ int sqlite3ValueFromExpr(
   u8 affinity,              /* Affinity to use */
   sqlite3_value **ppVal     /* Write the new value here */
 ){
-  return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0);
+  return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0;
 }
 
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
@@ -1528,9 +1509,9 @@ static int stat4ValueFromExpr(
 ** structures intended to be compared against sample index keys stored 
 ** in the sqlite_stat4 table.
 **
-** A single call to this function attempts to populates field iVal (leftmost 
-** is 0 etc.) of the unpacked record with a value extracted from expression
-** pExpr. Extraction of values is possible if:
+** A single call to this function populates zero or more fields of the
+** record starting with field iVal (fields are numbered from left to
+** right starting with 0). A single field is populated if:
 **
 **  * (pExpr==0). In this case the value is assumed to be an SQL NULL,
 **
@@ -1539,10 +1520,14 @@ static int stat4ValueFromExpr(
 **  * The sqlite3ValueFromExpr() function is able to extract a value 
 **    from the expression (i.e. the expression is a literal value).
 **
-** If a value can be extracted, the affinity passed as the 5th argument
-** is applied to it before it is copied into the UnpackedRecord. Output
-** parameter *pbOk is set to true if a value is extracted, or false 
-** otherwise.
+** Or, if pExpr is a TK_VECTOR, one field is populated for each of the
+** vector components that match either of the two latter criteria listed
+** above.
+**
+** Before any value is appended to the record, the affinity of the 
+** corresponding column within index pIdx is applied to it. Before
+** this function returns, output parameter *pnExtract is set to the
+** number of values appended to the record.
 **
 ** When this function is called, *ppRec must either point to an object
 ** allocated by an earlier call to this function, or must be NULL. If it
@@ -1558,22 +1543,33 @@ int sqlite3Stat4ProbeSetValue(
   Index *pIdx,                    /* Index being probed */
   UnpackedRecord **ppRec,         /* IN/OUT: Probe record */
   Expr *pExpr,                    /* The expression to extract a value from */
-  u8 affinity,                    /* Affinity to use */
+  int nElem,                      /* Maximum number of values to append */
   int iVal,                       /* Array element to populate */
-  int *pbOk                       /* OUT: True if value was extracted */
+  int *pnExtract                  /* OUT: Values appended to the record */
 ){
-  int rc;
-  sqlite3_value *pVal = 0;
-  struct ValueNewStat4Ctx alloc;
+  int rc = SQLITE_OK;
+  int nExtract = 0;
 
-  alloc.pParse = pParse;
-  alloc.pIdx = pIdx;
-  alloc.ppRec = ppRec;
-  alloc.iVal = iVal;
+  if( pExpr==0 || pExpr->op!=TK_SELECT ){
+    int i;
+    struct ValueNewStat4Ctx alloc;
 
-  rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal);
-  assert( pVal==0 || pVal->db==pParse->db );
-  *pbOk = (pVal!=0);
+    alloc.pParse = pParse;
+    alloc.pIdx = pIdx;
+    alloc.ppRec = ppRec;
+
+    for(i=0; idb, pIdx, iVal+i);
+      alloc.iVal = iVal+i;
+      rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal);
+      if( !pVal ) break;
+      nExtract++;
+    }
+  }
+
+  *pnExtract = nExtract;
   return rc;
 }
 
diff --git a/src/vdbesort.c b/src/vdbesort.c
index 5913cda6d0..631fb19616 100644
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -931,7 +931,6 @@ int sqlite3VdbeSorterInit(
 ){
   int pgsz;                       /* Page size of main database */
   int i;                          /* Used to iterate through aTask[] */
-  int mxCache;                    /* Cache size */
   VdbeSorter *pSorter;            /* The new sorter */
   KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
@@ -960,7 +959,7 @@ int sqlite3VdbeSorterInit(
   }
 #endif
 
-  assert( pCsr->pKeyInfo && pCsr->pBt==0 );
+  assert( pCsr->pKeyInfo && pCsr->pBtx==0 );
   assert( pCsr->eCurType==CURTYPE_SORTER );
   szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*);
   sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask);
@@ -988,11 +987,20 @@ int sqlite3VdbeSorterInit(
     }
 
     if( !sqlite3TempInMemory(db) ){
+      i64 mxCache;                /* Cache size in bytes*/
       u32 szPma = sqlite3GlobalConfig.szPma;
       pSorter->mnPmaSize = szPma * pgsz;
+
       mxCache = db->aDb[0].pSchema->cache_size;
-      if( mxCache<(int)szPma ) mxCache = (int)szPma;
-      pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_PMASZ);
+      if( mxCache<0 ){
+        /* A negative cache-size value C indicates that the cache is abs(C)
+        ** KiB in size.  */
+        mxCache = mxCache * -1024;
+      }else{
+        mxCache = mxCache * pgsz;
+      }
+      mxCache = MIN(mxCache, SQLITE_MAX_PMASZ);
+      pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache);
 
       /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of
       ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary
@@ -1319,12 +1327,8 @@ static int vdbeSorterOpenTempFile(
 */
 static int vdbeSortAllocUnpacked(SortSubtask *pTask){
   if( pTask->pUnpacked==0 ){
-    char *pFree;
-    pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(
-        pTask->pSorter->pKeyInfo, 0, 0, &pFree
-    );
-    assert( pTask->pUnpacked==(UnpackedRecord*)pFree );
-    if( pFree==0 ) return SQLITE_NOMEM_BKPT;
+    pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pTask->pSorter->pKeyInfo);
+    if( pTask->pUnpacked==0 ) return SQLITE_NOMEM_BKPT;
     pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField;
     pTask->pUnpacked->errCode = 0;
   }
@@ -1334,19 +1338,18 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){
 
 /*
 ** Merge the two sorted lists p1 and p2 into a single list.
-** Set *ppOut to the head of the new list.
 */
-static void vdbeSorterMerge(
+static SorterRecord *vdbeSorterMerge(
   SortSubtask *pTask,             /* Calling thread context */
   SorterRecord *p1,               /* First list to merge */
-  SorterRecord *p2,               /* Second list to merge */
-  SorterRecord **ppOut            /* OUT: Head of merged list */
+  SorterRecord *p2                /* Second list to merge */
 ){
   SorterRecord *pFinal = 0;
   SorterRecord **pp = &pFinal;
   int bCached = 0;
 
-  while( p1 && p2 ){
+  assert( p1!=0 && p2!=0 );
+  for(;;){
     int res;
     res = pTask->xCompare(
         pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal
@@ -1356,15 +1359,22 @@ static void vdbeSorterMerge(
       *pp = p1;
       pp = &p1->u.pNext;
       p1 = p1->u.pNext;
+      if( p1==0 ){
+        *pp = p2;
+        break;
+      }
     }else{
       *pp = p2;
       pp = &p2->u.pNext;
       p2 = p2->u.pNext;
       bCached = 0;
+      if( p2==0 ){
+        *pp = p1;
+        break;
+      }
     }
   }
-  *pp = p1 ? p1 : p2;
-  *ppOut = pFinal;
+  return pFinal;
 }
 
 /*
@@ -1417,7 +1427,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
     p->u.pNext = 0;
     for(i=0; aSlot[i]; i++){
-      vdbeSorterMerge(pTask, p, aSlot[i], &p);
+      p = vdbeSorterMerge(pTask, p, aSlot[i]);
       aSlot[i] = 0;
     }
     aSlot[i] = p;
@@ -1426,7 +1436,8 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){
 
   p = 0;
   for(i=0; i<64; i++){
-    vdbeSorterMerge(pTask, p, aSlot[i], &p);
+    if( aSlot[i]==0 ) continue;
+    p = p ? vdbeSorterMerge(pTask, p, aSlot[i]) : aSlot[i];
   }
   pList->pList = p;
 
@@ -2718,9 +2729,7 @@ int sqlite3VdbeSorterCompare(
   r2 = pSorter->pUnpacked;
   pKeyInfo = pCsr->pKeyInfo;
   if( r2==0 ){
-    char *p;
-    r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p);
-    assert( pSorter->pUnpacked==(UnpackedRecord*)p );
+    r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo);
     if( r2==0 ) return SQLITE_NOMEM_BKPT;
     r2->nField = nKeyCol;
   }
diff --git a/src/vdbetrace.c b/src/vdbetrace.c
index 07235c931b..7b47363991 100644
--- a/src/vdbetrace.c
+++ b/src/vdbetrace.c
@@ -81,10 +81,13 @@ char *sqlite3VdbeExpandSql(
   int i;                   /* Loop counter */
   Mem *pVar;               /* Value of a host parameter */
   StrAccum out;            /* Accumulate the output here */
+#ifndef SQLITE_OMIT_UTF16
+  Mem utf8;                /* Used to convert UTF16 parameters into UTF8 for display */
+#endif
   char zBase[100];         /* Initial working space */
 
   db = p->db;
-  sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), 
+  sqlite3StrAccumInit(&out, 0, zBase, sizeof(zBase), 
                       db->aLimit[SQLITE_LIMIT_LENGTH]);
   if( db->nVdbeExec>1 ){
     while( *zRawSql ){
@@ -135,12 +138,14 @@ char *sqlite3VdbeExpandSql(
         int nOut;  /* Number of bytes of the string text to include in output */
 #ifndef SQLITE_OMIT_UTF16
         u8 enc = ENC(db);
-        Mem utf8;
         if( enc!=SQLITE_UTF8 ){
           memset(&utf8, 0, sizeof(utf8));
           utf8.db = db;
           sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC);
-          sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8);
+          if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){
+            out.accError = STRACCUM_NOMEM;
+            out.nAlloc = 0;
+          }
           pVar = &utf8;
         }
 #endif
@@ -182,6 +187,7 @@ char *sqlite3VdbeExpandSql(
       }
     }
   }
+  if( out.accError ) sqlite3StrAccumReset(&out);
   return sqlite3StrAccumFinish(&out);
 }
 
diff --git a/src/vtab.c b/src/vtab.c
index ad8caef3bc..48891034d5 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -28,6 +28,41 @@ struct VtabCtx {
   int bDeclared;      /* True after sqlite3_declare_vtab() is called */
 };
 
+/*
+** Construct and install a Module object for a virtual table.  When this
+** routine is called, it is guaranteed that all appropriate locks are held
+** and the module is not already part of the connection.
+*/
+Module *sqlite3VtabCreateModule(
+  sqlite3 *db,                    /* Database in which module is registered */
+  const char *zName,              /* Name assigned to this module */
+  const sqlite3_module *pModule,  /* The definition of the module */
+  void *pAux,                     /* Context pointer for xCreate/xConnect */
+  void (*xDestroy)(void *)        /* Module destructor function */
+){
+  Module *pMod;
+  int nName = sqlite3Strlen30(zName);
+  pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1);
+  if( pMod ){
+    Module *pDel;
+    char *zCopy = (char *)(&pMod[1]);
+    memcpy(zCopy, zName, nName+1);
+    pMod->zName = zCopy;
+    pMod->pModule = pModule;
+    pMod->pAux = pAux;
+    pMod->xDestroy = xDestroy;
+    pMod->pEpoTab = 0;
+    pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod);
+    assert( pDel==0 || pDel==pMod );
+    if( pDel ){
+      sqlite3OomFault(db);
+      sqlite3DbFree(db, pDel);
+      pMod = 0;
+    }
+  }
+  return pMod;
+}
+
 /*
 ** The actual function that does the work of creating a new module.
 ** This function implements the sqlite3_create_module() and
@@ -41,35 +76,15 @@ static int createModule(
   void (*xDestroy)(void *)        /* Module destructor function */
 ){
   int rc = SQLITE_OK;
-  int nName;
 
   sqlite3_mutex_enter(db->mutex);
-  nName = sqlite3Strlen30(zName);
   if( sqlite3HashFind(&db->aModule, zName) ){
     rc = SQLITE_MISUSE_BKPT;
   }else{
-    Module *pMod;
-    pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1);
-    if( pMod ){
-      Module *pDel;
-      char *zCopy = (char *)(&pMod[1]);
-      memcpy(zCopy, zName, nName+1);
-      pMod->zName = zCopy;
-      pMod->pModule = pModule;
-      pMod->pAux = pAux;
-      pMod->xDestroy = xDestroy;
-      pMod->pEpoTab = 0;
-      pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod);
-      assert( pDel==0 || pDel==pMod );
-      if( pDel ){
-        sqlite3OomFault(db);
-        sqlite3DbFree(db, pDel);
-      }
-    }
+    (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy);
   }
   rc = sqlite3ApiExit(db, rc);
   if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux);
-
   sqlite3_mutex_leave(db->mutex);
   return rc;
 }
@@ -324,8 +339,7 @@ void sqlite3VtabBeginParse(
   iDb = sqlite3SchemaToIndex(db, pTable->pSchema);
   assert( iDb>=0 );
 
-  pTable->tabFlags |= TF_Virtual;
-  pTable->nModuleArg = 0;
+  assert( pTable->nModuleArg==0 );
   addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName));
   addModuleArgument(db, pTable, 0);
   addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName));
@@ -344,7 +358,7 @@ void sqlite3VtabBeginParse(
   */
   if( pTable->azModuleArg ){
     sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, 
-            pTable->azModuleArg[0], pParse->db->aDb[iDb].zName);
+            pTable->azModuleArg[0], pParse->db->aDb[iDb].zDbSName);
   }
 #endif
 }
@@ -408,7 +422,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
       "UPDATE %Q.%s "
          "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q "
        "WHERE rowid=#%d",
-      db->aDb[iDb].zName, SCHEMA_TABLE(iDb),
+      db->aDb[iDb].zDbSName, MASTER_NAME,
       pTab->zName,
       pTab->zName,
       zStmt,
@@ -418,7 +432,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){
     v = sqlite3GetVdbe(pParse);
     sqlite3ChangeCookie(pParse, iDb);
 
-    sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
+    sqlite3VdbeAddOp0(v, OP_Expire);
     zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
     sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
 
@@ -518,7 +532,7 @@ static int vtabCallConstructor(
   pVTable->pMod = pMod;
 
   iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-  pTab->azModuleArg[1] = db->aDb[iDb].zName;
+  pTab->azModuleArg[1] = db->aDb[iDb].zDbSName;
 
   /* Invoke the virtual table constructor */
   assert( &db->pVtabCtx );
@@ -564,22 +578,16 @@ static int vtabCallConstructor(
       pTab->pVTable = pVTable;
 
       for(iCol=0; iColnCol; iCol++){
-        char *zType = (char*)sqlite3StrNext(pTab->aCol[iCol].zName);
+        char *zType = sqlite3ColumnType(&pTab->aCol[iCol], "");
         int nType;
         int i = 0;
-        if( !zType[0] ){
-          pTab->tabFlags |= oooHidden;
-          continue;
-        }
         nType = sqlite3Strlen30(zType);
-        if( sqlite3StrNICmp("hidden", zType, 6)||(zType[6] && zType[6]!=' ') ){
-          for(i=0; itabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){
+  if( !IsVirtual(pTab) || sqlite3GetVTable(db, pTab) ){
     return SQLITE_OK;
   }
 
@@ -678,7 +686,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){
 ** This function is invoked by the vdbe to call the xCreate method
 ** of the virtual table named zTab in database iDb. 
 **
-** If an error occurs, *pzErr is set to point an an English language
+** If an error occurs, *pzErr is set to point to an English language
 ** description of the error and an SQLITE_XXX error code is returned.
 ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr.
 */
@@ -688,8 +696,8 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){
   Module *pMod;
   const char *zMod;
 
-  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName);
-  assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable );
+  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName);
+  assert( pTab && IsVirtual(pTab) && !pTab->pVTable );
 
   /* Locate the required virtual table module */
   zMod = pTab->azModuleArg[0];
@@ -743,7 +751,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
     return SQLITE_MISUSE_BKPT;
   }
   pTab = pCtx->pTab;
-  assert( (pTab->tabFlags & TF_Virtual)!=0 );
+  assert( IsVirtual(pTab) );
 
   pParse = sqlite3StackAllocZero(db, sizeof(*pParse));
   if( pParse==0 ){
@@ -757,13 +765,27 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){
      && pParse->pNewTable
      && !db->mallocFailed
      && !pParse->pNewTable->pSelect
-     && (pParse->pNewTable->tabFlags & TF_Virtual)==0
+     && !IsVirtual(pParse->pNewTable)
     ){
       if( !pTab->aCol ){
-        pTab->aCol = pParse->pNewTable->aCol;
-        pTab->nCol = pParse->pNewTable->nCol;
-        pParse->pNewTable->nCol = 0;
-        pParse->pNewTable->aCol = 0;
+        Table *pNew = pParse->pNewTable;
+        Index *pIdx;
+        pTab->aCol = pNew->aCol;
+        pTab->nCol = pNew->nCol;
+        pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid);
+        pNew->nCol = 0;
+        pNew->aCol = 0;
+        assert( pTab->pIndex==0 );
+        if( !HasRowid(pNew) && pCtx->pVTable->pMod->pModule->xUpdate!=0 ){
+          rc = SQLITE_ERROR;
+        }
+        pIdx = pNew->pIndex;
+        if( pIdx ){
+          assert( pIdx->pNext==0 );
+          pTab->pIndex = pIdx;
+          pNew->pIndex = 0;
+          pIdx->pTable = pTab;
+        }
       }
       pCtx->bDeclared = 1;
     }else{
@@ -798,8 +820,8 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){
   int rc = SQLITE_OK;
   Table *pTab;
 
-  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName);
-  if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){
+  pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName);
+  if( pTab!=0 && ALWAYS(pTab->pVTable!=0) ){
     VTable *p;
     int (*xDestroy)(sqlite3_vtab *);
     for(p=pTab->pVTable; p; p=p->pNext){
@@ -939,7 +961,10 @@ int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){
       if( rc==SQLITE_OK ){
         int iSvpt = db->nStatement + db->nSavepoint;
         addToVTrans(db, pVTab);
-        if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1);
+        if( iSvpt && pModule->xSavepoint ){
+          pVTab->iSavepoint = iSvpt;
+          rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1);
+        }
       }
     }
   }
@@ -1029,7 +1054,7 @@ FuncDef *sqlite3VtabOverloadFunction(
   if( pExpr->op!=TK_COLUMN ) return pDef;
   pTab = pExpr->pTab;
   if( NEVER(pTab==0) ) return pDef;
-  if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef;
+  if( !IsVirtual(pTab) ) return pDef;
   pVtab = sqlite3GetVTable(db, pTab)->pVtab;
   assert( pVtab!=0 );
   assert( pVtab->pModule!=0 );
@@ -1093,7 +1118,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){
 }
 
 /*
-** Check to see if virtual tale module pMod can be have an eponymous
+** Check to see if virtual table module pMod can be have an eponymous
 ** virtual table instance.  If it can, create one if one does not already
 ** exist. Return non-zero if the eponymous virtual table instance exists
 ** when this routine returns, and return zero if it does not exist.
@@ -1110,21 +1135,21 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
   const sqlite3_module *pModule = pMod->pModule;
   Table *pTab;
   char *zErr = 0;
-  int nName;
   int rc;
   sqlite3 *db = pParse->db;
   if( pMod->pEpoTab ) return 1;
   if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0;
-  nName = sqlite3Strlen30(pMod->zName) + 1;
-  pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName);
+  pTab = sqlite3DbMallocZero(db, sizeof(Table));
   if( pTab==0 ) return 0;
+  pTab->zName = sqlite3DbStrDup(db, pMod->zName);
+  if( pTab->zName==0 ){
+    sqlite3DbFree(db, pTab);
+    return 0;
+  }
   pMod->pEpoTab = pTab;
-  pTab->zName = (char*)&pTab[1];
-  memcpy(pTab->zName, pMod->zName, nName);
-  pTab->nRef = 1;
+  pTab->nTabRef = 1;
   pTab->pSchema = db->aDb[0].pSchema;
-  pTab->tabFlags |= TF_Virtual;
-  pTab->nModuleArg = 0;
+  assert( pTab->nModuleArg==0 );
   pTab->iPKey = -1;
   addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName));
   addModuleArgument(db, pTab, 0);
@@ -1146,9 +1171,11 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){
 void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){
   Table *pTab = pMod->pEpoTab;
   if( pTab!=0 ){
-    sqlite3DeleteColumnNames(db, pTab);
-    sqlite3VtabClear(db, pTab);
-    sqlite3DbFree(db, pTab);
+    /* Mark the table as Ephemeral prior to deleting it, so that the
+    ** sqlite3DeleteTable() routine will know that it is not stored in 
+    ** the schema. */
+    pTab->tabFlags |= TF_Ephemeral;
+    sqlite3DeleteTable(db, pTab);
     pMod->pEpoTab = 0;
   }
 }
@@ -1193,7 +1220,7 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){
       if( !p ){
         rc = SQLITE_MISUSE_BKPT;
       }else{
-        assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 );
+        assert( p->pTab==0 || IsVirtual(p->pTab) );
         p->pVTable->bConstraint = (u8)va_arg(ap, int);
       }
       break;
diff --git a/src/wal.c b/src/wal.c
index bbac1bd6d0..09f605fe57 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1723,6 +1723,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){
 */
 static int walCheckpoint(
   Wal *pWal,                      /* Wal connection */
+  sqlite3 *db,                    /* Check for interrupts on this handle */
   int eMode,                      /* One of PASSIVE, FULL or RESTART */
   int (*xBusy)(void*),            /* Function to call when busy */
   void *pBusyArg,                 /* Context argument for xBusyHandler */
@@ -1817,6 +1818,10 @@ static int walCheckpoint(
       while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){
         i64 iOffset;
         assert( walFramePgno(pWal, iFrame)==iDbpage );
+        if( db->u1.isInterrupted ){
+          rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT;
+          break;
+        }
         if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){
           continue;
         }
@@ -1921,6 +1926,7 @@ static void walLimitSize(Wal *pWal, i64 nMax){
 */
 int sqlite3WalClose(
   Wal *pWal,                      /* Wal to close */
+  sqlite3 *db,                    /* For interrupt flag */
   int sync_flags,                 /* Flags to pass to OsSync() (or 0) */
   int nBuf,
   u8 *zBuf                        /* Buffer of at least nBuf bytes */
@@ -1937,13 +1943,14 @@ int sqlite3WalClose(
     **
     ** The EXCLUSIVE lock is not released before returning.
     */
-    rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE);
-    if( rc==SQLITE_OK ){
+    if( zBuf!=0
+     && SQLITE_OK==(rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE))
+    ){
       if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = sqlite3WalCheckpoint(
-          pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
+      rc = sqlite3WalCheckpoint(pWal, db, 
+          SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
       );
       if( rc==SQLITE_OK ){
         int bPersist = -1;
@@ -2372,6 +2379,84 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){
   return rc;
 }
 
+#ifdef SQLITE_ENABLE_SNAPSHOT
+/*
+** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted 
+** variable so that older snapshots can be accessed. To do this, loop
+** through all wal frames from nBackfillAttempted to (nBackfill+1), 
+** comparing their content to the corresponding page with the database
+** file, if any. Set nBackfillAttempted to the frame number of the
+** first frame for which the wal file content matches the db file.
+**
+** This is only really safe if the file-system is such that any page 
+** writes made by earlier checkpointers were atomic operations, which 
+** is not always true. It is also possible that nBackfillAttempted
+** may be left set to a value larger than expected, if a wal frame
+** contains content that duplicate of an earlier version of the same
+** page.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code if an
+** error occurs. It is not an error if nBackfillAttempted cannot be
+** decreased at all.
+*/
+int sqlite3WalSnapshotRecover(Wal *pWal){
+  int rc;
+
+  assert( pWal->readLock>=0 );
+  rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
+  if( rc==SQLITE_OK ){
+    volatile WalCkptInfo *pInfo = walCkptInfo(pWal);
+    int szPage = (int)pWal->szPage;
+    i64 szDb;                   /* Size of db file in bytes */
+
+    rc = sqlite3OsFileSize(pWal->pDbFd, &szDb);
+    if( rc==SQLITE_OK ){
+      void *pBuf1 = sqlite3_malloc(szPage);
+      void *pBuf2 = sqlite3_malloc(szPage);
+      if( pBuf1==0 || pBuf2==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        u32 i = pInfo->nBackfillAttempted;
+        for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){
+          volatile ht_slot *dummy;
+          volatile u32 *aPgno;      /* Array of page numbers */
+          u32 iZero;                /* Frame corresponding to aPgno[0] */
+          u32 pgno;                 /* Page number in db file */
+          i64 iDbOff;               /* Offset of db file entry */
+          i64 iWalOff;              /* Offset of wal file entry */
+
+          rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero);
+          if( rc!=SQLITE_OK ) break;
+          pgno = aPgno[i-iZero];
+          iDbOff = (i64)(pgno-1) * szPage;
+
+          if( iDbOff+szPage<=szDb ){
+            iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE;
+            rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff);
+
+            if( rc==SQLITE_OK ){
+              rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff);
+            }
+
+            if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){
+              break;
+            }
+          }
+
+          pInfo->nBackfillAttempted = i-1;
+        }
+      }
+
+      sqlite3_free(pBuf1);
+      sqlite3_free(pBuf2);
+    }
+    walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
+  }
+
+  return rc;
+}
+#endif /* SQLITE_ENABLE_SNAPSHOT */
+
 /*
 ** Begin a read transaction on the database.
 **
@@ -2434,7 +2519,11 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){
       ** has not yet set the pInfo->nBackfillAttempted variable to indicate 
       ** its intent. To avoid the race condition this leads to, ensure that
       ** there is no checkpointer process by taking a shared CKPT lock 
-      ** before checking pInfo->nBackfillAttempted.  */
+      ** before checking pInfo->nBackfillAttempted.  
+      **
+      ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing
+      **       this already?
+      */
       rc = walLockShared(pWal, WAL_CKPT_LOCK);
 
       if( rc==SQLITE_OK ){
@@ -3109,16 +3198,21 @@ int sqlite3WalFrames(
   ** past the sector boundary is written after the sync.
   */
   if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){
+    int bSync = 1;
     if( pWal->padToSectorBoundary ){
       int sectorSize = sqlite3SectorSize(pWal->pWalFd);
       w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize;
+      bSync = (w.iSyncPoint==iOffset);
+      testcase( bSync );
       while( iOffsethdr.mxFrame && walPagesize(pWal)!=nBuf ){
       rc = SQLITE_CORRUPT_BKPT;
     }else{
-      rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf);
+      rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf);
     }
 
     /* If no error occurred, set the output variables. */
@@ -3380,9 +3475,14 @@ int sqlite3WalHeapMemory(Wal *pWal){
 int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){
   int rc = SQLITE_OK;
   WalIndexHdr *pRet;
+  static const u32 aZero[4] = { 0, 0, 0, 0 };
 
   assert( pWal->readLock>=0 && pWal->writeLock==0 );
 
+  if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){
+    *ppSnapshot = 0;
+    return SQLITE_ERROR;
+  }
   pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr));
   if( pRet==0 ){
     rc = SQLITE_NOMEM_BKPT;
@@ -3399,6 +3499,23 @@ int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){
 void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){
   pWal->pSnapshot = (WalIndexHdr*)pSnapshot;
 }
+
+/* 
+** Return a +ve value if snapshot p1 is newer than p2. A -ve value if
+** p1 is older than p2 and zero if p1 and p2 are the same snapshot.
+*/
+int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){
+  WalIndexHdr *pHdr1 = (WalIndexHdr*)p1;
+  WalIndexHdr *pHdr2 = (WalIndexHdr*)p2;
+
+  /* aSalt[0] is a copy of the value stored in the wal file header. It
+  ** is incremented each time the wal file is restarted.  */
+  if( pHdr1->aSalt[0]aSalt[0] ) return -1;
+  if( pHdr1->aSalt[0]>pHdr2->aSalt[0] ) return +1;
+  if( pHdr1->mxFramemxFrame ) return -1;
+  if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1;
+  return 0;
+}
 #endif /* SQLITE_ENABLE_SNAPSHOT */
 
 #ifdef SQLITE_ENABLE_ZIPVFS
diff --git a/src/wal.h b/src/wal.h
index 97e6ab4f10..4f6d01dad6 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -14,8 +14,8 @@
 ** the implementation of each function in log.c for further details.
 */
 
-#ifndef _WAL_H_
-#define _WAL_H_
+#ifndef SQLITE_WAL_H
+#define SQLITE_WAL_H
 
 #include "sqliteInt.h"
 
@@ -28,7 +28,7 @@
 #ifdef SQLITE_OMIT_WAL
 # define sqlite3WalOpen(x,y,z)                   0
 # define sqlite3WalLimit(x,y)
-# define sqlite3WalClose(w,x,y,z)                0
+# define sqlite3WalClose(v,w,x,y,z)              0
 # define sqlite3WalBeginReadTransaction(y,z)     0
 # define sqlite3WalEndReadTransaction(z)
 # define sqlite3WalDbsize(y)                     0
@@ -38,7 +38,7 @@
 # define sqlite3WalSavepoint(y,z)
 # define sqlite3WalSavepointUndo(y,z)            0
 # define sqlite3WalFrames(u,v,w,x,y,z)           0
-# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0
+# define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0
 # define sqlite3WalCallback(z)                   0
 # define sqlite3WalExclusiveMode(y,z)            0
 # define sqlite3WalHeapMemory(z)                 0
@@ -56,7 +56,7 @@ typedef struct Wal Wal;
 
 /* Open and close a connection to a write-ahead log. */
 int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**);
-int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *);
+int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *);
 
 /* Set the limiting size of a WAL file. */
 void sqlite3WalLimit(Wal*, i64);
@@ -99,6 +99,7 @@ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);
 /* Copy pages from the log to the database file */ 
 int sqlite3WalCheckpoint(
   Wal *pWal,                      /* Write-ahead log connection */
+  sqlite3 *db,                    /* Check this handle's interrupt flag */
   int eMode,                      /* One of PASSIVE, FULL and RESTART */
   int (*xBusy)(void*),            /* Function to call when busy */
   void *pBusyArg,                 /* Context argument for xBusyHandler */
@@ -130,6 +131,7 @@ int sqlite3WalHeapMemory(Wal *pWal);
 #ifdef SQLITE_ENABLE_SNAPSHOT
 int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot);
 void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot);
+int sqlite3WalSnapshotRecover(Wal *pWal);
 #endif
 
 #ifdef SQLITE_ENABLE_ZIPVFS
@@ -143,4 +145,4 @@ int sqlite3WalFramesize(Wal *pWal);
 sqlite3_file *sqlite3WalFile(Wal *pWal);
 
 #endif /* ifndef SQLITE_OMIT_WAL */
-#endif /* _WAL_H_ */
+#endif /* SQLITE_WAL_H */
diff --git a/src/walker.c b/src/walker.c
index 1e0ad32871..a7123d8ab0 100644
--- a/src/walker.c
+++ b/src/walker.c
@@ -27,11 +27,11 @@
 **
 **    WRC_Continue      Continue descending down the tree.
 **
-**    WRC_Prune         Do not descend into child nodes.  But allow
+**    WRC_Prune         Do not descend into child nodes, but allow
 **                      the walk to continue with sibling nodes.
 **
 **    WRC_Abort         Do no more callbacks.  Unwind the stack and
-**                      return the top-level walk call.
+**                      return from the top-level walk call.
 **
 ** The return value from this routine is WRC_Abort to abandon the tree walk
 ** and WRC_Continue to continue.
@@ -41,17 +41,17 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){
   testcase( ExprHasProperty(pExpr, EP_TokenOnly) );
   testcase( ExprHasProperty(pExpr, EP_Reduced) );
   rc = pWalker->xExprCallback(pWalker, pExpr);
-  if( rc==WRC_Continue
-              && !ExprHasProperty(pExpr,EP_TokenOnly) ){
-    if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
-    if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
-    if( ExprHasProperty(pExpr, EP_xIsSelect) ){
-      if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
-    }else{
-      if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
-    }
+  if( rc || ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){
+    return rc & WRC_Abort;
   }
-  return rc & WRC_Abort;
+  if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort;
+  if( pExpr->pRight && walkExpr(pWalker, pExpr->pRight) ) return WRC_Abort;
+  if( ExprHasProperty(pExpr, EP_xIsSelect) ){
+    if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort;
+  }else if( pExpr->x.pList ){
+    if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort;
+  }
+  return WRC_Continue;
 }
 int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){
   return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue;
diff --git a/src/where.c b/src/where.c
index dc3c3f10dd..c7e4ba200a 100644
--- a/src/where.c
+++ b/src/where.c
@@ -51,6 +51,18 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
   return pWInfo->nOBSat;
 }
 
+/*
+** Return TRUE if the innermost loop of the WHERE clause implementation
+** returns rows in ORDER BY order for complete run of the inner loop.
+**
+** Across multiple iterations of outer loops, the output rows need not be
+** sorted.  As long as rows are sorted for just the innermost loop, this
+** routine can return TRUE.
+*/
+int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){
+  return pWInfo->bOrderedInnerLoop;
+}
+
 /*
 ** Return the VDBE address or label to jump to in order to continue
 ** immediately with the next row of a WHERE clause.
@@ -186,16 +198,19 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
   WhereTerm *pTerm;    /* The term being tested */
   int k = pScan->k;    /* Where to start scanning */
 
-  while( pScan->iEquiv<=pScan->nEquiv ){
-    iCur = pScan->aiCur[pScan->iEquiv-1];
+  assert( pScan->iEquiv<=pScan->nEquiv );
+  pWC = pScan->pWC;
+  while(1){
     iColumn = pScan->aiColumn[pScan->iEquiv-1];
-    if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0;
-    while( (pWC = pScan->pWC)!=0 ){
+    iCur = pScan->aiCur[pScan->iEquiv-1];
+    assert( pWC!=0 );
+    do{
       for(pTerm=pWC->a+k; knTerm; k++, pTerm++){
         if( pTerm->leftCursor==iCur
          && pTerm->u.leftColumn==iColumn
          && (iColumn!=XN_EXPR
-             || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0)
+             || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft,
+                                       pScan->pIdxExpr,iCur)==0)
          && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin))
         ){
           if( (pTerm->eOperator & WO_EQUIV)!=0
@@ -240,15 +255,17 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
               testcase( pTerm->eOperator & WO_IS );
               continue;
             }
+            pScan->pWC = pWC;
             pScan->k = k+1;
             return pTerm;
           }
         }
       }
-      pScan->pWC = pScan->pWC->pOuter;
+      pWC = pWC->pOuter;
       k = 0;
-    }
-    pScan->pWC = pScan->pOrigWC;
+    }while( pWC!=0 );
+    if( pScan->iEquiv>=pScan->nEquiv ) break;
+    pWC = pScan->pOrigWC;
     k = 0;
     pScan->iEquiv++;
   }
@@ -261,7 +278,10 @@ static WhereTerm *whereScanNext(WhereScan *pScan){
 **
 ** The scanner will be searching the WHERE clause pWC.  It will look
 ** for terms of the form "X  " where X is column iColumn of table
-** iCur.  The  must be one of the operators described by opMask.
+** iCur.   Or if pIdx!=0 then X is column iColumn of index pIdx.  pIdx
+** must be one of the indexes of table iCur.
+**
+** The  must be one of the operators described by opMask.
 **
 ** If the search is for X and the WHERE clause contains terms of the
 ** form X=Y then this routine might also return terms of the form
@@ -279,23 +299,25 @@ static WhereTerm *whereScanInit(
   u32 opMask,             /* Operator(s) to scan for */
   Index *pIdx             /* Must be compatible with this index */
 ){
-  int j = 0;
-
-  /* memset(pScan, 0, sizeof(*pScan)); */
   pScan->pOrigWC = pWC;
   pScan->pWC = pWC;
   pScan->pIdxExpr = 0;
+  pScan->idxaff = 0;
+  pScan->zCollName = 0;
   if( pIdx ){
-    j = iColumn;
+    int j = iColumn;
     iColumn = pIdx->aiColumn[j];
-    if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
-  }
-  if( pIdx && iColumn>=0 ){
-    pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
-    pScan->zCollName = pIdx->azColl[j];
-  }else{
-    pScan->idxaff = 0;
-    pScan->zCollName = 0;
+    if( iColumn==XN_EXPR ){
+      pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr;
+      pScan->zCollName = pIdx->azColl[j];
+    }else if( iColumn==pIdx->pTable->iPKey ){
+      iColumn = XN_ROWID;
+    }else if( iColumn>=0 ){
+      pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity;
+      pScan->zCollName = pIdx->azColl[j];
+    }
+  }else if( iColumn==XN_EXPR ){
+    return 0;
   }
   pScan->opMask = opMask;
   pScan->k = 0;
@@ -308,11 +330,12 @@ static WhereTerm *whereScanInit(
 
 /*
 ** Search for a term in the WHERE clause that is of the form "X  "
-** where X is a reference to the iColumn of table iCur and  is one of
-** the WO_xx operator codes specified by the op parameter.
-** Return a pointer to the term.  Return 0 if not found.
+** where X is a reference to the iColumn of table iCur or of index pIdx
+** if pIdx!=0 and  is one of the WO_xx operator codes specified by
+** the op parameter.  Return a pointer to the term.  Return 0 if not found.
 **
-** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx
+** If pIdx!=0 then it must be one of the indexes of table iCur.  
+** Search for terms matching the iColumn-th column of pIdx
 ** rather than the iColumn-th column of table iCur.
 **
 ** The term returned might by Y= if there is another constraint in
@@ -494,14 +517,16 @@ static LogEst estLog(LogEst N){
 ** value stored in its output register.
 */
 static void translateColumnToCopy(
-  Vdbe *v,            /* The VDBE containing code to translate */
+  Parse *pParse,      /* Parsing context */
   int iStart,         /* Translate from this opcode to the end */
   int iTabCur,        /* OP_Column/OP_Rowid references to this table */
   int iRegister,      /* The first column is in this register */
   int bIncrRowid      /* If non-zero, transform OP_rowid to OP_AddImm(1) */
 ){
+  Vdbe *v = pParse->pVdbe;
   VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart);
   int iEnd = sqlite3VdbeCurrentAddr(v);
+  if( pParse->db->mallocFailed ) return;
   for(; iStartp1!=iTabCur ) continue;
     if( pOp->opcode==OP_Column ){
@@ -634,7 +659,7 @@ static void constructAutomaticIndex(
   ** transient index on 2nd and subsequent iterations of the loop. */
   v = pParse->pVdbe;
   assert( v!=0 );
-  addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v);
+  addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
 
   /* Count the number of columns that will be added to the index
   ** and used to match WHERE clause constraints */
@@ -779,7 +804,9 @@ static void constructAutomaticIndex(
   if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
   if( pTabItem->fg.viaCoroutine ){
     sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
-    translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1);
+    testcase( pParse->db->mallocFailed );
+    translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
+                          pTabItem->regResult, 1);
     sqlite3VdbeGoto(v, addrTop);
     pTabItem->fg.viaCoroutine = 0;
   }else{
@@ -809,7 +836,8 @@ static sqlite3_index_info *allocateIndexInfo(
   WhereClause *pWC,
   Bitmask mUnusable,              /* Ignore terms with these prereqs */
   struct SrcList_item *pSrc,
-  ExprList *pOrderBy
+  ExprList *pOrderBy,
+  u16 *pmNoOmit                   /* Mask of terms not to omit */
 ){
   int i, j;
   int nTerm;
@@ -819,6 +847,7 @@ static sqlite3_index_info *allocateIndexInfo(
   WhereTerm *pTerm;
   int nOrderBy;
   sqlite3_index_info *pIdxInfo;
+  u16 mNoOmit = 0;
 
   /* Count the number of possible WHERE clause constraints referring
   ** to this virtual table */
@@ -907,6 +936,15 @@ static sqlite3_index_info *allocateIndexInfo(
     assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE );
     assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH );
     assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) );
+
+    if( op & (WO_LT|WO_LE|WO_GT|WO_GE)
+     && sqlite3ExprIsVector(pTerm->pExpr->pRight) 
+    ){
+      if( i<16 ) mNoOmit |= (1 << i);
+      if( op==WO_LT ) pIdxCons[j].op = WO_LE;
+      if( op==WO_GT ) pIdxCons[j].op = WO_GE;
+    }
+
     j++;
   }
   for(i=0; ia[i].sortOrder;
   }
 
+  *pmNoOmit = mNoOmit;
   return pIdxInfo;
 }
 
@@ -1190,7 +1229,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){
 /*
 ** Return the affinity for a single column of an index.
 */
-static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
+char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){
   assert( iCol>=0 && iColnColumn );
   if( !pIdx->zColAff ){
     if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB;
@@ -1367,7 +1406,8 @@ static int whereRangeScanEst(
     if( nEq==pBuilder->nRecValid ){
       UnpackedRecord *pRec = pBuilder->pRec;
       tRowcnt a[2];
-      u8 aff;
+      int nBtm = pLoop->u.btree.nBtm;
+      int nTop = pLoop->u.btree.nTop;
 
       /* Variable iLower will be set to the estimate of the number of rows in 
       ** the index that are less than the lower bound of the range query. The
@@ -1397,8 +1437,6 @@ static int whereRangeScanEst(
         testcase( pRec->nField!=pBuilder->nRecValid );
         pRec->nField = pBuilder->nRecValid;
       }
-      aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq);
-      assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER );
       /* Determine iLower and iUpper using ($P) only. */
       if( nEq==0 ){
         iLower = 0;
@@ -1417,17 +1455,20 @@ static int whereRangeScanEst(
       if( p->aSortOrder[nEq] ){
         /* The roles of pLower and pUpper are swapped for a DESC index */
         SWAP(WhereTerm*, pLower, pUpper);
+        SWAP(int, nBtm, nTop);
       }
 
       /* If possible, improve on the iLower estimate using ($P:$L). */
       if( pLower ){
-        int bOk;                    /* True if value is extracted from pExpr */
+        int n;                    /* Values extracted from pExpr */
         Expr *pExpr = pLower->pExpr->pRight;
-        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-        if( rc==SQLITE_OK && bOk ){
+        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n);
+        if( rc==SQLITE_OK && n ){
           tRowcnt iNew;
+          u16 mask = WO_GT|WO_LE;
+          if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
           iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a);
-          iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
+          iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0);
           if( iNew>iLower ) iLower = iNew;
           nOut--;
           pLower = 0;
@@ -1436,13 +1477,15 @@ static int whereRangeScanEst(
 
       /* If possible, improve on the iUpper estimate using ($P:$U). */
       if( pUpper ){
-        int bOk;                    /* True if value is extracted from pExpr */
+        int n;                    /* Values extracted from pExpr */
         Expr *pExpr = pUpper->pExpr->pRight;
-        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-        if( rc==SQLITE_OK && bOk ){
+        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n);
+        if( rc==SQLITE_OK && n ){
           tRowcnt iNew;
+          u16 mask = WO_GT|WO_LE;
+          if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT);
           iUprIdx = whereKeyStats(pParse, p, pRec, 1, a);
-          iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
+          iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0);
           if( iNewpNew->u.btree.pIndex;
   int nEq = pBuilder->pNew->u.btree.nEq;
   UnpackedRecord *pRec = pBuilder->pRec;
-  u8 aff;                   /* Column affinity */
   int rc;                   /* Subfunction return code */
   tRowcnt a[2];             /* Statistics */
   int bOk;
@@ -1556,8 +1598,7 @@ static int whereEqualScanEst(
     return SQLITE_OK;
   }
 
-  aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1);
-  rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk);
+  rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk);
   pBuilder->pRec = pRec;
   if( rc!=SQLITE_OK ) return rc;
   if( bOk==0 ) return SQLITE_NOTFOUND;
@@ -1631,30 +1672,58 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){
     sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
   }else{
     char zType[4];
+    char zLeft[50];
     memcpy(zType, "...", 4);
     if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
     if( pTerm->eOperator & WO_EQUIV  ) zType[1] = 'E';
     if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
+    if( pTerm->eOperator & WO_SINGLE ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}",
+                       pTerm->leftCursor, pTerm->u.leftColumn);
+    }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%lld", 
+                       pTerm->u.pOrInfo->indexable);
+    }else{
+      sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor);
+    }
     sqlite3DebugPrintf(
-       "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n",
-       iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb,
+       "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x",
+       iTerm, pTerm, zType, zLeft, pTerm->truthProb,
        pTerm->eOperator, pTerm->wtFlags);
+    if( pTerm->iField ){
+      sqlite3DebugPrintf(" iField=%d\n", pTerm->iField);
+    }else{
+      sqlite3DebugPrintf("\n");
+    }
     sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
   }
 }
 #endif
 
+#ifdef WHERETRACE_ENABLED
+/*
+** Show the complete content of a WhereClause
+*/
+void sqlite3WhereClausePrint(WhereClause *pWC){
+  int i;
+  for(i=0; inTerm; i++){
+    whereTermPrint(&pWC->a[i], i);
+  }
+}
+#endif
+
 #ifdef WHERETRACE_ENABLED
 /*
 ** Print a WhereLoop object for debugging purposes
 */
 static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){
   WhereInfo *pWInfo = pWC->pWInfo;
-  int nb = 1+(pWInfo->pTabList->nSrc+7)/8;
+  int nb = 1+(pWInfo->pTabList->nSrc+3)/4;
   struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab;
   Table *pTab = pItem->pTab;
+  Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1;
   sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId,
-                     p->iTab, nb, p->maskSelf, nb, p->prereq);
+                     p->iTab, nb, p->maskSelf, nb, p->prereq & mAll);
   sqlite3DebugPrintf(" %12s",
                      pItem->zAlias ? pItem->zAlias : pTab->zName);
   if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){
@@ -2147,6 +2216,72 @@ static void whereLoopOutputAdjust(
   if( pLoop->nOut > nRow-iReduce )  pLoop->nOut = nRow - iReduce;
 }
 
+/* 
+** Term pTerm is a vector range comparison operation. The first comparison
+** in the vector can be optimized using column nEq of the index. This
+** function returns the total number of vector elements that can be used
+** as part of the range comparison.
+**
+** For example, if the query is:
+**
+**   WHERE a = ? AND (b, c, d) > (?, ?, ?)
+**
+** and the index:
+**
+**   CREATE INDEX ... ON (a, b, c, d, e)
+**
+** then this function would be invoked with nEq=1. The value returned in
+** this case is 3.
+*/
+static int whereRangeVectorLen(
+  Parse *pParse,       /* Parsing context */
+  int iCur,            /* Cursor open on pIdx */
+  Index *pIdx,         /* The index to be used for a inequality constraint */
+  int nEq,             /* Number of prior equality constraints on same index */
+  WhereTerm *pTerm     /* The vector inequality constraint */
+){
+  int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft);
+  int i;
+
+  nCmp = MIN(nCmp, (pIdx->nColumn - nEq));
+  for(i=1; ipExpr->pLeft->x.pList->a[i].pExpr;
+    Expr *pRhs = pTerm->pExpr->pRight;
+    if( pRhs->flags & EP_xIsSelect ){
+      pRhs = pRhs->x.pSelect->pEList->a[i].pExpr;
+    }else{
+      pRhs = pRhs->x.pList->a[i].pExpr;
+    }
+
+    /* Check that the LHS of the comparison is a column reference to
+    ** the right column of the right source table. And that the sort
+    ** order of the index column is the same as the sort order of the
+    ** leftmost index column.  */
+    if( pLhs->op!=TK_COLUMN 
+     || pLhs->iTable!=iCur 
+     || pLhs->iColumn!=pIdx->aiColumn[i+nEq] 
+     || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq]
+    ){
+      break;
+    }
+
+    testcase( pLhs->iColumn==XN_ROWID );
+    aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs));
+    idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn);
+    if( aff!=idxaff ) break;
+
+    pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
+    if( pColl==0 ) break;
+    if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break;
+  }
+  return i;
+}
+
 /*
 ** Adjust the cost C by the costMult facter T.  This only occurs if
 ** compiled with -DSQLITE_ENABLE_COSTMULT
@@ -2185,6 +2320,8 @@ static int whereLoopAddBtreeIndex(
   Bitmask saved_prereq;           /* Original value of pNew->prereq */
   u16 saved_nLTerm;               /* Original value of pNew->nLTerm */
   u16 saved_nEq;                  /* Original value of pNew->u.btree.nEq */
+  u16 saved_nBtm;                 /* Original value of pNew->u.btree.nBtm */
+  u16 saved_nTop;                 /* Original value of pNew->u.btree.nTop */
   u16 saved_nSkip;                /* Original value of pNew->nSkip */
   u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
   LogEst saved_nOut;              /* Original value of pNew->nOut */
@@ -2195,14 +2332,15 @@ static int whereLoopAddBtreeIndex(
 
   pNew = pBuilder->pNew;
   if( db->mallocFailed ) return SQLITE_NOMEM_BKPT;
+  WHERETRACE(0x800, ("BEGIN addBtreeIdx(%s), nEq=%d\n",
+                     pProbe->zName, pNew->u.btree.nEq));
 
   assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 );
   assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 );
   if( pNew->wsFlags & WHERE_BTM_LIMIT ){
     opMask = WO_LT|WO_LE;
-  }else if( /*pProbe->tnum<=0 ||*/ (pSrc->fg.jointype & JT_LEFT)!=0 ){
-    opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE;
   }else{
+    assert( pNew->u.btree.nBtm==0 );
     opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS;
   }
   if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE);
@@ -2210,6 +2348,8 @@ static int whereLoopAddBtreeIndex(
   assert( pNew->u.btree.nEqnColumn );
 
   saved_nEq = pNew->u.btree.nEq;
+  saved_nBtm = pNew->u.btree.nBtm;
+  saved_nTop = pNew->u.btree.nTop;
   saved_nSkip = pNew->nSkip;
   saved_nLTerm = pNew->nLTerm;
   saved_wsFlags = pNew->wsFlags;
@@ -2239,8 +2379,27 @@ static int whereLoopAddBtreeIndex(
     ** to mix with a lower range bound from some other source */
     if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue;
 
+    /* Do not allow IS constraints from the WHERE clause to be used by the
+    ** right table of a LEFT JOIN.  Only constraints in the ON clause are
+    ** allowed */
+    if( (pSrc->fg.jointype & JT_LEFT)!=0
+     && !ExprHasProperty(pTerm->pExpr, EP_FromJoin)
+     && (eOp & (WO_IS|WO_ISNULL))!=0
+    ){
+      testcase( eOp & WO_IS );
+      testcase( eOp & WO_ISNULL );
+      continue;
+    }
+
+    if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
+      pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE;
+    }else{
+      pBuilder->bldFlags |= SQLITE_BLDF_INDEXED;
+    }
     pNew->wsFlags = saved_wsFlags;
     pNew->u.btree.nEq = saved_nEq;
+    pNew->u.btree.nBtm = saved_nBtm;
+    pNew->u.btree.nTop = saved_nTop;
     pNew->nLTerm = saved_nLTerm;
     if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
     pNew->aLTerm[pNew->nLTerm++] = pTerm;
@@ -2257,14 +2416,23 @@ static int whereLoopAddBtreeIndex(
       pNew->wsFlags |= WHERE_COLUMN_IN;
       if( ExprHasProperty(pExpr, EP_xIsSelect) ){
         /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
+        int i;
         nIn = 46;  assert( 46==sqlite3LogEst(25) );
+
+        /* The expression may actually be of the form (x, y) IN (SELECT...).
+        ** In this case there is a separate term for each of (x) and (y).
+        ** However, the nIn multiplier should only be applied once, not once
+        ** for each such term. The following loop checks that pTerm is the
+        ** first such term in use, and sets nIn back to 0 if it is not. */
+        for(i=0; inLTerm-1; i++){
+          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
+        }
       }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
         /* "x IN (value, value, ...)" */
         nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
+        assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
+                          ** changes "x IN (?)" into "x=?". */
       }
-      assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
-                        ** changes "x IN (?)" into "x=?". */
-
     }else if( eOp & (WO_EQ|WO_IS) ){
       int iCol = pProbe->aiColumn[saved_nEq];
       pNew->wsFlags |= WHERE_COLUMN_EQ;
@@ -2284,6 +2452,9 @@ static int whereLoopAddBtreeIndex(
       testcase( eOp & WO_GT );
       testcase( eOp & WO_GE );
       pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT;
+      pNew->u.btree.nBtm = whereRangeVectorLen(
+          pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
+      );
       pBtm = pTerm;
       pTop = 0;
       if( pTerm->wtFlags & TERM_LIKEOPT ){
@@ -2296,12 +2467,16 @@ static int whereLoopAddBtreeIndex(
         if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */
         pNew->aLTerm[pNew->nLTerm++] = pTop;
         pNew->wsFlags |= WHERE_TOP_LIMIT;
+        pNew->u.btree.nTop = 1;
       }
     }else{
       assert( eOp & (WO_LT|WO_LE) );
       testcase( eOp & WO_LT );
       testcase( eOp & WO_LE );
       pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
+      pNew->u.btree.nTop = whereRangeVectorLen(
+          pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm
+      );
       pTop = pTerm;
       pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ?
                      pNew->aLTerm[pNew->nLTerm-2] : 0;
@@ -2401,6 +2576,8 @@ static int whereLoopAddBtreeIndex(
   }
   pNew->prereq = saved_prereq;
   pNew->u.btree.nEq = saved_nEq;
+  pNew->u.btree.nBtm = saved_nBtm;
+  pNew->u.btree.nTop = saved_nTop;
   pNew->nSkip = saved_nSkip;
   pNew->wsFlags = saved_wsFlags;
   pNew->nOut = saved_nOut;
@@ -2440,6 +2617,8 @@ static int whereLoopAddBtreeIndex(
     pNew->wsFlags = saved_wsFlags;
   }
 
+  WHERETRACE(0x800, ("END addBtreeIdx(%s), nEq=%d, rc=%d\n",
+                      pProbe->zName, saved_nEq, rc));
   return rc;
 }
 
@@ -2522,7 +2701,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){
 
 /*
 ** Add all WhereLoop objects for a single table of the join where the table
-** is idenfied by pBuilder->pNew->iTab.  That table is guaranteed to be
+** is identified by pBuilder->pNew->iTab.  That table is guaranteed to be
 ** a b-tree table, not a virtual table.
 **
 ** The costs (WhereLoop.rRun) of the b-tree loops added by this function
@@ -2619,7 +2798,7 @@ static int whereLoopAddBtree(
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
   /* Automatic indexes */
   if( !pBuilder->pOrSet      /* Not part of an OR optimization */
-   && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0
+   && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0
    && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0
    && pSrc->pIBIndex==0      /* Has no INDEXED BY clause */
    && !pSrc->fg.notIndexed   /* Has no NOT INDEXED clause */
@@ -2651,6 +2830,7 @@ static int whereLoopAddBtree(
           pNew->rSetup += 24;
         }
         ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
+        if( pNew->rSetup<0 ) pNew->rSetup = 0;
         /* TUNING: Each index lookup yields 20 rows in the table.  This
         ** is more than the usual guess of 10 rows, since we have no way
         ** of knowing how selective the index will ultimately be.  It would
@@ -2675,6 +2855,8 @@ static int whereLoopAddBtree(
     }
     rSize = pProbe->aiRowLogEst[0];
     pNew->u.btree.nEq = 0;
+    pNew->u.btree.nBtm = 0;
+    pNew->u.btree.nTop = 0;
     pNew->nSkip = 0;
     pNew->nLTerm = 0;
     pNew->iSortIdx = 0;
@@ -2711,6 +2893,7 @@ static int whereLoopAddBtree(
       /* Full scan via index */
       if( b
        || !HasRowid(pTab)
+       || pProbe->pPartIdxWhere!=0
        || ( m==0
          && pProbe->bUnordered==0
          && (pProbe->szIdxRowszTabRow)
@@ -2723,11 +2906,34 @@ static int whereLoopAddBtree(
 
         /* The cost of visiting the index rows is N*K, where K is
         ** between 1.1 and 3.0, depending on the relative sizes of the
-        ** index and table rows. If this is a non-covering index scan,
-        ** also add the cost of visiting table rows (N*3.0).  */
+        ** index and table rows. */
         pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
         if( m!=0 ){
-          pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16);
+          /* If this is a non-covering index scan, add in the cost of
+          ** doing table lookups.  The cost will be 3x the number of
+          ** lookups.  Take into account WHERE clause terms that can be
+          ** satisfied using just the index, and that do not require a
+          ** table lookup. */
+          LogEst nLookup = rSize + 16;  /* Base cost:  N*3 */
+          int ii;
+          int iCur = pSrc->iCursor;
+          WhereClause *pWC2 = &pWInfo->sWC;
+          for(ii=0; iinTerm; ii++){
+            WhereTerm *pTerm = &pWC2->a[ii];
+            if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){
+              break;
+            }
+            /* pTerm can be evaluated using just the index.  So reduce
+            ** the expected number of table lookups accordingly */
+            if( pTerm->truthProb<=0 ){
+              nLookup += pTerm->truthProb;
+            }else{
+              nLookup--;
+              if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19;
+            }
+          }
+          
+          pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup);
         }
         ApplyCostMultiplier(pNew->rRun, pTab->costMult);
         whereLoopOutputAdjust(pWC, pNew, rSize);
@@ -2737,7 +2943,15 @@ static int whereLoopAddBtree(
       }
     }
 
+    pBuilder->bldFlags = 0;
     rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
+    if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){
+      /* If a non-unique index is used, or if a prefix of the key for
+      ** unique index is used (making the index functionally non-unique)
+      ** then the sqlite_stat1 data becomes important for scoring the
+      ** plan */
+      pTab->tabFlags |= TF_StatsUsed;
+    }
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
     sqlite3Stat4ProbeFree(pBuilder->pRec);
     pBuilder->nRecValid = 0;
@@ -2779,6 +2993,7 @@ static int whereLoopAddVirtualOne(
   Bitmask mUsable,                /* Mask of usable tables */
   u16 mExclude,                   /* Exclude terms using these operators */
   sqlite3_index_info *pIdxInfo,   /* Populated object for xBestIndex */
+  u16 mNoOmit,                    /* Do not omit these constraints */
   int *pbIn                       /* OUT: True if plan uses an IN(...) op */
 ){
   WhereClause *pWC = pBuilder->pWC;
@@ -2867,6 +3082,7 @@ static int whereLoopAddVirtualOne(
       }
     }
   }
+  pNew->u.vtab.omitMask &= ~mNoOmit;
 
   pNew->nLTerm = mxTerm+1;
   assert( pNew->nLTerm<=pNew->nLSlot );
@@ -2940,6 +3156,7 @@ static int whereLoopAddVirtual(
   int bIn;                     /* True if plan uses IN(...) operator */
   WhereLoop *pNew;
   Bitmask mBest;               /* Tables used by best possible plan */
+  u16 mNoOmit;
 
   assert( (mPrereq & mUnusable)==0 );
   pWInfo = pBuilder->pWInfo;
@@ -2948,7 +3165,8 @@ static int whereLoopAddVirtual(
   pNew = pBuilder->pNew;
   pSrc = &pWInfo->pTabList->a[pNew->iTab];
   assert( IsVirtual(pSrc->pTab) );
-  p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy);
+  p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, 
+      &mNoOmit);
   if( p==0 ) return SQLITE_NOMEM_BKPT;
   pNew->rSetup = 0;
   pNew->wsFlags = WHERE_VIRTUALTABLE;
@@ -2962,7 +3180,7 @@ static int whereLoopAddVirtual(
 
   /* First call xBestIndex() with all constraints usable. */
   WHERETRACE(0x40, ("  VirtualOne: all usable\n"));
-  rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, &bIn);
+  rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn);
 
   /* If the call to xBestIndex() with all terms enabled produced a plan
   ** that does not require any source tables (IOW: a plan with mBest==0),
@@ -2979,7 +3197,8 @@ static int whereLoopAddVirtual(
     ** xBestIndex again, this time with IN(...) terms disabled. */
     if( bIn ){
       WHERETRACE(0x40, ("  VirtualOne: all usable w/o IN\n"));
-      rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, WO_IN, p, &bIn);
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn);
       assert( bIn==0 );
       mBestNoIn = pNew->prereq & ~mPrereq;
       if( mBestNoIn==0 ){
@@ -3005,7 +3224,8 @@ static int whereLoopAddVirtual(
       if( mNext==mBest || mNext==mBestNoIn ) continue;
       WHERETRACE(0x40, ("  VirtualOne: mPrev=%04llx mNext=%04llx\n",
                        (sqlite3_uint64)mPrev, (sqlite3_uint64)mNext));
-      rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mNext|mPrereq, 0, p, &bIn);
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, mNext|mPrereq, 0, p, mNoOmit, &bIn);
       if( pNew->prereq==mPrereq ){
         seenZero = 1;
         if( bIn==0 ) seenZeroNoIN = 1;
@@ -3017,7 +3237,8 @@ static int whereLoopAddVirtual(
     ** usable), make a call here with all source tables disabled */
     if( rc==SQLITE_OK && seenZero==0 ){
       WHERETRACE(0x40, ("  VirtualOne: all disabled\n"));
-      rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, 0, p, &bIn);
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn);
       if( bIn==0 ) seenZeroNoIN = 1;
     }
 
@@ -3026,7 +3247,8 @@ static int whereLoopAddVirtual(
     ** operator, make a final call to obtain one here.  */
     if( rc==SQLITE_OK && seenZeroNoIN==0 ){
       WHERETRACE(0x40, ("  VirtualOne: all disabled and w/o IN\n"));
-      rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, WO_IN, p, &bIn);
+      rc = whereLoopAddVirtualOne(
+          pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn);
     }
   }
 
@@ -3096,9 +3318,7 @@ static int whereLoopAddOr(
         WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", 
                    (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
         if( sqlite3WhereTrace & 0x400 ){
-          for(i=0; inTerm; i++){
-            whereTermPrint(&sSubBuild.pWC->a[i], i);
-          }
+          sqlite3WhereClausePrint(sSubBuild.pWC);
         }
 #endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -3191,6 +3411,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
       mPrereq = mPrior;
     }
     priorJointype = pItem->fg.jointype;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
     if( IsVirtual(pItem->pTab) ){
       struct SrcList_item *p;
       for(p=&pItem[1]; paLoop[] */
   WhereLoop *pLast,     /* Add this WhereLoop to the end of pPath->aLoop[] */
   Bitmask *pRevMask     /* OUT: Mask of WhereLoops to run in reverse order */
@@ -3245,6 +3468,7 @@ static i8 wherePathSatisfiesOrderBy(
   u8 isOrderDistinct;   /* All prior WhereLoops are order-distinct */
   u8 distinctColumns;   /* True if the loop has UNIQUE NOT NULL columns */
   u8 isMatch;           /* iColumn matches a term of the ORDER BY clause */
+  u16 eqOpMask;         /* Allowed equality operators */
   u16 nKeyCol;          /* Number of key columns in pIndex */
   u16 nColumn;          /* Total number of ordered columns in the index */
   u16 nOrderBy;         /* Number terms in the ORDER BY clause */
@@ -3295,9 +3519,16 @@ static i8 wherePathSatisfiesOrderBy(
   obDone = MASKBIT(nOrderBy)-1;
   orderDistinctMask = 0;
   ready = 0;
+  eqOpMask = WO_EQ | WO_IS | WO_ISNULL;
+  if( wctrlFlags & WHERE_ORDERBY_LIMIT ) eqOpMask |= WO_IN;
   for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf;
-    pLoop = iLoopaLoop[iLoop] : pLast;
+    if( iLoopaLoop[iLoop];
+      if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue;
+    }else{
+      pLoop = pLast;
+    }
     if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
       if( pLoop->u.vtab.isOrdered ) obSat = obDone;
       break;
@@ -3315,8 +3546,16 @@ static i8 wherePathSatisfiesOrderBy(
       if( pOBExpr->op!=TK_COLUMN ) continue;
       if( pOBExpr->iTable!=iCur ) continue;
       pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
-                       ~ready, WO_EQ|WO_ISNULL|WO_IS, 0);
+                       ~ready, eqOpMask, 0);
       if( pTerm==0 ) continue;
+      if( pTerm->eOperator==WO_IN ){
+        /* IN terms are only valid for sorting in the ORDER BY LIMIT 
+        ** optimization, and then only if they are actually used
+        ** by the query plan */
+        assert( wctrlFlags & WHERE_ORDERBY_LIMIT );
+        for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){}
+        if( j>=pLoop->nLTerm ) continue;
+      }
       if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){
         const char *z1, *z2;
         pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr);
@@ -3353,18 +3592,42 @@ static i8 wherePathSatisfiesOrderBy(
       rev = revSet = 0;
       distinctColumns = 0;
       for(j=0; ju.btree.nEq
-         && pLoop->nSkip==0
-         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0
-        ){
-          if( i & WO_ISNULL ){
-            testcase( isOrderDistinct );
-            isOrderDistinct = 0;
+        assert( j>=pLoop->u.btree.nEq 
+            || (pLoop->aLTerm[j]==0)==(jnSkip)
+        );
+        if( ju.btree.nEq && j>=pLoop->nSkip ){
+          u16 eOp = pLoop->aLTerm[j]->eOperator;
+
+          /* Skip over == and IS and ISNULL terms.  (Also skip IN terms when
+          ** doing WHERE_ORDERBY_LIMIT processing). 
+          **
+          ** If the current term is a column of an ((?,?) IN (SELECT...)) 
+          ** expression for which the SELECT returns more than one column,
+          ** check that it is the only column used by this loop. Otherwise,
+          ** if it is one of two or more, none of the columns can be
+          ** considered to match an ORDER BY term.  */
+          if( (eOp & eqOpMask)!=0 ){
+            if( eOp & WO_ISNULL ){
+              testcase( isOrderDistinct );
+              isOrderDistinct = 0;
+            }
+            continue;  
+          }else if( ALWAYS(eOp & WO_IN) ){
+            /* ALWAYS() justification: eOp is an equality operator due to the
+            ** ju.btree.nEq constraint above.  Any equality other
+            ** than WO_IN is captured by the previous "if".  So this one
+            ** always has to be WO_IN. */
+            Expr *pX = pLoop->aLTerm[j]->pExpr;
+            for(i=j+1; iu.btree.nEq; i++){
+              if( pLoop->aLTerm[i]->pExpr==pX ){
+                assert( (pLoop->aLTerm[i]->eOperator & WO_IN) );
+                bOnce = 0;
+                break;
+              }
+            }
           }
-          continue;  
         }
 
         /* Get the column number in the table (iColumn) and sort order
@@ -3393,7 +3656,6 @@ static i8 wherePathSatisfiesOrderBy(
         /* Find the ORDER BY term that corresponds to the j-th column
         ** of the index and mark that ORDER BY term off 
         */
-        bOnce = 1;
         isMatch = 0;
         for(i=0; bOnce && inOBSat = pFrom->isOrdered;
-      if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0;
       pWInfo->revMask = pFrom->revLoop;
+      if( pWInfo->nOBSat<=0 ){
+        pWInfo->nOBSat = 0;
+        if( nLoop>0 ){
+          u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags;
+          if( (wsFlags & WHERE_ONEROW)==0 
+           && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN)
+          ){
+            Bitmask m = 0;
+            int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom,
+                      WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m);
+            testcase( wsFlags & WHERE_IPK );
+            testcase( wsFlags & WHERE_COLUMN_IN );
+            if( rc==pWInfo->pOrderBy->nExpr ){
+              pWInfo->bOrderedInnerLoop = 1;
+              pWInfo->revMask = m;
+            }
+          }
+        }
+      }
     }
     if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
         && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0
@@ -3929,9 +4209,9 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
   int j;
   Table *pTab;
   Index *pIdx;
-  
+
   pWInfo = pBuilder->pWInfo;
-  if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0;
+  if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0;
   assert( pWInfo->pTabList->nSrc>=1 );
   pItem = pWInfo->pTabList->a;
   pTab = pItem->pTab;
@@ -4184,7 +4464,7 @@ static void whereTraceBuilder(
 ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL.
 **
 ** The iIdxCur parameter is the cursor number of an index.  If 
-** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index
+** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index
 ** to use for OR clause processing.  The WHERE clause should use this
 ** specific cursor.  If WHERE_ONEPASS_DESIRED is set, then iIdxCur is
 ** the first cursor in an array of cursors for all indices.  iIdxCur should
@@ -4192,14 +4472,14 @@ static void whereTraceBuilder(
 ** used.
 */
 WhereInfo *sqlite3WhereBegin(
-  Parse *pParse,        /* The parser context */
-  SrcList *pTabList,    /* FROM clause: A list of all tables to be scanned */
-  Expr *pWhere,         /* The WHERE clause */
-  ExprList *pOrderBy,   /* An ORDER BY (or GROUP BY) clause, or NULL */
-  ExprList *pResultSet, /* Result set of the query */
-  u16 wctrlFlags,       /* One of the WHERE_* flags defined in sqliteInt.h */
-  int iAuxArg           /* If WHERE_ONETABLE_ONLY is set, index cursor number,
-                        ** If WHERE_USE_LIMIT, then the limit amount */
+  Parse *pParse,          /* The parser context */
+  SrcList *pTabList,      /* FROM clause: A list of all tables to be scanned */
+  Expr *pWhere,           /* The WHERE clause */
+  ExprList *pOrderBy,     /* An ORDER BY (or GROUP BY) clause, or NULL */
+  ExprList *pResultSet,   /* Query result set.  Req'd for DISTINCT */
+  u16 wctrlFlags,         /* The WHERE_* flags defined in sqliteInt.h */
+  int iAuxArg             /* If WHERE_OR_SUBCLAUSE is set, index cursor number
+                          ** If WHERE_USE_LIMIT, then the limit amount */
 ){
   int nByteWInfo;            /* Num. bytes allocated for WhereInfo struct */
   int nTabList;              /* Number of elements in pTabList */
@@ -4217,11 +4497,11 @@ WhereInfo *sqlite3WhereBegin(
 
   assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || (
         (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 
-     && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 
+     && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 
   ));
 
-  /* Only one of WHERE_ONETABLE_ONLY or WHERE_USE_LIMIT */
-  assert( (wctrlFlags & WHERE_ONETABLE_ONLY)==0
+  /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */
+  assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0
             || (wctrlFlags & WHERE_USE_LIMIT)==0 );
 
   /* Variable initialization */
@@ -4249,11 +4529,11 @@ WhereInfo *sqlite3WhereBegin(
   }
 
   /* This function normally generates a nested loop for all tables in 
-  ** pTabList.  But if the WHERE_ONETABLE_ONLY flag is set, then we should
+  ** pTabList.  But if the WHERE_OR_SUBCLAUSE flag is set, then we should
   ** only generate code for the first table in pTabList and assume that
   ** any cursors associated with subsequent tables are uninitialized.
   */
-  nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc;
+  nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc;
 
   /* Allocate and initialize the WhereInfo structure that will become the
   ** return value. A single allocation is used to store the WhereInfo
@@ -4263,22 +4543,25 @@ WhereInfo *sqlite3WhereBegin(
   ** some architectures. Hence the ROUND8() below.
   */
   nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel));
-  pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop));
+  pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop));
   if( db->mallocFailed ){
     sqlite3DbFree(db, pWInfo);
     pWInfo = 0;
     goto whereBeginError;
   }
-  pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
-  pWInfo->nLevel = nTabList;
   pWInfo->pParse = pParse;
   pWInfo->pTabList = pTabList;
   pWInfo->pOrderBy = pOrderBy;
   pWInfo->pResultSet = pResultSet;
+  pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1;
+  pWInfo->nLevel = nTabList;
   pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
   pWInfo->wctrlFlags = wctrlFlags;
   pWInfo->iLimit = iAuxArg;
   pWInfo->savedNQueryLoop = pParse->nQueryLoop;
+  memset(&pWInfo->nOBSat, 0, 
+         offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat));
+  memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel));
   assert( pWInfo->eOnePass==ONEPASS_OFF );  /* ONEPASS defaults to OFF */
   pMaskSet = &pWInfo->sMaskSet;
   sWLB.pWInfo = pWInfo;
@@ -4329,7 +4612,7 @@ WhereInfo *sqlite3WhereBegin(
   ** Note that bitmasks are created for all pTabList->nSrc tables in
   ** pTabList, not just the first nTabList tables.  nTabList is normally
   ** equal to pTabList->nSrc but might be shortened to 1 if the
-  ** WHERE_ONETABLE_ONLY flag is set.
+  ** WHERE_OR_SUBCLAUSE flag is set.
   */
   for(ii=0; iinSrc; ii++){
     createMask(pMaskSet, pTabList->a[ii].iCursor);
@@ -4367,10 +4650,7 @@ WhereInfo *sqlite3WhereBegin(
     sqlite3DebugPrintf(")\n");
   }
   if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */
-    int i;
-    for(i=0; inTerm; i++){
-      whereTermPrint(&sWLB.pWC->a[i], i);
-    }
+    sqlite3WhereClausePrint(sWLB.pWC);
   }
 #endif
 
@@ -4515,7 +4795,7 @@ WhereInfo *sqlite3WhereBegin(
     }else
 #endif
     if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-         && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
+         && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){
       int op = OP_OpenRead;
       if( pWInfo->eOnePass!=ONEPASS_OFF ){
         op = OP_OpenWrite;
@@ -4554,7 +4834,7 @@ WhereInfo *sqlite3WhereBegin(
       /* iAuxArg is always set if to a positive value if ONEPASS is possible */
       assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 );
       if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx)
-       && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0
+       && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0
       ){
         /* This is one term of an OR-optimization using the PRIMARY KEY of a
         ** WITHOUT ROWID table.  No need for a separate index */
@@ -4570,9 +4850,9 @@ WhereInfo *sqlite3WhereBegin(
         }
         op = OP_OpenWrite;
         pWInfo->aiCurOnePass[1] = iIndexCur;
-      }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){
+      }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){
         iIndexCur = iAuxArg;
-        if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx;
+        op = OP_ReopenIdx;
       }else{
         iIndexCur = pParse->nTab++;
       }
@@ -4634,7 +4914,7 @@ WhereInfo *sqlite3WhereBegin(
     pLevel->addrBody = sqlite3VdbeCurrentAddr(v);
     notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady);
     pWInfo->iContinue = pLevel->addrCont;
-    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){
+    if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){
       sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain);
     }
   }
@@ -4688,10 +4968,12 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
       for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
         sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
-        sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
-        VdbeCoverage(v);
-        VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen);
-        VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen);
+        if( pIn->eEndLoopOp!=OP_Noop ){
+          sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
+          VdbeCoverage(v);
+          VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen);
+          VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen);
+        }
         sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
       }
     }
@@ -4704,24 +4986,21 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     }
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
     if( pLevel->addrLikeRep ){
-      int op;
-      if( sqlite3VdbeGetOp(v, pLevel->addrLikeRep-1)->p1 ){
-        op = OP_DecrJumpZero;
-      }else{
-        op = OP_JumpZeroIncr;
-      }
-      sqlite3VdbeAddOp2(v, op, pLevel->iLikeRepCntr, pLevel->addrLikeRep);
+      sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1),
+                        pLevel->addrLikeRep);
       VdbeCoverage(v);
     }
 #endif
     if( pLevel->iLeftJoin ){
+      int ws = pLoop->wsFlags;
       addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v);
-      assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
-           || (pLoop->wsFlags & WHERE_INDEXED)!=0 );
-      if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){
+      assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 );
+      if( (ws & WHERE_IDX_ONLY)==0 ){
         sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
       }
-      if( pLoop->wsFlags & WHERE_INDEXED ){
+      if( (ws & WHERE_INDEXED) 
+       || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) 
+      ){
         sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
       }
       if( pLevel->op==OP_Return ){
@@ -4754,33 +5033,13 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
     ** the co-routine into OP_Copy of result contained in a register.
     ** OP_Rowid becomes OP_Null.
     */
-    if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){
-      translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur,
+    if( pTabItem->fg.viaCoroutine ){
+      testcase( pParse->db->mallocFailed );
+      translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur,
                             pTabItem->regResult, 0);
       continue;
     }
 
-    /* Close all of the cursors that were opened by sqlite3WhereBegin.
-    ** Except, do not close cursors that will be reused by the OR optimization
-    ** (WHERE_OMIT_OPEN_CLOSE).  And do not close the OP_OpenWrite cursors
-    ** created for the ONEPASS optimization.
-    */
-    if( (pTab->tabFlags & TF_Ephemeral)==0
-     && pTab->pSelect==0
-     && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
-    ){
-      int ws = pLoop->wsFlags;
-      if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){
-        sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor);
-      }
-      if( (ws & WHERE_INDEXED)!=0
-       && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 
-       && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1]
-      ){
-        sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur);
-      }
-    }
-
     /* If this scan uses an index, make VDBE code substitutions to read data
     ** from the index instead of from the table where possible.  In some cases
     ** this optimization prevents the table from ever being read, which can
@@ -4819,7 +5078,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
             pOp->p2 = x;
             pOp->p1 = pLevel->iIdxCur;
           }
-          assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 );
+          assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 
+              || pWInfo->eOnePass );
         }else if( pOp->opcode==OP_Rowid ){
           pOp->p1 = pLevel->iIdxCur;
           pOp->opcode = OP_IdxRowid;
diff --git a/src/whereInt.h b/src/whereInt.h
index eb6ca326f5..f065fae6ba 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -70,7 +70,7 @@ struct WhereLevel {
   int addrFirst;        /* First instruction of interior of the loop */
   int addrBody;         /* Beginning of the body of this loop */
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
-  int iLikeRepCntr;     /* LIKE range processing counter register */
+  u32 iLikeRepCntr;     /* LIKE range processing counter register (times 2) */
   int addrLikeRep;      /* LIKE range processing address */
 #endif
   u8 iFrom;             /* Which entry in the FROM clause */
@@ -122,6 +122,8 @@ struct WhereLoop {
   union {
     struct {               /* Information for internal btree tables */
       u16 nEq;               /* Number of equality constraints */
+      u16 nBtm;              /* Size of BTM vector */
+      u16 nTop;              /* Size of TOP vector */
       Index *pIndex;         /* Index used, or NULL */
     } btree;
     struct {               /* Information for virtual tables */
@@ -244,19 +246,20 @@ struct WherePath {
 */
 struct WhereTerm {
   Expr *pExpr;            /* Pointer to the subexpression that is this term */
+  WhereClause *pWC;       /* The clause this term is part of */
+  LogEst truthProb;       /* Probability of truth for this expression */
+  u16 wtFlags;            /* TERM_xxx bit flags.  See below */
+  u16 eOperator;          /* A WO_xx value describing  */
+  u8 nChild;              /* Number of children that must disable us */
+  u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
   int iParent;            /* Disable pWC->a[iParent] when this term disabled */
   int leftCursor;         /* Cursor number of X in "X  " */
+  int iField;             /* Field in (?,?,?) IN (SELECT...) vector */
   union {
     int leftColumn;         /* Column number of X in "X  " */
     WhereOrInfo *pOrInfo;   /* Extra information if (eOperator & WO_OR)!=0 */
     WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */
   } u;
-  LogEst truthProb;       /* Probability of truth for this expression */
-  u16 eOperator;          /* A WO_xx value describing  */
-  u16 wtFlags;            /* TERM_xxx bit flags.  See below */
-  u8 nChild;              /* Number of children that must disable us */
-  u8 eMatchOp;            /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */
-  WhereClause *pWC;       /* The clause this term is part of */
   Bitmask prereqRight;    /* Bitmask of tables used by pExpr->pRight */
   Bitmask prereqAll;      /* Bitmask of tables referenced by pExpr */
 };
@@ -392,8 +395,13 @@ struct WhereLoopBuilder {
   UnpackedRecord *pRec;     /* Probe for stat4 (if required) */
   int nRecValid;            /* Number of valid fields currently in pRec */
 #endif
+  unsigned int bldFlags;    /* SQLITE_BLDF_* flags */
 };
 
+/* Allowed values for WhereLoopBuider.bldFlags */
+#define SQLITE_BLDF_INDEXED  0x0001   /* An index is used */
+#define SQLITE_BLDF_UNIQUE   0x0002   /* All keys of a UNIQUE index used */
+
 /*
 ** The WHERE clause processing routine has two halves.  The
 ** first part does the start of the WHERE loop and the second
@@ -408,25 +416,26 @@ struct WhereInfo {
   Parse *pParse;            /* Parsing and code generating context */
   SrcList *pTabList;        /* List of tables in the join */
   ExprList *pOrderBy;       /* The ORDER BY clause or NULL */
-  ExprList *pResultSet;     /* Result set. DISTINCT operates on these */
-  WhereLoop *pLoops;        /* List of all WhereLoop objects */
-  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
-  LogEst nRowOut;           /* Estimated number of output rows */
+  ExprList *pResultSet;     /* Result set of the query */
   LogEst iLimit;            /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */
+  int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
+  int iContinue;            /* Jump here to continue with next record */
+  int iBreak;               /* Jump here to break out of the loop */
+  int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
   u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
+  u8 nLevel;                /* Number of nested loop */
   i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
   u8 sorted;                /* True if really sorted (not just grouped) */
   u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
-  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
-  u8 nLevel;                /* Number of nested loop */
+  u8 eDistinct;             /* One of the WHERE_DISTINCT_* values */
+  u8 bOrderedInnerLoop;     /* True if only the inner-most loop is ordered */
   int iTop;                 /* The very beginning of the WHERE loop */
-  int iContinue;            /* Jump here to continue with next record */
-  int iBreak;               /* Jump here to break out of the loop */
-  int savedNQueryLoop;      /* pParse->nQueryLoop outside the WHERE loop */
-  int aiCurOnePass[2];      /* OP_OpenWrite cursors for the ONEPASS opt */
-  WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
+  WhereLoop *pLoops;        /* List of all WhereLoop objects */
+  Bitmask revMask;          /* Mask of ORDER BY terms that need reversing */
+  LogEst nRowOut;           /* Estimated number of output rows */
   WhereClause sWC;          /* Decomposition of the WHERE clause */
+  WhereMaskSet sMaskSet;    /* Map cursor numbers to bitmasks */
   WhereLevel a[1];          /* Information about each nest loop in WHERE */
 };
 
@@ -436,6 +445,9 @@ struct WhereInfo {
 ** where.c:
 */
 Bitmask sqlite3WhereGetMask(WhereMaskSet*,int);
+#ifdef WHERETRACE_ENABLED
+void sqlite3WhereClausePrint(WhereClause *pWC);
+#endif
 WhereTerm *sqlite3WhereFindTerm(
   WhereClause *pWC,     /* The WHERE clause to be searched */
   int iCur,             /* Cursor number of LHS */
@@ -492,6 +504,14 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*);
 ** operators that are of interest to the query planner.  An
 ** OR-ed combination of these values can be used when searching for
 ** particular WhereTerms within a WhereClause.
+**
+** Value constraints:
+**     WO_EQ    == SQLITE_INDEX_CONSTRAINT_EQ
+**     WO_LT    == SQLITE_INDEX_CONSTRAINT_LT
+**     WO_LE    == SQLITE_INDEX_CONSTRAINT_LE
+**     WO_GT    == SQLITE_INDEX_CONSTRAINT_GT
+**     WO_GE    == SQLITE_INDEX_CONSTRAINT_GE
+**     WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH
 */
 #define WO_IN     0x0001
 #define WO_EQ     0x0002
diff --git a/src/wherecode.c b/src/wherecode.c
index 3705aab51d..4fd5e16fac 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -21,6 +21,17 @@
 #include "whereInt.h"
 
 #ifndef SQLITE_OMIT_EXPLAIN
+
+/*
+** Return the name of the i-th column of the pIdx index.
+*/
+static const char *explainIndexColumnName(Index *pIdx, int i){
+  i = pIdx->aiColumn[i];
+  if( i==XN_EXPR ) return "";
+  if( i==XN_ROWID ) return "rowid";
+  return pIdx->pTable->aCol[i].zName;
+}
+
 /*
 ** This routine is a helper for explainIndexRange() below
 **
@@ -31,24 +42,32 @@
 */
 static void explainAppendTerm(
   StrAccum *pStr,             /* The text expression being built */
-  int iTerm,                  /* Index of this term.  First is zero */
-  const char *zColumn,        /* Name of the column */
+  Index *pIdx,                /* Index to read column names from */
+  int nTerm,                  /* Number of terms */
+  int iTerm,                  /* Zero-based index of first term. */
+  int bAnd,                   /* Non-zero to append " AND " */
   const char *zOp             /* Name of the operator */
 ){
-  if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5);
-  sqlite3StrAccumAppendAll(pStr, zColumn);
-  sqlite3StrAccumAppend(pStr, zOp, 1);
-  sqlite3StrAccumAppend(pStr, "?", 1);
-}
+  int i;
 
-/*
-** Return the name of the i-th column of the pIdx index.
-*/
-static const char *explainIndexColumnName(Index *pIdx, int i){
-  i = pIdx->aiColumn[i];
-  if( i==XN_EXPR ) return "";
-  if( i==XN_ROWID ) return "rowid";
-  return pIdx->pTable->aCol[i].zName;
+  assert( nTerm>=1 );
+  if( bAnd ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+
+  if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1);
+  for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1);
+
+  sqlite3StrAccumAppend(pStr, zOp, 1);
+
+  if( nTerm>1 ) sqlite3StrAccumAppend(pStr, "(", 1);
+  for(i=0; i1 ) sqlite3StrAccumAppend(pStr, ")", 1);
 }
 
 /*
@@ -81,12 +100,11 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){
 
   j = i;
   if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
-    const char *z = explainIndexColumnName(pIndex, i);
-    explainAppendTerm(pStr, i++, z, ">");
+    explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">");
+    i = 1;
   }
   if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
-    const char *z = explainIndexColumnName(pIndex, j);
-    explainAppendTerm(pStr, i, z, "<");
+    explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<");
   }
   sqlite3StrAccumAppend(pStr, ")", 1);
 }
@@ -126,7 +144,7 @@ int sqlite3WhereExplainOneScan(
 
     pLoop = pLevel->pWLoop;
     flags = pLoop->wsFlags;
-    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0;
+    if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0;
 
     isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0
             || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
@@ -276,7 +294,7 @@ void sqlite3WhereAddScanStatus(
 */
 static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){
   int nLoop = 0;
-  while( pTerm
+  while( ALWAYS(pTerm!=0)
       && (pTerm->wtFlags & TERM_CODED)==0
       && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin))
       && (pLevel->notReady & pTerm->prereqAll)==0
@@ -332,16 +350,45 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){
   }
 }
 
+/*
+** Expression pRight, which is the RHS of a comparison operation, is 
+** either a vector of n elements or, if n==1, a scalar expression.
+** Before the comparison operation, affinity zAff is to be applied
+** to the pRight values. This function modifies characters within the
+** affinity string to SQLITE_AFF_BLOB if either:
+**
+**   * the comparison will be performed with no affinity, or
+**   * the affinity change in zAff is guaranteed not to change the value.
+*/
+static void updateRangeAffinityStr(
+  Expr *pRight,                   /* RHS of comparison */
+  int n,                          /* Number of vector elements in comparison */
+  char *zAff                      /* Affinity string to modify */
+){
+  int i;
+  for(i=0; ipVdbe;
   int iReg;                  /* Register holding results */
 
+  assert( pLevel->pWLoop->aLTerm[iEq]==pTerm );
   assert( iTarget>0 );
   if( pX->op==TK_EQ || pX->op==TK_IS ){
     iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget);
@@ -364,10 +412,13 @@ static int codeEqualityTerm(
     sqlite3VdbeAddOp2(v, OP_Null, 0, iReg);
 #ifndef SQLITE_OMIT_SUBQUERY
   }else{
-    int eType;
+    int eType = IN_INDEX_NOOP;
     int iTab;
     struct InLoop *pIn;
     WhereLoop *pLoop = pLevel->pWLoop;
+    int i;
+    int nEq = 0;
+    int *aiMap = 0;
 
     if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0
       && pLoop->u.btree.pIndex!=0
@@ -379,7 +430,78 @@ static int codeEqualityTerm(
     }
     assert( pX->op==TK_IN );
     iReg = iTarget;
-    eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0);
+
+    for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){
+        disableTerm(pLevel, pTerm);
+        return iTarget;
+      }
+    }
+    for(i=iEq;inLTerm; i++){
+      if( ALWAYS(pLoop->aLTerm[i]) && pLoop->aLTerm[i]->pExpr==pX ) nEq++;
+    }
+
+    if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){
+      eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0);
+    }else{
+      Select *pSelect = pX->x.pSelect;
+      sqlite3 *db = pParse->db;
+      u16 savedDbOptFlags = db->dbOptFlags;
+      ExprList *pOrigRhs = pSelect->pEList;
+      ExprList *pOrigLhs = pX->pLeft->x.pList;
+      ExprList *pRhs = 0;         /* New Select.pEList for RHS */
+      ExprList *pLhs = 0;         /* New pX->pLeft vector */
+
+      for(i=iEq;inLTerm; i++){
+        if( pLoop->aLTerm[i]->pExpr==pX ){
+          int iField = pLoop->aLTerm[i]->iField - 1;
+          Expr *pNewRhs = sqlite3ExprDup(db, pOrigRhs->a[iField].pExpr, 0);
+          Expr *pNewLhs = sqlite3ExprDup(db, pOrigLhs->a[iField].pExpr, 0);
+
+          pRhs = sqlite3ExprListAppend(pParse, pRhs, pNewRhs);
+          pLhs = sqlite3ExprListAppend(pParse, pLhs, pNewLhs);
+        }
+      }
+      if( !db->mallocFailed ){
+        Expr *pLeft = pX->pLeft;
+
+        if( pSelect->pOrderBy ){
+          /* If the SELECT statement has an ORDER BY clause, zero the 
+          ** iOrderByCol variables. These are set to non-zero when an 
+          ** ORDER BY term exactly matches one of the terms of the 
+          ** result-set. Since the result-set of the SELECT statement may
+          ** have been modified or reordered, these variables are no longer 
+          ** set correctly.  Since setting them is just an optimization, 
+          ** it's easiest just to zero them here.  */
+          ExprList *pOrderBy = pSelect->pOrderBy;
+          for(i=0; inExpr; i++){
+            pOrderBy->a[i].u.x.iOrderByCol = 0;
+          }
+        }
+
+        /* Take care here not to generate a TK_VECTOR containing only a
+        ** single value. Since the parser never creates such a vector, some
+        ** of the subroutines do not handle this case.  */
+        if( pLhs->nExpr==1 ){
+          pX->pLeft = pLhs->a[0].pExpr;
+        }else{
+          pLeft->x.pList = pLhs;
+          aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int) * nEq);
+          testcase( aiMap==0 );
+        }
+        pSelect->pEList = pRhs;
+        db->dbOptFlags |= SQLITE_QueryFlattener;
+        eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap);
+        db->dbOptFlags = savedDbOptFlags;
+        testcase( aiMap!=0 && aiMap[0]!=0 );
+        pSelect->pEList = pOrigRhs;
+        pLeft->x.pList = pOrigLhs;
+        pX->pLeft = pLeft;
+      }
+      sqlite3ExprListDelete(pParse->db, pLhs);
+      sqlite3ExprListDelete(pParse->db, pRhs);
+    }
+
     if( eType==IN_INDEX_INDEX_DESC ){
       testcase( bRev );
       bRev = !bRev;
@@ -389,28 +511,45 @@ static int codeEqualityTerm(
     VdbeCoverageIf(v, bRev);
     VdbeCoverageIf(v, !bRev);
     assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 );
+
     pLoop->wsFlags |= WHERE_IN_ABLE;
     if( pLevel->u.in.nIn==0 ){
       pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
     }
-    pLevel->u.in.nIn++;
+
+    i = pLevel->u.in.nIn;
+    pLevel->u.in.nIn += nEq;
     pLevel->u.in.aInLoop =
        sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop,
                               sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn);
     pIn = pLevel->u.in.aInLoop;
     if( pIn ){
-      pIn += pLevel->u.in.nIn - 1;
-      pIn->iCur = iTab;
-      if( eType==IN_INDEX_ROWID ){
-        pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg);
-      }else{
-        pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg);
+      int iMap = 0;               /* Index in aiMap[] */
+      pIn += i;
+      for(i=iEq;inLTerm; i++){
+        if( pLoop->aLTerm[i]->pExpr==pX ){
+          int iOut = iReg + i - iEq;
+          if( eType==IN_INDEX_ROWID ){
+            testcase( nEq>1 );  /* Happens with a UNIQUE index on ROWID */
+            pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut);
+          }else{
+            int iCol = aiMap ? aiMap[iMap++] : 0;
+            pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut);
+          }
+          sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v);
+          if( i==iEq ){
+            pIn->iCur = iTab;
+            pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen;
+          }else{
+            pIn->eEndLoopOp = OP_Noop;
+          }
+          pIn++;
+        }
       }
-      pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen;
-      sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v);
     }else{
       pLevel->u.in.nIn = 0;
     }
+    sqlite3DbFree(pParse->db, aiMap);
 #endif
   }
   disableTerm(pLevel, pTerm);
@@ -536,9 +675,15 @@ static int codeAllEqualityTerms(
         sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j);
       }
     }
-    testcase( pTerm->eOperator & WO_ISNULL );
-    testcase( pTerm->eOperator & WO_IN );
-    if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){
+    if( pTerm->eOperator & WO_IN ){
+      if( pTerm->pExpr->flags & EP_xIsSelect ){
+        /* No affinity ever needs to be (or should be) applied to a value
+        ** from the RHS of an "? IN (SELECT ...)" expression. The 
+        ** sqlite3FindInIndex() routine has already ensured that the 
+        ** affinity of the comparison has been applied to the value.  */
+        if( zAff ) zAff[j] = SQLITE_AFF_BLOB;
+      }
+    }else if( (pTerm->eOperator & WO_ISNULL)==0 ){
       Expr *pRight = pTerm->pExpr->pRight;
       if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){
         sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
@@ -560,15 +705,16 @@ static int codeAllEqualityTerms(
 
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
 /*
-** If the most recently coded instruction is a constant range contraint
-** that originated from the LIKE optimization, then change the P3 to be
-** pLoop->iLikeRepCntr and set P5.
+** If the most recently coded instruction is a constant range constraint
+** (a string literal) that originated from the LIKE optimization, then 
+** set P3 and P5 on the OP_String opcode so that the string will be cast
+** to a BLOB at appropriate times.
 **
 ** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range
 ** expression: "x>='ABC' AND x<'abd'".  But this requires that the range
 ** scan loop run twice, once for strings and a second time for BLOBs.
 ** The OP_String opcodes on the second pass convert the upper and lower
-** bound string contants to blobs.  This routine makes the necessary changes
+** bound string constants to blobs.  This routine makes the necessary changes
 ** to the OP_String opcodes for that to happen.
 **
 ** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then
@@ -587,8 +733,8 @@ static void whereLikeOptimizationStringFixup(
     assert( pOp!=0 );
     assert( pOp->opcode==OP_String8 
             || pTerm->pWC->pWInfo->pParse->db->mallocFailed );
-    pOp->p3 = pLevel->iLikeRepCntr;
-    pOp->p5 = 1;
+    pOp->p3 = (int)(pLevel->iLikeRepCntr>>1);  /* Register holding counter */
+    pOp->p5 = (u8)(pLevel->iLikeRepCntr&1);    /* ASC or DESC */
   }
 }
 #else
@@ -625,6 +771,38 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 
+/*
+** Test whether or not expression pExpr, which was part of a WHERE clause,
+** should be included in the cursor-hint for a table that is on the rhs
+** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the 
+** expression is not suitable.
+**
+** An expression is unsuitable if it might evaluate to non NULL even if
+** a TK_COLUMN node that does affect the value of the expression is set
+** to NULL. For example:
+**
+**   col IS NULL
+**   col IS NOT NULL
+**   coalesce(col, 1)
+**   CASE WHEN col THEN 0 ELSE 1 END
+*/
+static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_IS 
+   || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT 
+   || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE 
+  ){
+    pWalker->eCode = 1;
+  }else if( pExpr->op==TK_FUNCTION ){
+    int d1;
+    char d2[3];
+    if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){
+      pWalker->eCode = 1;
+    }
+  }
+
+  return WRC_Continue;
+}
+
 
 /*
 ** This function is called on every node of an expression tree used as an
@@ -677,6 +855,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
 ** Insert an OP_CursorHint instruction if it is appropriate to do so.
 */
 static void codeCursorHint(
+  struct SrcList_item *pTabItem,  /* FROM clause item */
   WhereInfo *pWInfo,    /* The where clause */
   WhereLevel *pLevel,   /* Which loop to provide hints for */
   WhereTerm *pEndRange  /* Hint this end-of-scan boundary term if not NULL */
@@ -707,7 +886,42 @@ static void codeCursorHint(
     pTerm = &pWC->a[i];
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( pTerm->prereqAll & pLevel->notReady ) continue;
-    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+
+    /* Any terms specified as part of the ON(...) clause for any LEFT 
+    ** JOIN for which the current table is not the rhs are omitted
+    ** from the cursor-hint. 
+    **
+    ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms 
+    ** that were specified as part of the WHERE clause must be excluded.
+    ** This is to address the following:
+    **
+    **   SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL;
+    **
+    ** Say there is a single row in t2 that matches (t1.a=t2.b), but its
+    ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is 
+    ** pushed down to the cursor, this row is filtered out, causing
+    ** SQLite to synthesize a row of NULL values. Which does match the
+    ** WHERE clause, and so the query returns a row. Which is incorrect.
+    **
+    ** For the same reason, WHERE terms such as:
+    **
+    **   WHERE 1 = (t2.c IS NULL)
+    **
+    ** are also excluded. See codeCursorHintIsOrFunction() for details.
+    */
+    if( pTabItem->fg.jointype & JT_LEFT ){
+      Expr *pExpr = pTerm->pExpr;
+      if( !ExprHasProperty(pExpr, EP_FromJoin) 
+       || pExpr->iRightJoinTable!=pTabItem->iCursor
+      ){
+        sWalker.eCode = 0;
+        sWalker.xExprCallback = codeCursorHintIsOrFunction;
+        sqlite3WalkExpr(&sWalker, pTerm->pExpr);
+        if( sWalker.eCode ) continue;
+      }
+    }else{
+      if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue;
+    }
 
     /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize
     ** the cursor.  These terms are not needed as hints for a pure range
@@ -741,7 +955,7 @@ static void codeCursorHint(
   }
 }
 #else
-# define codeCursorHint(A,B,C)  /* No-op */
+# define codeCursorHint(A,B,C,D)  /* No-op */
 #endif /* SQLITE_ENABLE_CURSOR_HINTS */
 
 /*
@@ -775,7 +989,7 @@ static void codeDeferredSeek(
   assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 );
   
   sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur);
-  if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)
+  if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)
    && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask)
   ){
     int i;
@@ -792,6 +1006,39 @@ static void codeDeferredSeek(
   }
 }
 
+/*
+** If the expression passed as the second argument is a vector, generate
+** code to write the first nReg elements of the vector into an array
+** of registers starting with iReg.
+**
+** If the expression is not a vector, then nReg must be passed 1. In
+** this case, generate code to evaluate the expression and leave the
+** result in register iReg.
+*/
+static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){
+  assert( nReg>0 );
+  if( sqlite3ExprIsVector(p) ){
+#ifndef SQLITE_OMIT_SUBQUERY
+    if( (p->flags & EP_xIsSelect) ){
+      Vdbe *v = pParse->pVdbe;
+      int iSelect = sqlite3CodeSubselect(pParse, p, 0, 0);
+      sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1);
+    }else
+#endif
+    {
+      int i;
+      ExprList *pList = p->x.pList;
+      assert( nReg<=pList->nExpr );
+      for(i=0; ia[i].pExpr, iReg+i);
+      }
+    }
+  }else{
+    assert( nReg==1 );
+    sqlite3ExprCode(pParse, p, iReg);
+  }
+}
+
 /*
 ** Generate code for the start of the iLevel-th loop in the WHERE clause
 ** implementation described by pWInfo.
@@ -815,6 +1062,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   Vdbe *v;                        /* The prepared stmt under constructions */
   struct SrcList_item *pTabItem;  /* FROM clause term being coded */
   int addrBrk;                    /* Jump here to break out of the loop */
+  int addrHalt;                   /* addrBrk for the outermost loop */
   int addrCont;                   /* Jump here to continue with next cycle */
   int iRowidReg = 0;        /* Rowid is stored in this register, if not zero */
   int iReleaseReg = 0;      /* Temp register to free before returning */
@@ -830,7 +1078,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur);
   bRev = (pWInfo->revMask>>iLevel)&1;
   omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 
-           && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0;
+           && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0;
   VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName));
 
   /* Create labels for the "break" and "continue" instructions
@@ -856,6 +1104,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     VdbeComment((v, "init LEFT JOIN no-match flag"));
   }
 
+  /* Compute a safe address to jump to if we discover that the table for
+  ** this loop is empty and can never contribute content. */
+  for(j=iLevel; j>0 && pWInfo->a[j].iLeftJoin==0; j--){}
+  addrHalt = pWInfo->a[j].addrBrk;
+
   /* Special case of a FROM clause subquery implemented as a co-routine */
   if( pTabItem->fg.viaCoroutine ){
     int regYield = pTabItem->regReturn;
@@ -887,14 +1140,15 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget);
         addrNotFound = pLevel->addrNxt;
       }else{
-        sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget);
+        Expr *pRight = pTerm->pExpr->pRight;
+        codeExprOrVector(pParse, pRight, iTarget, 1);
       }
     }
     sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg);
     sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1);
     sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg,
                       pLoop->u.vtab.idxStr,
-                      pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC);
+                      pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC);
     VdbeCoverage(v);
     pLoop->u.vtab.needFree = 0;
     pLevel->p1 = iCur;
@@ -927,7 +1181,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
 
         /* Generate code that will continue to the next row if 
         ** the IN constraint is not satisfied */
-        pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0);
+        pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0);
         assert( pCompare!=0 || db->mallocFailed );
         if( pCompare ){
           pCompare->pLeft = pTerm->pExpr->pLeft;
@@ -941,7 +1195,13 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         }
       }
     }
-    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+    /* These registers need to be preserved in case there is an IN operator
+    ** loop.  So we could deallocate the registers here (and potentially
+    ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0.  But it seems
+    ** simpler and safer to simply not reuse the registers.
+    **
+    **    sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
+    */
     sqlite3ExprCachePop(pParse);
   }else
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -964,8 +1224,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg);
     if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg);
     addrNxt = pLevel->addrNxt;
-    sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v);
-    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg);
+    sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg);
     VdbeCoverage(v);
     sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1);
     sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
@@ -992,10 +1251,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pStart = pEnd;
       pEnd = pTerm;
     }
-    codeCursorHint(pWInfo, pLevel, pEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pEnd);
     if( pStart ){
       Expr *pX;             /* The expression that defines the start bound */
       int r1, rTemp;        /* Registers for holding the start boundary */
+      int op;               /* Cursor seek operation */
 
       /* The following constant maps TK_xx codes into corresponding 
       ** seek opcodes.  It depends on a particular ordering of TK_xx
@@ -1015,8 +1275,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       pX = pStart->pExpr;
       assert( pX!=0 );
       testcase( pStart->leftCursor!=iCur ); /* transitive constraints */
-      r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
-      sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1);
+      if( sqlite3ExprIsVector(pX->pRight) ){
+        r1 = rTemp = sqlite3GetTempReg(pParse);
+        codeExprOrVector(pParse, pX->pRight, r1, 1);
+        op = aMoveOp[(pX->op - TK_GT) | 0x0001];
+      }else{
+        r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp);
+        disableTerm(pLevel, pStart);
+        op = aMoveOp[(pX->op - TK_GT)];
+      }
+      sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1);
       VdbeComment((v, "pk"));
       VdbeCoverageIf(v, pX->op==TK_GT);
       VdbeCoverageIf(v, pX->op==TK_LE);
@@ -1024,9 +1292,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       VdbeCoverageIf(v, pX->op==TK_GE);
       sqlite3ExprCacheAffinityChange(pParse, r1, 1);
       sqlite3ReleaseTempReg(pParse, rTemp);
-      disableTerm(pLevel, pStart);
     }else{
-      sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk);
+      sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt);
       VdbeCoverageIf(v, bRev==0);
       VdbeCoverageIf(v, bRev!=0);
     }
@@ -1038,13 +1305,17 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */
       testcase( pEnd->wtFlags & TERM_VIRTUAL );
       memEndValue = ++pParse->nMem;
-      sqlite3ExprCode(pParse, pX->pRight, memEndValue);
-      if( pX->op==TK_LT || pX->op==TK_GT ){
+      codeExprOrVector(pParse, pX->pRight, memEndValue, 1);
+      if( 0==sqlite3ExprIsVector(pX->pRight) 
+       && (pX->op==TK_LT || pX->op==TK_GT) 
+      ){
         testOp = bRev ? OP_Le : OP_Ge;
       }else{
         testOp = bRev ? OP_Lt : OP_Gt;
       }
-      disableTerm(pLevel, pEnd);
+      if( 0==sqlite3ExprIsVector(pX->pRight) ){
+        disableTerm(pLevel, pEnd);
+      }
     }
     start = sqlite3VdbeCurrentAddr(v);
     pLevel->op = bRev ? OP_Prev : OP_Next;
@@ -1111,6 +1382,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       OP_IdxLT,            /* 3: (end_constraints &&  bRev &&  endEq) */
     };
     u16 nEq = pLoop->u.btree.nEq;     /* Number of == or IN terms */
+    u16 nBtm = pLoop->u.btree.nBtm;   /* Length of BTM vector */
+    u16 nTop = pLoop->u.btree.nTop;   /* Length of TOP vector */
     int regBase;                 /* Base register holding constraint values */
     WhereTerm *pRangeStart = 0;  /* Inequality constraint at range start */
     WhereTerm *pRangeEnd = 0;    /* Inequality constraint at range end */
@@ -1123,7 +1396,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     int nExtraReg = 0;           /* Number of extra registers needed */
     int op;                      /* Instruction opcode */
     char *zStartAff;             /* Affinity for start of range constraint */
-    char cEndAff = 0;            /* Affinity for end of range constraint */
+    char *zEndAff = 0;           /* Affinity for end of range constraint */
     u8 bSeekPastNull = 0;        /* True to seek past initial nulls */
     u8 bStopAtNull = 0;          /* Add condition to terminate at NULLs */
 
@@ -1157,33 +1430,36 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     j = nEq;
     if( pLoop->wsFlags & WHERE_BTM_LIMIT ){
       pRangeStart = pLoop->aLTerm[j++];
-      nExtraReg = 1;
+      nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm);
       /* Like optimization range constraints always occur in pairs */
       assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || 
               (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 );
     }
     if( pLoop->wsFlags & WHERE_TOP_LIMIT ){
       pRangeEnd = pLoop->aLTerm[j++];
-      nExtraReg = 1;
+      nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop);
 #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS
       if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){
         assert( pRangeStart!=0 );                     /* LIKE opt constraints */
         assert( pRangeStart->wtFlags & TERM_LIKEOPT );   /* occur in pairs */
-        pLevel->iLikeRepCntr = ++pParse->nMem;
-        testcase( bRev );
-        testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC );
-        sqlite3VdbeAddOp2(v, OP_Integer,
-                          bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC),
-                          pLevel->iLikeRepCntr);
+        pLevel->iLikeRepCntr = (u32)++pParse->nMem;
+        sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr);
         VdbeComment((v, "LIKE loop counter"));
         pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v);
+        /* iLikeRepCntr actually stores 2x the counter register number.  The
+        ** bottom bit indicates whether the search order is ASC or DESC. */
+        testcase( bRev );
+        testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC );
+        assert( (bRev & ~1)==0 );
+        pLevel->iLikeRepCntr <<=1;
+        pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC);
       }
 #endif
-      if( pRangeStart==0
-       && (j = pIdx->aiColumn[nEq])>=0 
-       && pIdx->pTable->aCol[j].notNull==0
-      ){
-        bSeekPastNull = 1;
+      if( pRangeStart==0 ){
+        j = pIdx->aiColumn[nEq];
+        if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){
+          bSeekPastNull = 1;
+        }
       }
     }
     assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 );
@@ -1197,16 +1473,19 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ){
       SWAP(WhereTerm *, pRangeEnd, pRangeStart);
       SWAP(u8, bSeekPastNull, bStopAtNull);
+      SWAP(u8, nBtm, nTop);
     }
 
     /* Generate code to evaluate all constraint terms using == or IN
     ** and store the values of those terms in an array of registers
     ** starting at regBase.
     */
-    codeCursorHint(pWInfo, pLevel, pRangeEnd);
+    codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd);
     regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff);
     assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq );
-    if( zStartAff ) cEndAff = zStartAff[nEq];
+    if( zStartAff && nTop ){
+      zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]);
+    }
     addrNxt = pLevel->addrNxt;
 
     testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 );
@@ -1221,7 +1500,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     nConstraint = nEq;
     if( pRangeStart ){
       Expr *pRight = pRangeStart->pExpr->pRight;
-      sqlite3ExprCode(pParse, pRight, regBase+nEq);
+      codeExprOrVector(pParse, pRight, regBase+nEq, nBtm);
       whereLikeOptimizationStringFixup(v, pLevel, pRangeStart);
       if( (pRangeStart->wtFlags & TERM_VNULL)==0
        && sqlite3ExprCanBeNull(pRight)
@@ -1230,18 +1509,16 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         VdbeCoverage(v);
       }
       if( zStartAff ){
-        if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_BLOB){
-          /* Since the comparison is to be performed with no conversions
-          ** applied to the operands, set the affinity to apply to pRight to 
-          ** SQLITE_AFF_BLOB.  */
-          zStartAff[nEq] = SQLITE_AFF_BLOB;
-        }
-        if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){
-          zStartAff[nEq] = SQLITE_AFF_BLOB;
-        }
+        updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]);
       }  
-      nConstraint++;
+      nConstraint += nBtm;
       testcase( pRangeStart->wtFlags & TERM_VIRTUAL );
+      if( sqlite3ExprIsVector(pRight)==0 ){
+        disableTerm(pLevel, pRangeStart);
+      }else{
+        startEq = 1;
+      }
+      bSeekPastNull = 0;
     }else if( bSeekPastNull ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
       nConstraint++;
@@ -1273,7 +1550,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pRangeEnd ){
       Expr *pRight = pRangeEnd->pExpr->pRight;
       sqlite3ExprCacheRemove(pParse, regBase+nEq, 1);
-      sqlite3ExprCode(pParse, pRight, regBase+nEq);
+      codeExprOrVector(pParse, pRight, regBase+nEq, nTop);
       whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd);
       if( (pRangeEnd->wtFlags & TERM_VNULL)==0
        && sqlite3ExprCanBeNull(pRight)
@@ -1281,19 +1558,27 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
         VdbeCoverage(v);
       }
-      if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_BLOB
-       && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff)
-      ){
-        codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff);
+      if( zEndAff ){
+        updateRangeAffinityStr(pRight, nTop, zEndAff);
+        codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff);
+      }else{
+        assert( pParse->db->mallocFailed );
       }
-      nConstraint++;
+      nConstraint += nTop;
       testcase( pRangeEnd->wtFlags & TERM_VIRTUAL );
+
+      if( sqlite3ExprIsVector(pRight)==0 ){
+        disableTerm(pLevel, pRangeEnd);
+      }else{
+        endEq = 1;
+      }
     }else if( bStopAtNull ){
       sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
       endEq = 0;
       nConstraint++;
     }
     sqlite3DbFree(db, zStartAff);
+    sqlite3DbFree(db, zEndAff);
 
     /* Top of the loop body */
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
@@ -1309,12 +1594,13 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     }
 
     /* Seek the table cursor, if required */
-    disableTerm(pLevel, pRangeStart);
-    disableTerm(pLevel, pRangeEnd);
     if( omitTable ){
       /* pIdx is a covering index.  No need to access the main table. */
     }else if( HasRowid(pIdx->pTable) ){
-      if( pWInfo->eOnePass!=ONEPASS_OFF ){
+      if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE) || (
+          (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE) 
+       && (pWInfo->eOnePass==ONEPASS_SINGLE)
+      )){
         iRowidReg = ++pParse->nMem;
         sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
         sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
@@ -1334,9 +1620,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
                            iRowidReg, pPk->nKeyCol); VdbeCoverage(v);
     }
 
-    /* Record the instruction used to terminate the loop. Disable 
-    ** WHERE clause terms made redundant by the index range scan.
-    */
+    /* Record the instruction used to terminate the loop. */
     if( pLoop->wsFlags & WHERE_ONEROW ){
       pLevel->op = OP_Noop;
     }else if( bRev ){
@@ -1413,7 +1697,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     u16 wctrlFlags;                    /* Flags for sub-WHERE clause */
     Expr *pAndExpr = 0;                /* An ".. AND (...)" expression */
     Table *pTab = pTabItem->pTab;
-   
+
     pTerm = pLoop->aLTerm[0];
     assert( pTerm!=0 );
     assert( pTerm->eOperator & WO_OR );
@@ -1499,7 +1783,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
         pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr);
       }
       if( pAndExpr ){
-        pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0);
+        pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr);
       }
     }
 
@@ -1507,10 +1791,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     ** eliminating duplicates from other WHERE clauses, the action for each
     ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
     */
-    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE
-                | WHERE_FORCE_TABLE
-                | WHERE_ONETABLE_ONLY
-                | WHERE_NO_AUTOINDEX;
+    wctrlFlags =  WHERE_OR_SUBCLAUSE | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE);
     for(ii=0; iinTerm; ii++){
       WhereTerm *pOrTerm = &pOrWc->a[ii];
       if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
@@ -1575,7 +1856,8 @@ Bitmask sqlite3WhereCodeOneLoopStart(
               }
               if( iSet>=0 ){
                 sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid);
-                sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0);
+                sqlite3VdbeAddOp4Int(v, OP_IdxInsert, regRowset, regRowid,
+                                     r, nPk);
                 if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
               }
 
@@ -1618,7 +1900,6 @@ Bitmask sqlite3WhereCodeOneLoopStart(
           ){
             assert( pSubWInfo->a[0].iIdxCur==iCovCur );
             pCov = pSubLoop->u.btree.pIndex;
-            wctrlFlags |= WHERE_REOPEN_IDX;
           }else{
             pCov = 0;
           }
@@ -1655,10 +1936,10 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       ** a pseudo-cursor.  No need to Rewind or Next such cursors. */
       pLevel->op = OP_Noop;
     }else{
-      codeCursorHint(pWInfo, pLevel, 0);
+      codeCursorHint(pTabItem, pWInfo, pLevel, 0);
       pLevel->op = aStep[bRev];
       pLevel->p1 = iCur;
-      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
+      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt);
       VdbeCoverageIf(v, bRev==0);
       VdbeCoverageIf(v, bRev!=0);
       pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
@@ -1680,7 +1961,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( (pTerm->prereqAll & pLevel->notReady)!=0 ){
       testcase( pWInfo->untestedTerms==0
-               && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 );
+               && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 );
       pWInfo->untestedTerms = 1;
       continue;
     }
@@ -1690,11 +1971,17 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       continue;
     }
     if( pTerm->wtFlags & TERM_LIKECOND ){
+      /* If the TERM_LIKECOND flag is set, that means that the range search
+      ** is sufficient to guarantee that the LIKE operator is true, so we
+      ** can skip the call to the like(A,B) function.  But this only works
+      ** for strings.  So do not skip the call to the function on the pass
+      ** that compares BLOBs. */
 #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS
       continue;
 #else
-      assert( pLevel->iLikeRepCntr>0 );
-      skipLikeAddr = sqlite3VdbeAddOp1(v, OP_IfNot, pLevel->iLikeRepCntr);
+      u32 x = pLevel->iLikeRepCntr;
+      assert( x>0 );
+      skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1));
       VdbeCoverage(v);
 #endif
     }
@@ -1712,7 +1999,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
   ** the implied "t1.a=123" constraint.
   */
   for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
-    Expr *pE, *pEAlt;
+    Expr *pE, sEAlt;
     WhereTerm *pAlt;
     if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue;
     if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue;
@@ -1730,13 +2017,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
     testcase( pAlt->eOperator & WO_IS );
     testcase( pAlt->eOperator & WO_IN );
     VdbeModuleComment((v, "begin transitive constraint"));
-    pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt));
-    if( pEAlt ){
-      *pEAlt = *pAlt->pExpr;
-      pEAlt->pLeft = pE->pLeft;
-      sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL);
-      sqlite3StackFree(db, pEAlt);
-    }
+    sEAlt = *pAlt->pExpr;
+    sEAlt.pLeft = pE->pLeft;
+    sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL);
   }
 
   /* For a LEFT OUTER JOIN, generate code that will record the fact that
diff --git a/src/whereexpr.c b/src/whereexpr.c
index 0ad1f6a0dc..248b5349db 100644
--- a/src/whereexpr.c
+++ b/src/whereexpr.c
@@ -77,7 +77,6 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){
       sqlite3DbFree(db, pOld);
     }
     pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]);
-    memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm));
   }
   pTerm = &pWC->a[idx = pWC->nTerm++];
   if( p && ExprHasProperty(p, EP_Unlikely) ){
@@ -89,13 +88,15 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){
   pTerm->wtFlags = wtFlags;
   pTerm->pWC = pWC;
   pTerm->iParent = -1;
+  memset(&pTerm->eOperator, 0,
+         sizeof(WhereTerm) - offsetof(WhereTerm,eOperator));
   return idx;
 }
 
 /*
 ** Return TRUE if the given operator is one of the operators that is
 ** allowed for an indexable WHERE clause term.  The allowed operators are
-** "=", "<", ">", "<=", ">=", "IN", and "IS NULL"
+** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL"
 */
 static int allowedOp(int op){
   assert( TK_GT>TK_EQ && TK_GTx.pList;
   pLeft = pList->a[1].pExpr;
-  if( pLeft->op!=TK_COLUMN 
-   || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
-   || IsVirtual(pLeft->pTab)  /* Value might be numeric */
-  ){
-    /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must
-    ** be the name of an indexed column with TEXT affinity. */
-    return 0;
-  }
-  assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */
 
   pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr);
   op = pRight->op;
@@ -237,6 +229,23 @@ static int isLikeOrGlob(
     z = pRight->u.zToken;
   }
   if( z ){
+
+    /* If the RHS begins with a digit or a minus sign, then the LHS must
+    ** be an ordinary column (not a virtual table column) with TEXT affinity.
+    ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false
+    ** even though "lhs LIKE rhs" is true.  But if the RHS does not start
+    ** with a digit or '-', then "lhs LIKE rhs" will always be false if
+    ** the LHS is numeric and so the optimization still works.
+    */
+    if( sqlite3Isdigit(z[0]) || z[0]=='-' ){
+      if( pLeft->op!=TK_COLUMN 
+       || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT 
+       || IsVirtual(pLeft->pTab)  /* Value might be numeric */
+      ){
+        sqlite3ValueFree(pVal);
+        return 0;
+      }
+    }
     cnt = 0;
     while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){
       cnt++;
@@ -290,7 +299,7 @@ static int isMatchOfColumn(
   Expr *pExpr,                    /* Test this expression */
   unsigned char *peOp2            /* OUT: 0 for MATCH, or else an op2 value */
 ){
-  struct Op2 {
+  static const struct Op2 {
     const char *zOp;
     unsigned char eOp2;
   } aOp[] = {
@@ -568,7 +577,9 @@ static void exprAnalyzeOrTerm(
         if( !db->mallocFailed ){
           for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){
             assert( pAndTerm->pExpr );
-            if( allowedOp(pAndTerm->pExpr->op) ){
+            if( allowedOp(pAndTerm->pExpr->op) 
+             || pAndTerm->eOperator==WO_MATCH 
+            ){
               b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor);
             }
           }
@@ -731,7 +742,7 @@ static void exprAnalyzeOrTerm(
       }
       assert( pLeft!=0 );
       pDup = sqlite3ExprDup(db, pLeft, 0);
-      pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0);
+      pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0);
       if( pNew ){
         int idxNew;
         transferJoinMarkings(pNew, pExpr);
@@ -783,12 +794,10 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){
   pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight);
   if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft);
-  /* Since pLeft and pRight are both a column references, their collating
-  ** sequence should always be defined. */
-  zColl1 = ALWAYS(pColl) ? pColl->zName : 0;
+  zColl1 = pColl ? pColl->zName : 0;
   pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight);
-  zColl2 = ALWAYS(pColl) ? pColl->zName : 0;
-  return sqlite3StrICmp(zColl1, zColl2)==0;
+  zColl2 = pColl ? pColl->zName : 0;
+  return sqlite3_stricmp(zColl1, zColl2)==0;
 }
 
 /*
@@ -823,7 +832,8 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 ** in any index.  Return TRUE (1) if pExpr is an indexed term and return
 ** FALSE (0) if not.  If TRUE is returned, also set *piCur to the cursor
 ** number of the table that is indexed and *piColumn to the column number
-** of the column that is indexed, or -2 if an expression is being indexed.
+** of the column that is indexed, or XN_EXPR (-2) if an expression is being
+** indexed.
 **
 ** If pExpr is a TK_COLUMN column reference, then this routine always returns
 ** true even if that particular column is not indexed, because the column
@@ -831,6 +841,7 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){
 */
 static int exprMightBeIndexed(
   SrcList *pFrom,        /* The FROM clause */
+  int op,                /* The specific comparison operator */
   Bitmask mPrereq,       /* Bitmask of FROM clause terms referenced by pExpr */
   Expr *pExpr,           /* An operand of a comparison operator */
   int *piCur,            /* Write the referenced table cursor number here */
@@ -839,6 +850,17 @@ static int exprMightBeIndexed(
   Index *pIdx;
   int i;
   int iCur;
+
+  /* If this expression is a vector to the left or right of a 
+  ** inequality constraint (>, <, >= or <=), perform the processing 
+  ** on the first element of the vector.  */
+  assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE );
+  assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){
+    pExpr = pExpr->x.pList->a[0].pExpr;
+  }
+
   if( pExpr->op==TK_COLUMN ){
     *piCur = pExpr->iTable;
     *piColumn = pExpr->iColumn;
@@ -851,10 +873,10 @@ static int exprMightBeIndexed(
   for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){
     if( pIdx->aColExpr==0 ) continue;
     for(i=0; inKeyCol; i++){
-      if( pIdx->aiColumn[i]!=(-2) ) continue;
-      if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
+      if( pIdx->aiColumn[i]!=XN_EXPR ) continue;
+      if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){
         *piCur = iCur;
-        *piColumn = -2;
+        *piColumn = XN_EXPR;
         return 1;
       }
     }
@@ -899,6 +921,7 @@ static void exprAnalyze(
   Parse *pParse = pWInfo->pParse;  /* Parsing context */
   sqlite3 *db = pParse->db;        /* Database connection */
   unsigned char eOp2;              /* op2 value for LIKE/REGEXP/GLOB */
+  int nLeft;                       /* Number of elements on left side vector */
 
   if( db->mallocFailed ){
     return;
@@ -911,6 +934,7 @@ static void exprAnalyze(
   op = pExpr->op;
   if( op==TK_IN ){
     assert( pExpr->pRight==0 );
+    if( sqlite3ExprCheckIN(pParse, pExpr) ) return;
     if( ExprHasProperty(pExpr, EP_xIsSelect) ){
       pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect);
     }else{
@@ -927,6 +951,10 @@ static void exprAnalyze(
     prereqAll |= x;
     extraRight = x-1;  /* ON clause terms may not be used with an index
                        ** on left table of a LEFT JOIN.  Ticket #3015 */
+    if( (prereqAll>>1)>=x ){
+      sqlite3ErrorMsg(pParse, "ON clause references tables to its right");
+      return;
+    }
   }
   pTerm->prereqAll = prereqAll;
   pTerm->leftCursor = -1;
@@ -937,18 +965,26 @@ static void exprAnalyze(
     Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft);
     Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight);
     u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV;
-    if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){
+
+    if( pTerm->iField>0 ){
+      assert( op==TK_IN );
+      assert( pLeft->op==TK_VECTOR );
+      pLeft = pLeft->x.pList->a[pTerm->iField-1].pExpr;
+    }
+
+    if( exprMightBeIndexed(pSrc, op, prereqLeft, pLeft, &iCur, &iColumn) ){
       pTerm->leftCursor = iCur;
       pTerm->u.leftColumn = iColumn;
       pTerm->eOperator = operatorMask(op) & opMask;
     }
     if( op==TK_IS ) pTerm->wtFlags |= TERM_IS;
     if( pRight 
-     && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn)
+     && exprMightBeIndexed(pSrc, op, pTerm->prereqRight, pRight, &iCur,&iColumn)
     ){
       WhereTerm *pNew;
       Expr *pDup;
       u16 eExtraOp = 0;        /* Extra bits for pNew->eOperator */
+      assert( pTerm->iField==0 );
       if( pTerm->leftCursor>=0 ){
         int idxNew;
         pDup = sqlite3ExprDup(db, pExpr, 0);
@@ -1009,7 +1045,7 @@ static void exprAnalyze(
       int idxNew;
       pNewExpr = sqlite3PExpr(pParse, ops[i], 
                              sqlite3ExprDup(db, pExpr->pLeft, 0),
-                             sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0);
+                             sqlite3ExprDup(db, pList->a[i].pExpr, 0));
       transferJoinMarkings(pNewExpr, pExpr);
       idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
       testcase( idxNew==0 );
@@ -1094,7 +1130,7 @@ static void exprAnalyze(
     pNewExpr1 = sqlite3ExprDup(db, pLeft, 0);
     pNewExpr1 = sqlite3PExpr(pParse, TK_GE,
            sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName),
-           pStr1, 0);
+           pStr1);
     transferJoinMarkings(pNewExpr1, pExpr);
     idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags);
     testcase( idxNew1==0 );
@@ -1102,7 +1138,7 @@ static void exprAnalyze(
     pNewExpr2 = sqlite3ExprDup(db, pLeft, 0);
     pNewExpr2 = sqlite3PExpr(pParse, TK_LT,
            sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName),
-           pStr2, 0);
+           pStr2);
     transferJoinMarkings(pNewExpr2, pExpr);
     idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags);
     testcase( idxNew2==0 );
@@ -1122,7 +1158,7 @@ static void exprAnalyze(
   ** virtual tables.  The native query optimizer does not attempt
   ** to do anything with MATCH functions.
   */
-  if( isMatchOfColumn(pExpr, &eOp2) ){
+  if( pWC->op==TK_AND && isMatchOfColumn(pExpr, &eOp2) ){
     int idxNew;
     Expr *pRight, *pLeft;
     WhereTerm *pNewTerm;
@@ -1135,7 +1171,7 @@ static void exprAnalyze(
     if( (prereqExpr & prereqColumn)==0 ){
       Expr *pNewExpr;
       pNewExpr = sqlite3PExpr(pParse, TK_MATCH, 
-                              0, sqlite3ExprDup(db, pRight, 0), 0);
+                              0, sqlite3ExprDup(db, pRight, 0));
       idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC);
       testcase( idxNew==0 );
       pNewTerm = &pWC->a[idxNew];
@@ -1152,6 +1188,59 @@ static void exprAnalyze(
   }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+  /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create
+  ** new terms for each component comparison - "a = ?" and "b = ?".  The
+  ** new terms completely replace the original vector comparison, which is
+  ** no longer used.
+  **
+  ** This is only required if at least one side of the comparison operation
+  ** is not a sub-select.  */
+  if( pWC->op==TK_AND 
+  && (pExpr->op==TK_EQ || pExpr->op==TK_IS)
+  && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1
+  && sqlite3ExprVectorSize(pExpr->pRight)==nLeft
+  && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 
+    || (pExpr->pRight->flags & EP_xIsSelect)==0)
+  ){
+    int i;
+    for(i=0; ipLeft, i);
+      Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i);
+
+      pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight);
+      transferJoinMarkings(pNew, pExpr);
+      idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC);
+      exprAnalyze(pSrc, pWC, idxNew);
+    }
+    pTerm = &pWC->a[idxTerm];
+    pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL;  /* Disable the original */
+    pTerm->eOperator = 0;
+  }
+
+  /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create
+  ** a virtual term for each vector component. The expression object
+  ** used by each such virtual term is pExpr (the full vector IN(...) 
+  ** expression). The WhereTerm.iField variable identifies the index within
+  ** the vector on the LHS that the virtual term represents.
+  **
+  ** This only works if the RHS is a simple SELECT, not a compound
+  */
+  if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0
+   && pExpr->pLeft->op==TK_VECTOR
+   && pExpr->x.pSelect->pPrior==0
+  ){
+    int i;
+    for(i=0; ipLeft); i++){
+      int idxNew;
+      idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL);
+      pWC->a[idxNew].iField = i+1;
+      exprAnalyze(pSrc, pWC, idxNew);
+      markTermAsChild(pWC, idxNew, idxTerm);
+    }
+  }
+
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   /* When sqlite_stat3 histogram data is available an operator of the
   ** form "x IS NOT NULL" can sometimes be evaluated more efficiently
@@ -1172,7 +1261,7 @@ static void exprAnalyze(
 
     pNewExpr = sqlite3PExpr(pParse, TK_GT,
                             sqlite3ExprDup(db, pLeft, 0),
-                            sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0);
+                            sqlite3ExprAlloc(db, TK_NULL, 0, 0));
 
     idxNew = whereClauseInsert(pWC, pNewExpr,
                               TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
@@ -1193,6 +1282,8 @@ static void exprAnalyze(
   /* Prevent ON clause terms of a LEFT JOIN from being used to drive
   ** an index for tables to the left of the join.
   */
+  testcase( pTerm!=&pWC->a[idxTerm] );
+  pTerm = &pWC->a[idxTerm];
   pTerm->prereqRight |= extraRight;
 }
 
@@ -1275,17 +1366,18 @@ void sqlite3WhereClauseClear(WhereClause *pWC){
 ** tree.
 */
 Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){
-  Bitmask mask = 0;
+  Bitmask mask;
   if( p==0 ) return 0;
   if( p->op==TK_COLUMN ){
     mask = sqlite3WhereGetMask(pMaskSet, p->iTable);
     return mask;
   }
-  mask = sqlite3WhereExprUsage(pMaskSet, p->pRight);
-  mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft);
+  assert( !ExprHasProperty(p, EP_TokenOnly) );
+  mask = p->pRight ? sqlite3WhereExprUsage(pMaskSet, p->pRight) : 0;
+  if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft);
   if( ExprHasProperty(p, EP_xIsSelect) ){
     mask |= exprSelectUsage(pMaskSet, p->x.pSelect);
-  }else{
+  }else if( p->x.pList ){
     mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList);
   }
   return mask;
@@ -1349,13 +1441,13 @@ void sqlite3WhereTabFuncArgs(
                       pTab->zName, j);
       return;
     }
-    pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0);
+    pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0);
     if( pColRef==0 ) return;
     pColRef->iTable = pItem->iCursor;
     pColRef->iColumn = k++;
     pColRef->pTab = pTab;
     pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef,
-                         sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0);
+                         sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0));
     whereClauseInsert(pWC, pTerm, TERM_DYNAMIC);
   }
 }
diff --git a/test/affinity3.test b/test/affinity3.test
new file mode 100644
index 0000000000..a335618a74
--- /dev/null
+++ b/test/affinity3.test
@@ -0,0 +1,91 @@
+# 2017-01-16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for bugs:
+#
+#    https://www.sqlite.org/src/info/91e2e8ba6ff2e2
+#    https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Ticket https://www.sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19)
+# Automatic index causes undesired type conversions
+#
+do_execsql_test affinity3-100 {
+  CREATE TABLE customer (id INT PRIMARY KEY);
+  CREATE TABLE apr (id INT PRIMARY KEY, apr REAL);
+  
+  CREATE VIEW v1 AS
+  SELECT c.id, i.apr
+  FROM customer c
+  LEFT JOIN apr i ON i.id=c.id;
+  
+  CREATE VIEW v2 AS
+  SELECT c.id, v1.apr
+  FROM customer c
+  LEFT JOIN v1 ON v1.id=c.id;
+  
+  INSERT INTO customer (id) VALUES (1);
+  INSERT INTO apr (id, apr) VALUES (1, 12);
+  INSERT INTO customer (id) VALUES (2);
+  INSERT INTO apr (id, apr) VALUES (2, 12.01);
+}
+do_execsql_test affinity3-110 {
+  PRAGMA automatic_index=ON;
+  SELECT id, (apr / 100), typeof(apr) apr_type  FROM v1;
+} {1 0.12 real 2 0.1201 real}
+do_execsql_test affinity3-120 {
+  SELECT id, (apr / 100), typeof(apr) apr_type  FROM v2;
+} {1 0.12 real 2 0.1201 real}
+do_execsql_test affinity3-130 {
+  PRAGMA automatic_index=OFF;
+  SELECT id, (apr / 100), typeof(apr) apr_type  FROM v1;
+} {1 0.12 real 2 0.1201 real}
+do_execsql_test affinity3-140 {
+  SELECT id, (apr / 100), typeof(apr) apr_type  FROM v2;
+} {1 0.12 real 2 0.1201 real}
+
+# Ticket https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf  (2017-01-16)
+# Incorrect affinity when using automatic indexes 
+#
+do_execsql_test affinity3-200 {
+  CREATE TABLE map_integer (id INT, name);
+  INSERT INTO map_integer VALUES(1,'a');
+  CREATE TABLE map_text (id TEXT, name);
+  INSERT INTO map_text VALUES('4','e');
+  CREATE TABLE data (id TEXT, name);
+  INSERT INTO data VALUES(1,'abc');
+  INSERT INTO data VALUES('4','xyz');
+  CREATE VIEW idmap as
+      SELECT * FROM map_integer
+      UNION SELECT * FROM map_text;
+  CREATE TABLE mzed AS SELECT * FROM idmap;
+}
+
+do_execsql_test affinity3-210 {
+  PRAGMA automatic_index=ON;
+  SELECT * FROM data JOIN idmap USING(id);
+} {1 abc a 4 xyz e}
+do_execsql_test affinity3-220 {
+  SELECT * FROM data JOIN mzed USING(id);
+} {1 abc a 4 xyz e}
+
+do_execsql_test affinity3-250 {
+  PRAGMA automatic_index=OFF;
+  SELECT * FROM data JOIN idmap USING(id);
+} {1 abc a 4 xyz e}
+do_execsql_test affinity3-260 {
+  SELECT * FROM data JOIN mzed USING(id);
+} {1 abc a 4 xyz e}
+
+finish_test
diff --git a/test/alter.test b/test/alter.test
index ebfe97a764..76c18cedc9 100644
--- a/test/alter.test
+++ b/test/alter.test
@@ -77,7 +77,7 @@ do_test alter-1.2 {
   ifcapable tempdb {
     execsql {
       INSERT INTO objlist SELECT type, name, tbl_name 
-          FROM sqlite_temp_master WHERE NAME!='objlist';
+          FROM temp.sqlite_master WHERE NAME!='objlist';
     }
   }
 
@@ -153,7 +153,7 @@ ifcapable tempdb {
       CREATE TEMP TABLE objlist(type, name, tbl_name);
       INSERT INTO objlist SELECT type, name, tbl_name FROM sqlite_master;
       INSERT INTO objlist 
-          SELECT type, name, tbl_name FROM sqlite_temp_master 
+          SELECT type, name, tbl_name FROM temp.sqlite_master 
           WHERE NAME!='objlist';
       SELECT type, name, tbl_name FROM objlist 
           ORDER BY tbl_name, type desc, name;
@@ -524,7 +524,7 @@ do_test alter-3.3.7 {
 ifcapable tempdb {
   do_test alter-3.3.8 {
     execsql {
-      SELECT * FROM sqlite_temp_master WHERE type = 'trigger';
+      SELECT * FROM temp.sqlite_master WHERE type = 'trigger';
     }
   } {}
 }
diff --git a/test/alter3.test b/test/alter3.test
index f8ebe056f2..44b31c9833 100644
--- a/test/alter3.test
+++ b/test/alter3.test
@@ -184,7 +184,7 @@ do_test alter3-3.2 {
 if {!$has_codec} {
   do_test alter3-3.3 {
     get_file_format
-  } {4}
+  } {3}
 }
 ifcapable schema_version {
   do_test alter3-3.4 {
@@ -220,7 +220,7 @@ do_test alter3-4.2 {
 if {!$has_codec} {
   do_test alter3-4.3 {
     get_file_format
-  } {4}
+  } {3}
 }
 ifcapable schema_version {
   do_test alter3-4.4 {
@@ -270,7 +270,7 @@ ifcapable attach {
   if {!$has_codec} {
     do_test alter3-5.5 {
       list [get_file_format test2.db] [get_file_format]
-    } {4 4}
+    } {3 3}
   }
   do_test alter3-5.6 {
     execsql {
@@ -347,19 +347,19 @@ if {!$has_codec} {
         ALTER TABLE abc ADD d DEFAULT NULL;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.3 {
       execsql {
         ALTER TABLE abc ADD e DEFAULT 10;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.4 {
       execsql {
         ALTER TABLE abc ADD f DEFAULT NULL;
       }
       get_file_format
-    } {4}
+    } {3}
     do_test alter3-7.5 {
       execsql {
         VACUUM;
diff --git a/test/alter4.test b/test/alter4.test
index ac39d614a5..5155110cf5 100644
--- a/test/alter4.test
+++ b/test/alter4.test
@@ -45,18 +45,33 @@ do_test alter4-1.1 {
     SELECT sql FROM sqlite_temp_master;
   }
 } {{CREATE TABLE abc(a, b, c)}}
+do_test alter4-1.1b {
+  execsql {
+    SELECT sql FROM temp.sqlite_master;
+  }
+} {{CREATE TABLE abc(a, b, c)}}
 do_test alter4-1.2 {
   execsql {ALTER TABLE abc ADD d INTEGER;}
   execsql {
     SELECT sql FROM sqlite_temp_master;
   }
 } {{CREATE TABLE abc(a, b, c, d INTEGER)}}
+do_test alter4-1.2b {
+  execsql {
+    SELECT sql FROM temp.sqlite_master;
+  }
+} {{CREATE TABLE abc(a, b, c, d INTEGER)}}
 do_test alter4-1.3 {
   execsql {ALTER TABLE abc ADD e}
   execsql {
     SELECT sql FROM sqlite_temp_master;
   }
 } {{CREATE TABLE abc(a, b, c, d INTEGER, e)}}
+do_test alter4-1.3b {
+  execsql {
+    SELECT sql FROM temp.sqlite_master;
+  }
+} {{CREATE TABLE abc(a, b, c, d INTEGER, e)}}
 do_test alter4-1.4 {
   execsql {
     CREATE TABLE temp.t1(a, b);
@@ -64,6 +79,11 @@ do_test alter4-1.4 {
     SELECT sql FROM sqlite_temp_master WHERE tbl_name = 't1';
   }
 } {{CREATE TABLE t1(a, b, c)}}
+do_test alter4-1.4b {
+  execsql {
+    SELECT sql FROM temp.sqlite_master WHERE tbl_name = 't1';
+  }
+} {{CREATE TABLE t1(a, b, c)}}
 do_test alter4-1.5 {
   execsql {
     ALTER TABLE t1 ADD d CHECK (a>d);
@@ -355,4 +375,23 @@ do_execsql_test alter4-9.3 {
   SELECT typeof(c), c FROM t5;
 } {real 9.22337203685478e+18}
 
+# Confirm that doing an ALTER TABLE on a legacy format database
+# does not corrupt DESC indexes.
+#
+# Ticket https://www.sqlite.org/src/tktview/f68bf68513a1c
+#
+do_test alter4-10.1 {
+  db close
+  sqlite3 db :memory:
+  db eval {
+    PRAGMA legacy_file_format=on;
+    CREATE TABLE t1(a,b,c);
+    CREATE INDEX t1a ON t1(a DESC);
+    INSERT INTO t1 VALUES(1,2,3);
+    INSERT INTO t1 VALUES(2,3,4);
+    ALTER TABLE t1 ADD COLUMN d;
+    PRAGMA integrity_check;
+  }
+} {ok}
+
 finish_test
diff --git a/test/analyzeF.test b/test/analyzeF.test
index 3cbc5f47be..76891ddfe3 100644
--- a/test/analyzeF.test
+++ b/test/analyzeF.test
@@ -120,5 +120,31 @@ do_catchsql_test 4.4 {
   SELECT * FROM t1 WHERE x = test_zeroblob(1100000) AND y = 4;
 } {1 {string or blob too big}}
 
+# 2016-12-08: Constraints of the form "x=? AND x IS NOT NULL" were being
+# mishandled.  The sqlite3Stat4ProbeSetValue() routine was assuming that
+# valueNew() was returning a Mem object that was preset to NULL, which is
+# not the case.  The consequence was the the "x IS NOT NULL" constraint
+# was used to drive the index (via the "x>NULL" pseudo-constraint) rather
+# than the "x=?" constraint.
+#
+do_execsql_test 5.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c INT);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<10000)
+    INSERT INTO t1(a, c) SELECT x, x FROM c;
+  UPDATE t1 SET b=printf('x%02x',a/500) WHERE a>4000;
+  UPDATE t1 SET b='xyz' where a>=9998;
+  CREATE INDEX t1b ON t1(b);
+  ANALYZE;
+  SELECT count(*), b FROM t1 GROUP BY 2 ORDER BY 2;
+} {4000 {} 499 x08 500 x09 500 x0a 500 x0b 500 x0c 500 x0d 500 x0e 500 x0f 500 x10 500 x11 500 x12 498 x13 3 xyz}
+do_execsql_test 5.2 {
+  explain query plan
+  SELECT * FROM t1 WHERE b='xyz' AND b IS NOT NULL ORDER BY +a;
+  /*                  v---- Should be "=", not ">"  */
+} {/USING INDEX t1b .b=/}
+do_execsql_test 5.3 {
+  SELECT * FROM t1 WHERE b='xyz' AND b IS NOT NULL ORDER BY +a;
+} {9998 xyz 9998 9999 xyz 9999 10000 xyz 10000}
 
 finish_test
diff --git a/test/attach.test b/test/attach.test
index 31c24e61d9..54be6a41ac 100644
--- a/test/attach.test
+++ b/test/attach.test
@@ -193,7 +193,7 @@ do_test attach-1.20.2 {
 } ;# ifcapable schema_pragmas
 integrity_check attach-1.20.3
 ifcapable tempdb {
-  execsql {select * from sqlite_temp_master}
+  execsql {select * from temp.sqlite_master}
 }
 do_test attach-1.21 {
   catchsql {
diff --git a/test/attach3.test b/test/attach3.test
index 1ac10d97a4..1c8601c7b3 100644
--- a/test/attach3.test
+++ b/test/attach3.test
@@ -207,7 +207,7 @@ ifcapable tempdb {
       CREATE TEMP TRIGGER tst_trigger BEFORE INSERT ON aux.t4 BEGIN 
         SELECT 'hello world';
       END;
-      SELECT count(*) FROM sqlite_temp_master;
+      SELECT count(*) FROM temp.sqlite_master;
     }
   } {1}
   do_test attach3-9.1 {
@@ -219,7 +219,7 @@ ifcapable tempdb {
   do_test attach3-9.2 {
     execsql {
       DROP TABLE aux.t4;
-      SELECT count(*) FROM sqlite_temp_master;
+      SELECT count(*) FROM temp.sqlite_master;
     }
   } {0}
 }
diff --git a/test/auth.test b/test/auth.test
index f3c2fa79e8..0044fddebe 100644
--- a/test/auth.test
+++ b/test/auth.test
@@ -91,7 +91,7 @@ ifcapable tempdb {
     catchsql {CREATE TEMP TABLE t1(a,b,c)}
   } {1 {not authorized}}
   do_test auth-1.6 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {}
   do_test auth-1.7.1 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -148,7 +148,7 @@ ifcapable tempdb {
     catchsql {CREATE TEMP TABLE t1(a,b,c)}
   } {0 {}}
   do_test auth-1.14 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {}
   do_test auth-1.15 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -561,7 +561,7 @@ ifcapable tempdb {
     catchsql {DROP TABLE t1}
   } {0 {}}
   do_test auth-1.78 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1}
 }
 
@@ -632,7 +632,7 @@ ifcapable tempdb {
     set ::authargs
   } {v1 {} temp {}}
   do_test auth-1.90 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1}
 }
 
@@ -779,7 +779,7 @@ ifcapable tempdb {
     }
   } {1 {not authorized}}
   do_test auth-1.113 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1 v1}
   do_test auth-1.114 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -823,7 +823,7 @@ ifcapable tempdb {
     set ::authargs
   } {v1 {} temp {}}
   do_test auth-1.121 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1 v1}
   do_test auth-1.122 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -980,7 +980,7 @@ do_test auth-1.139 {
   set ::authargs
 } {r1 t1 temp {}}
 do_test auth-1.140 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1}
 do_test auth-1.141 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1016,7 +1016,7 @@ do_test auth-1.144 {
   set ::authargs
 } {r1 t1 temp {}}
 do_test auth-1.145 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1}
 do_test auth-1.146 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1052,7 +1052,7 @@ do_test auth-1.149 {
   set ::authargs
 } {r1 t1 temp {}}
 do_test auth-1.150 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1 r1}
 
 do_test auth-1.151 {
@@ -1142,7 +1142,7 @@ do_test auth-1.164 {
   catchsql {DROP TRIGGER r1}
 } {1 {not authorized}}
 do_test auth-1.165 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1 r1}
 do_test auth-1.166 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1170,7 +1170,7 @@ do_test auth-1.169 {
   catchsql {DROP TRIGGER r1}
 } {0 {}}
 do_test auth-1.170 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1 r1}
 do_test auth-1.171 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1202,7 +1202,7 @@ do_test auth-1.175 {
   set ::authargs
 } {r1 t1 temp {}}
 do_test auth-1.176 {
-  execsql {SELECT name FROM sqlite_temp_master}
+  execsql {SELECT name FROM temp.sqlite_master}
 } {t1}
 } ;# ifcapable trigger
 
@@ -1306,7 +1306,7 @@ ifcapable tempdb {
     catchsql {CREATE INDEX i1 ON t1(b)}
   } {1 {not authorized}}
   do_test auth-1.194 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1}
   do_test auth-1.195 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1350,7 +1350,7 @@ ifcapable tempdb {
     set ::authargs
   } {i1 t1 temp {}}
   do_test auth-1.202 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1 i1}
 }
 
@@ -1466,7 +1466,7 @@ ifcapable tempdb {
     catchsql {DROP INDEX i1}
   } {0 {}}
   do_test auth-1.222 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1 i1}
   do_test auth-1.223 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1482,7 +1482,7 @@ ifcapable tempdb {
     set ::authargs
   } {i1 t1 temp {}}
   do_test auth-1.225 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1 i1}
   do_test auth-1.226 {
     proc auth {code arg1 arg2 arg3 arg4 args} {
@@ -1498,7 +1498,7 @@ ifcapable tempdb {
     set ::authargs
   } {i1 t1 temp {}}
   do_test auth-1.228 {
-    execsql {SELECT name FROM sqlite_temp_master}
+    execsql {SELECT name FROM temp.sqlite_master}
   } {t1}
 }
 
@@ -1765,7 +1765,7 @@ ifcapable attach {
         }
       } {0 {}}
       do_test auth-1.267 {
-        execsql {SELECT name FROM sqlite_temp_master WHERE type='table'}
+        execsql {SELECT name FROM temp.sqlite_master WHERE type='table'}
       } {t1x}
       do_test auth-1.268 {
         set authargs
@@ -2070,7 +2070,7 @@ ifcapable {altertable} {
     }
   } {1 {not authorized}}
   do_test auth-1.307 {
-    set x [execsql {SELECT sql FROM sqlite_temp_master WHERE type='t5'}]
+    set x [execsql {SELECT sql FROM temp.sqlite_master WHERE type='t5'}]
     regexp new_col_3 $x
   } {0}
 
@@ -2373,7 +2373,7 @@ ifcapable compound&&subquery {
   do_test auth-5.2 {
     execsql {
       SELECT name FROM (
-        SELECT * FROM sqlite_master UNION ALL SELECT * FROM sqlite_temp_master)
+        SELECT * FROM sqlite_master UNION ALL SELECT * FROM temp.sqlite_master)
       WHERE type='table'
       ORDER BY name
     }
@@ -2432,8 +2432,53 @@ do_test auth-6.3 {
   execsql {SELECT rowid, * FROM t6}
 } {101 1 2 3 4 5 6 7 8}
 
+#-------------------------------------------------------------------------
+# Test that view names are included as zArg4.
+#
+do_execsql_test auth-7.1 {
+  CREATE TABLE t7(a, b, c);
+  CREATE VIEW v7 AS SELECT * FROM t7;
+} {}
+set ::authargs [list]
+proc auth {args} {
+  eval lappend ::authargs [lrange $args 0 4]
+  return SQLITE_OK
+}
+
+do_test auth-7.2 {
+  execsql {SELECT a, c FROM v7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main v7       \
+  SQLITE_READ t7 b main v7       \
+  SQLITE_READ t7 c main v7       \
+  SQLITE_READ v7 a main {}       \
+  SQLITE_READ v7 c main {}       \
+  SQLITE_SELECT {} {} {} v7      \
+]
+
+set ::authargs [list]
+do_test auth-7.3 {
+  execsql {SELECT a, c FROM t7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main {}       \
+  SQLITE_READ t7 c main {}       \
+]
+
+set ::authargs [list]
+do_test auth-7.4 {
+  execsql {SELECT a, c FROM t7 AS v7}
+  set ::authargs
+} [list                          \
+  SQLITE_SELECT {} {} {} {}      \
+  SQLITE_READ t7 a main {}       \
+  SQLITE_READ t7 c main {}       \
+]
+
+
 rename proc {}
 rename proc_real proc
-
-
 finish_test
diff --git a/test/auth2.test b/test/auth2.test
index a9d64d08af..08d46cac57 100644
--- a/test/auth2.test
+++ b/test/auth2.test
@@ -98,12 +98,6 @@ SQLITE_UPDATE sqlite_master tbl_name main {}
 SQLITE_UPDATE sqlite_master rootpage main {}
 SQLITE_UPDATE sqlite_master sql main {}
 SQLITE_READ sqlite_master ROWID main {}
-SQLITE_READ sqlite_master name main {}
-SQLITE_READ sqlite_master rootpage main {}
-SQLITE_READ sqlite_master sql main {}
-SQLITE_READ sqlite_master tbl_name main {}
-SQLITE_READ sqlite_master type main {}
-SQLITE_READ sqlite_master ROWID main {}
 }
 do_test auth2-2.2 {
   set ::authargs {}
@@ -119,12 +113,6 @@ SQLITE_UPDATE sqlite_master tbl_name main {}
 SQLITE_UPDATE sqlite_master rootpage main {}
 SQLITE_UPDATE sqlite_master sql main {}
 SQLITE_READ sqlite_master ROWID main {}
-SQLITE_READ sqlite_master name main {}
-SQLITE_READ sqlite_master rootpage main {}
-SQLITE_READ sqlite_master sql main {}
-SQLITE_READ sqlite_master tbl_name main {}
-SQLITE_READ sqlite_master type main {}
-SQLITE_READ sqlite_master ROWID main {}
 }
 do_test auth2-2.3 {
   set ::authargs {}
diff --git a/test/auth3.test b/test/auth3.test
index eef10b398f..013486e5a0 100644
--- a/test/auth3.test
+++ b/test/auth3.test
@@ -12,8 +12,7 @@
 # Test that the truncate optimization is disabled if the SQLITE_DELETE
 # authorization callback returns SQLITE_IGNORE.
 #
-# $Id: auth3.test,v 1.2 2009/05/04 01:58:31 drh Exp $
-#
+# Test that authorizer is disabled during schema parsing.
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -108,4 +107,22 @@ do_test auth3-2.2 {
   set sqlite_search_count
 } {1}
 
+# 2016-07-28.  A problem report from a private client complaining about
+# an authorizer failure during an ALTER TABLE.  The solution (I think) is
+# to disable the authorizer during schema parsing.
+#
+proc auth {code args} {
+  if {$code=="SQLITE_READ" && [regexp {DoNotRead} $args]} {
+    return SQLITE_DENY
+  }
+  return SQLITE_OK
+}
+do_execsql_test auth3-3.0 {
+  CREATE TEMPORARY TABLE TempTable (
+      key TEXT NOT NULL ON CONFLICT FAIL UNIQUE ON CONFLICT REPLACE,
+      value TEXT NOT NULL ON CONFLICT FAIL);
+  ALTER TABLE TempTable RENAME TO DoNotRead;
+  SELECT name FROM temp.sqlite_master;
+} {DoNotRead sqlite_autoindex_DoNotRead_1}
+
 finish_test
diff --git a/test/autoanalyze1.test b/test/autoanalyze1.test
new file mode 100644
index 0000000000..71e5d6c163
--- /dev/null
+++ b/test/autoanalyze1.test
@@ -0,0 +1,123 @@
+# 2017-02-17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements tests for the logic used to estimate when
+# running ANALYZE would be beneficial.
+#
+# Note that this test uses some hard-coded bitmask values from sqliteInt.h.
+# If any of the following constants changes:
+#
+#    define TF_HasStat1   0x0010
+#    define TF_StatsUsed  0x0100
+#
+# then some of the magic numbers in test results below might need to be
+# adjusted.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# There is nothing to test if ANALYZE is disable for this build.
+# These tests also use "PRAGMA stats" which are only enabled for
+# debugging builds.
+#
+ifcapable {!debug || !analyze || !vtab} {
+  finish_test
+  return
+}
+
+do_execsql_test autoanalyze1-100 {
+  -- Build up a test table with some indexes
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d);
+  CREATE UNIQUE INDEX t1bc ON t1(b,c);
+  CREATE INDEX t1d ON t1(d);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t1(a,b,c,d) SELECT x, x, x, x FROM c;
+  -- Verify that the hasStat1 flag is clear on on indexes
+  SELECT idx, flgs FROM pragma_stats
+   WHERE idx IS NOT NULL
+   ORDER BY idx;
+  -- Verify that the TF_HasStat1 flag is clear on the table
+  SELECT tbl, (flgs & 0x10)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {t1bc 0 t1d 0 t1 0}
+
+# No use of stat1 recorded so far
+do_execsql_test autoanalyze1-110 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}
+
+# Access using a unique index does not set the TF_StatsUsed flag.
+#
+do_execsql_test autoanalyze1-200 {
+  SELECT * FROM t1 WHERE a=55;
+} {55 55 55 55}
+do_execsql_test autoanalyze1-201 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}
+
+do_execsql_test autoanalyze1-210 {
+  SELECT * FROM t1 WHERE a IN (55,199,299);
+} {55 55 55 55}
+do_execsql_test autoanalyze1-211 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}
+
+do_execsql_test autoanalyze1-220 {
+  SELECT * FROM t1 WHERE (b,c)=(45,45);
+} {45 45 45 45}
+do_execsql_test autoanalyze1-221 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}
+
+# Any use of the non-unique t1d index triggers the use of stats.
+#
+sqlite3 db test.db
+do_execsql_test autoanalyze1-300 {
+  SELECT * FROM t1 WHERE d=45;
+} {45 45 45 45}
+do_execsql_test autoanalyze1-301 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {1}
+
+sqlite3 db test.db
+do_execsql_test autoanalyze1-310 {
+  SELECT * FROM t1 WHERE d=45 AND a=45;
+} {45 45 45 45}
+do_execsql_test autoanalyze1-311 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}  ;# The ROWID lookup short-circuits the d=45 constraint
+
+sqlite3 db test.db
+do_execsql_test autoanalyze1-320 {
+  SELECT * FROM t1 WHERE d=45 AND a IN (45,46);
+} {45 45 45 45}
+do_execsql_test autoanalyze1-321 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {1}
+
+# Any use of prefix of a unique index triggers the use of stats
+#
+sqlite3 db test.db
+do_execsql_test autoanalyze1-400 {
+  SELECT * FROM t1 WHERE b=45;
+} {45 45 45 45}
+do_execsql_test autoanalyze1-401 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {1}
+
+# The TF_StatsUsed flag is reset when the database is reopened
+#
+sqlite3 db test.db
+do_execsql_test autoanalyze1-500 {
+  SELECT (flgs & 0x0100)!=0 FROM pragma_stats WHERE tbl='t1' AND idx IS NULL;
+} {0}
+
+finish_test
diff --git a/test/autoinc.test b/test/autoinc.test
index 239600616b..8ac12b1772 100644
--- a/test/autoinc.test
+++ b/test/autoinc.test
@@ -344,7 +344,7 @@ ifcapable tempdb {
   do_test autoinc-4.1 {
     execsql {
       SELECT 1, name FROM sqlite_master WHERE type='table';
-      SELECT 2, name FROM sqlite_temp_master WHERE type='table';
+      SELECT 2, name FROM temp.sqlite_master WHERE type='table';
     }
   } {1 sqlite_sequence}
   do_test autoinc-4.2 {
@@ -663,6 +663,18 @@ ifcapable trigger {
   } {1 124 2 10123}
 }
 
+# 2016-10-03 ticket https://www.sqlite.org/src/tktview/7b3328086a5c1
+# Make sure autoincrement plays nicely with the xfer optimization
+#
+do_execsql_test autoinc-10.1 {
+  DELETE FROM sqlite_sequence;
+  CREATE TABLE t10a(a INTEGER PRIMARY KEY AUTOINCREMENT, b UNIQUE);
+  INSERT INTO t10a VALUES(888,9999);
+  CREATE TABLE t10b(x INTEGER PRIMARY KEY AUTOINCREMENT, y UNIQUE);
+  INSERT INTO t10b SELECT * FROM t10a;
+  SELECT * FROM sqlite_sequence;
+} {t10a 888 t10b 888}
+
 
 
 finish_test
diff --git a/test/backup.test b/test/backup.test
index 3b1e1db9e5..829b32452d 100644
--- a/test/backup.test
+++ b/test/backup.test
@@ -164,7 +164,7 @@ foreach zOpenScript [list {
   set file_dest temp
 }] {
 foreach rows_dest {0 3 10} {
-foreach pgsz_dest {512 1024 2048} {
+foreach pgsz_dest {512 1024 2048 4096} {
 foreach nPagePerStep {1 200} {
 
   # Open the databases.
@@ -176,17 +176,16 @@ foreach nPagePerStep {1 200} {
   # in-memory destination is only possible if the initial destination
   # page size is the same as the source page size (in this case 1024 bytes).
   #
-  set isMemDest [expr {
-    $zDestFile eq ":memory:" || $file_dest eq "temp" && $TEMP_STORE>=2
-  }]
+  set isMemDest [expr { $zDestFile eq ":memory:" || $file_dest eq "temp" }]
 
-  if { $isMemDest==0 || $pgsz_dest == 1024 } {
-    if 0 {
-      puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile"
-      puts -nonewline " (as $db_dest.$file_dest)"
-      puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest"
-      puts ""
-    }
+  if 0 {
+    puts -nonewline "Test $iTest: src=$zSrcFile dest=$zDestFile"
+    puts -nonewline " (as $db_dest.$file_dest)"
+    puts -nonewline " rows_dest=$rows_dest pgsz_dest=$pgsz_dest"
+    puts ""
+  }
+
+  if { $isMemDest==0 || $pgsz_dest==1024 || $rows_dest==0 } {
 
     # Set up the content of the source database.
     execsql {
diff --git a/test/backup_malloc.test b/test/backup_malloc.test
index f556861fb1..5444a70bdf 100644
--- a/test/backup_malloc.test
+++ b/test/backup_malloc.test
@@ -84,4 +84,36 @@ do_malloc_test backup_malloc-2 -tclprep {
   db2 close
 }
 
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA page_size = 16384;
+  BEGIN;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep {
+  catch { db close }
+  catch { db2 close }
+
+  forcedelete test2.db
+  sqlite3 db2 test2.db
+  sqlite3 db test.db
+  sqlite3_backup B db2 main db main
+} -body {
+
+  set rc [B step 50]
+  if {$rc == "SQLITE_NOMEM" || $rc == "SQLITE_IOERR_NOMEM"} {
+    error "out of memory"
+  }
+
+} -test {
+  faultsim_test_result {0 {}} 
+  faultsim_integrity_check
+  
+  # Finalize the backup.
+  catch { B finish }
+}
+
 finish_test
diff --git a/test/bestindex1.test b/test/bestindex1.test
index 97ae4b0f81..f90f96bf44 100644
--- a/test/bestindex1.test
+++ b/test/bestindex1.test
@@ -165,4 +165,111 @@ foreach {tn mode} {
   } $plan($mode)
 }
 
+# 2016-04-09.
+# Demonstrate a register overwrite problem when using two virtual
+# tables where the outer loop uses the IN operator.
+#
+set G(collist) [list PrimaryKey flagA columnA]
+set G(cols) [join $G(collist) ,]
+set G(nulls) "NULL"
+
+proc vtab_command {method args} {
+  global G
+
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1($G(cols))"
+    }
+
+    xBestIndex {
+      set clist [lindex $args 0]
+      #puts $clist
+      set W [list]
+      set U [list]
+
+      set i 0
+      for {set idx 0} {$idx < [llength $clist]} {incr idx} {
+        array set c [lindex $clist $idx]
+        if {$c(op)=="eq" && $c(usable)} {
+          lappend W "[lindex $G(collist) $c(column)] = %$i%"
+          lappend U use $idx
+          incr i
+        }
+      }
+
+      if {$W==""} {
+        set sql "SELECT rowid, * FROM t1"
+      } else {
+        set sql "SELECT rowid, * FROM t1 WHERE [join $W { AND }]"
+      }
+
+      return [concat [list idxstr $sql] $U]
+    }
+
+    xFilter {
+      foreach {idxnum idxstr vals} $args {}
+
+      set map [list]
+      for {set i 0} {$i < [llength $vals]} {incr i} {
+        lappend map "%$i%" 
+        set v [lindex $vals $i]
+        if {[string is integer $v]} { 
+          lappend map $v 
+        } else {
+          lappend map "'$v'"
+        }
+      }
+      set sql [string map $map $idxstr]
+
+      #puts "SQL: $sql"
+      return [list sql $sql]
+    }
+  }
+
+  return {}
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+register_tcl_module db
+
+do_execsql_test 3.1 "
+  CREATE TABLE t1($G(cols));
+  INSERT INTO t1 VALUES(1, 0, 'ValueA');
+  INSERT INTO t1 VALUES(2, 0, 'ValueA');
+  INSERT INTO t1 VALUES(3, 0, 'ValueB');
+  INSERT INTO t1 VALUES(4, 0, 'ValueB');
+"
+
+do_execsql_test 3.2 {
+  CREATE VIRTUAL TABLE VirtualTableA USING tcl(vtab_command);
+  CREATE VIRTUAL TABLE VirtualTableB USING tcl(vtab_command);
+}
+
+do_execsql_test 3.3 { SELECT primarykey FROM VirtualTableA } {1 2 3 4}
+
+do_execsql_test 3.4 {
+  SELECT * FROM 
+  VirtualTableA a CROSS JOIN VirtualTableB b ON b.PrimaryKey=a.PrimaryKey
+  WHERE a.ColumnA IN ('ValueA', 'ValueB') AND a.FlagA=0
+} {
+  1 0 ValueA 1 0 ValueA
+  2 0 ValueA 2 0 ValueA
+  3 0 ValueB 3 0 ValueB
+  4 0 ValueB 4 0 ValueB
+}
+
+do_execsql_test 3.5 {
+  SELECT * FROM 
+  VirtualTableA a CROSS JOIN VirtualTableB b ON b.PrimaryKey=a.PrimaryKey
+  WHERE a.FlagA=0 AND a.ColumnA IN ('ValueA', 'ValueB') 
+} {
+  1 0 ValueA 1 0 ValueA
+  2 0 ValueA 2 0 ValueA
+  3 0 ValueB 3 0 ValueB
+  4 0 ValueB 4 0 ValueB
+}
+
+
 finish_test
diff --git a/test/bestindex3.test b/test/bestindex3.test
new file mode 100644
index 0000000000..70c60d66dd
--- /dev/null
+++ b/test/bestindex3.test
@@ -0,0 +1,174 @@
+# 2016 May 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix bestindex3
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Virtual table callback for a virtual table named $tbl.
+#
+# The table created is:
+#
+#      "CREATE TABLE t1 (a, b, c)"
+#
+# This virtual table supports both LIKE and = operators on all columns.
+#  
+proc vtab_cmd {bOmit method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c)"
+    }
+
+    xBestIndex {
+      foreach {clist orderby mask} $args {}
+
+      set ret [list]
+      set use use
+      if {$bOmit} {set use omit}
+
+      for {set i 0} {$i < [llength $clist]} {incr i} {
+        array unset C
+        array set C [lindex $clist $i]
+        if {$C(usable) && ($C(op)=="like" || $C(op)=="eq")} {
+          lappend ret $use $i
+          lappend ret idxstr 
+          lappend ret "[lindex {a b c} $C(column)] [string toupper $C(op)] ?"
+          break
+        }
+      }
+
+      if {$ret==""} {
+        lappend ret cost 1000000 rows 1000000
+      } else {
+        lappend ret cost 100 rows 10
+      }
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr param} $args {}
+      set where ""
+      if {$bOmit && $idxstr != ""} {
+        set where " WHERE [string map [list ? '$param' EQ =] $idxstr]"
+      }
+      return [list sql "SELECT rowid, * FROM ttt$where"]
+    }
+  }
+  return ""
+}
+
+register_tcl_module db
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING tcl("vtab_cmd 0");
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a LIKE 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a = 'abc';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a = 'abc' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a EQ ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a LIKE 'abc%' OR b = 'def';
+} {
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:a LIKE ?}
+  0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 0:b EQ ?}
+}
+
+do_execsql_test 1.5 {
+  CREATE TABLE ttt(a, b, c);
+
+  INSERT INTO ttt VALUES(1, 'two',   'three');
+  INSERT INTO ttt VALUES(2, 'one',   'two');
+  INSERT INTO ttt VALUES(3, 'three', 'one');
+  INSERT INTO ttt VALUES(4, 'y',     'one');
+  INSERT INTO ttt VALUES(5, 'x',     'two');
+  INSERT INTO ttt VALUES(6, 'y',     'three');
+}
+
+foreach omit {0 1} {
+  do_execsql_test 1.6.$omit.0 "
+    DROP TABLE t1;
+    CREATE VIRTUAL TABLE t1 USING tcl('vtab_cmd $omit');
+  "
+  do_execsql_test 1.6.$omit.1 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%'
+  } {3 4}
+
+  do_execsql_test 1.6.$omit.2 { 
+    SELECT rowid FROM t1 WHERE c LIKE 'o%' OR b='y'
+  } {3 4 6}
+
+  do_execsql_test 1.6.$omit.3 { 
+    SELECT rowid FROM t1 WHERE c = 'three' OR c LIKE 'o%'
+  } {1 6 3 4}
+}
+
+#-------------------------------------------------------------------------
+# Test the same pattern works with ordinary tables.
+#
+# This test does not work if the ICU extension is enabled. ICU overrides
+# LIKE - and this optimization only works with the built-in LIKE function.
+#
+ifcapable !icu {
+  do_execsql_test 2.1 {
+    CREATE TABLE t2(x TEXT COLLATE nocase, y TEXT);
+    CREATE INDEX t2x ON t2(x COLLATE nocase);
+    CREATE INDEX t2y ON t2(y);
+  }
+
+  do_eqp_test 2.2 {
+    SELECT * FROM t2 WHERE x LIKE 'abc%' OR y = 'def'
+  } {
+    0 0 0 {SEARCH TABLE t2 USING INDEX t2x (x>? AND x=0
+} $res
 
 finish_test
diff --git a/test/cursorhint2.test b/test/cursorhint2.test
new file mode 100644
index 0000000000..616235b376
--- /dev/null
+++ b/test/cursorhint2.test
@@ -0,0 +1,184 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The
+# focus is on testing that cursor-hints are correct for queries
+# involving LEFT JOIN.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix cursorhint2
+
+ifcapable !cursorhints {
+  finish_test
+  return
+}
+
+proc extract_hints {sql} {
+
+  db eval "SELECT tbl_name, rootpage FROM sqlite_master where rootpage" {
+    set lookup($rootpage) $tbl_name
+  }
+
+  set ret [list]
+  db eval "EXPLAIN $sql" a {
+    switch -- $a(opcode) {
+      OpenRead {
+        set csr($a(p1)) $lookup($a(p2))
+      }
+      CursorHint { 
+        lappend ret $csr($a(p1)) $a(p4) 
+      }
+    }
+  }
+
+  set ret
+}
+
+proc do_extract_hints_test {tn sql ret} {
+  uplevel [list do_test $tn [list extract_hints $sql] [list {*}$ret]]
+}
+
+do_execsql_test 1.0 {
+  PRAGMA automatic_index = 0;
+  CREATE TABLE t1(a, b);
+  CREATE TABLE t2(c, d);
+  CREATE TABLE t3(e, f);
+}
+
+do_extract_hints_test 1.1 {
+  SELECT * FROM t1 WHERE a=1;
+} {
+  t1 EQ(c0,1)
+}
+
+do_extract_hints_test 1.2 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {AND(ISNULL(c1),EQ(r[1],c0))}
+}
+
+do_extract_hints_test 1.3 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) WHERE d IS NULL;
+} {
+  t2 {EQ(r[2],c0)}
+}
+
+do_extract_hints_test 1.4 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t2 {AND(EQ(r[2],c0),EQ(r[3],10))}
+}
+
+do_extract_hints_test 1.5 {
+  SELECT * FROM t1 CROSS JOIN t2 ON (a=c AND a=10) WHERE d IS NULL;
+} {
+  t1 EQ(c0,10) t2 {AND(ISNULL(c1),EQ(r[3],c0))}
+}
+
+do_extract_hints_test 1.6 {
+  SELECT * FROM t1 LEFT JOIN t2 ON (a=c) LEFT JOIN t3 ON (d=f);
+} {
+  t2 {EQ(r[2],c0)} t3 {EQ(r[6],c1)}
+}
+
+if 0 {
+  do_extract_hints_test 1.7 {
+    SELECT * FROM t1 LEFT JOIN t2 ON (a=c AND d=e) LEFT JOIN t3 ON (d=f);
+  } {
+    t2 {EQ(r[2],c0)} t3 {AND(EQ(r[6],c0),EQ(r[7],c1))}
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+  CREATE TABLE x1(x, y);
+  CREATE TABLE x2(a, b);
+}
+
+do_extract_hints_test 2.1 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.2 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS +NULL;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.3 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = (b IS NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.4 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.5 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 1 = coalesce(b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.6 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.7 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE 0 = (b IS NOT +NULL)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.8 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE b IS NOT +NULL
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.9 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE CASE b WHEN 0 THEN 0 ELSE 1 END;
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+do_extract_hints_test 2.10 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b = 32+32
+} {
+  x2 {AND(EQ(c1,ADD(32,32)),EQ(c0,r[2]))}
+}
+
+ifcapable !icu {
+  # This test only works using the built-in LIKE, not the ICU LIKE extension.
+  do_extract_hints_test 2.11 {
+    SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE x2.b LIKE 'abc%'
+  } {
+    x2 {AND(expr,EQ(c0,r[2]))}
+  }
+}
+
+do_extract_hints_test 2.12 {
+  SELECT * FROM x1 LEFT JOIN x2 ON (a=x) WHERE coalesce(x2.b, 1)
+} {
+  x2 {EQ(c0,r[2])}
+}
+
+finish_test
diff --git a/test/date.test b/test/date.test
index 2f48b111e6..f002334d17 100644
--- a/test/date.test
+++ b/test/date.test
@@ -62,7 +62,7 @@ datetest 1.20 {julianday('2000-01-01 12:00:00.01')}  2451545.00000012
 datetest 1.21 {julianday('2000-01-01 12:00:00.001')} 2451545.00000001
 datetest 1.22 {julianday('2000-01-01 12:00:00.')} NULL
 datetest 1.23 julianday(12345.6) 12345.6
-datetest 1.23b julianday('12345.6') 12345.6
+datetest 1.23b julianday(1721059.5) 1721059.5
 datetest 1.24 {julianday('2001-01-01 12:00:00 bogus')} NULL
 datetest 1.25 {julianday('2001-01-01 bogus')} NULL
 datetest 1.26 {julianday('2001-01-01 12:60:00')} NULL
@@ -560,4 +560,53 @@ do_test date-15.2 {
   }
 } {1}
 
+# Tests of extreme values in date/time functions.  Run with UBSan or the
+# equivalent to verify no signed interger overflow warnings.
+#
+datetest 16.1 {date(147483649)} NULL
+datetest 16.2 {datetime(0)} {-4713-11-24 12:00:00}
+datetest 16.3 {datetime(5373484.49999999)} {9999-12-31 23:59:59}
+datetest 16.4 {julianday('-4713-11-24 12:00:00')} 0.0
+datetest 16.5 {julianday('9999-12-31 23:59:59.999')} 5373484.49999999
+datetest 16.6 {datetime(0,'+464269060799 seconds')} {9999-12-31 23:59:59}
+datetest 16.7 {datetime(0,'+464269060800 seconds')} NULL
+datetest 16.8 {datetime(0,'+7737817679 minutes')} {9999-12-31 23:59:00}
+datetest 16.9 {datetime(0,'+7737817680 minutes')} NULL
+datetest 16.10 {datetime(0,'+128963627 hours')} {9999-12-31 23:00:00}
+datetest 16.11 {datetime(0,'+128963628 hours')} NULL
+datetest 16.12 {datetime(0,'+5373484 days')} {9999-12-31 12:00:00}
+datetest 16.13 {datetime(0,'+5373485 days')} NULL
+datetest 16.14 {datetime(0,'+176545 months')} {9999-12-24 12:00:00}
+datetest 16.15 {datetime(0,'+176546 months')} NULL
+datetest 16.16 {datetime(0,'+14712 years')} {9999-11-24 12:00:00}
+datetest 16.17 {datetime(0,'+14713 years')} NULL
+datetest 16.20 {datetime(5373484.4999999,'-464269060799 seconds')} \
+                {-4713-11-24 12:00:00}
+datetest 16.21 {datetime(5373484,'-464269060800 seconds')} NULL
+datetest 16.22 {datetime(5373484.4999999,'-7737817679 minutes')} \
+               {-4713-11-24 12:00:59}
+datetest 16.23 {datetime(5373484,'-7737817680 minutes')} NULL
+datetest 16.24 {datetime(5373484.4999999,'-128963627 hours')} \
+               {-4713-11-24 12:59:59}
+datetest 16.25 {datetime(5373484,'-128963628 hours')} NULL
+datetest 16.26 {datetime(5373484,'-5373484 days')} {-4713-11-24 12:00:00}
+datetest 16.27 {datetime(5373484,'-5373485 days')} NULL
+datetest 16.28 {datetime(5373484,'-176545 months')} {-4713-12-01 12:00:00}
+datetest 16.29 {datetime(5373484,'-176546 months')} NULL
+datetest 16.30 {datetime(5373484,'-14712 years')} {-4713-12-31 12:00:00}
+datetest 16.31 {datetime(5373484,'-14713 years')} NULL
+
+# 2017-03-02: Wrong 'start of day' computation.
+# https://www.sqlite.org/src/info/6097cb92745327a1
+#
+datetest 17.1 {datetime(2457754, 'start of day')} {2016-12-31 00:00:00}
+datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00}
+datetest 17.3 {datetime(2457828,'start of day')} {2017-03-15 00:00:00}
+datetest 17.4 {datetime(2457828,'start of month')} {2017-03-01 00:00:00}
+datetest 17.5 {datetime(2457828,'start of year')} {2017-01-01 00:00:00}
+datetest 17.6 {datetime(37,'start of year')} NULL
+datetest 17.7 {datetime(38,'start of year')} {-4712-01-01 00:00:00}
+
+
+
 finish_test
diff --git a/test/dbfuzz.c b/test/dbfuzz.c
new file mode 100644
index 0000000000..ca1c6ea217
--- /dev/null
+++ b/test/dbfuzz.c
@@ -0,0 +1,754 @@
+/*
+** 2016-12-17
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This program is designed for fuzz-testing SQLite database files.
+**
+** This program reads fuzzed database files from the disk files named
+** on the command-line.  Each database is loaded into an in-memory
+** filesystem so that the original database file is unmolested.
+**
+** The fuzzed database is then opened, and series of SQL statements
+** are run against the database to ensure that SQLite can safely handle
+** the fuzzed database.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#define ISSPACE(X) isspace((unsigned char)(X))
+#define ISDIGIT(X) isdigit((unsigned char)(X))
+#include "sqlite3.h"
+#ifdef __unix__
+# include 
+# include 
+#endif
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(const char *zArgv0){
+  printf("Usage: %s [options] DATABASE ...\n", zArgv0);
+  printf(
+"Read databases into an in-memory filesystem.  Run test SQL as specified\n"
+"by command-line arguments or from\n"
+"\n"
+"    SELECT group_concat(sql) FROM autoexec;\n"
+"\n"
+"Options:\n"
+"  --help              Show this help text\n"
+"  -q|--quiet          Reduced output\n"
+"  --limit-mem N       Limit memory used by test SQLite instances to N bytes\n"
+"  --limit-vdbe        Panic if any test runs for more than 100,000 cycles\n"
+"  --no-lookaside      Disable the lookaside memory allocator\n"
+"  --timeout N         Timeout after N seconds.\n"
+"  --trace             Show the results of each SQL command\n"
+"  -v|--verbose        Increased output.  Repeat for more output.\n"
+  );
+  exit(0);
+}
+
+/*
+** Print an error message and quit.
+*/
+static void fatalError(const char *zFormat, ...){
+  va_list ap;
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Files in the virtual file system.
+*/
+typedef struct VFile VFile;
+typedef struct VHandle VHandle;
+struct VFile {
+  char *zFilename;      /* Filename. NULL for delete-on-close. From malloc() */
+  int sz;               /* Size of the file in bytes */
+  int nRef;             /* Number of references to this file */
+  unsigned char *a;     /* Content of the file.  From malloc() */
+};
+struct VHandle {
+  sqlite3_file base;    /* Base class.  Must be first */
+  VFile *pVFile;        /* The underlying file */
+};
+
+/*
+** Maximum number of files in the in-memory virtual filesystem.
+*/
+#define MX_FILE  10
+
+/*
+** Maximum allowed file size
+*/
+#define MX_FILE_SZ 1000000
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+static struct GlobalVars {
+  VFile aFile[MX_FILE];            /* The virtual filesystem */
+} g;
+
+
+/*
+** Initialize the virtual file system.
+*/
+static void formatVfs(void){
+  int i;
+  for(i=0; i0 ){
+      fatalError("file %d still open.  nRef=%d", i, g.aFile[i].nRef);
+    }
+    g.aFile[i].sz = -1;
+    free(g.aFile[i].a);
+    g.aFile[i].a = 0;
+    g.aFile[i].nRef = 0;
+  }
+}
+
+/*
+** Find a VFile by name
+*/
+static VFile *findVFile(const char *zName){
+  int i;
+  if( zName==0 ) return 0;
+  for(i=0; i=0; i++){}
+  if( i>=MX_FILE ) return 0;
+  if( zDiskFile ){
+    in = fopen(zDiskFile, "rb");
+    if( in==0 ) fatalError("no such file: \"%s\"", zDiskFile);
+    fseek(in, 0, SEEK_END);
+    sz = ftell(in);
+    rewind(in);
+  }
+  pNew = &g.aFile[i];
+  if( zName ){
+    int nName = (int)strlen(zName)+1;
+    pNew->zFilename = malloc(nName);
+    if( pNew->zFilename==0 ){
+      if( in ) fclose(in);
+      return 0;
+    }
+    memcpy(pNew->zFilename, zName, nName);
+  }else{
+    pNew->zFilename = 0;
+  }
+  pNew->nRef = 0;
+  pNew->sz = sz;
+  pNew->a = malloc(sz);
+  if( sz>0 ){
+    if( pNew->a==0 || fread(pNew->a, sz, 1, in)<1 ){
+      free(pNew->zFilename);
+      free(pNew->a);
+      pNew->a = 0;
+      pNew->zFilename = 0;
+      pNew->sz = -1;
+      pNew = 0;
+    }
+  }
+  if( in ) fclose(in);
+  return pNew;
+}
+
+/* Methods for the VHandle object
+*/
+static int inmemClose(sqlite3_file *pFile){
+  VHandle *p = (VHandle*)pFile;
+  VFile *pVFile = p->pVFile;
+  pVFile->nRef--;
+  if( pVFile->nRef==0 && pVFile->zFilename==0 ){
+    pVFile->sz = -1;
+    free(pVFile->a);
+    pVFile->a = 0;
+  }
+  return SQLITE_OK;
+}
+static int inmemRead(
+  sqlite3_file *pFile,   /* Read from this open file */
+  void *pData,           /* Store content in this buffer */
+  int iAmt,              /* Bytes of content */
+  sqlite3_int64 iOfst    /* Start reading here */
+){
+  VHandle *pHandle = (VHandle*)pFile;
+  VFile *pVFile = pHandle->pVFile;
+  if( iOfst<0 || iOfst>=pVFile->sz ){
+    memset(pData, 0, iAmt);
+    return SQLITE_IOERR_SHORT_READ;
+  }
+  if( iOfst+iAmt>pVFile->sz ){
+    memset(pData, 0, iAmt);
+    iAmt = (int)(pVFile->sz - iOfst);
+    memcpy(pData, pVFile->a, iAmt);
+    return SQLITE_IOERR_SHORT_READ;
+  }
+  memcpy(pData, pVFile->a + iOfst, iAmt);
+  return SQLITE_OK;
+}
+static int inmemWrite(
+  sqlite3_file *pFile,   /* Write to this file */
+  const void *pData,     /* Content to write */
+  int iAmt,              /* bytes to write */
+  sqlite3_int64 iOfst    /* Start writing here */
+){
+  VHandle *pHandle = (VHandle*)pFile;
+  VFile *pVFile = pHandle->pVFile;
+  if( iOfst+iAmt > pVFile->sz ){
+    unsigned char *aNew;
+    if( iOfst+iAmt >= MX_FILE_SZ ){
+      return SQLITE_FULL;
+    }
+    aNew = realloc(pVFile->a, (int)(iOfst+iAmt));
+    if( aNew==0 ){
+      return SQLITE_FULL;
+    }
+    pVFile->a = aNew;
+    if( iOfst > pVFile->sz ){
+      memset(pVFile->a + pVFile->sz, 0, (int)(iOfst - pVFile->sz));
+    }
+    pVFile->sz = (int)(iOfst + iAmt);
+  }
+  memcpy(pVFile->a + iOfst, pData, iAmt);
+  return SQLITE_OK;
+}
+static int inmemTruncate(sqlite3_file *pFile, sqlite3_int64 iSize){
+  VHandle *pHandle = (VHandle*)pFile;
+  VFile *pVFile = pHandle->pVFile;
+  if( pVFile->sz>iSize && iSize>=0 ) pVFile->sz = (int)iSize;
+  return SQLITE_OK;
+}
+static int inmemSync(sqlite3_file *pFile, int flags){
+  return SQLITE_OK;
+}
+static int inmemFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){
+  *pSize = ((VHandle*)pFile)->pVFile->sz;
+  return SQLITE_OK;
+}
+static int inmemLock(sqlite3_file *pFile, int type){
+  return SQLITE_OK;
+}
+static int inmemUnlock(sqlite3_file *pFile, int type){
+  return SQLITE_OK;
+}
+static int inmemCheckReservedLock(sqlite3_file *pFile, int *pOut){
+  *pOut = 0;
+  return SQLITE_OK;
+}
+static int inmemFileControl(sqlite3_file *pFile, int op, void *pArg){
+  return SQLITE_NOTFOUND;
+}
+static int inmemSectorSize(sqlite3_file *pFile){
+  return 512;
+}
+static int inmemDeviceCharacteristics(sqlite3_file *pFile){
+  return
+      SQLITE_IOCAP_SAFE_APPEND |
+      SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN |
+      SQLITE_IOCAP_POWERSAFE_OVERWRITE;
+}
+
+
+/* Method table for VHandle
+*/
+static sqlite3_io_methods VHandleMethods = {
+  /* iVersion    */  1,
+  /* xClose      */  inmemClose,
+  /* xRead       */  inmemRead,
+  /* xWrite      */  inmemWrite,
+  /* xTruncate   */  inmemTruncate,
+  /* xSync       */  inmemSync,
+  /* xFileSize   */  inmemFileSize,
+  /* xLock       */  inmemLock,
+  /* xUnlock     */  inmemUnlock,
+  /* xCheck...   */  inmemCheckReservedLock,
+  /* xFileCtrl   */  inmemFileControl,
+  /* xSectorSz   */  inmemSectorSize,
+  /* xDevchar    */  inmemDeviceCharacteristics,
+  /* xShmMap     */  0,
+  /* xShmLock    */  0,
+  /* xShmBarrier */  0,
+  /* xShmUnmap   */  0,
+  /* xFetch      */  0,
+  /* xUnfetch    */  0
+};
+
+/*
+** Open a new file in the inmem VFS.  All files are anonymous and are
+** delete-on-close.
+*/
+static int inmemOpen(
+  sqlite3_vfs *pVfs,
+  const char *zFilename,
+  sqlite3_file *pFile,
+  int openFlags,
+  int *pOutFlags
+){
+  VFile *pVFile = createVFile(zFilename, 0);
+  VHandle *pHandle = (VHandle*)pFile;
+  if( pVFile==0 ){
+    return SQLITE_FULL;
+  }
+  pHandle->pVFile = pVFile;
+  pVFile->nRef++;
+  pFile->pMethods = &VHandleMethods;
+  if( pOutFlags ) *pOutFlags = openFlags;
+  return SQLITE_OK;
+}
+
+/*
+** Delete a file by name
+*/
+static int inmemDelete(
+  sqlite3_vfs *pVfs,
+  const char *zFilename,
+  int syncdir
+){
+  VFile *pVFile = findVFile(zFilename);
+  if( pVFile==0 ) return SQLITE_OK;
+  if( pVFile->nRef==0 ){
+    free(pVFile->zFilename);
+    pVFile->zFilename = 0;
+    pVFile->sz = -1;
+    free(pVFile->a);
+    pVFile->a = 0;
+    return SQLITE_OK;
+  }
+  return SQLITE_IOERR_DELETE;
+}
+
+/* Check for the existance of a file
+*/
+static int inmemAccess(
+  sqlite3_vfs *pVfs,
+  const char *zFilename,
+  int flags,
+  int *pResOut
+){
+  VFile *pVFile = findVFile(zFilename);
+  *pResOut =  pVFile!=0;
+  return SQLITE_OK;
+}
+
+/* Get the canonical pathname for a file
+*/
+static int inmemFullPathname(
+  sqlite3_vfs *pVfs,
+  const char *zFilename,
+  int nOut,
+  char *zOut
+){
+  sqlite3_snprintf(nOut, zOut, "%s", zFilename);
+  return SQLITE_OK;
+}
+
+/*
+** Register the VFS that reads from the g.aFile[] set of files.
+*/
+static void inmemVfsRegister(void){
+  static sqlite3_vfs inmemVfs;
+  sqlite3_vfs *pDefault = sqlite3_vfs_find(0);
+  inmemVfs.iVersion = 3;
+  inmemVfs.szOsFile = sizeof(VHandle);
+  inmemVfs.mxPathname = 200;
+  inmemVfs.zName = "inmem";
+  inmemVfs.xOpen = inmemOpen;
+  inmemVfs.xDelete = inmemDelete;
+  inmemVfs.xAccess = inmemAccess;
+  inmemVfs.xFullPathname = inmemFullPathname;
+  inmemVfs.xRandomness = pDefault->xRandomness;
+  inmemVfs.xSleep = pDefault->xSleep;
+  inmemVfs.xCurrentTimeInt64 = pDefault->xCurrentTimeInt64;
+  sqlite3_vfs_register(&inmemVfs, 0);
+};
+
+/*
+** Timeout handler
+*/
+#ifdef __unix__
+static void timeoutHandler(int NotUsed){
+  (void)NotUsed;
+  fatalError("timeout\n");
+}
+#endif
+
+/*
+** Set the an alarm to go off after N seconds.  Disable the alarm
+** if N==0
+*/
+static void setAlarm(int N){
+#ifdef __unix__
+  alarm(N);
+#else
+  (void)N;
+#endif
+}
+/***************************************************************************
+** String accumulator object
+*/
+typedef struct Str Str;
+struct Str {
+  char *z;                /* The string.  Memory from malloc() */
+  sqlite3_uint64 n;       /* Bytes of input used */
+  sqlite3_uint64 nAlloc;  /* Bytes allocated to z[] */
+  int oomErr;             /* OOM error has been seen */
+};
+
+/* Initialize a Str object */
+static void StrInit(Str *p){
+  memset(p, 0, sizeof(*p));
+}
+
+/* Append text to the end of a Str object */
+static void StrAppend(Str *p, const char *z){
+  sqlite3_uint64 n = strlen(z);
+  if( p->n + n >= p->nAlloc ){
+    char *zNew;
+    sqlite3_uint64 nNew;
+    if( p->oomErr ) return;
+    nNew = p->nAlloc*2 + 100 + n;
+    zNew = sqlite3_realloc64(p->z, nNew);
+    if( zNew==0 ){
+      sqlite3_free(p->z);
+      memset(p, 0, sizeof(*p));
+      p->oomErr = 1;
+      return;
+    }
+    p->z = zNew;
+    p->nAlloc = nNew;
+  }
+  memcpy(p->z + p->n, z, (int)n);
+  p->n += n;
+  p->z[p->n] = 0;
+}
+
+/* Return the current string content */
+static char *StrStr(Str *p){
+ return p->z;
+}
+
+/* Free the string */
+static void StrFree(Str *p){
+  sqlite3_free(p->z);
+  StrInit(p);
+}
+
+/*
+** Return the value of a hexadecimal digit.  Return -1 if the input
+** is not a hex digit.
+*/
+static int hexDigitValue(char c){
+  if( c>='0' && c<='9' ) return c - '0';
+  if( c>='a' && c<='f' ) return c - 'a' + 10;
+  if( c>='A' && c<='F' ) return c - 'A' + 10;
+  return -1;
+}
+
+/*
+** Interpret zArg as an integer value, possibly with suffixes.
+*/
+static int integerValue(const char *zArg){
+  sqlite3_int64 v = 0;
+  static const struct { char *zSuffix; int iMult; } aMult[] = {
+    { "KiB", 1024 },
+    { "MiB", 1024*1024 },
+    { "GiB", 1024*1024*1024 },
+    { "KB",  1000 },
+    { "MB",  1000000 },
+    { "GB",  1000000000 },
+    { "K",   1000 },
+    { "M",   1000000 },
+    { "G",   1000000000 },
+  };
+  int i;
+  int isNeg = 0;
+  if( zArg[0]=='-' ){
+    isNeg = 1;
+    zArg++;
+  }else if( zArg[0]=='+' ){
+    zArg++;
+  }
+  if( zArg[0]=='0' && zArg[1]=='x' ){
+    int x;
+    zArg += 2;
+    while( (x = hexDigitValue(zArg[0]))>=0 ){
+      v = (v<<4) + x;
+      zArg++;
+    }
+  }else{
+    while( ISDIGIT(zArg[0]) ){
+      v = v*10 + zArg[0] - '0';
+      zArg++;
+    }
+  }
+  for(i=0; i0x7fffffff ) fatalError("parameter too large - max 2147483648");
+  return (int)(isNeg? -v : v);
+}
+
+/*
+** This callback is invoked by sqlite3_log().
+*/
+static void sqlLog(void *pNotUsed, int iErrCode, const char *zMsg){
+  printf("LOG: (%d) %s\n", iErrCode, zMsg);
+  fflush(stdout);
+}
+
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+/*
+** This an SQL progress handler.  After an SQL statement has run for
+** many steps, we want to interrupt it.  This guards against infinite
+** loops from recursive common table expressions.
+**
+** *pVdbeLimitFlag is true if the --limit-vdbe command-line option is used.
+** In that case, hitting the progress handler is a fatal error.
+*/
+static int progressHandler(void *pVdbeLimitFlag){
+  if( *(int*)pVdbeLimitFlag ) fatalError("too many VDBE cycles");
+  return 1;
+}
+#endif
+
+/*
+** Allowed values for the runFlags parameter to runSql()
+*/
+#define SQL_TRACE  0x0001     /* Print each SQL statement as it is prepared */
+#define SQL_OUTPUT 0x0002     /* Show the SQL output */
+
+/*
+** Run multiple commands of SQL.  Similar to sqlite3_exec(), but does not
+** stop if an error is encountered.
+*/
+static void runSql(sqlite3 *db, const char *zSql, unsigned  runFlags){
+  const char *zMore;
+  const char *zEnd = &zSql[strlen(zSql)];
+  sqlite3_stmt *pStmt;
+
+  while( zSql && zSql[0] ){
+    zMore = 0;
+    pStmt = 0;
+    sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zMore);
+    assert( zMore<=zEnd );
+    if( zMore==zSql ) break;
+    if( runFlags & SQL_TRACE ){
+      const char *z = zSql;
+      int n;
+      while( z0 && ISSPACE(z[n-1]) ) n--;
+      if( n==0 ) break;
+      if( pStmt==0 ){
+        printf("TRACE: %.*s (error: %s)\n", n, z, sqlite3_errmsg(db));
+      }else{
+        printf("TRACE: %.*s\n", n, z);
+      }
+    }
+    zSql = zMore;
+    if( pStmt ){
+      if( (runFlags & SQL_OUTPUT)==0 ){
+        while( SQLITE_ROW==sqlite3_step(pStmt) ){}
+      }else{
+        int nCol = -1;
+        int nRow;
+        for(nRow=0; SQLITE_ROW==sqlite3_step(pStmt); nRow++){
+          int i;
+          if( nCol<0 ){
+            nCol = sqlite3_column_count(pStmt);
+          }
+          for(i=0; i0 ){
+    void *pHeap = malloc( nHeap );
+    if( pHeap==0 ) fatalError("cannot allocate %d-byte heap\n", nHeap);
+    rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, 32);
+    if( rc ) fatalError("heap configuration failed: %d\n", rc);
+  }
+  if( noLookaside ){
+    sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);
+  }
+  inmemVfsRegister();
+  formatVfs();
+  StrInit(&sql);
+#ifdef __unix__
+  signal(SIGALRM, timeoutHandler);
+#endif
+  for(i=0; i1 ){
+      printf("DATABASE-FILE: %s\n", azDb[i]);
+      fflush(stdout);
+    }
+    if( iTimeout ) setAlarm(iTimeout);
+    createVFile("test.db", azDb[i]);
+    rc = sqlite3_open_v2("test.db", &db, SQLITE_OPEN_READWRITE, "inmem");
+    if( rc ){
+      printf("cannot open test.db for \"%s\"\n", azDb[i]);
+      reformatVfs();
+      continue;
+    }
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+    if( vdbeLimitFlag ){
+      sqlite3_progress_handler(db, 100000, progressHandler, &vdbeLimitFlag);
+    }
+#endif
+    rc = sqlite3_prepare_v2(db, "SELECT sql FROM autoexec", -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      while( SQLITE_ROW==sqlite3_step(pStmt) ){
+        StrAppend(&sql, (const char*)sqlite3_column_text(pStmt, 0));
+        StrAppend(&sql, "\n");
+      }
+    }
+    sqlite3_finalize(pStmt);
+    StrAppend(&sql, "PRAGMA integrity_check;\n");
+    runSql(db, StrStr(&sql), runFlags);
+    sqlite3_close(db);
+    reformatVfs();
+    StrFree(&sql);
+    if( sqlite3_memory_used()>0 ){
+      free(azDb);
+      reformatVfs();
+      fatalError("memory leak of %lld bytes", sqlite3_memory_used());
+    }
+  }
+  StrFree(&sql);
+  reformatVfs();
+  return 0;
+}
diff --git a/test/dbstatus.test b/test/dbstatus.test
index 368c6b28e5..711d66ebb3 100644
--- a/test/dbstatus.test
+++ b/test/dbstatus.test
@@ -14,6 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix dbstatus
 
 ifcapable !compound {
   finish_test
@@ -24,6 +25,7 @@ ifcapable !compound {
 db close
 sqlite3_shutdown
 sqlite3_config_memstatus 1
+sqlite3_config_uri 1
 sqlite3_initialize
 sqlite3 db test.db
 
@@ -67,16 +69,12 @@ ifcapable stat4||stat3 {
   set STAT3 0
 }
 
-ifcapable malloc_usable_size {
-  finish_test
-  return
-}
-
 #---------------------------------------------------------------------------
 # Run the dbstatus-2 and dbstatus-3 tests with several of different
 # lookaside buffer sizes.
 #
 foreach ::lookaside_buffer_size {0 64 120} {
+  ifcapable malloc_usable_size break
 
   # Do not run any of these tests if there is SQL configured to run
   # as part of the [sqlite3] command. This prevents the script from
@@ -377,4 +375,44 @@ foreach ::lookaside_buffer_size {0 64 120} {
   }
 }
 
+#-------------------------------------------------------------------------
+# The following tests focus on DBSTATUS_CACHE_USED_SHARED
+#
+ifcapable shared_cache {
+  if {[permutation]=="memsys3"
+      || [permutation]=="memsys5"
+      || $::tcl_platform(os)=="Linux"} {
+    proc do_cacheused_test {tn db res} {
+      set cu [sqlite3_db_status $db SQLITE_DBSTATUS_CACHE_USED 0]
+      set pcu [sqlite3_db_status $db SQLITE_DBSTATUS_CACHE_USED_SHARED 0]
+      set cu [lindex $cu 1]
+      set pcu [lindex $pcu 1]
+      uplevel [list do_test $tn [list list $cu $pcu] "#/$res/"]
+    }
+    reset_db
+    sqlite3 db file:test.db?cache=shared
+  
+    do_execsql_test 4.0 {
+      PRAGMA auto_vacuum=NONE;
+      CREATE TABLE t1(a, b, c);
+      INSERT INTO t1 VALUES(1, 2, 3);
+    }
+    do_cacheused_test 4.0.1 db { 4568 4568 }
+    do_execsql_test 4.1 {
+      CREATE TEMP TABLE tt(a, b, c);
+      INSERT INTO tt VALUES(1, 2, 3);
+    }
+    do_cacheused_test 4.1.1 db { 9000 9000 }
+  
+    sqlite3 db2 file:test.db?cache=shared
+    do_cacheused_test 4.2.1 db2 { 4568 2284 }
+    do_cacheused_test 4.2.2 db { 9000 6716 }
+    db close
+    do_cacheused_test 4.2.3 db2 { 4568 4568 }
+    sqlite3 db file:test.db?cache=shared
+    do_cacheused_test 4.2.4 db2 { 4568 2284 }
+    db2 close
+  }
+}
+
 finish_test
diff --git a/test/dbstatus2.test b/test/dbstatus2.test
index 2541a1a823..eff4b0207f 100644
--- a/test/dbstatus2.test
+++ b/test/dbstatus2.test
@@ -86,7 +86,7 @@ do_test 2.3 { db_write db 1 } {0 4 0}
 do_test 2.4 { db_write db 0 } {0 0 0}
 do_test 2.5 { db_write db 1 } {0 0 0}
 
-ifcapable wal {
+if {[wal_is_capable]} {
   do_test 2.6 { 
     execsql { PRAGMA journal_mode = WAL }
     db_write db 1
diff --git a/test/delete.test b/test/delete.test
index d2dc106495..8e84ed4665 100644
--- a/test/delete.test
+++ b/test/delete.test
@@ -389,5 +389,19 @@ do_test delete-9.5 {
   set res
 } {1 a b 1 c d 2 a b 2 c d}
 
+do_execsql_test delete-10.0 {
+  CREATE TABLE t1(a INT UNIQUE, b INT);
+  INSERT INTO t1(a,b) VALUES('1','2');
+  SELECT * FROM t1 WHERE a='1' AND b='2';
+} {1 2}
+
+do_execsql_test delete-10.1 {
+  DELETE FROM t1 WHERE a='1' AND b='2';
+}
+
+do_execsql_test delete-10.2 {
+  SELECT * FROM t1 WHERE a='1' AND b='2';
+}
+
 
 finish_test
diff --git a/test/delete4.test b/test/delete4.test
index f3598a9496..34ba9aaef8 100644
--- a/test/delete4.test
+++ b/test/delete4.test
@@ -139,7 +139,49 @@ do_execsql_test 4.12 {
   PRAGMA integrity_check;
 } {ok}
 
+# 2016-04-09
+# Ticket https://sqlite.org/src/info/a306e56ff68b8fa5
+# Failure to completely delete when reverse_unordered_selects is
+# engaged.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_execsql_test 5.0 {
+  PRAGMA page_size=1024;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX x1 ON t1(b, c);
+  INSERT INTO t1(a,b,c) VALUES(1, 1, zeroblob(80));
+  INSERT INTO t1(a,b,c) SELECT a+1, 1, c FROM t1;
+  INSERT INTO t1(a,b,c) SELECT a+2, 1, c FROM t1;
+  INSERT INTO t1(a,b,c) SELECT a+10, 2, c FROM t1 WHERE b=1;
+  INSERT INTO t1(a,b,c) SELECT a+20, 3, c FROM t1 WHERE b=1;
+  PRAGMA reverse_unordered_selects = ON;
+  DELETE FROM t1 WHERE b=2;
+  SELECT a FROM t1 WHERE b=2;
+} {}
 
+# 2016-05-02
+# Ticket https://www.sqlite.org/src/tktview/dc6ebeda93960877
+# A subquery in the WHERE clause of a one-pass DELETE can cause an
+# incorrect answer.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_execsql_test 6.0 {
+  CREATE TABLE t2(x INT);
+  INSERT INTO t2(x) VALUES(1),(2),(3),(4),(5);
+  DELETE FROM t2 WHERE EXISTS(SELECT 1 FROM t2 AS v WHERE v.x=t2.x-1);
+  SELECT x FROM t2;
+} {1}
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(x INT);
+  INSERT INTO t2(x) VALUES(1),(2),(3),(4),(5);
+  DELETE FROM t2 WHERE EXISTS(SELECT 1 FROM t2 AS v WHERE v.x=t2.x+1);
+  SELECT x FROM t2;
+} {5}
 
 
 finish_test
diff --git a/test/delete_db.test b/test/delete_db.test
new file mode 100644
index 0000000000..09c44ff9f3
--- /dev/null
+++ b/test/delete_db.test
@@ -0,0 +1,217 @@
+# 2016 September 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the code in test_delete.c (the
+# sqlite3_delete_database() API).
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix delete_db
+
+proc delete_all {} {
+  foreach f [glob -nocomplain test2*] { file delete $f }
+  foreach f [glob -nocomplain test3*] { file delete $f }
+}
+
+proc copydb {} {
+  foreach f [glob -nocomplain test3*] { file delete $f }
+  foreach f [glob -nocomplain test2*] { 
+    set p [string range $f 5 end]
+    file copy "test2$p" "test3$p"
+  }
+}
+
+proc files {} {
+  lsort [glob -nocomplain test3*]
+}
+
+db close
+delete_all
+sqlite3 db test2.database
+
+#-------------------------------------------------------------------------
+#
+# 1.1: Journal files.
+# 1.2: Wal files.
+# 1.3: Multiplexor with journal file.
+# 1.4: Multiplexor with wal file.
+#
+# 2.* are a copy of 1.* with the multiplexor enabled.
+#
+# 3.* tests errors.
+#
+
+do_test 1.1.0 {
+  execsql {
+    CREATE TABLE t1(x, y);
+    BEGIN;
+      INSERT INTO t1 VALUES(1, 2);
+  }
+  copydb
+  files
+} {test3.database test3.database-journal}
+
+do_test 1.1.1 {
+  sqlite3_delete_database test3.database
+  files
+} {}
+
+do_test 1.2.0 {
+  execsql {
+    COMMIT;
+    PRAGMA journal_mode = wal;
+    INSERT INTO t1 VALUES(3, 4);
+  }
+  copydb
+  files
+} {test3.database test3.database-shm test3.database-wal}
+do_test 1.2.1 {
+  sqlite3_delete_database test3.database
+  files
+} {}
+
+db close
+delete_all
+sqlite3_multiplex_initialize "" 0
+sqlite3 db test2.database -vfs multiplex
+sqlite3_multiplex_control db "main" chunk_size 32768
+
+do_test 1.3.0 {
+  execsql { PRAGMA auto_vacuum = 0; }
+  execsql {
+    CREATE TABLE x1(a, b);
+    WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 )
+    INSERT INTO x1 SELECT randomblob(100), randomblob(100) FROM s;
+    BEGIN;
+      UPDATE x1 SET a=randomblob(101)
+  }
+  copydb
+  files
+} [list {*}{
+  test3.database test3.database-journal test3.database001 
+  test3.database002 test3.database003
+}]
+do_test 1.3.1 {
+  sqlite3_delete_database test3.database
+  files
+} {}
+
+
+do_test 1.4.0 {
+  execsql {
+    COMMIT;
+    PRAGMA journal_mode = wal;
+    UPDATE x1 SET a=randomblob(102)
+  }
+  copydb
+  files
+} [list {*}{
+  test3.database test3.database-shm test3.database-wal test3.database001 
+  test3.database002 test3.database003
+}]
+do_test 1.4.1 {
+  sqlite3_delete_database test3.database
+  files
+} {}
+
+
+ifcapable 8_3_names {
+  db close
+  delete_all
+  sqlite3 db file:test2.db?8_3_names=1 -uri 1
+
+  do_test 2.1.0 {
+    execsql {
+      CREATE TABLE t1(x, y);
+      BEGIN;
+        INSERT INTO t1 VALUES(1, 2);
+    }
+    copydb
+    files
+  } {test3.db test3.nal}
+  
+  do_test 2.1.1 {
+    sqlite3_delete_database test3.db
+    files
+  } {}
+  
+  do_test 2.2.0 {
+    execsql {
+      COMMIT;
+      PRAGMA journal_mode = wal;
+      INSERT INTO t1 VALUES(3, 4);
+    }
+    copydb
+    files
+  } {test3.db test3.shm test3.wal}
+  do_test 2.2.1 {
+    sqlite3_delete_database test3.db
+    files
+  } {}
+
+
+  db close
+  delete_all
+  sqlite3_multiplex_initialize "" 0
+  sqlite3 db file:test2.db?8_3_names=1 -uri 1 -vfs multiplex
+  sqlite3_multiplex_control db "main" chunk_size 32768
+  
+  do_test 2.3.0 {
+    execsql { PRAGMA auto_vacuum = 0; }
+    execsql {
+      CREATE TABLE x1(a, b);
+      WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 )
+      INSERT INTO x1 SELECT randomblob(100), randomblob(100) FROM s;
+      BEGIN;
+        UPDATE x1 SET a=randomblob(101)
+    }
+    copydb
+    files
+  } [list {*}{
+    test3.001 test3.002 test3.003 test3.db test3.nal 
+  }]
+  do_test 2.3.1 {
+    sqlite3_delete_database test3.db
+    files
+  } {}
+  
+  
+  do_test 2.4.0 {
+    execsql {
+      COMMIT;
+      PRAGMA journal_mode = wal;
+      UPDATE x1 SET a=randomblob(102)
+    }
+    copydb
+    files
+  } [list {*}{
+    test3.001 test3.002 test3.003 test3.db test3.db-shm test3.wal 
+  }]
+  do_test 2.4.1 {
+    sqlite3_delete_database test3.db
+    files
+  } {}
+}
+
+db close
+delete_all
+sqlite3_multiplex_shutdown 
+
+do_test 3.0 {
+  file mkdir dir2.db
+  sqlite3_delete_database dir2.db
+} {SQLITE_ERROR}
+do_test 3.1 {
+  sqlite3_delete_database dir2.db/test.db
+} {SQLITE_OK}
+
+finish_test
diff --git a/test/e_dropview.test b/test/e_dropview.test
index 143dce2907..04c4ad8c4b 100644
--- a/test/e_dropview.test
+++ b/test/e_dropview.test
@@ -45,7 +45,7 @@ proc list_all_views {{db db}} {
   set res [list]
   $db eval { PRAGMA database_list } {
     set tbl "$name.sqlite_master"
-    if {$name == "temp"} { set tbl sqlite_temp_master }
+    if {$name == "temp"} { set tbl temp.sqlite_master }
 
     set sql "SELECT '$name.' || name FROM $tbl WHERE type = 'view'"
     lappend res {*}[$db eval $sql]
diff --git a/test/e_expr.test b/test/e_expr.test
index 8c0957f8d3..de7b43c92e 100644
--- a/test/e_expr.test
+++ b/test/e_expr.test
@@ -847,6 +847,9 @@ foreach {tn x expr res nEval} {
 
   3   5  "x() >= 5 AND x() <= 5"   1  2
   4   5  "x() BETWEEN 5 AND 5"     1  1
+
+  5   9  "(x(),8) >= (9,7) AND (x(),8)<=(9,10)"  1 2
+  6   9  "(x(),8) BETWEEN (9,7) AND (9,10)"      1 1
 } {
   do_test e_expr-13.1.$tn {
     set ::xcount 0
@@ -1434,6 +1437,13 @@ do_expr_test e_expr-27.2.2 { CAST(NULL AS text) }    null {}
 do_expr_test e_expr-27.2.3 { CAST(NULL AS blob) }    null {}
 do_expr_test e_expr-27.2.4 { CAST(NULL AS number) }  null {}
 
+# EVIDENCE-OF: R-29283-15561 Otherwise, the storage class of the result
+# is determined by applying the rules for determining column affinity to
+# the type-name.
+#
+# The R-29283-15561 requirement above is demonstrated by all of the 
+# subsequent e_expr-26 tests.
+#
 # EVIDENCE-OF: R-43522-35548 Casting a value to a type-name with no
 # affinity causes the value to be converted into a BLOB.
 #
@@ -1783,8 +1793,8 @@ do_test e_expr-35.0 {
   }
 } {}
 
-# EVIDENCE-OF: R-00980-39256 A SELECT statement enclosed in parentheses
-# may appear as a scalar quantity.
+# EVIDENCE-OF: R-43573-23448 A SELECT statement enclosed in parentheses
+# is a subquery.
 #
 # EVIDENCE-OF: R-56294-03966 All types of SELECT statement, including
 # aggregate and compound SELECT queries (queries with keywords like
@@ -1803,13 +1813,14 @@ do_expr_test e_expr-35.1.6 {
   (SELECT a FROM t2 UNION SELECT COALESCE(b, 55) FROM t2 ORDER BY 1)
 } integer 4
 
-# EVIDENCE-OF: R-46899-53765 A SELECT used as a scalar quantity must
-# return a result set with a single column.
+# EVIDENCE-OF: R-22239-33740 A subquery that returns two or more columns
+# is a row value subquery and can only be used as the operand of a
+# comparison operator.
 #
 # The following block tests that errors are returned in a bunch of cases
 # where a subquery returns more than one column.
 #
-set M {only a single result allowed for a SELECT that is part of an expression}
+set M {/1 {sub-select returns [23] columns - expected 1}/}
 foreach {tn sql} {
   1     { SELECT (SELECT * FROM t2 UNION SELECT a+1, b+1 FROM t2) }
   2     { SELECT (SELECT * FROM t2 UNION SELECT a+1, b+1 FROM t2 ORDER BY 1) }
@@ -1818,14 +1829,14 @@ foreach {tn sql} {
   5     { SELECT (SELECT * FROM t2) }
   6     { SELECT (SELECT * FROM (SELECT 1, 2, 3)) }
 } {
-  do_catchsql_test e_expr-35.2.$tn $sql [list 1 $M]
+  do_catchsql_test e_expr-35.2.$tn $sql $M
 }
 
-# EVIDENCE-OF: R-35764-28041 The result of the expression is the value
-# of the only column in the first row returned by the SELECT statement.
+# EVIDENCE-OF: R-18318-14995 The value of a subquery expression is the
+# first row of the result from the enclosed SELECT statement.
 #
-# EVIDENCE-OF: R-41898-06686 If the SELECT yields more than one result
-# row, all rows after the first are ignored.
+# EVIDENCE-OF: R-15900-52156 In other words, an implied "LIMIT 1" is
+# added to the subquery, overriding an explicitly coded LIMIT.
 #
 do_execsql_test e_expr-36.3.1 {
   CREATE TABLE t4(x, y);
@@ -1849,8 +1860,8 @@ foreach {tn expr restype resval} {
   do_expr_test e_expr-36.3.$tn $expr $restype $resval
 }
 
-# EVIDENCE-OF: R-25492-41572 If the SELECT yields no rows, then the
-# value of the expression is NULL.
+# EVIDENCE-OF: R-52325-25449 The value of a subquery expression is NULL
+# if the enclosed SELECT statement returns no rows.
 #
 foreach {tn expr} {
     1  { ( SELECT x FROM t4 WHERE x>3 ORDER BY x )      }
diff --git a/test/e_fkey.test b/test/e_fkey.test
index 09756505c3..99f1ae8e62 100644
--- a/test/e_fkey.test
+++ b/test/e_fkey.test
@@ -52,11 +52,11 @@ ifcapable trigger&&foreignkey {
 #-------------------------------------------------------------------------
 # Test the effects of defining OMIT_TRIGGER but not OMIT_FOREIGN_KEY.
 #
-# EVIDENCE-OF: R-44697-61543 If SQLITE_OMIT_TRIGGER is defined but
+# EVIDENCE-OF: R-10109-20452 If SQLITE_OMIT_TRIGGER is defined but
 # SQLITE_OMIT_FOREIGN_KEY is not, then SQLite behaves as it did prior to
-# version 3.6.19 - foreign key definitions are parsed and may be queried
-# using PRAGMA foreign_key_list, but foreign key constraints are not
-# enforced.
+# version 3.6.19 (2009-10-14) - foreign key definitions are parsed and
+# may be queried using PRAGMA foreign_key_list, but foreign key
+# constraints are not enforced.
 #
 # Specifically, test that "PRAGMA foreign_keys" is a no-op in this case.
 # When using the pragma to query the current setting, 0 rows are returned.
diff --git a/test/e_select.test b/test/e_select.test
index 89d61b53bc..9aa2de6038 100644
--- a/test/e_select.test
+++ b/test/e_select.test
@@ -1252,8 +1252,8 @@ do_select_tests e_select-5.5 {
   1  "SELECT DISTINCT d FROM h3" {{} 2 2,3 2,4 3}
 }
 
-# EVIDENCE-OF: R-58359-52112 The normal rules for selecting a collation
-# sequence to compare text values with apply.
+# EVIDENCE-OF: R-47709-27231 The usual rules apply for selecting a
+# collation sequence to compare text values.
 #
 do_select_tests e_select-5.6 {
   1  "SELECT DISTINCT b FROM h1"                  {one I i four IV iv}
diff --git a/test/e_vacuum.test b/test/e_vacuum.test
index 1113a1fb6e..3fd34afd43 100644
--- a/test/e_vacuum.test
+++ b/test/e_vacuum.test
@@ -176,7 +176,7 @@ if {![nonzero_reserved_bytes]} {
   # EVIDENCE-OF: R-48521-51450 When in write-ahead log mode, only the
   # auto_vacuum support property can be changed using VACUUM.
   #
-  ifcapable wal {
+  if {[wal_is_capable]} {
     do_test e_vacuum-1.3.3.1 {
       execsql { PRAGMA journal_mode = wal }
       execsql { PRAGMA page_size ; PRAGMA auto_vacuum }
@@ -190,8 +190,8 @@ if {![nonzero_reserved_bytes]} {
   }
 }
   
-# EVIDENCE-OF: R-38001-03952 VACUUM only works on the main database. It
-# is not possible to VACUUM an attached database file.
+# EVIDENCE-OF: R-55119-57913 By default, VACUUM only works only on the
+# main database.
 forcedelete test.db2
 create_db { PRAGMA auto_vacuum = NONE }
 do_execsql_test e_vacuum-2.1.1 {
@@ -202,14 +202,15 @@ do_execsql_test e_vacuum-2.1.1 {
 } {}
 set original_size [file size test.db2]
 
-# Try everything we can think of to get the aux database vacuumed:
+# Vacuuming the main database does not affect aux
 do_execsql_test e_vacuum-2.1.3 { VACUUM } {}
-do_execsql_test e_vacuum-2.1.4 { VACUUM aux } {}
-do_execsql_test e_vacuum-2.1.5 { VACUUM 'test.db2' } {}
-
-# Despite our efforts, space in the aux database has not been reclaimed:
 do_test e_vacuum-2.1.6 { expr {[file size test.db2]==$::original_size} } 1
 
+# EVIDENCE-OF: R-36598-60500 Attached databases can be vacuumed by
+# appending the appropriate schema-name to the VACUUM statement.
+do_execsql_test e_vacuum-2.1.7 { VACUUM aux; } {}
+do_test e_vacuum-2.1.8 { expr {[file size test.db2]<$::original_size} } 1
+
 # EVIDENCE-OF: R-17495-17419 The VACUUM command may change the ROWIDs of
 # entries in any tables that do not have an explicit INTEGER PRIMARY
 # KEY.
@@ -271,9 +272,9 @@ do_test e_vacuum-3.2.2.1 {
 } {1 {cannot VACUUM - SQL statements in progress}}
 
 
-# EVIDENCE-OF: R-38735-12540 As of SQLite version 3.1, an alternative to
-# using the VACUUM command to reclaim space after data has been deleted
-# is auto-vacuum mode, enabled using the auto_vacuum pragma.
+# EVIDENCE-OF: R-55138-13241 An alternative to using the VACUUM command
+# to reclaim space after data has been deleted is auto-vacuum mode,
+# enabled using the auto_vacuum pragma.
 #
 do_test e_vacuum-3.3.1 {
   create_db { PRAGMA auto_vacuum = FULL }
diff --git a/test/emptytable.test b/test/emptytable.test
new file mode 100644
index 0000000000..79cd16e913
--- /dev/null
+++ b/test/emptytable.test
@@ -0,0 +1,50 @@
+# 2017-02-15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases to show that a join involving an empty table is very fast.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Build some test data
+#
+do_execsql_test emptytable-100 {
+  CREATE TABLE t1(a);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t1(a) SELECT x FROM c;
+  CREATE TABLE empty(x);
+  SELECT count(*) FROM t1;
+} {100}
+
+# Interrupt queries after 1M cycles to prevent burning excess CPU
+proc stopDb {args} {
+  db interrupt
+}
+db progress 1000000 {stopDb}
+
+# Prior to the query planner optimization on 2017-02-15, this query would
+# take a ridiculous amount of time.  If that optimization stops working,
+# the result here will be in interrupt for running too long.
+#
+do_catchsql_test emptytable-110 {
+  SELECT count(*) FROM t1, t1, t1, t1, t1, t1, empty;
+} {0 0}
+
+do_catchsql_test emptytable-120 {
+  SELECT count(*) FROM t1, t1 LEFT JOIN empty;
+} {0 10000}
+do_catchsql_test emptytable-121 {
+  SELECT count(*) FROM t1, t1 LEFT JOIN t1, empty;
+} {0 0}
+
+
+finish_test
diff --git a/test/exclusive.test b/test/exclusive.test
index c7b88cfbca..45f9318205 100644
--- a/test/exclusive.test
+++ b/test/exclusive.test
@@ -423,7 +423,7 @@ do_test exclusive-5.1 {
   # (2016-03-04) The statement-journal is now opened lazily
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 2 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.2 {
   execsql {
     COMMIT;
@@ -450,7 +450,7 @@ do_test exclusive-5.4 {
   # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 2 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.5 {
   execsql {
     COMMIT;
@@ -459,7 +459,7 @@ do_test exclusive-5.5 {
   # 2016-03-04: The statement-journal open is deferred
   set sqlite_open_file_count
   expr $sqlite_open_file_count-$extrafds
-} [expr 2 - ($TEMP_STORE>=2)]
+} {2}
 do_test exclusive-5.6 {
   execsql {
     PRAGMA locking_mode = normal;
diff --git a/test/exclusive2.test b/test/exclusive2.test
index 712363e762..92fcd76ab3 100644
--- a/test/exclusive2.test
+++ b/test/exclusive2.test
@@ -121,13 +121,13 @@ do_test exclusive2-1.1 {
   execsql {
     BEGIN;
     CREATE TABLE t1(a, b);
-    INSERT INTO t1(a) VALUES(randstr(10, 400));
-    INSERT INTO t1(a) VALUES(randstr(10, 400));
-    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
-    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
-    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
-    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
-    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
+    INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0);
+    INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0);
+    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
+    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
+    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
+    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
+    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
     COMMIT;
     SELECT count(*) FROM t1;
   }
@@ -154,7 +154,7 @@ do_test exclusive2-1.4 {
 } $::sig
 do_test exclusive2-1.5 {
   execsql {
-    UPDATE t1 SET b=a, a=NULL;
+    UPDATE t1 SET b=a, a=0;
   } db2
   expr {[t1sig db2] eq $::sig}
 } 0
diff --git a/test/exists.test b/test/exists.test
index fb73797d29..5f9e36eb00 100644
--- a/test/exists.test
+++ b/test/exists.test
@@ -19,6 +19,7 @@ source $testdir/lock_common.tcl
 
 
 foreach jm {rollback wal} {
+  if {![wal_is_capable] && $jm=="wal"} continue
 
   set testprefix exists-$jm
 
diff --git a/test/expr.test b/test/expr.test
index 7d7b8ce5a7..7a6d477259 100644
--- a/test/expr.test
+++ b/test/expr.test
@@ -308,6 +308,33 @@ ifcapable floatingpoint {if {[working_64bit_int]} {
   test_realnum_expr expr-1.257\
       {i1=-4294967296, i2=-2147483647} {i1*i2}    9223372032559808512
 
+  test_realnum_expr expr-1.260\
+      {i1=3037000500, i2=3037000500} {i1*i2}      9.22337203700025e+18
+  test_realnum_expr expr-1.261\
+      {i1=3037000500, i2=-3037000500} {i1*i2}     -9.22337203700025e+18
+  test_realnum_expr expr-1.262\
+      {i1=-3037000500, i2=3037000500} {i1*i2}     -9.22337203700025e+18
+  test_realnum_expr expr-1.263\
+      {i1=-3037000500, i2=-3037000500} {i1*i2}    9.22337203700025e+18
+
+  test_realnum_expr expr-1.264\
+      {i1=3037000500, i2=3037000499} {i1*i2}      9223372033963249500
+  test_realnum_expr expr-1.265\
+      {i1=3037000500, i2=-3037000499} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.266\
+      {i1=-3037000500, i2=3037000499} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.267\
+      {i1=-3037000500, i2=-3037000499} {i1*i2}    9223372033963249500
+
+  test_realnum_expr expr-1.268\
+      {i1=3037000499, i2=3037000500} {i1*i2}      9223372033963249500
+  test_realnum_expr expr-1.269\
+      {i1=3037000499, i2=-3037000500} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.270\
+      {i1=-3037000499, i2=3037000500} {i1*i2}     -9223372033963249500
+  test_realnum_expr expr-1.271\
+      {i1=-3037000499, i2=-3037000500} {i1*i2}    9223372033963249500
+
 }}
 
 ifcapable floatingpoint {
diff --git a/test/filectrl.test b/test/filectrl.test
index 28fecee92f..460b71e257 100644
--- a/test/filectrl.test
+++ b/test/filectrl.test
@@ -43,4 +43,30 @@ do_test filectrl-1.6 {
 } {/etilqs_/}
 db close
 forcedelete .test_control_lockproxy.db-conch test.proxy
+forcedelete test.db test2.db
+
+if {$tcl_platform(platform)=="windows"} {
+  do_test filectrl-2.1 {
+    sqlite3 db test2.db
+    set size [file size test2.db]
+    set handle [file_control_win32_get_handle db]
+    db close
+    forcedelete test2.db
+    list $size $handle [expr {$handle != 0}]
+  } {/^0 \{0 [0-9A-Fa-f]+\} 1$/}
+
+  do_test filectrl-2.2 {
+    sqlite3 db test2.db
+    execsql {
+      CREATE TABLE t1(x);
+      INSERT INTO t1 (x) VALUES(RANDOMBLOB(1048576));
+    }
+    set size [file size test2.db]
+    set handle [file_control_win32_get_handle db]
+    db close
+    forcedelete test2.db
+    list $size $handle [expr {$handle != 0}]
+  } {/^1\d+ \{0 [0-9A-Fa-f]+\} 1$/}
+}
+
 finish_test
diff --git a/test/filefmt.test b/test/filefmt.test
index 2df1424436..087a955c39 100644
--- a/test/filefmt.test
+++ b/test/filefmt.test
@@ -144,9 +144,11 @@ do_execsql_test filefmt-2.1.1 {
   CREATE TABLE t2(a);
   INSERT INTO t2 VALUES(1);
 } {}
-do_test filefmt-2.1.2 {
-  hexio_read test.db 28 4
-} {00000009}
+if {![nonzero_reserved_bytes]} {
+  do_test filefmt-2.1.2 {
+    hexio_read test.db 28 4
+  } {00000009}
+}
 
 do_test filefmt-2.1.3 {
   sql36231 { INSERT INTO t1 VALUES(a_string(3000)) }
@@ -170,9 +172,11 @@ do_execsql_test filefmt-2.2.1 {
   CREATE TABLE t2(a);
   INSERT INTO t2 VALUES(1);
 } {}
-do_test filefmt-2.2.2 {
-  hexio_read test.db 28 4
-} {00000009}
+if {![nonzero_reserved_bytes]} {
+  do_test filefmt-2.2.2 {
+    hexio_read test.db 28 4
+  } {00000009}
+}
 
 do_test filefmt-2.2.3 {
   sql36231 { INSERT INTO t1 VALUES(a_string(3000)) }
diff --git a/test/fkey1.test b/test/fkey1.test
index e10781ac52..d9b038a022 100644
--- a/test/fkey1.test
+++ b/test/fkey1.test
@@ -15,6 +15,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix fkey1
 
 ifcapable {!foreignkey} {
   finish_test
@@ -185,4 +186,25 @@ do_catchsql_test fkey1-5.4 {
   INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (2, 3, 'A-2-3');
 } {1 {FOREIGN KEY constraint failed}}
 
+#-------------------------------------------------------------------------
+# Check that foreign key processing is not fooled by partial indexes
+# on the parent table.
+#
+do_execsql_test 6.0 {
+  CREATE TABLE p1(x, y);
+  CREATE UNIQUE INDEX p1x ON p1(x) WHERE y<2;
+  INSERT INTO p1 VALUES(1, 1);
+  CREATE TABLE c1(a REFERENCES p1(x));
+}
+
+do_catchsql_test 6.1 {
+  INSERT INTO c1 VALUES(1);
+} {1 {foreign key mismatch - "c1" referencing "p1"}}
+
+do_execsql_test 6.2 {
+  CREATE UNIQUE INDEX p1x2 ON p1(x);
+  INSERT INTO c1 VALUES(1);
+} {}
+
+
 finish_test
diff --git a/test/fkey2.test b/test/fkey2.test
index aec75ed693..c2ae3788f5 100644
--- a/test/fkey2.test
+++ b/test/fkey2.test
@@ -1062,7 +1062,7 @@ ifcapable altertable {
       PRAGMA foreign_keys = off;
       ALTER TABLE t2 ADD COLUMN h DEFAULT 'text' REFERENCES t1;
       PRAGMA foreign_keys = on;
-      SELECT sql FROM sqlite_temp_master WHERE name='t2';
+      SELECT sql FROM temp.sqlite_master WHERE name='t2';
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
@@ -1093,7 +1093,7 @@ ifcapable altertable {
   ]
   do_test fkey2-14.2tmp.2.2 {
     execsql { ALTER TABLE t1 RENAME TO t4 }
-    execsql { SELECT sql FROM sqlite_temp_master WHERE type = 'table'}
+    execsql { SELECT sql FROM temp.sqlite_master WHERE type = 'table'}
   } [list \
     {CREATE TABLE "t4"(a PRIMARY KEY, b REFERENCES "t4")}                    \
     {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES "t4", c REFERENCES t2)}     \
diff --git a/test/fkey6.test b/test/fkey6.test
index 45ad1ef2cc..b658f20fea 100644
--- a/test/fkey6.test
+++ b/test/fkey6.test
@@ -1,4 +1,4 @@
-# 2013-07-11
+# 2012 December 17
 #
 # The author disclaims copyright to this source code.  In place of
 # a legal notice, here is a blessing:
diff --git a/test/fkey8.test b/test/fkey8.test
index b4b6bb75e7..4269c20a92 100644
--- a/test/fkey8.test
+++ b/test/fkey8.test
@@ -101,5 +101,67 @@ foreach {tn use_stmt sql schema} {
   } $use_stmt
 }
 
+#-------------------------------------------------------------------------
+# The following tests check that foreign key constaint counters are
+# correctly updated for any implicit DELETE operations that occur
+# when a REPLACE command is executed against a WITHOUT ROWID table
+# that has no triggers or auxiliary indexes.
+#
+reset_db
+do_execsql_test 2.1.0 {
+  PRAGMA foreign_keys = on;
+  CREATE TABLE p1(a PRIMARY KEY, b) WITHOUT ROWID;
+  CREATE TABLE c1(x REFERENCES p1 DEFERRABLE INITIALLY DEFERRED);
+
+  INSERT INTO p1 VALUES(1, 'one');
+  INSERT INTO p1 VALUES(2, 'two');
+  INSERT INTO c1 VALUES(1);
+  INSERT INTO c1 VALUES(2);
+}
+
+do_catchsql_test 2.1.2 {
+  BEGIN;
+    DELETE FROM p1 WHERE a=1;
+    INSERT OR REPLACE INTO p1 VALUES(2, 'two');
+  COMMIT;
+} {1 {FOREIGN KEY constraint failed}}
+
+reset_db
+do_execsql_test 2.2.0 {
+  PRAGMA foreign_keys = on;
+  CREATE TABLE p2(a PRIMARY KEY, b);
+  CREATE TABLE c2(
+    x PRIMARY KEY,
+    y REFERENCES p2 DEFERRABLE INITIALLY DEFERRED
+  ) WITHOUT ROWID;
+}
+
+do_catchsql_test 2.2.1 {
+  BEGIN;
+    INSERT INTO c2 VALUES(13, 13);
+    INSERT OR REPLACE INTO c2 VALUES(13, 13);
+    DELETE FROM c2;
+  COMMIT;
+} {0 {}}
+
+reset_db
+do_execsql_test 2.3.0 {
+  PRAGMA foreign_keys = on;
+  CREATE TABLE p3(a PRIMARY KEY, b) WITHOUT ROWID;
+  CREATE TABLE c3(x REFERENCES p3);
+
+  INSERT INTO p3 VALUES(1, 'one');
+  INSERT INTO p3 VALUES(2, 'two');
+  INSERT INTO c3 VALUES(1);
+  INSERT INTO c3 VALUES(2);
+
+  CREATE TRIGGER p3d AFTER DELETE ON p3 WHEN old.a=1 BEGIN
+    INSERT OR REPLACE INTO p3 VALUES(2, 'three');
+  END;
+}
+
+do_catchsql_test 2.3.1 {
+  DELETE FROM p3 WHERE a=1
+} {1 {FOREIGN KEY constraint failed}}
 
 finish_test
diff --git a/test/fts3aux1.test b/test/fts3aux1.test
index d17ac85df5..b6111f8470 100644
--- a/test/fts3aux1.test
+++ b/test/fts3aux1.test
@@ -117,12 +117,12 @@ do_test 2.1.2.1 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND term='braid' }
   set cnt
-} {2}
+} {1}
 do_test 2.1.2.2 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND +term='braid' }
   set cnt
-} {38}
+} {19}
 
 # Similar to the test immediately above, but using a term ("breakfast") that 
 # is not featured in the dataset.
@@ -136,7 +136,7 @@ do_test 2.1.3.2 {
   set cnt 0
   execsql { SELECT * FROM terms_v WHERE rec('cnt', term) AND +term='breakfast' }
   set cnt
-} {38}
+} {19}
 
 do_execsql_test 2.1.4.1 { SELECT * FROM terms_v WHERE term='braid' } {braid 1 1}
 do_execsql_test 2.1.4.2 { SELECT * FROM terms_v WHERE +term='braid'} {braid 1 1}
diff --git a/test/fts3conf.test b/test/fts3conf.test
index f9698770cc..6766a95408 100644
--- a/test/fts3conf.test
+++ b/test/fts3conf.test
@@ -211,4 +211,42 @@ do_execsql_test 4.2.2 {
   INSERT INTO t01(t01) VALUES('integrity-check');
 } {}
 
+do_execsql_test 4.3.1 {
+  CREATE VIRTUAL TABLE t02 USING fts4;
+  INSERT INTO t01 VALUES('1 1 1');
+  INSERT INTO t02 VALUES('2 2 2');
+  BEGIN;
+    SAVEPOINT abc;
+      INSERT INTO t01 VALUES('a b c');
+      INSERT INTO t02 VALUES('a b c');
+    ROLLBACK TO abc;
+  COMMIT;
+}
+do_execsql_test 4.3.2 {
+  SELECT * FROM t01 WHERE t01 MATCH 'b';
+  INSERT INTO t01(t01) VALUES('integrity-check');
+} {}
+
+do_execsql_test 4.4.1 {
+  CREATE TABLE A(ID INTEGER PRIMARY KEY, AnotherID INTEGER, Notes TEXT);
+  CREATE VIRTUAL TABLE AFTS USING FTS4 (Notes);
+  CREATE TRIGGER A_DeleteTrigger AFTER DELETE ON A FOR EACH ROW BEGIN 
+    DELETE FROM AFTS WHERE rowid=OLD.ID; 
+  END;
+  CREATE TABLE B(ID INTEGER PRIMARY KEY,Notes TEXT);
+  CREATE VIRTUAL TABLE BFTS USING FTS3 (Notes);
+  CREATE TRIGGER B_DeleteTrigger AFTER DELETE ON B FOR EACH ROW BEGIN 
+    DELETE FROM BFTS WHERE rowid=OLD.ID; 
+  END;
+}
+		
+do_execsql_test 4.4.2 {
+  BEGIN TRANSACTION;
+    DELETE FROM A WHERE AnotherID=1;
+    DELETE FROM B WHERE ID=1;
+  COMMIT;
+}
+		
+
+
 finish_test
diff --git a/test/fts3expr.test b/test/fts3expr.test
index 6e23faf633..b186a157d1 100644
--- a/test/fts3expr.test
+++ b/test/fts3expr.test
@@ -514,4 +514,8 @@ do_test fts3expr-9.1 {
   test_fts3expr "f (e NEAR/2 a)"
 } {AND {PHRASE 3 0 f} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}}
 
+do_test fts3expr-10.1 { test_fts3expr "abc *" } {PHRASE 3 0 abc}
+do_test fts3expr-10.2 { test_fts3expr "*" } {}
+do_test fts3expr-10.3 { test_fts3expr "abc*" } {PHRASE 3 0 abc+}
+
 finish_test
diff --git a/test/fts4lastrowid.test b/test/fts4lastrowid.test
new file mode 100644
index 0000000000..b5720a2ef3
--- /dev/null
+++ b/test/fts4lastrowid.test
@@ -0,0 +1,73 @@
+# 2017 Feb 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests of the last_insert_rowid functionality with fts4.
+#
+
+set testdir [file dirname $argv0]
+source [file join [file dirname [info script]] tester.tcl]
+set testprefix fts4lastrowid
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts4(str);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('one string');
+  INSERT INTO t1 VALUES('two string');
+  INSERT INTO t1 VALUES('three string');
+  SELECT last_insert_rowid();
+} {3}
+
+do_execsql_test 1.2 {
+  BEGIN;
+    INSERT INTO t1 VALUES('one string');
+    INSERT INTO t1 VALUES('two string');
+    INSERT INTO t1 VALUES('three string');
+  COMMIT;
+  SELECT last_insert_rowid();
+} {6}
+
+do_execsql_test 1.3 {
+  INSERT INTO t1(rowid, str) VALUES(-22, 'some more text');
+  SELECT last_insert_rowid();
+} {-22}
+
+do_execsql_test 1.4 {
+  BEGIN;
+    INSERT INTO t1(rowid, str) VALUES(45, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(46, 'some more text');
+    INSERT INTO t1(rowid, str) VALUES(222, 'some more text');
+    SELECT last_insert_rowid();
+  COMMIT;
+  SELECT last_insert_rowid();
+} {222 222}
+
+do_execsql_test 1.5 {
+  CREATE TABLE x1(x);
+  INSERT INTO x1 VALUES('john'), ('paul'), ('george'), ('ringo');
+  INSERT INTO t1 SELECT x FROM x1;
+  SELECT last_insert_rowid();
+} {226}
+
+do_execsql_test 1.6 {
+  INSERT INTO t1(rowid, str) SELECT rowid+10, x FROM x1;
+  SELECT last_insert_rowid();
+} {14}
+
+
+finish_test
+
diff --git a/test/fuzz-oss1.test b/test/fuzz-oss1.test
index 08bc670845..e77b7ed0f8 100644
--- a/test/fuzz-oss1.test
+++ b/test/fuzz-oss1.test
@@ -1997,5 +1997,9 @@ NULL AND ("9_u" COLLATE NOCASE = ? COLLATE NOCASE))) FROM (SELECT
   }
 } {/.* Goto .*/}
 
+# Crash reported by OSS-FUZZ on 2016-11-10
+do_catchsql_test fuzz-oss1-detach {
+  DETACH x IS #1;
+} {1 {near "#1": syntax error}}
 
 finish_test
diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c
index 1e0ce86e66..fd72273f34 100644
--- a/test/fuzzcheck.c
+++ b/test/fuzzcheck.c
@@ -70,7 +70,6 @@
 #include 
 #include 
 #include "sqlite3.h"
-#include 
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
 
@@ -80,6 +79,11 @@
 # include 
 #endif
 
+#ifdef SQLITE_OSS_FUZZ
+# include 
+# include 
+#endif
+
 /*
 ** Files in the virtual file system.
 */
@@ -128,6 +132,7 @@ static struct GlobalVars {
   Blob *pFirstDb;                  /* Content of first template database */
   int nSql;                        /* Number of SQL scripts */
   Blob *pFirstSql;                 /* First SQL script */
+  unsigned int uRandom;            /* Seed for the SQLite PRNG */
   char zTestName[100];             /* Name of current test */
 } g;
 
@@ -189,7 +194,7 @@ static int progressHandler(void *pVdbeLimitFlag){
 ** Reallocate memory.  Show and error and quit if unable.
 */
 static void *safe_realloc(void *pOld, int szNew){
-  void *pNew = realloc(pOld, szNew);
+  void *pNew = realloc(pOld, szNew<=0 ? 1 : szNew);
   if( pNew==0 ) fatalError("unable to realloc for %d bytes", szNew);
   return pNew;
 }
@@ -590,10 +595,18 @@ static int inmemFullPathname(
   return SQLITE_OK;
 }
 
+/* Always use the same random see, for repeatability.
+*/
+static int inmemRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){
+  memset(zBuf, 0, nBuf);
+  memcpy(zBuf, &g.uRandom, nBufxRandomness;
+  inmemVfs.xRandomness = inmemRandomness;
   inmemVfs.xSleep = pDefault->xSleep;
   inmemVfs.xCurrentTimeInt64 = pDefault->xCurrentTimeInt64;
-  sqlite3_vfs_register(&inmemVfs, 0);
+  sqlite3_vfs_register(&inmemVfs, makeDefault);
 };
 
 /*
@@ -622,14 +635,12 @@ static void inmemVfsRegister(void){
 */
 static void runSql(sqlite3 *db, const char *zSql, unsigned  runFlags){
   const char *zMore;
-  const char *zEnd = &zSql[strlen(zSql)];
   sqlite3_stmt *pStmt;
 
   while( zSql && zSql[0] ){
     zMore = 0;
     pStmt = 0;
     sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zMore);
-    assert( zMore<=zEnd );
     if( zMore==zSql ) break;
     if( runFlags & SQL_TRACE ){
       const char *z = zSql;
@@ -782,23 +793,26 @@ static void showHelp(void){
 "Read databases and SQL scripts from SOURCE-DB and execute each script against\n"
 "each database, checking for crashes and memory leaks.\n"
 "Options:\n"
-"  --cell-size-check     Set the PRAGMA cell_size_check=ON\n"
-"  --dbid N              Use only the database where dbid=N\n"
-"  --export-db DIR       Write databases to files(s) in DIR. Works with --dbid\n"
-"  --export-sql DIR      Write SQL to file(s) in DIR. Also works with --sqlid\n"
-"  --help                Show this help text\n"
-"  -q|--quiet            Reduced output\n"
-"  --limit-mem N         Limit memory used by test SQLite instance to N bytes\n"
-"  --limit-vdbe          Panic if an sync SQL runs for more than 100,000 cycles\n"
-"  --load-sql ARGS...    Load SQL scripts fro files into SOURCE-DB\n"
-"  --load-db ARGS...     Load template databases from files into SOURCE_DB\n"
-"  -m TEXT               Add a description to the database\n"
-"  --native-vfs          Use the native VFS for initially empty database files\n"
-"  --rebuild             Rebuild and vacuum the database file\n"
-"  --result-trace        Show the results of each SQL command\n"
-"  --sqlid N             Use only SQL where sqlid=N\n"
-"  --timeout N           Abort if any single test case needs more than N seconds\n"
-"  -v|--verbose          Increased output.  Repeat for more output.\n"
+"  --cell-size-check    Set the PRAGMA cell_size_check=ON\n"
+"  --dbid N             Use only the database where dbid=N\n"
+"  --export-db DIR      Write databases to files(s) in DIR. Works with --dbid\n"
+"  --export-sql DIR     Write SQL to file(s) in DIR. Also works with --sqlid\n"
+"  --help               Show this help text\n"
+"  -q|--quiet           Reduced output\n"
+"  --limit-mem N        Limit memory used by test SQLite instance to N bytes\n"
+"  --limit-vdbe         Panic if any test runs for more than 100,000 cycles\n"
+"  --load-sql ARGS...   Load SQL scripts fro files into SOURCE-DB\n"
+"  --load-db ARGS...    Load template databases from files into SOURCE_DB\n"
+"  -m TEXT              Add a description to the database\n"
+"  --native-vfs         Use the native VFS for initially empty database files\n"
+"  --native-malloc      Turn off MEMSYS3/5 and Lookaside\n"
+"  --oss-fuzz           Enable OSS-FUZZ testing\n"
+"  --prng-seed N        Seed value for the PRGN inside of SQLite\n"
+"  --rebuild            Rebuild and vacuum the database file\n"
+"  --result-trace       Show the results of each SQL command\n"
+"  --sqlid N            Use only SQL where sqlid=N\n"
+"  --timeout N          Abort if any single test needs more than N seconds\n"
+"  -v|--verbose         Increased output.  Repeat for more output.\n"
   );
 }
 
@@ -832,9 +846,14 @@ int main(int argc, char **argv){
   int sqlFuzz = 0;             /* True for SQL fuzz testing. False for DB fuzz */
   int iTimeout = 120;          /* Default 120-second timeout */
   int nMem = 0;                /* Memory limit */
+  int nMemThisDb = 0;          /* Memory limit set by the CONFIG table */
   char *zExpDb = 0;            /* Write Databases to files in this directory */
   char *zExpSql = 0;           /* Write SQL to files in this directory */
   void *pHeap = 0;             /* Heap for use by SQLite */
+  int ossFuzz = 0;             /* enable OSS-FUZZ testing */
+  int ossFuzzThisDb = 0;       /* ossFuzz value for this particular database */
+  int nativeMalloc = 0;        /* Turn off MEMSYS3/5 and lookaside if true */
+  sqlite3_vfs *pDfltVfs;       /* The default VFS */
 
   iBegin = timeOfDay();
 #ifdef __unix__
@@ -842,6 +861,8 @@ int main(int argc, char **argv){
 #endif
   g.zArgv0 = argv[0];
   zFailCode = getenv("TEST_FAILURE");
+  pDfltVfs = sqlite3_vfs_find(0);
+  inmemVfsRegister(1);
   for(i=1; i=argc-1 ) fatalError("missing arguments on %s", argv[i]);
         zMsg = argv[++i];
       }else
+      if( strcmp(z,"native-malloc")==0 ){
+        nativeMalloc = 1;
+      }else
       if( strcmp(z,"native-vfs")==0 ){
         nativeFlag = 1;
       }else
+      if( strcmp(z,"oss-fuzz")==0 ){
+        ossFuzz = 1;
+      }else
+      if( strcmp(z,"prng-seed")==0 ){
+        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
+        g.uRandom = atoi(argv[++i]);
+      }else
       if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){
         quietFlag = 1;
         verboseFlag = 0;
@@ -945,7 +976,8 @@ int main(int argc, char **argv){
 
   /* Process each source database separately */
   for(iSrcDb=0; iSrcDbzName);
     if( rc ){
       fatalError("cannot open source database %s - %s",
       azSrcDb[iSrcDb], sqlite3_errmsg(db));
@@ -971,6 +1003,34 @@ int main(int argc, char **argv){
       sqlite3_free(zSql);
       if( rc ) fatalError("cannot change description: %s", sqlite3_errmsg(db));
     }
+    ossFuzzThisDb = ossFuzz;
+
+    /* If the CONFIG(name,value) table exists, read db-specific settings
+    ** from that table */
+    if( sqlite3_table_column_metadata(db,0,"config",0,0,0,0,0,0)==SQLITE_OK ){
+      rc = sqlite3_prepare_v2(db, "SELECT name, value FROM config", -1, &pStmt, 0);
+      if( rc ) fatalError("cannot prepare query of CONFIG table: %s",
+                          sqlite3_errmsg(db));
+      while( SQLITE_ROW==sqlite3_step(pStmt) ){
+        const char *zName = (const char *)sqlite3_column_text(pStmt,0);
+        if( zName==0 ) continue;
+        if( strcmp(zName, "oss-fuzz")==0 ){
+          ossFuzzThisDb = sqlite3_column_int(pStmt,1);
+          if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb);
+        }
+        if( strcmp(zName, "limit-mem")==0 && !nativeMalloc ){
+#if !defined(SQLITE_ENABLE_MEMSYS3) && !defined(SQLITE_ENABLE_MEMSYS5)
+          fatalError("the limit-mem option requires -DSQLITE_ENABLE_MEMSYS5"
+                     " or _MEMSYS3");
+#else
+          nMemThisDb = sqlite3_column_int(pStmt,1);
+          if( verboseFlag ) printf("Config: limit-mem=%d\n", nMemThisDb);
+#endif
+        }
+      }
+      sqlite3_finalize(pStmt);
+    }
+
     if( zInsSql ){
       sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
                               readfileFunc, 0, 0);
@@ -992,6 +1052,8 @@ int main(int argc, char **argv){
       sqlite3_close(db);
       return 0;
     }
+    rc = sqlite3_exec(db, "PRAGMA query_only=1;", 0, 0, 0);
+    if( rc ) fatalError("cannot set database to query-only");
     if( zExpDb!=0 || zExpSql!=0 ){
       sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
                               writefileFunc, 0, 0);
@@ -1086,19 +1148,22 @@ int main(int argc, char **argv){
     }
 
     /* Limit available memory, if requested */
-    if( nMem>0 ){
-      sqlite3_shutdown();
-      pHeap = malloc(nMem);
+    sqlite3_shutdown();
+    if( nMemThisDb>0 && !nativeMalloc ){
+      pHeap = realloc(pHeap, nMemThisDb);
       if( pHeap==0 ){
         fatalError("failed to allocate %d bytes of heap memory", nMem);
       }
-      sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMem, 128);
+      sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128);
+    }
+
+    /* Disable lookaside with the --native-malloc option */
+    if( nativeMalloc ){
+      sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);
     }
   
-    /* Register the in-memory virtual filesystem
-    */
+    /* Reset the in-memory virtual filesystem */
     formatVfs();
-    inmemVfsRegister();
     
     /* Run a test using each SQL script against each database.
     */
@@ -1123,26 +1188,42 @@ int main(int argc, char **argv){
           }
         }
         createVFile("main.db", pDb->sz, pDb->a);
-        openFlags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE;
-        if( nativeFlag && pDb->sz==0 ){
-          openFlags |= SQLITE_OPEN_MEMORY;
-          zVfs = 0;
-        }
-        rc = sqlite3_open_v2("main.db", &db, openFlags, zVfs);
-        if( rc ) fatalError("cannot open inmem database");
-        if( cellSzCkFlag ) runSql(db, "PRAGMA cell_size_check=ON", runFlags);
-        setAlarm(iTimeout);
-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
-        if( sqlFuzz || vdbeLimitFlag ){
-          sqlite3_progress_handler(db, 100000, progressHandler, &vdbeLimitFlag);
-        }
+        sqlite3_randomness(0,0);
+        if( ossFuzzThisDb ){
+#ifndef SQLITE_OSS_FUZZ
+          fatalError("--oss-fuzz not supported: recompile with -DSQLITE_OSS_FUZZ");
+#else
+          extern int LLVMFuzzerTestOneInput(const uint8_t*, size_t);
+          LLVMFuzzerTestOneInput((const uint8_t*)pSql->a, (size_t)pSql->sz);
 #endif
-        do{
-          runSql(db, (char*)pSql->a, runFlags);
-        }while( timeoutTest );
-        setAlarm(0);
-        sqlite3_close(db);
-        if( sqlite3_memory_used()>0 ) fatalError("memory leak");
+        }else{
+          openFlags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE;
+          if( nativeFlag && pDb->sz==0 ){
+            openFlags |= SQLITE_OPEN_MEMORY;
+            zVfs = 0;
+          }
+          rc = sqlite3_open_v2("main.db", &db, openFlags, zVfs);
+          if( rc ) fatalError("cannot open inmem database");
+          sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 100000000);
+          sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 50);
+          if( cellSzCkFlag ) runSql(db, "PRAGMA cell_size_check=ON", runFlags);
+          setAlarm(iTimeout);
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+          if( sqlFuzz || vdbeLimitFlag ){
+            sqlite3_progress_handler(db, 100000, progressHandler, &vdbeLimitFlag);
+          }
+#endif
+          do{
+            runSql(db, (char*)pSql->a, runFlags);
+          }while( timeoutTest );
+          setAlarm(0);
+          sqlite3_exec(db, "PRAGMA temp_store_directory=''", 0, 0, 0);
+          sqlite3_close(db);
+        }
+        if( sqlite3_memory_used()>0 ){
+           fatalError("memory leak: %lld bytes outstanding",
+                      sqlite3_memory_used());
+        }
         reformatVfs();
         nTest++;
         g.zTestName[0] = 0;
diff --git a/test/fuzzdata5.db b/test/fuzzdata5.db
new file mode 100644
index 0000000000..4645b1921b
Binary files /dev/null and b/test/fuzzdata5.db differ
diff --git a/test/gcfault.test b/test/gcfault.test
new file mode 100644
index 0000000000..d54b78fafc
--- /dev/null
+++ b/test/gcfault.test
@@ -0,0 +1,56 @@
+# 2016 December 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing OOM error handling within the built-in 
+# group_concat() function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix gcfault
+
+
+foreach {enc} {
+  utf16
+  utf8
+} {
+  reset_db
+  sqlite3_db_config_lookaside db 0 0 0
+  execsql "PRAGMA encoding = $enc"
+
+  do_execsql_test 1.$enc.1 {
+    CREATE TABLE s(i, s);
+    INSERT INTO s VALUES(1, ',0123456789,');
+    INSERT INTO s VALUES(2, X'2c303132333435363738392c');
+
+    CREATE TABLE e(e);
+    INSERT INTO e VALUES('v1'), ('v2');
+  } {}
+
+  do_faultsim_test 1.$enc.1 -faults oom* -body {
+    execsql { SELECT group_concat(e, (SELECT s FROM s WHERE i=1)) FROM e }
+  }
+
+  do_faultsim_test 1.$enc.2 -faults oom-t* -body {
+    execsql { SELECT group_concat(e, (SELECT s FROM s WHERE i=2)) FROM e }
+  }
+
+  do_faultsim_test 1.$enc.3 -faults oom-t* -prep {
+    set ::STMT [sqlite3_prepare db {SELECT group_concat(e, ?) FROM e} -1 dummy]
+    sqlite3_bind_text $::STMT 1 ",0123456789," 12
+  } -body {
+    while { "SQLITE_ROW"==[sqlite3_step $::STMT] } { }
+  } -test {
+    sqlite3_finalize $::STMT
+  }
+}
+
+finish_test
diff --git a/test/hexlit.test b/test/hexlit.test
index c48930b49b..2161f27a12 100644
--- a/test/hexlit.test
+++ b/test/hexlit.test
@@ -112,6 +112,9 @@ do_catchsql_test hexlist-400 {
 do_catchsql_test hexlist-401 {
   SELECT DISTINCT 0x10000000000000000;
 } {1 {hex literal too big: 0x10000000000000000}}
+do_catchsql_test hexlist-402 {
+  SELECT DISTINCT -0x08000000000000000;
+} {1 {hex literal too big: -0x08000000000000000}}
 do_catchsql_test hexlist-410 {
   DROP TABLE IF EXISTS t1;
   CREATE TABLE t1(x);
diff --git a/test/hook.test b/test/hook.test
index de6fbdd254..9ba220cded 100644
--- a/test/hook.test
+++ b/test/hook.test
@@ -21,6 +21,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set ::testprefix hook
 
 do_test hook-1.2 {
   db commit_hook
@@ -438,4 +439,472 @@ do_test hook-6.2 {
 } {COMMIT ROLLBACK}
 unset ::hooks
 
+#----------------------------------------------------------------------------
+# The following tests - hook-7.* - test the pre-update hook.
+#
+ifcapable !preupdate {
+  finish_test
+  return
+}
+#
+# 7.1.1 - INSERT statement.
+# 7.1.2 - INSERT INTO ... SELECT statement.
+# 7.1.3 - REPLACE INTO ... (rowid conflict)
+# 7.1.4 - REPLACE INTO ... (other index conflicts)
+# 7.1.5 - REPLACE INTO ... (both rowid and other index conflicts)
+#
+# 7.2.1 - DELETE statement.
+# 7.2.2 - DELETE statement that uses the truncate optimization.
+#
+# 7.3.1 - UPDATE statement.
+# 7.3.2 - UPDATE statement that modifies the rowid.
+# 7.3.3 - UPDATE OR REPLACE ... (rowid conflict).
+# 7.3.4 - UPDATE OR REPLACE ... (other index conflicts)
+# 7.3.4 - UPDATE OR REPLACE ... (both rowid and other index conflicts)
+#
+# 7.4.1 - Test that the pre-update-hook is invoked only once if a row being
+#         deleted is removed by a BEFORE trigger.
+#
+# 7.4.2 - Test that the pre-update-hook is invoked if a BEFORE trigger 
+#         removes a row being updated. In this case the update hook should
+#         be invoked with SQLITE_INSERT as the opcode when inserting the
+#         new version of the row.
+#         
+# TODO: Short records (those created before a column is added to a table 
+#       using ALTER TABLE)
+#
+
+proc do_preupdate_test {tn sql x} {
+  set X [list]
+  foreach elem $x {lappend X $elem}
+  uplevel do_test $tn [list "
+    set ::preupdate \[list\]
+    execsql { $sql }
+    set ::preupdate
+  "] [list $X]
+}
+
+proc preupdate_hook {args} {
+  set type [lindex $args 0]
+  eval lappend ::preupdate $args
+  if {$type != "INSERT"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      lappend ::preupdate [db preupdate old $i]
+    }
+  }
+  if {$type != "DELETE"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      set rc [catch { db preupdate new $i } v]
+      lappend ::preupdate $v
+    }
+  }
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db preupdate hook preupdate_hook
+
+# Set up a schema to use for tests 7.1.* to 7.3.*.
+do_execsql_test 7.0 { 
+  CREATE TABLE t1(a, b); 
+  CREATE TABLE t2(x, y); 
+  CREATE TABLE t3(i, j, UNIQUE(i));
+
+  INSERT INTO t2 VALUES('a', 'b');
+  INSERT INTO t2 VALUES('c', 'd');
+
+  INSERT INTO t3 VALUES(4, 16);
+  INSERT INTO t3 VALUES(5, 25);
+  INSERT INTO t3 VALUES(6, 36);
+} 
+
+do_preupdate_test 7.1.1 {
+  INSERT INTO t1 VALUES('x', 'y')
+} {INSERT main t1 1 1  x y}
+
+# 7.1.2.1 does not use the xfer optimization. 7.1.2.2 does.
+do_preupdate_test 7.1.2.1 {
+  INSERT INTO t1 SELECT y, x FROM t2;
+} {INSERT main t1 2 2 b a   INSERT main t1 3 3 d c}
+do_preupdate_test 7.1.2.2 {
+  INSERT INTO t1 SELECT * FROM t2;
+} {INSERT main t1 4 4 a b   INSERT main t1 5 5 c d}
+
+do_preupdate_test 7.1.3 {
+  REPLACE INTO t1(rowid, a, b) VALUES(1, 1, 1);
+} {
+  DELETE main t1 1 1   x y
+  INSERT main t1 1 1   1 1
+}
+
+do_preupdate_test 7.1.4 {
+  REPLACE INTO t3 VALUES(4, NULL);
+} {
+  DELETE main t3 1 1   4 16
+  INSERT main t3 4 4   4 {}
+}
+
+do_preupdate_test 7.1.5 {
+  REPLACE INTO t3(rowid, i, j) VALUES(2, 6, NULL);
+} {
+  DELETE main t3 2 2  5 25
+  DELETE main t3 3 3  6 36
+  INSERT main t3 2 2  6 {}
+}
+
+do_execsql_test 7.2.0 { SELECT rowid FROM t1 } {1 2 3 4 5}
+
+do_preupdate_test 7.2.1 {
+  DELETE FROM t1 WHERE rowid = 3
+} {
+  DELETE main t1 3 3  d c
+}
+do_preupdate_test 7.2.2 {
+  DELETE FROM t1
+} {
+  DELETE main t1 1 1   1 1
+  DELETE main t1 2 2   b a
+  DELETE main t1 4 4   a b
+  DELETE main t1 5 5   c d
+}
+
+do_execsql_test 7.3.0 { 
+  DELETE FROM t1;
+  DELETE FROM t2;
+  DELETE FROM t3;
+
+  INSERT INTO t2 VALUES('a', 'b');
+  INSERT INTO t2 VALUES('c', 'd');
+
+  INSERT INTO t3 VALUES(4, 16);
+  INSERT INTO t3 VALUES(5, 25);
+  INSERT INTO t3 VALUES(6, 36);
+}
+
+do_preupdate_test 7.3.1 {
+  UPDATE t2 SET y = y||y;
+} {
+  UPDATE main t2 1 1   a b  a bb
+  UPDATE main t2 2 2   c d  c dd
+}
+
+do_preupdate_test 7.3.2 {
+  UPDATE t2 SET rowid = rowid-1;
+} {
+  UPDATE main t2 1 0   a bb  a bb
+  UPDATE main t2 2 1   c dd  c dd
+}
+
+do_preupdate_test 7.3.3 {
+  UPDATE OR REPLACE t2 SET rowid = 1 WHERE x = 'a'
+} {
+  DELETE main t2 1 1   c dd
+  UPDATE main t2 0 1   a bb  a bb
+}
+
+do_preupdate_test 7.3.4.1 {
+  UPDATE OR REPLACE t3 SET i = 5 WHERE i = 6
+} {
+  DELETE main t3 2 2   5 25
+  UPDATE main t3 3 3   6 36  5 36
+}
+
+do_execsql_test 7.3.4.2 {
+  INSERT INTO t3 VALUES(10, 100);
+  SELECT rowid, * FROM t3;
+} {1 4 16   3 5 36   4 10 100}
+
+do_preupdate_test 7.3.5 {
+  UPDATE OR REPLACE t3 SET rowid = 1, i = 5 WHERE j = 100;
+} {
+  DELETE main t3 1 1    4  16
+  DELETE main t3 3 3    5  36
+  UPDATE main t3 4 1   10 100  5 100
+}
+
+do_execsql_test 7.4.1.0 {
+  CREATE TABLE t4(a, b);
+  INSERT INTO t4 VALUES('a', 1);
+  INSERT INTO t4 VALUES('b', 2);
+  INSERT INTO t4 VALUES('c', 3);
+
+  CREATE TRIGGER t4t BEFORE DELETE ON t4 BEGIN
+    DELETE FROM t4 WHERE b = 1;
+  END;
+}
+
+do_preupdate_test 7.4.1.1 {
+  DELETE FROM t4 WHERE b = 3
+} {
+  DELETE main t4 1 1   a 1
+  DELETE main t4 3 3   c 3
+}
+
+do_execsql_test 7.4.1.2 {
+  INSERT INTO t4(rowid, a, b) VALUES(1, 'a', 1);
+  INSERT INTO t4(rowid, a, b) VALUES(3, 'c', 3);
+}
+do_preupdate_test 7.4.1.3 {
+  DELETE FROM t4 WHERE b = 1
+} {
+  DELETE main t4 1 1   a 1
+}
+
+do_execsql_test 7.4.2.0 {
+  CREATE TABLE t5(a, b);
+  INSERT INTO t5 VALUES('a', 1);
+  INSERT INTO t5 VALUES('b', 2);
+  INSERT INTO t5 VALUES('c', 3);
+
+  CREATE TRIGGER t5t BEFORE UPDATE ON t5 BEGIN
+    DELETE FROM t5 WHERE b = 1;
+  END;
+}
+do_preupdate_test 7.4.2.1 {
+  UPDATE t5 SET b = 4 WHERE a = 'c'
+} {
+  DELETE main t5 1 1   a 1
+  UPDATE main t5 3 3   c 3  c 4
+}
+
+do_execsql_test 7.4.2.2 {
+  INSERT INTO t5(rowid, a, b) VALUES(1, 'a', 1);
+}
+
+do_preupdate_test 7.4.2.3 {
+  UPDATE t5 SET b = 5 WHERE a = 'a'
+} {
+  DELETE main t5 1 1   a 1
+}
+
+do_execsql_test 7.5.1.0 {
+  CREATE TABLE t7(a, b);
+  INSERT INTO t7 VALUES('one', 'two');
+  INSERT INTO t7 VALUES('three', 'four');
+  ALTER TABLE t7 ADD COLUMN c DEFAULT NULL;
+}
+
+do_preupdate_test 7.5.1.1 {
+  DELETE FROM t7 WHERE a = 'one'
+} {
+  DELETE main t7 1 1   one two {}
+}
+
+do_preupdate_test 7.5.1.2 {
+  UPDATE t7 SET b = 'five'
+} {
+  UPDATE main t7 2 2   three four {}  three five {}
+}
+
+do_execsql_test 7.5.2.0 {
+  CREATE TABLE t8(a, b);
+  INSERT INTO t8 VALUES('one', 'two');
+  INSERT INTO t8 VALUES('three', 'four');
+  ALTER TABLE t8 ADD COLUMN c DEFAULT 'xxx';
+}
+
+if 0 {
+  # At time of writing, these two are broken. They demonstrate that the
+  # sqlite3_preupdate_old() method does not handle the case where ALTER TABLE
+  # has been used to add a column with a default value other than NULL.
+  #
+  do_preupdate_test 7.5.2.1 {
+    DELETE FROM t8 WHERE a = 'one'
+  } {
+    DELETE main t8 1 1   one two xxx
+  }
+  do_preupdate_test 7.5.2.2 {
+    UPDATE t8 SET b = 'five'
+  } {
+    UPDATE main t8 2 2   three four xxx  three five xxx
+  }
+}
+
+# This block of tests verifies that IPK values are correctly reported
+# by the sqlite3_preupdate_old() and sqlite3_preupdate_new() functions.
+#
+do_execsql_test 7.6.1 { CREATE TABLE t9(a, b INTEGER PRIMARY KEY, c) }
+do_preupdate_test 7.6.2 {
+  INSERT INTO t9 VALUES(1, 2, 3);
+  UPDATE t9 SET b = b+1, c = c+1;
+  DELETE FROM t9 WHERE a = 1;
+} {
+  INSERT main t9 2 2   1 2 3
+  UPDATE main t9 2 3   1 2 3   1 3 4
+  DELETE main t9 3 3   1 3 4
+}
+
+#--------------------------------------------------------------------------
+# Test that the sqlite3_preupdate_depth() API seems to work.
+#
+proc preupdate_hook {args} {
+  set type [lindex $args 0]
+  eval lappend ::preupdate $args
+  eval lappend ::preupdate [db preupdate depth]
+
+  if {$type != "INSERT"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      lappend ::preupdate [db preupdate old $i]
+    }
+  }
+  if {$type != "DELETE"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      set rc [catch { db preupdate new $i } v]
+      lappend ::preupdate $v
+    }
+  }
+}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db preupdate hook preupdate_hook
+
+do_execsql_test 7.6.1 { 
+  CREATE TABLE t1(x PRIMARY KEY);
+  CREATE TABLE t2(x PRIMARY KEY);
+  CREATE TABLE t3(x PRIMARY KEY);
+  CREATE TABLE t4(x PRIMARY KEY);
+
+  CREATE TRIGGER a AFTER INSERT ON t1 BEGIN INSERT INTO t2 VALUES(new.x); END;
+  CREATE TRIGGER b AFTER INSERT ON t2 BEGIN INSERT INTO t3 VALUES(new.x); END;
+  CREATE TRIGGER c AFTER INSERT ON t3 BEGIN INSERT INTO t4 VALUES(new.x); END;
+
+  CREATE TRIGGER d AFTER UPDATE ON t1 BEGIN UPDATE t2 SET x = new.x; END;
+  CREATE TRIGGER e AFTER UPDATE ON t2 BEGIN UPDATE t3 SET x = new.x; END;
+  CREATE TRIGGER f AFTER UPDATE ON t3 BEGIN UPDATE t4 SET x = new.x; END;
+
+  CREATE TRIGGER g AFTER DELETE ON t1 BEGIN DELETE FROM t2 WHERE 1; END;
+  CREATE TRIGGER h AFTER DELETE ON t2 BEGIN DELETE FROM t3 WHERE 1; END;
+  CREATE TRIGGER i AFTER DELETE ON t3 BEGIN DELETE FROM t4 WHERE 1; END;
+}
+
+do_preupdate_test 7.6.2 {
+  INSERT INTO t1 VALUES('xyz');
+} {
+  INSERT main t1 1 1   0      xyz
+  INSERT main t2 1 1   1      xyz
+  INSERT main t3 1 1   2      xyz
+  INSERT main t4 1 1   3      xyz
+}
+do_preupdate_test 7.6.3 {
+  UPDATE t1 SET x = 'abc';
+} {
+  UPDATE main t1 1 1   0      xyz abc
+  UPDATE main t2 1 1   1      xyz abc
+  UPDATE main t3 1 1   2      xyz abc
+  UPDATE main t4 1 1   3      xyz abc
+}
+do_preupdate_test 7.6.4 {
+  DELETE FROM t1 WHERE 1;
+} {
+  DELETE main t1 1 1   0      abc
+  DELETE main t2 1 1   1      abc
+  DELETE main t3 1 1   2      abc
+  DELETE main t4 1 1   3      abc
+}
+
+do_execsql_test 7.6.5 { 
+  DROP TRIGGER a; DROP TRIGGER b; DROP TRIGGER c;
+  DROP TRIGGER d; DROP TRIGGER e; DROP TRIGGER f;
+  DROP TRIGGER g; DROP TRIGGER h; DROP TRIGGER i;
+
+  CREATE TRIGGER a BEFORE INSERT ON t1 BEGIN INSERT INTO t2 VALUES(new.x); END;
+  CREATE TRIGGER b BEFORE INSERT ON t2 BEGIN INSERT INTO t3 VALUES(new.x); END;
+  CREATE TRIGGER c BEFORE INSERT ON t3 BEGIN INSERT INTO t4 VALUES(new.x); END;
+
+  CREATE TRIGGER d BEFORE UPDATE ON t1 BEGIN UPDATE t2 SET x = new.x; END;
+  CREATE TRIGGER e BEFORE UPDATE ON t2 BEGIN UPDATE t3 SET x = new.x; END;
+  CREATE TRIGGER f BEFORE UPDATE ON t3 BEGIN UPDATE t4 SET x = new.x; END;
+
+  CREATE TRIGGER g BEFORE DELETE ON t1 BEGIN DELETE FROM t2 WHERE 1; END;
+  CREATE TRIGGER h BEFORE DELETE ON t2 BEGIN DELETE FROM t3 WHERE 1; END;
+  CREATE TRIGGER i BEFORE DELETE ON t3 BEGIN DELETE FROM t4 WHERE 1; END;
+}
+
+do_preupdate_test 7.6.6 {
+  INSERT INTO t1 VALUES('xyz');
+} {
+  INSERT main t4 1 1   3      xyz
+  INSERT main t3 1 1   2      xyz
+  INSERT main t2 1 1   1      xyz
+  INSERT main t1 1 1   0      xyz
+}
+do_preupdate_test 7.6.3 {
+  UPDATE t1 SET x = 'abc';
+} {
+  UPDATE main t4 1 1   3      xyz abc
+  UPDATE main t3 1 1   2      xyz abc
+  UPDATE main t2 1 1   1      xyz abc
+  UPDATE main t1 1 1   0      xyz abc
+}
+do_preupdate_test 7.6.4 {
+  DELETE FROM t1 WHERE 1;
+} {
+  DELETE main t4 1 1   3      abc
+  DELETE main t3 1 1   2      abc
+  DELETE main t2 1 1   1      abc
+  DELETE main t1 1 1   0      abc
+}
+
+# No preupdate callbacks for modifying sqlite_master.
+do_preupdate_test 8.1 { CREATE TABLE x1(x, y); } { }
+do_preupdate_test 8.2 { ALTER TABLE x1 ADD COLUMN z } { }
+do_preupdate_test 8.3 { ALTER TABLE x1 RENAME TO y1 } { }
+do_preupdate_test 8.4 { CREATE INDEX y1x ON y1(x) } { }
+do_preupdate_test 8.5 { CREATE VIEW v1 AS SELECT * FROM y1 } { }
+do_preupdate_test 8.6 { DROP TABLE y1 } { }
+
+#-------------------------------------------------------------------------
+reset_db
+db preupdate hook preupdate_hook
+do_execsql_test 9.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
+  CREATE TABLE t2(a, b INTEGER PRIMARY KEY);
+}
+do_preupdate_test 9.1 {
+  INSERT INTO t1 VALUES(456, NULL, NULL);
+} {
+  INSERT main t1 456 456  0  456 {} {}
+}
+do_execsql_test 9.2 {
+  ALTER TABLE t1 ADD COLUMN d;
+}
+do_preupdate_test 9.3 {
+  INSERT INTO t1(a, b, c) VALUES(457, NULL, NULL);
+} {
+  INSERT main t1 457 457  0  457 {} {} {}
+}
+do_preupdate_test 9.4 {
+  DELETE FROM t1 WHERE a=456
+} {
+  DELETE main t1 456 456  0  456 {} {} {}
+}
+do_preupdate_test 9.5 {
+  INSERT INTO t2 DEFAULT VALUES;
+} {
+  INSERT main t2 1 1  0  {} 1
+} 
+do_preupdate_test 9.6 {
+  INSERT INTO t1 DEFAULT VALUES;
+} {
+  INSERT main t1 458 458  0  458 {} {} {}
+} 
+
+
+do_execsql_test 10.0 {
+  CREATE TABLE t3(a, b INTEGER PRIMARY KEY);
+}
+do_preupdate_test 10.1 {
+  INSERT INTO t3 DEFAULT VALUES
+} {
+  INSERT main t3 1 1 0 {} 1
+}
+do_execsql_test 10.2 { SELECT * FROM t3 } {{} 1}
+do_preupdate_test 10.3 {
+  DELETE FROM t3 WHERE b=1
+} {DELETE main t3 1 1 0 {} 1}
+
+
 finish_test
diff --git a/test/hook2.test b/test/hook2.test
new file mode 100644
index 0000000000..9ae1103a7d
--- /dev/null
+++ b/test/hook2.test
@@ -0,0 +1,218 @@
+# 2017 Jan 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# The tests in this file focus on the pre-update hook.
+# 
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix hook2
+
+ifcapable !preupdate {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+proc do_preupdate_test {tn sql x} {
+  set X [list]
+  foreach elem $x {lappend X $elem}
+  uplevel do_test $tn [list "
+    set ::preupdate \[list\]
+    execsql { $sql }
+    set ::preupdate
+  "] [list $X]
+}
+
+proc preupdate_hook {args} {
+  set type [lindex $args 0]
+  eval lappend ::preupdate $args
+  if {$type != "INSERT"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      lappend ::preupdate [db preupdate old $i]
+    }
+  }
+  if {$type != "DELETE"} {
+    for {set i 0} {$i < [db preupdate count]} {incr i} {
+      set rc [catch { db preupdate new $i } v]
+      lappend ::preupdate $v
+    }
+  }
+}
+
+#-------------------------------------------------------------------------
+# Simple tests - INSERT, UPDATE and DELETE on a WITHOUT ROWID table.
+#
+db preupdate hook preupdate_hook
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b) WITHOUT ROWID;
+}
+do_preupdate_test 1.1 {
+  INSERT INTO t1 VALUES('one', 1);
+} {
+  INSERT main t1 0 0  one 1
+}
+do_preupdate_test 1.2 {
+  UPDATE t1 SET b=2 WHERE a='one';
+} {
+  UPDATE main t1 0 0  one 1 one 2
+}
+do_preupdate_test 1.3 {
+  DELETE FROM t1 WHERE a='one';
+} {
+  DELETE main t1 0 0  one 2
+}
+
+#-------------------------------------------------------------------------
+# Some more complex tests for the pre-update callback on WITHOUT ROWID
+# tables.
+#
+#   2.1.1 - INSERT statement.
+#   2.1.2 - INSERT INTO ... SELECT statement.
+#   2.1.3 - REPLACE INTO ... (PK conflict)
+#   2.1.4 - REPLACE INTO ... (other index conflicts)
+#   2.1.5 - REPLACE INTO ... (both PK and other index conflicts)
+#
+#   2.2.1 - DELETE statement.
+#   2.2.2 - DELETE statement that uses the truncate optimization.
+#
+#   2.3.1 - UPDATE statement.
+#   2.3.2 - UPDATE statement that modifies the PK.
+#   2.3.3 - UPDATE OR REPLACE ... (PK conflict).
+#   2.3.4 - UPDATE OR REPLACE ... (other index conflicts)
+#   2.3.4 - UPDATE OR REPLACE ... (both PK and other index conflicts)
+#
+do_execsql_test 2.0 {
+  CREATE TABLE t2(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID;
+  CREATE UNIQUE INDEX t2a ON t2(a);
+}
+
+do_preupdate_test 2.1.1 {
+  INSERT INTO t2(b, c) VALUES(1, 1);
+} {
+  INSERT main t2 0 0  4 1 1
+}
+
+do_execsql_test 2.1.2.0 {
+  CREATE TABLE d1(a DEFAULT 4, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID;
+  CREATE UNIQUE INDEX d1a ON d1(a);
+  INSERT INTO d1 VALUES(1, 2, 3);
+  INSERT INTO d1 VALUES(11, 12, 13);
+}
+do_preupdate_test 2.1.2.1 {
+  INSERT INTO t2 SELECT * FROM d1;
+} {
+  INSERT main t2 0 0  1 2 3
+  INSERT main t2 0 0  11 12 13
+}
+do_preupdate_test 2.1.2.2 {
+  INSERT INTO t2 SELECT a+20, b+20, c+20 FROM d1;
+} {
+  INSERT main t2 0 0  21 22 23
+  INSERT main t2 0 0  31 32 33
+}
+do_execsql_test 2.1.2.3 {
+  SELECT * FROM t2 ORDER BY b, c;
+} {
+  4 1 1
+  1 2 3
+  11 12 13
+  21 22 23
+  31 32 33
+}
+do_preupdate_test 2.1.3 {
+  REPLACE INTO t2 VALUES(45, 22, 23);
+} {
+  DELETE main t2 0 0 21 22 23
+  INSERT main t2 0 0 45 22 23
+}
+do_preupdate_test 2.1.4 {
+  REPLACE INTO t2 VALUES(11, 100, 100);
+} {
+  DELETE main t2 0 0 11 12 13
+  INSERT main t2 0 0 11 100 100
+}
+do_preupdate_test 2.1.5 {
+  REPLACE INTO t2(c, b) VALUES(33, 32)
+} {
+  DELETE main t2 0 0 4 1 1 
+  DELETE main t2 0 0 31 32 33
+  INSERT main t2 0 0 4 32 33
+}
+
+do_execsql_test 2.2.0 {
+  SELECT * FROM t2 ORDER BY b,c;
+} {
+  1    2   3 
+  45  22  23 
+  4   32  33 
+  11 100 100
+}
+do_preupdate_test 2.2.1 {
+  DELETE FROM t2 WHERE b=22;
+} {
+  DELETE main t2 0 0  45 22 23
+}
+do_preupdate_test 2.2.2 {
+  DELETE FROM t2;
+} {
+  DELETE main t2 0 0 1 2 3 
+  DELETE main t2 0 0 4 32 33 
+  DELETE main t2 0 0 11 100 100
+}
+
+do_execsql_test 2.3.0 {
+  CREATE TABLE t3(x, y PRIMARY KEY, z UNIQUE) WITHOUT ROWID;
+  INSERT INTO t3 VALUES('a', 'b', 'c');
+  INSERT INTO t3 VALUES('d', 'e', 'f');
+
+  INSERT INTO t3 VALUES(1, 1, 1);
+  INSERT INTO t3 VALUES(2, 2, 2);
+  INSERT INTO t3 VALUES(3, 3, 3);
+}
+
+do_preupdate_test 2.3.1 {
+  UPDATE t3 SET x=4 WHERE y IN ('b', 'e', 'x');
+} {
+  UPDATE main t3 0 0  a b c   4 b c
+  UPDATE main t3 0 0  d e f   4 e f
+}
+
+do_preupdate_test 2.3.2 {
+  UPDATE t3 SET y=y||y WHERE z IN('c', 'f');
+} {
+  UPDATE main t3 0 0  4 b c   4 bb c
+  UPDATE main t3 0 0  4 e f   4 ee f
+}
+
+do_preupdate_test 2.3.3 {
+  UPDATE OR REPLACE t3 SET y='bb' WHERE z='f'
+} {
+  DELETE main t3 0 0  4 bb c
+  UPDATE main t3 0 0  4 ee f   4 bb f
+}
+
+do_preupdate_test 2.3.4 {
+  UPDATE OR REPLACE t3 SET z=2 WHERE y=1;
+} {
+  DELETE main t3 0 0  2 2 2
+  UPDATE main t3 0 0  1 1 1  1 1 2
+}
+
+do_preupdate_test 2.3.5 {
+  UPDATE OR REPLACE t3 SET z=2, y='bb' WHERE y=3;
+} {
+  DELETE main t3 0 0  1 1 2
+  DELETE main t3 0 0  4 bb f
+  UPDATE main t3 0 0  3 3 3  3 bb 2
+}
+  
+
+finish_test
diff --git a/test/in.test b/test/in.test
index 3a42e84b9a..4e10800481 100644
--- a/test/in.test
+++ b/test/in.test
@@ -314,7 +314,7 @@ do_test in-9.4 {
   catchsql {
     SELECT b FROM t1 WHERE a NOT IN tb;
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 
 # IN clauses in CHECK constraints.  Ticket #1645
 #
@@ -391,28 +391,28 @@ do_test in-12.2 {
       SELECT a, b FROM t3 UNION ALL SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.3 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 UNION SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.4 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 EXCEPT SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.5 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
       SELECT a, b FROM t3 INTERSECT SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.6 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
@@ -478,7 +478,7 @@ do_test in-12.14 {
       SELECT a, b FROM t3 UNION ALL SELECT a, b FROM t2
     );
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 do_test in-12.15 {
   catchsql {
     SELECT * FROM t2 WHERE a IN (
@@ -629,15 +629,27 @@ do_test in-13.14 {
   }
 } {}
 
+breakpoint
 do_test in-13.15 {
   catchsql {
     SELECT 0 WHERE (SELECT 0,0) OR (0 IN (1,2));
   }
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {sub-select returns 2 columns - expected 1}}
 
 
 do_test in-13.X {
   db nullvalue ""
 } {}
 
+# At one point the following was causing valgrind to report a "jump
+# depends on unitialized location" problem.
+#
+do_execsql_test in-14.0 {
+  CREATE TABLE c1(a);
+  INSERT INTO c1 VALUES(1), (2), (4), (3);
+}
+do_execsql_test in-14.1 {
+  SELECT * FROM c1 WHERE a IN (SELECT a FROM c1) ORDER BY 1
+} {1 2 3 4}
+
 finish_test
diff --git a/test/in5.test b/test/in5.test
index b49c93e15b..ac60cfe58e 100644
--- a/test/in5.test
+++ b/test/in5.test
@@ -214,8 +214,27 @@ do_execsql_test 7.3.2 {
   SELECT a FROM y1 WHERE b IN (SELECT a FROM y2);
 } {two}
 
-finish_test
-
+#-------------------------------------------------------------------------
+# Tests to confirm that indexes on the rowid column do not confuse
+# the query planner. See ticket [0eab1ac7591f511d].
+#
+do_execsql_test 8.0 {
+  CREATE TABLE n1(a INTEGER PRIMARY KEY, b VARCHAR(500));
+  CREATE UNIQUE INDEX n1a ON n1(a);
+}
 
+do_execsql_test 8.1 {
+  SELECT count(*) FROM n1 WHERE a IN (1, 2, 3)
+} 0
+do_execsql_test 8.2 {
+  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
+} 0
+do_execsql_test 8.3 {
+  INSERT INTO n1 VALUES(1, NULL), (2, NULL), (3, NULL);
+  SELECT count(*) FROM n1 WHERE a IN (1, 2, 3)
+} 3
+do_execsql_test 8.4 {
+  SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1)
+} 3
 
 finish_test
diff --git a/test/incrblob4.test b/test/incrblob4.test
index a92e373536..dbff8eb7d5 100644
--- a/test/incrblob4.test
+++ b/test/incrblob4.test
@@ -86,4 +86,24 @@ do_test 3.3 {
   execsql { INSERT INTO t1(v) VALUES($new) }
 } {}
 
+#-------------------------------------------------------------------------
+# Test that it is not possible to DROP a table with an incremental blob
+# cursor open on it.
+#
+do_execsql_test 4.1 {
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t2 VALUES(456, '0123456789');
+}
+do_test 4.2 {
+  set blob [db incrblob -readonly t2 b 456]
+  read $blob 5
+} {01234}
+do_catchsql_test 4.3 {
+  DROP TABLE t2
+} {1 {database table is locked}}
+do_test 4.4 {
+  sqlite3_extended_errcode db
+} {SQLITE_LOCKED}
+close $blob
+
 finish_test
diff --git a/test/incrblobfault.test b/test/incrblobfault.test
index 0c1d93d460..10c2c8ecb4 100644
--- a/test/incrblobfault.test
+++ b/test/incrblobfault.test
@@ -52,7 +52,7 @@ do_faultsim_test 2 -prep {
     error [sqlite3_errmsg db]
   }
 } -test {
-  faultsim_test_result {1 {no such rowid: -1}}
+  faultsim_test_result {1 {no such rowid: -1}} {1 {disk I/O error}}
   close $::blob
 }
 
diff --git a/test/incrvacuum2.test b/test/incrvacuum2.test
index 2219d54195..6ec66c9deb 100644
--- a/test/incrvacuum2.test
+++ b/test/incrvacuum2.test
@@ -134,7 +134,7 @@ do_test incrvacuum2-3.2 {
 
 integrity_check incrvacuum2-3.3
 
-ifcapable wal {
+if {[wal_is_capable]} {
   # At one point, when a specific page was being extracted from the b-tree
   # free-list (e.g. during an incremental-vacuum), all trunk pages that
   # occurred before the specific page in the free-list trunk were being
diff --git a/test/index6.test b/test/index6.test
index 33ae3d1cb1..f6177b44e9 100644
--- a/test/index6.test
+++ b/test/index6.test
@@ -65,12 +65,15 @@ do_test index6-1.5 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a!=random();
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {1 {non-deterministic functions prohibited in partial index WHERE clauses}}
 do_test index6-1.6 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a NOT LIKE 'abc%';
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {0 {}}
+do_execsql_test index6-1.7 {
+  DROP INDEX IF EXISTS bad1;
+}
 
 do_test index6-1.10 {
   execsql {
@@ -376,4 +379,15 @@ do_execsql_test index6-10.3eqp {
   SELECT e FROM t10 WHERE a=1 AND b=2 ORDER BY d DESC;
 } {~/USING INDEX t10x/}
 
+# A partial index will be used for a full table scan, where possible
+do_execsql_test index6-11.1 {
+  CREATE TABLE t11(a,b,c);
+  CREATE INDEX t11x ON t11(a) WHERE b<>99;
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99;
+} {/USING INDEX t11x/}
+do_execsql_test index6-11.2 {
+  EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98;
+} {/USING INDEX t11x/}
+  
+
 finish_test
diff --git a/test/index7.test b/test/index7.test
index 557fe21324..0037a8a44d 100644
--- a/test/index7.test
+++ b/test/index7.test
@@ -99,12 +99,25 @@ do_test index7-1.5 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a!=random();
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {1 {non-deterministic functions prohibited in partial index WHERE clauses}}
 do_test index7-1.6 {
   catchsql {
     CREATE INDEX bad1 ON t1(a,b) WHERE a NOT LIKE 'abc%';
   }
-} {1 {functions prohibited in partial index WHERE clauses}}
+} {0 {}}
+do_execsql_test index7-1.7 {
+  INSERT INTO t1(a,b,c)
+     VALUES('abcde',1,101),('abdef',2,102),('xyz',3,103),('abcz',4,104);
+  SELECT c FROM t1 WHERE a NOT LIKE 'abc%' AND a=7 ORDER BY +b;
+} {7}
+do_execsql_test index7-1.7eqp {
+  EXPLAIN QUERY PLAN
+  SELECT b FROM t1 WHERE a NOT LIKE 'abc%' AND a=7 ORDER BY +b;
+} {/SEARCH TABLE t1 USING COVERING INDEX bad1 /}
+do_execsql_test index7-1.8 {
+  DELETE FROM t1 WHERE c>=101;
+  DROP INDEX IF EXISTS bad1;
+} {}
 
 do_test index7-1.10 {
   execsql {
diff --git a/test/index8.test b/test/index8.test
new file mode 100644
index 0000000000..bb58228527
--- /dev/null
+++ b/test/index8.test
@@ -0,0 +1,60 @@
+# 2016-07-27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for ORDER BY and LIMIT on an index scan.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Performance regression reported at
+# http://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg98615.html
+#
+# Caused by the ORDER BY LIMIT optionation for check-in
+# https://sqlite.org/src/info/bf46179d44843769
+#
+# Fixed on approximately 2016-07-27 by changes that compute a better score
+# for index scans by taking into account WHERE clause constraints that can
+# be handled by the index and do not require a table lookup.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a,b,c,d);
+  WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b,c,d)
+     SELECT x/10, x%10, x%19, x FROM c;
+  CREATE INDEX t1abc ON t1(a,b,c);
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {0 4 4 4 2 3 4 23}
+
+# Prior to the fix, the following EQP would show a table scan and a sort
+# rather than an index scan.
+#
+do_execsql_test 1.0eqp {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {/SCAN TABLE t1 USING INDEX t1abc/}
+
+# If we change the index so that it no longer covers the WHERE clause,
+# then we should (correctly) revert to using a table scan.
+#
+do_execsql_test 1.1 {
+  DROP INDEX t1abc;
+  CREATE INDEX t1abd ON t1(a,b,d);
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {0 4 4 4 2 3 4 23}
+do_execsql_test 1.1eqp {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2;
+} {~/USING INDEX/}
+
+
+finish_test
diff --git a/test/indexexpr1.test b/test/indexexpr1.test
index a8a74f259e..2b375a5855 100644
--- a/test/indexexpr1.test
+++ b/test/indexexpr1.test
@@ -323,5 +323,61 @@ do_execsql_test indexexpr1-1010 {
   SELECT *, '|' FROM t0 ORDER BY +a;
 } {0 88 2 | 2 99 4 | 5 99 7 |}
 
+# 2016-10-10
+# Make sure indexes on expressions skip over initial NULL values in the
+# index as they are suppose to do.
+# Ticket https://www.sqlite.org/src/tktview/4baa46491212947
+#
+do_execsql_test indexexpr1-1100 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a);
+  INSERT INTO t1 VALUES(NULL),(1);
+  SELECT '1:', typeof(a), a FROM t1 WHERE a<10;
+  SELECT '2:', typeof(a), a FROM t1 WHERE a+0<10;
+  CREATE INDEX t1x1 ON t1(a);
+  CREATE INDEX t1x2 ON t1(a+0);
+  SELECT '3:', typeof(a), a FROM t1 WHERE a<10;
+  SELECT '4:', typeof(a), a FROM t1 WHERE a+0<10;
+} {1: integer 1 2: integer 1 3: integer 1 4: integer 1}
+
+do_execsql_test indexexpr1-1200 {
+  CREATE TABLE t10(a int, b int, c int, d int);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 2, 2);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 0, 0);
+  INSERT INTO t10(a, b, c, d) VALUES(0, 0, 1, 1);
+  INSERT INTO t10(a, b, c, d) VALUES(1, 1, 1, 1);
+  INSERT INTO t10(a, b, c, d) VALUES(1, 1, 0, 0);
+  INSERT INTO t10(a, b, c, d) VALUES(2, 2, 0, 0);
+
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
+do_execsql_test indexexpr1-1200.1 {
+  CREATE INDEX t10_ab ON t10(a+b);
+}
+do_execsql_test indexexpr1-1200.2 {
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
+do_execsql_test indexexpr1-1200.3 {
+  CREATE INDEX t10_abcd ON t10(a+b,c+d);
+}
+do_execsql_test indexexpr1-1200.4 {
+  SELECT a+b, c+d FROM t10 ORDER BY a+b, c+d;
+} {
+  0 0 0 2 0 4 2 0 2 2 4 0
+}
+
+# Ticket https://www.sqlite.org/src/tktview/eb703ba7b50c1a
+# Incorrect result using an index on an expression with a collating function
+#
+do_execsql_test indexexpr1-1300.1 {
+  CREATE TABLE t1300(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1300 VALUES(1,'coffee'),(2,'COFFEE'),(3,'stress'),(4,'STRESS');
+  CREATE INDEX t1300bexpr ON t1300( substr(b,4) );
+  SELECT a FROM t1300 WHERE substr(b,4)='ess' COLLATE nocase ORDER BY +a;
+} {3 4}
 
 finish_test
diff --git a/test/instr.test b/test/instr.test
index c8be4862b9..2caf3bce3b 100644
--- a/test/instr.test
+++ b/test/instr.test
@@ -248,4 +248,13 @@ do_execsql_test instr-1.62 {
   SELECT coalesce(instr(NULL,NULL), 999);
 } {999}
 
+do_execsql_test instr-1.63 {
+  SELECT instr(X'', 'abc')
+} 0
+do_execsql_test instr-1.64 {
+  CREATE TABLE x1(a, b);
+  INSERT INTO x1 VALUES(X'', 'abc');
+  SELECT instr(a, b) FROM x1;
+} 0
+
 finish_test
diff --git a/test/instrfault.test b/test/instrfault.test
new file mode 100644
index 0000000000..0ddb12c84b
--- /dev/null
+++ b/test/instrfault.test
@@ -0,0 +1,92 @@
+# 2016 November 4
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing OOM error handling within the built-in 
+# INSTR() function.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix instrfault
+
+# Use big NEEDLE and HAYSTACK strings. Strings so large they cannot
+# use lookaside buffers.
+#
+set ::NEEDLE [string repeat "abcdefghijklmnopqrstuvwxyz" 10]
+set ::HAYSTACK "[string repeat 123 10]$NEEDLE[string repeat 456 10]"
+
+foreach {enc} {
+  utf8
+  utf16
+} {
+  reset_db
+  sqlite3_db_config_lookaside db 0 0 0
+
+  execsql "PRAGMA encoding = $enc"
+  do_execsql_test 1.$enc.1 {
+    CREATE TABLE t1(n, h);
+    INSERT INTO t1 VALUES($::NEEDLE, $::HAYSTACK);
+  } {}
+
+  do_faultsim_test 1.$enc.1 -faults oom-t* -prep {
+    execsql { SELECT instr(h, n) FROM t1 }
+  } -body {
+    execsql { SELECT instr(h, n) FROM t1 }
+  } -test {
+    faultsim_test_result {0 31}
+  }
+
+  do_faultsim_test 1.$enc.2 -faults oom-t* -prep {
+    execsql { SELECT instr($::HAYSTACK, $::NEEDLE) FROM t1 }
+  } -body {
+    execsql { SELECT instr($::HAYSTACK, $::NEEDLE) FROM t1 }
+  } -test {
+    faultsim_test_result {0 31}
+  }
+
+  do_faultsim_test 1.$enc.3 -faults oom-t* -prep {
+    set ::stmt [sqlite3_prepare_v2 db "SELECT instr(?, ?)" -1 dummy]
+    sqlite3_bind_text $::stmt 1 $::HAYSTACK [string length $::HAYSTACK]
+    sqlite3_bind_text $::stmt 2 $::NEEDLE [string length $::NEEDLE]
+  } -body {
+    set rc [sqlite3_step $::stmt]
+    if {$rc=="SQLITE_NOMEM"} { error "out of memory" }
+    sqlite3_column_int $::stmt 0
+  } -test {
+    faultsim_test_result {0 31}
+    sqlite3_finalize $::stmt
+  }
+
+  do_faultsim_test 1.$enc.4 -faults oom-t* -prep {
+    set ::stmt [sqlite3_prepare_v2 db "SELECT instr(?, ?)" -1 dummy]
+    sqlite3_bind_blob $::stmt 1 $::HAYSTACK [string length $::HAYSTACK]
+    sqlite3_bind_text $::stmt 2 $::NEEDLE [string length $::NEEDLE]
+  } -body {
+    set rc [sqlite3_step $::stmt]
+    if {$rc=="SQLITE_NOMEM"} { error "out of memory" }
+    sqlite3_column_int $::stmt 0
+  } -test {
+    faultsim_test_result {0 31}
+    sqlite3_finalize $::stmt
+  }
+
+  do_execsql_test 1.$enc.5.0 {
+    CREATE TABLE h1(a, b);
+    INSERT INTO h1 VALUES('abcdefg%200hijkl', randomblob(200));
+    INSERT INTO h1 SELECT b, a FROM h1;
+  }
+  do_faultsim_test 1.$enc.5 -faults oom-t* -body {
+    execsql { SELECT rowid FROM h1 WHERE instr(a,b) }
+  } -test {}
+}
+
+finish_test
diff --git a/test/intarray.test b/test/intarray.test
index 2aba080bd4..5c70ebaf1a 100644
--- a/test/intarray.test
+++ b/test/intarray.test
@@ -42,7 +42,7 @@ do_test intarray-1.1 {
   set ia3 [sqlite3_intarray_create db ia3]
   set ia4 [sqlite3_intarray_create db ia4]
   db eval {
-    SELECT type, name FROM sqlite_temp_master
+    SELECT type, name FROM temp.sqlite_master
      ORDER BY name
   }
 } {table ia1 table ia2 table ia3 table ia4}
diff --git a/test/interrupt.test b/test/interrupt.test
index 92ab4c3265..8774aee860 100644
--- a/test/interrupt.test
+++ b/test/interrupt.test
@@ -128,7 +128,7 @@ ifcapable tempdb {
     } {1 interrupted}
     do_test interrupt-3.$i.3 {
       execsql {
-        SELECT name FROM sqlite_temp_master;
+        SELECT name FROM temp.sqlite_master;
       }
     } {}
     do_test interrupt-3.$i.4 {
@@ -139,7 +139,7 @@ ifcapable tempdb {
     do_test interrupt-3.$i.5 {
       catchsql {SELECT name FROM sqlite_temp_master};
       execsql {
-        SELECT name FROM sqlite_temp_master;
+        SELECT name FROM temp.sqlite_master;
       }
     } {}
   }
diff --git a/test/interrupt2.test b/test/interrupt2.test
new file mode 100644
index 0000000000..b9c66bd1f4
--- /dev/null
+++ b/test/interrupt2.test
@@ -0,0 +1,159 @@
+# 2016 Aug 12
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is using the sqlite_interrupt() API to 
+# interrupt WAL checkpoint operations.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/wal_common.tcl
+set testprefix interrupt2
+
+if {[permutation]=="journaltest" || [permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
+db close
+testvfs tvfs -default 1
+
+tvfs filter xWrite
+tvfs script write_cb
+
+set ::trigger_interrupt 0
+proc write_cb {method args} {
+  set filename [lindex $args 0]
+  if {[file tail $filename]=="test.db" && $::trigger_interrupt} {
+    if {$::trigger_interrupt} {
+      incr ::trigger_interrupt -1
+      if {$::trigger_interrupt==0} { sqlite3_interrupt db }
+    }
+  }
+  return 0
+}
+
+sqlite3 db test.db 
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  PRAGMA journal_mode = wal;
+
+  WITH ii(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM ii WHERE i<1000 )
+  INSERT INTO t1 SELECT i, i FROM ii;
+} {wal}
+
+foreach idelay {
+  5
+  10
+  15
+  20
+} {
+
+  set ::trigger_interrupt $idelay
+  do_catchsql_test 1.$idelay.1 { PRAGMA wal_checkpoint; } {1 interrupted}
+  do_execsql_test  1.$idelay.2 { SELECT count(*) FROM t1 } 1000
+
+  set ::trigger_interrupt $idelay
+  do_test 1.$idelay.3 { 
+    list [catch { sqlite3_wal_checkpoint_v2 db truncate } msg] $msg
+  } {1 {SQLITE_INTERRUPT - interrupted}}
+  do_execsql_test  1.$idelay.4 { SELECT count(*) FROM t1 } 1000
+}
+
+#-------------------------------------------------------------------------
+# Check that if there are other SQL statements running, a checkpoint does
+# not clear the isInterrupted flag.
+#
+do_execsql_test 2.0 {
+  CREATE TEMP TABLE z1(a, b);
+  INSERT INTO z1 SELECT * FROM t1;
+}
+
+do_test 2.1 {
+  set i 10
+  set res [list [catch {
+    set i 10
+    db eval {SELECT * FROM z1} {
+      incr i -1
+      if {$i==0} {
+        set ::trigger_interrupt 10
+        set cres [catch { sqlite3_wal_checkpoint_v2 db truncate } msg] 
+        lappend cres $msg
+      }
+    }
+  } msg] $msg]
+
+  list $cres $res
+} {{1 {SQLITE_INTERRUPT - interrupted}} {1 interrupted}}
+
+do_execsql_test 2.0 {
+  SELECT count(*) FROM t1
+  UNION ALL
+  SELECT count(*) FROM z1
+} {1000 1000}
+
+#-------------------------------------------------------------------------
+# Check the effect of an interrupt during sqlite3_close().
+#
+db_save_and_close
+
+db_restore_and_reopen
+do_test 3.1.1 {
+  set ::trigger_interrupt 10
+  db eval { SELECT * FROM sqlite_master }
+  db close
+  set {} {}
+} {}
+do_test 3.1.2 {
+  list [file exists test.db] [file exists test.db-wal]
+} {1 1}
+
+db_restore_and_reopen
+do_test 3.2.1 {
+  db eval { SELECT * FROM sqlite_master }
+  db close
+  set {} {}
+} {}
+do_test 3.2.2 {
+  list [file exists test.db] [file exists test.db-wal]
+} {1 0}
+
+#-------------------------------------------------------------------------
+# Check the effect of an interrupt during an automatic checkpoint
+#
+db_restore_and_reopen
+do_test 4.0 { 
+  execsql { PRAGMA wal_autocheckpoint = 10 }
+  set ::trigger_interrupt 10
+  execsql { CREATE TABLE t2(x, y) }
+} {}
+
+# The auto-checkpoint in test 4.0 should have been interrupted. So this
+# db write should cause the wal file to grow.
+do_test 4.1 {
+  set nFrame1 [wal_frame_count test.db-wal 1024]
+  execsql { CREATE TABLE t3(x, y) }
+  set nFrame2 [wal_frame_count test.db-wal 1024]
+  expr $nFrame2 > $nFrame1
+} {1}
+
+# The auto-checkpoint in test 4.0 should not have been interrupted. So 
+# this db write should not cause the wal file to grow.
+do_test 4.2 {
+  set nFrame1 [wal_frame_count test.db-wal 1024]
+  execsql { CREATE TABLE t4(x, y) }
+  set nFrame2 [wal_frame_count test.db-wal 1024]
+  expr $nFrame2 == $nFrame1
+} {1}
+
+finish_test
diff --git a/test/intpkey.test b/test/intpkey.test
index 41858e5d46..a56cab4c69 100644
--- a/test/intpkey.test
+++ b/test/intpkey.test
@@ -296,7 +296,7 @@ do_test intpkey-3.8 {
   count {
     SELECT * FROM t1 WHERE c=='world' AND a>7;
   }
-} {11 hello world 4}
+} {11 hello world 3}
 do_test intpkey-3.9 {
   count {
     SELECT * FROM t1 WHERE 7 "');
+} {0}
+do_execsql_test json-10.32 {
+  SELECT json_valid('" \? "');
+} {0}
+do_execsql_test json-10.33 {
+  SELECT json_valid('" \@ "');
+} {0}
+do_execsql_test json-10.34 {
+  SELECT json_valid('" \A "');
+} {0}
+do_execsql_test json-10.35 {
+  SELECT json_valid('" \B "');
+} {0}
+do_execsql_test json-10.36 {
+  SELECT json_valid('" \C "');
+} {0}
+do_execsql_test json-10.37 {
+  SELECT json_valid('" \D "');
+} {0}
+do_execsql_test json-10.38 {
+  SELECT json_valid('" \E "');
+} {0}
+do_execsql_test json-10.39 {
+  SELECT json_valid('" \F "');
+} {0}
+do_execsql_test json-10.40 {
+  SELECT json_valid('" \G "');
+} {0}
+do_execsql_test json-10.41 {
+  SELECT json_valid('" \H "');
+} {0}
+do_execsql_test json-10.42 {
+  SELECT json_valid('" \I "');
+} {0}
+do_execsql_test json-10.43 {
+  SELECT json_valid('" \J "');
+} {0}
+do_execsql_test json-10.44 {
+  SELECT json_valid('" \K "');
+} {0}
+do_execsql_test json-10.45 {
+  SELECT json_valid('" \L "');
+} {0}
+do_execsql_test json-10.46 {
+  SELECT json_valid('" \M "');
+} {0}
+do_execsql_test json-10.47 {
+  SELECT json_valid('" \N "');
+} {0}
+do_execsql_test json-10.48 {
+  SELECT json_valid('" \O "');
+} {0}
+do_execsql_test json-10.49 {
+  SELECT json_valid('" \P "');
+} {0}
+do_execsql_test json-10.50 {
+  SELECT json_valid('" \Q "');
+} {0}
+do_execsql_test json-10.51 {
+  SELECT json_valid('" \R "');
+} {0}
+do_execsql_test json-10.52 {
+  SELECT json_valid('" \S "');
+} {0}
+do_execsql_test json-10.53 {
+  SELECT json_valid('" \T "');
+} {0}
+do_execsql_test json-10.54 {
+  SELECT json_valid('" \U "');
+} {0}
+do_execsql_test json-10.55 {
+  SELECT json_valid('" \V "');
+} {0}
+do_execsql_test json-10.56 {
+  SELECT json_valid('" \W "');
+} {0}
+do_execsql_test json-10.57 {
+  SELECT json_valid('" \X "');
+} {0}
+do_execsql_test json-10.58 {
+  SELECT json_valid('" \Y "');
+} {0}
+do_execsql_test json-10.59 {
+  SELECT json_valid('" \Z "');
+} {0}
+do_execsql_test json-10.60 {
+  SELECT json_valid('" \[ "');
+} {0}
+do_execsql_test json-10.61 {
+  SELECT json_valid('" \\ "');
+} {1}
+do_execsql_test json-10.62 {
+  SELECT json_valid('" \] "');
+} {0}
+do_execsql_test json-10.63 {
+  SELECT json_valid('" \^ "');
+} {0}
+do_execsql_test json-10.64 {
+  SELECT json_valid('" \_ "');
+} {0}
+do_execsql_test json-10.65 {
+  SELECT json_valid('" \` "');
+} {0}
+do_execsql_test json-10.66 {
+  SELECT json_valid('" \a "');
+} {0}
+do_execsql_test json-10.67 {
+  SELECT json_valid('" \b "');
+} {1}
+do_execsql_test json-10.68 {
+  SELECT json_valid('" \c "');
+} {0}
+do_execsql_test json-10.69 {
+  SELECT json_valid('" \d "');
+} {0}
+do_execsql_test json-10.70 {
+  SELECT json_valid('" \e "');
+} {0}
+do_execsql_test json-10.71 {
+  SELECT json_valid('" \f "');
+} {1}
+do_execsql_test json-10.72 {
+  SELECT json_valid('" \g "');
+} {0}
+do_execsql_test json-10.73 {
+  SELECT json_valid('" \h "');
+} {0}
+do_execsql_test json-10.74 {
+  SELECT json_valid('" \i "');
+} {0}
+do_execsql_test json-10.75 {
+  SELECT json_valid('" \j "');
+} {0}
+do_execsql_test json-10.76 {
+  SELECT json_valid('" \k "');
+} {0}
+do_execsql_test json-10.77 {
+  SELECT json_valid('" \l "');
+} {0}
+do_execsql_test json-10.78 {
+  SELECT json_valid('" \m "');
+} {0}
+do_execsql_test json-10.79 {
+  SELECT json_valid('" \n "');
+} {1}
+do_execsql_test json-10.80 {
+  SELECT json_valid('" \o "');
+} {0}
+do_execsql_test json-10.81 {
+  SELECT json_valid('" \p "');
+} {0}
+do_execsql_test json-10.82 {
+  SELECT json_valid('" \q "');
+} {0}
+do_execsql_test json-10.83 {
+  SELECT json_valid('" \r "');
+} {1}
+do_execsql_test json-10.84 {
+  SELECT json_valid('" \s "');
+} {0}
+do_execsql_test json-10.85 {
+  SELECT json_valid('" \t "');
+} {1}
+do_execsql_test json-10.86.0 {
+  SELECT json_valid('" \u "');
+} {0}
+do_execsql_test json-10.86.1 {
+  SELECT json_valid('" \ua "');
+} {0}
+do_execsql_test json-10.86.2 {
+  SELECT json_valid('" \uab "');
+} {0}
+do_execsql_test json-10.86.3 {
+  SELECT json_valid('" \uabc "');
+} {0}
+do_execsql_test json-10.86.4 {
+  SELECT json_valid('" \uabcd "');
+} {1}
+do_execsql_test json-10.86.5 {
+  SELECT json_valid('" \uFEDC "');
+} {1}
+do_execsql_test json-10.86.6 {
+  SELECT json_valid('" \u1234 "');
+} {1}
+do_execsql_test json-10.87 {
+  SELECT json_valid('" \v "');
+} {0}
+do_execsql_test json-10.88 {
+  SELECT json_valid('" \w "');
+} {0}
+do_execsql_test json-10.89 {
+  SELECT json_valid('" \x "');
+} {0}
+do_execsql_test json-10.90 {
+  SELECT json_valid('" \y "');
+} {0}
+do_execsql_test json-10.91 {
+  SELECT json_valid('" \z "');
+} {0}
+do_execsql_test json-10.92 {
+  SELECT json_valid('" \{ "');
+} {0}
+do_execsql_test json-10.93 {
+  SELECT json_valid('" \| "');
+} {0}
+do_execsql_test json-10.94 {
+  SELECT json_valid('" \} "');
+} {0}
+do_execsql_test json-10.95 {
+  SELECT json_valid('" \~ "');
+} {0}
+
 
 finish_test
diff --git a/test/json104.test b/test/json104.test
new file mode 100644
index 0000000000..b5313f01b5
--- /dev/null
+++ b/test/json104.test
@@ -0,0 +1,130 @@
+# 2017-03-22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for json_patch(A,B) SQL function.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !json1 {
+  finish_test
+  return
+}
+
+# This is the example from pages 2 and 3 of RFC-7396
+do_execsql_test json104-100 {
+  SELECT json_patch('{
+       "a": "b",
+       "c": {
+         "d": "e",
+         "f": "g"
+       }
+     }','{
+       "a":"z",
+       "c": {
+         "f": null
+       }
+     }');
+} {{{"a":"z","c":{"d":"e"}}}}
+
+
+# This is the example from pages 4 and 5 of RFC-7396 
+do_execsql_test json104-110 {
+  SELECT json_patch('{
+       "title": "Goodbye!",
+       "author" : {
+         "givenName" : "John",
+         "familyName" : "Doe"
+       },
+       "tags":[ "example", "sample" ],
+       "content": "This will be unchanged"
+     }','{
+       "title": "Hello!",
+       "phoneNumber": "+01-123-456-7890",
+       "author": {
+         "familyName": null
+       },
+       "tags": [ "example" ]
+     }');
+} {{{"title":"Hello!","author":{"givenName":"John"},"tags":["example"],"content":"This will be unchanged","phoneNumber":"+01-123-456-7890"}}}
+
+do_execsql_test json104-200 {
+  SELECT json_patch('[1,2,3]','{"x":null}');
+} {{{}}}
+do_execsql_test json104-210 {
+  SELECT json_patch('[1,2,3]','{"x":null,"y":1,"z":null}');
+} {{{"y":1}}}
+do_execsql_test json104-220 {
+  SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}');
+} {{{"a":{"bb":{}}}}}
+do_execsql_test json104-221 {
+  SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,null,3]}}}');
+} {{{"a":{"bb":{"ccc":[1,null,3]}}}}}
+do_execsql_test json104-222 {
+  SELECT json_patch('{}','{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}');
+} {{{"a":{"bb":{"ccc":[1,{"dddd":null},3]}}}}}
+
+# Example test cases at the end of the RFC-7396 document
+do_execsql_test json104-300 {
+  SELECT json_patch('{"a":"b"}','{"a":"c"}');
+} {{{"a":"c"}}}
+do_execsql_test json104-300a {
+  SELECT coalesce(json_patch(null,'{"a":"c"}'), 'real-null');
+} {{real-null}}
+do_execsql_test json104-301 {
+  SELECT json_patch('{"a":"b"}','{"b":"c"}');
+} {{{"a":"b","b":"c"}}}
+do_execsql_test json104-302 {
+  SELECT json_patch('{"a":"b"}','{"a":null}');
+} {{{}}}
+do_execsql_test json104-303 {
+  SELECT json_patch('{"a":"b","b":"c"}','{"a":null}');
+} {{{"b":"c"}}}
+do_execsql_test json104-304 {
+  SELECT json_patch('{"a":["b"]}','{"a":"c"}');
+} {{{"a":"c"}}}
+do_execsql_test json104-305 {
+  SELECT json_patch('{"a":"c"}','{"a":["b"]}');
+} {{{"a":["b"]}}}
+do_execsql_test json104-306 {
+  SELECT json_patch('{"a":{"b":"c"}}','{"a":{"b":"d","c":null}}');
+} {{{"a":{"b":"d"}}}}
+do_execsql_test json104-307 {
+  SELECT json_patch('{"a":[{"b":"c"}]}','{"a":[1]}');
+} {{{"a":[1]}}}
+do_execsql_test json104-308 {
+  SELECT json_patch('["a","b"]','["c","d"]');
+} {{["c","d"]}}
+do_execsql_test json104-309 {
+  SELECT json_patch('{"a":"b"}','["c"]');
+} {{["c"]}}
+do_execsql_test json104-310 {
+  SELECT json_patch('{"a":"foo"}','null');
+} {{null}}
+do_execsql_test json104-310a {
+  SELECT coalesce(json_patch('{"a":"foo"}',null), 'real-null');
+} {{real-null}}
+do_execsql_test json104-311 {
+  SELECT json_patch('{"a":"foo"}','"bar"');
+} {{"bar"}}
+do_execsql_test json104-312 {
+  SELECT json_patch('{"e":null}','{"a":1}');
+} {{{"e":null,"a":1}}}
+do_execsql_test json104-313 {
+  SELECT json_patch('[1,2]','{"a":"b","c":null}');
+} {{{"a":"b"}}}
+do_execsql_test json104-314 {
+  SELECT json_patch('{}','{"a":{"bb":{"ccc":null}}}');
+} {{{"a":{"bb":{}}}}}
+
+
+
+finish_test
diff --git a/test/kvtest.c b/test/kvtest.c
new file mode 100644
index 0000000000..4fa733f1a7
--- /dev/null
+++ b/test/kvtest.c
@@ -0,0 +1,880 @@
+/*
+** 2016-12-28
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file implements "key-value" performance test for SQLite.  The
+** purpose is to compare the speed of SQLite for accessing large BLOBs
+** versus reading those same BLOB values out of individual files in the
+** filesystem.
+**
+** Run "kvtest" with no arguments for on-line help, or see comments below.
+**
+** HOW TO COMPILE:
+**
+** (1) Gather this source file and a recent SQLite3 amalgamation with its
+**     header into the working directory.  You should have:
+**
+**          kvtest.c       >--- this file
+**          sqlite3.c      \___ SQLite
+**          sqlite3.h      /    amlagamation & header
+**
+** (2) Run you compiler against the two C source code files.
+**
+**    (a) On linux or mac:
+**
+**        OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION"
+**        gcc -Os -I. $OPTS kvtest.c sqlite3.c -o kvtest
+**
+**             The $OPTS options can be omitted.  The $OPTS merely omit
+**             the need to link against -ldl and -lpthread, or whatever
+**             the equivalent libraries are called on your system.
+**
+**    (b) Windows with MSVC:
+**
+**        cl -I. kvtest.c sqlite3.c
+**
+** USAGE:
+**
+** (1) Create a test database by running "kvtest init" with appropriate
+**     options.  See the help message for available options.
+**
+** (2) Construct the corresponding pile-of-files database on disk using
+**     the "kvtest export" command.
+**
+** (3) Run tests using "kvtest run" against either the SQLite database or
+**     the pile-of-files database and with appropriate options.
+**
+** For example:
+**
+**       ./kvtest init x1.db --count 100000 --size 10000
+**       mkdir x1
+**       ./kvtest export x1.db x1
+**       ./kvtest run x1.db --count 10000 --max-id 1000000
+**       ./kvtest run x1 --count 10000 --max-id 1000000
+*/
+static const char zHelp[] = 
+"Usage: kvtest COMMAND ARGS...\n"
+"\n"
+"   kvtest init DBFILE --count N --size M --pagesize X\n"
+"\n"
+"        Generate a new test database file named DBFILE containing N\n"
+"        BLOBs each of size M bytes.  The page size of the new database\n"
+"        file will be X.  Additional options:\n"
+"\n"
+"           --variance V           Randomly vary M by plus or minus V\n"
+"\n"
+"   kvtest export DBFILE DIRECTORY\n"
+"\n"
+"        Export all the blobs in the kv table of DBFILE into separate\n"
+"        files in DIRECTORY.\n"
+"\n"
+"   kvtest stat DBFILE\n"
+"\n"
+"        Display summary information about DBFILE\n"
+"\n"
+"   kvtest run DBFILE [options]\n"
+"\n"
+"        Run a performance test.  DBFILE can be either the name of a\n"
+"        database or a directory containing sample files.  Options:\n"
+"\n"
+"           --asc                  Read blobs in ascending order\n"
+"           --blob-api             Use the BLOB API\n"
+"           --cache-size N         Database cache size\n"
+"           --count N              Read N blobs\n"
+"           --desc                 Read blobs in descending order\n"
+"           --max-id N             Maximum blob key to use\n"
+"           --mmap N               Mmap as much as N bytes of DBFILE\n"
+"           --jmode MODE           Set MODE journal mode prior to starting\n"
+"           --random               Read blobs in a random order\n"
+"           --start N              Start reading with this blob key\n"
+"           --stats                Output operating stats before exiting\n"
+;
+
+/* Reference resources used */
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "sqlite3.h"
+
+#ifndef _WIN32
+# include 
+#else
+  /* Provide Windows equivalent for the needed parts of unistd.h */
+# include 
+# define R_OK 2
+# define S_ISREG(m) (((m) & S_IFMT) == S_IFREG)
+# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+# define access _access
+#endif
+
+
+/*
+** Show thqe help text and quit.
+*/
+static void showHelp(void){
+  fprintf(stdout, "%s", zHelp);
+  exit(1);
+}
+
+/*
+** Show an error message an quit.
+*/
+static void fatalError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stdout, "ERROR: ");
+  va_start(ap, zFormat);
+  vfprintf(stdout, zFormat, ap);
+  va_end(ap);
+  fprintf(stdout, "\n");
+  exit(1);
+}
+
+/*
+** Return the value of a hexadecimal digit.  Return -1 if the input
+** is not a hex digit.
+*/
+static int hexDigitValue(char c){
+  if( c>='0' && c<='9' ) return c - '0';
+  if( c>='a' && c<='f' ) return c - 'a' + 10;
+  if( c>='A' && c<='F' ) return c - 'A' + 10;
+  return -1;
+}
+
+/*
+** Interpret zArg as an integer value, possibly with suffixes.
+*/
+static int integerValue(const char *zArg){
+  int v = 0;
+  static const struct { char *zSuffix; int iMult; } aMult[] = {
+    { "KiB", 1024 },
+    { "MiB", 1024*1024 },
+    { "GiB", 1024*1024*1024 },
+    { "KB",  1000 },
+    { "MB",  1000000 },
+    { "GB",  1000000000 },
+    { "K",   1000 },
+    { "M",   1000000 },
+    { "G",   1000000000 },
+  };
+  int i;
+  int isNeg = 0;
+  if( zArg[0]=='-' ){
+    isNeg = 1;
+    zArg++;
+  }else if( zArg[0]=='+' ){
+    zArg++;
+  }
+  if( zArg[0]=='0' && zArg[1]=='x' ){
+    int x;
+    zArg += 2;
+    while( (x = hexDigitValue(zArg[0]))>=0 ){
+      v = (v<<4) + x;
+      zArg++;
+    }
+  }else{
+    while( zArg[0]>='0' && zArg[0]<='9' ){
+      v = v*10 + zArg[0] - '0';
+      zArg++;
+    }
+  }
+  for(i=0; i>1) ^ ((1+~(x&1)) & 0xd0000001);
+  y = y*1103515245 + 12345;
+  return x^y;
+}
+
+/*
+** Do database initialization.
+*/
+static int initMain(int argc, char **argv){
+  char *zDb;
+  int i, rc;
+  int nCount = 1000;
+  int sz = 10000;
+  int iVariance = 0;
+  int pgsz = 4096;
+  sqlite3 *db;
+  char *zSql;
+  char *zErrMsg = 0;
+
+  assert( strcmp(argv[1],"init")==0 );
+  assert( argc>=3 );
+  zDb = argv[2];
+  for(i=3; i65536 || ((pgsz-1)&pgsz)!=0 ){
+        fatalError("the --pagesize must be power of 2 between 512 and 65536");
+      }
+      continue;
+    }
+    fatalError("unknown option: \"%s\"", argv[i]);
+  }
+  rc = sqlite3_open(zDb, &db);
+  if( rc ){
+    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
+  }
+  zSql = sqlite3_mprintf(
+    "DROP TABLE IF EXISTS kv;\n"
+    "PRAGMA page_size=%d;\n"
+    "VACUUM;\n"
+    "BEGIN;\n"
+    "CREATE TABLE kv(k INTEGER PRIMARY KEY, v BLOB);\n"
+    "WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<%d)"
+    " INSERT INTO kv(k,v) SELECT x, randomblob(%d+(random()%%(%d))) FROM c;\n"
+    "COMMIT;\n",
+    pgsz, nCount, sz, iVariance+1
+  );
+  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
+  if( rc ) fatalError("database create failed: %s", zErrMsg);
+  sqlite3_free(zSql);
+  sqlite3_close(db);
+  return 0;
+}
+
+/*
+** Analyze an existing database file.  Report its content.
+*/
+static int statMain(int argc, char **argv){
+  char *zDb;
+  int i, rc;
+  sqlite3 *db;
+  char *zSql;
+  sqlite3_stmt *pStmt;
+
+  assert( strcmp(argv[1],"stat")==0 );
+  assert( argc>=3 );
+  zDb = argv[2];
+  for(i=3; i=3 );
+  zDb = argv[2];
+  if( argc!=4 ) fatalError("Usage: kvtest export DATABASE DIRECTORY");
+  zDir = argv[3];
+  if( pathType(zDir)!=PATH_DIR ){
+    fatalError("object \"%s\" is not a directory", zDir);
+  }
+  rc = sqlite3_open(zDb, &db);
+  if( rc ){
+    fatalError("cannot open database \"%s\": %s", zDb, sqlite3_errmsg(db));
+  }
+  sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
+                          writefileFunc, 0, 0);
+  zSql = sqlite3_mprintf(
+    "SELECT writefile(printf('%s/%%06d',k),v) FROM kv;",
+    zDir
+  );
+  rc = sqlite3_exec(db, zSql, 0, 0, &zErrMsg);
+  if( rc ) fatalError("database create failed: %s", zErrMsg);
+  sqlite3_free(zSql);
+  sqlite3_close(db);
+  printf("\n");
+  return 0;
+}
+
+/*
+** Read the content of file zName into memory obtained from sqlite3_malloc64()
+** and return a pointer to the buffer. The caller is responsible for freeing 
+** the memory. 
+**
+** If parameter pnByte is not NULL, (*pnByte) is set to the number of bytes
+** read.
+**
+** For convenience, a nul-terminator byte is always appended to the data read
+** from the file before the buffer is returned. This byte is not included in
+** the final value of (*pnByte), if applicable.
+**
+** NULL is returned if any error is encountered. The final value of *pnByte
+** is undefined in this case.
+*/
+static unsigned char *readFile(const char *zName, int *pnByte){
+  FILE *in;               /* FILE from which to read content of zName */
+  sqlite3_int64 nIn;      /* Size of zName in bytes */
+  size_t nRead;           /* Number of bytes actually read */
+  unsigned char *pBuf;    /* Content read from disk */
+
+  nIn = fileSize(zName);
+  if( nIn<0 ) return 0;
+  in = fopen(zName, "rb");
+  if( in==0 ) return 0;
+  pBuf = sqlite3_malloc64( nIn );
+  if( pBuf==0 ) return 0;
+  nRead = fread(pBuf, (size_t)nIn, 1, in);
+  fclose(in);
+  if( nRead!=1 ){
+    sqlite3_free(pBuf);
+    return 0;
+  }
+  if( pnByte ) *pnByte = (int)nIn;
+  return pBuf;
+}
+
+/*
+** Return the current time in milliseconds since the beginning of
+** the Julian epoch.
+*/
+static sqlite3_int64 timeOfDay(void){
+  static sqlite3_vfs *clockVfs = 0;
+  sqlite3_int64 t;
+  if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
+  if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
+    clockVfs->xCurrentTimeInt64(clockVfs, &t);
+  }else{
+    double r;
+    clockVfs->xCurrentTime(clockVfs, &r);
+    t = (sqlite3_int64)(r*86400000.0);
+  }
+  return t;
+}
+
+#ifdef __linux__
+/*
+** Attempt to display I/O stats on Linux using /proc/PID/io
+*/
+static void displayLinuxIoStats(FILE *out){
+  FILE *in;
+  char z[200];
+  sqlite3_snprintf(sizeof(z), z, "/proc/%d/io", getpid());
+  in = fopen(z, "rb");
+  if( in==0 ) return;
+  while( fgets(z, sizeof(z), in)!=0 ){
+    static const struct {
+      const char *zPattern;
+      const char *zDesc;
+    } aTrans[] = {
+      { "rchar: ",                  "Bytes received by read():" },
+      { "wchar: ",                  "Bytes sent to write():"    },
+      { "syscr: ",                  "Read() system calls:"      },
+      { "syscw: ",                  "Write() system calls:"     },
+      { "read_bytes: ",             "Bytes read from storage:"  },
+      { "write_bytes: ",            "Bytes written to storage:" },
+      { "cancelled_write_bytes: ",  "Cancelled write bytes:"    },
+    };
+    int i;
+    for(i=0; i=3 );
+  zDb = argv[2];
+  eType = pathType(zDb);
+  if( eType==PATH_OTHER ) fatalError("unknown object type: \"%s\"", zDb);
+  if( eType==PATH_NEXIST ) fatalError("object does not exist: \"%s\"", zDb);
+  for(i=3; iiMax ) iKey = 1;
+    }else if( eOrder==ORDER_DESC ){
+      iKey--;
+      if( iKey<=0 ) iKey = iMax;
+    }else{
+      iKey = (randInt()%iMax)+1;
+    }
+    nTotal += nData;
+    if( nData==0 ){ nCount++; nExtra++; }
+  }
+  if( nAlloc ) sqlite3_free(pData);
+  if( pStmt ) sqlite3_finalize(pStmt);
+  if( pBlob ) sqlite3_blob_close(pBlob);
+  if( bStats ){
+    display_stats(db, 0);
+  }
+  if( db ) sqlite3_close(db);
+  tmElapsed = timeOfDay() - tmStart;
+  if( nExtra ){
+    printf("%d cycles due to %d misses\n", nCount, nExtra);
+  }
+  if( eType==PATH_DB ){
+    printf("SQLite version: %s\n", sqlite3_libversion());
+  }
+  printf("--count %d --max-id %d", nCount-nExtra, iMax);
+  switch( eOrder ){
+    case ORDER_RANDOM:  printf(" --random\n");  break;
+    case ORDER_DESC:    printf(" --desc\n");    break;
+    default:            printf(" --asc\n");     break;
+  }
+  if( eType==PATH_DB ){
+    printf("--cache-size %d --jmode %s\n", iCache, zJMode);
+    printf("--mmap %d%s\n", mmapSize, bBlobApi ? " --blob-api" : "");
+  }
+  if( iPagesize ) printf("Database page size: %d\n", iPagesize);
+  printf("Total elapsed time: %.3f\n", tmElapsed/1000.0);
+  printf("Microseconds per BLOB read: %.3f\n", tmElapsed*1000.0/nCount);
+  printf("Content read rate: %.1f MB/s\n", nTotal/(1000.0*tmElapsed));
+  return 0;
+}
+
+
+int main(int argc, char **argv){
+  if( argc<3 ) showHelp();
+  if( strcmp(argv[1],"init")==0 ){
+    return initMain(argc, argv);
+  }
+  if( strcmp(argv[1],"export")==0 ){
+    return exportMain(argc, argv);
+  }
+  if( strcmp(argv[1],"run")==0 ){
+    return runMain(argc, argv);
+  }
+  if( strcmp(argv[1],"stat")==0 ){
+    return statMain(argc, argv);
+  }
+  showHelp();
+  return 0;
+}
diff --git a/test/like.test b/test/like.test
index fba89e9037..8082b35123 100644
--- a/test/like.test
+++ b/test/like.test
@@ -980,6 +980,24 @@ do_execsql_test like-13.4 {
   SELECT char(0x4d) LIKE char(0x6d);
 } {1}
 
-
+# Performance testing for patterns with many wildcards.  These LIKE and GLOB
+# patterns were quite slow with SQLite 3.15.2 and earlier.
+#
+do_test like-14.1 {
+  set x [lindex [time {
+    db one {SELECT 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz'GLOB'*a*a*a*a*a*a*a*a*y'}
+  }] 0]
+  puts -nonewline " ($x ms - want less than 1000) "
+  expr {$x<1000}
+} {1}
+ifcapable !icu {
+  do_test like-14.2 {
+    set x [lindex [time {
+      db one {SELECT 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz'LIKE'%a%a%a%a%a%a%a%a%y'}
+    }] 0]
+    puts -nonewline " ($x ms - want less than 1000) "
+    expr {$x<1000}
+  } {1}
+}
 
 finish_test
diff --git a/test/limit2.test b/test/limit2.test
new file mode 100644
index 0000000000..f6eaefb8aa
--- /dev/null
+++ b/test/limit2.test
@@ -0,0 +1,153 @@
+# 2016-05-20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the LIMIT in combination with ORDER BY
+# and in particular, the optimizations in the inner loop that cause an
+# early exit of the inner loop when the LIMIT is reached and the inner
+# loop is emitting rows in ORDER BY order.
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix limit2
+
+do_execsql_test limit2-100 {
+  CREATE TABLE t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t1(a,b) SELECT 1, (x*17)%1000 + 1000 FROM c;
+  INSERT INTO t1(a,b) VALUES(2,2),(3,1006),(4,4),(5,9999);
+  CREATE INDEX t1ab ON t1(a,b);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-100.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-100.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+do_test limit2-100.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-110 {
+  CREATE TABLE t2(x,y);
+  INSERT INTO t2(x,y) VALUES('a',1),('a',2),('a',3),('a',4);
+  INSERT INTO t2(x,y) VALUES('b',1),('c',2),('d',3),('e',4);
+  CREATE INDEX t2xy ON t2(x,y);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-110.1 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-110.2 {
+  SELECT a, b, '|' FROM t2, t1 WHERE t2.x='a' AND t1.a=t2.y ORDER BY +t1.b LIMIT 5;
+} {2 2 | 4 4 | 1 1000 | 1 1001 | 1 1002 |}
+set slow_count $sqlite_search_count
+do_test limit2-110.3 {
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+do_execsql_test limit2-120 {
+  DROP INDEX t1ab;
+  CREATE INDEX t1ab ON t1(a,b DESC);
+}
+set sqlite_search_count 0
+do_execsql_test limit2-120.1 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+set fast_count $sqlite_search_count
+set sqlite_search_count 0
+do_execsql_test limit2-120.2 {
+  SELECT a, b, '|' FROM t1 WHERE a IN (2,4,5,3,1) ORDER BY +b DESC LIMIT 5;
+} {5 9999 | 1 1999 | 1 1998 | 1 1997 | 1 1996 |}
+do_test limit2-120.3 {
+  set slow_count $sqlite_search_count
+  expr {$fast_count < 0.02*$slow_count}
+} {1}
+
+# Bug report against the new ORDER BY LIMIT optimization just prior to
+# release.  (Unreleased so there is no ticket).
+#
+# Make sure the optimization is not applied if the inner loop can only
+# provide a single row of output.
+#
+do_execsql_test limit2-200 {
+  CREATE TABLE t200(a, b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t200(a,b) SELECT x, x FROM c;
+  CREATE TABLE t201(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t201(x,y) VALUES(2,12345);
+
+  SELECT *, '|' FROM t200, t201 WHERE x=b ORDER BY y LIMIT 3;
+} {2 2 2 12345 |}
+do_execsql_test limit2-210 {
+  SELECT *, '|' FROM t200 LEFT JOIN t201 ON x=b ORDER BY y LIMIT 3;
+} {1 1 {} {} | 3 3 {} {} | 4 4 {} {} |}
+
+# Bug in the ORDER BY LIMIT optimization reported on 2016-09-06.
+# Ticket https://www.sqlite.org/src/info/559733b09e96
+#
+do_execsql_test limit2-300 {
+  CREATE TABLE t300(a,b,c);
+  CREATE INDEX t300x ON t300(a,b,c);
+  INSERT INTO t300 VALUES(0,1,99),(0,1,0),(0,0,0);
+  SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC;
+} {0 1 99 . 0 0 0 . 0 1 0 .}
+do_execsql_test limit2-310 {
+  SELECT *,'.' FROM t300 WHERE a=0 AND (c=0 OR c=99) ORDER BY c DESC LIMIT 1;
+} {0 1 99 .}
+
+# Make sure the SELECT loop is ordered correctly for the direction of
+# the ORDER BY
+#
+do_execsql_test limit2-400 {
+  CREATE TABLE t400(a,b);
+  CREATE INDEX t400_ab ON t400(a,b);
+  INSERT INTO t400(a,b) VALUES(1,90),(1,40),(2,80),(2,30),(3,70),(3,20);
+  SELECT *,'x' FROM t400 WHERE a IN (1,2,3) ORDER BY b DESC LIMIT 3;
+  SELECT *,'y' FROM t400 WHERE a IN (1,2,3) ORDER BY +b DESC LIMIT 3;
+} {1 90 x 2 80 x 3 70 x 1 90 y 2 80 y 3 70 y}
+
+
+do_execsql_test 500 {
+  CREATE TABLE t500(i INTEGER PRIMARY KEY, j);
+  INSERT INTO t500 VALUES(1, 1);
+  INSERT INTO t500 VALUES(2, 2);
+  INSERT INTO t500 VALUES(3, 3);
+  INSERT INTO t500 VALUES(4, 0);
+  INSERT INTO t500 VALUES(5, 5);
+  SELECT j FROM t500 WHERE i IN (1,2,3,4,5) ORDER BY j DESC LIMIT 3;
+} {5 3 2}
+do_execsql_test 501 {
+  CREATE TABLE t501(i INTEGER PRIMARY KEY, j);
+  INSERT INTO t501 VALUES(1, 5);
+  INSERT INTO t501 VALUES(2, 4);
+  INSERT INTO t501 VALUES(3, 3);
+  INSERT INTO t501 VALUES(4, 6);
+  INSERT INTO t501 VALUES(5, 1);
+  SELECT j FROM t501 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
+} {1 3 4}
+do_execsql_test 502 {
+  CREATE TABLE t502(i INT PRIMARY KEY, j);
+  INSERT INTO t502 VALUES(1, 5);
+  INSERT INTO t502 VALUES(2, 4);
+  INSERT INTO t502 VALUES(3, 3);
+  INSERT INTO t502 VALUES(4, 6);
+  INSERT INTO t502 VALUES(5, 1);
+  SELECT j FROM t502 WHERE i IN (1,2,3,4,5) ORDER BY j LIMIT 3;
+} {1 3 4}
+
+finish_test
diff --git a/test/loadext.test b/test/loadext.test
index 7ba4c0cf77..24ee7a6802 100644
--- a/test/loadext.test
+++ b/test/loadext.test
@@ -81,7 +81,7 @@ if {![file exists $testextension]} {
   set testextsrc $srcdir/test_loadext.c
 
   set cmdline [concat exec gcc $gcc_shared]
-  lappend cmdline -Wall -I$srcdir -I. -g $testextsrc -o $testextension
+  lappend cmdline -Wall -I$srcdir -I. -I.. -g $testextsrc -o $testextension
   
   if {[catch $cmdline msg]} {
     puts "Skipping loadext tests: Test extension not built..."
@@ -111,7 +111,7 @@ do_test loadext-1.2 {
 #
 do_test loadext-1.3 {
   sqlite3 db2 test.db
-  sqlite3_enable_load_extension db2 1
+  sqlite3_db_config db2 SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1
   catchsql {
     SELECT half(1.0);
   } db2
@@ -256,6 +256,7 @@ do_test loadext-4.2 {
   }
 } {0 {{}}}
 
+# disable all extension loading
 do_test loadext-4.3 {
   sqlite3_enable_load_extension db 0
   catchsql {
@@ -263,6 +264,15 @@ do_test loadext-4.3 {
   }
 } {1 {not authorized}}
 
+# enable C-api extension loading only.  Show that the SQL function
+# still does not work.
+do_test loadext-4.4 {
+  sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1
+  catchsql {
+    SELECT load_extension($::testextension,'testloadext_init')
+  }
+} {1 {not authorized}}
+
 source $testdir/malloc_common.tcl
 
 
diff --git a/test/lock.test b/test/lock.test
index f785c4bb23..534aa3b9a4 100644
--- a/test/lock.test
+++ b/test/lock.test
@@ -423,8 +423,9 @@ do_test lock-6.5 {
 #   * there exists one or more active read-only statements, and
 #   * a transaction that modified zero database pages is committed.
 # 
-set temp_status unlocked
-if {$TEMP_STORE>=2} {set temp_status unknown}
+#set temp_status unlocked
+#if {$TEMP_STORE>=2} {set temp_status unknown}
+set temp_status unknown
 do_test lock-7.1 {
   set STMT [sqlite3_prepare $DB "SELECT * FROM sqlite_master" -1 TAIL]
   sqlite3_step $STMT
diff --git a/test/malloc5.test b/test/malloc5.test
index 8f0db04c2a..e529874b51 100644
--- a/test/malloc5.test
+++ b/test/malloc5.test
@@ -39,11 +39,27 @@ ifcapable !memorymanage {
    return
 }
 
+# The sizes of memory allocations from system malloc() might vary,
+# depending on the memory allocator algorithms used.  The following
+# routine is designed to support answers that fall within a range
+# of values while also supplying easy-to-understand "expected" values
+# when errors occur.
+#
+proc value_in_range {target x args} {
+  set v [lindex $args 0]
+  if {$v!=""} {
+    if {$v<$target*$x} {return $v}
+    if {$v>$target/$x} {return $v}
+  }
+  return "number between [expr {int($target*$x)}] and [expr {int($target/$x)}]"
+}
+set mrange 0.98   ;#  plus or minus 2%
+
 test_set_config_pagecache 0 100
 
 sqlite3_soft_heap_limit 0
 sqlite3 db test.db
-db eval {PRAGMA cache_size=1}
+# db eval {PRAGMA cache_size=1}
 
 do_test malloc5-1.1 {
   # Simplest possible test. Call sqlite3_release_memory when there is exactly
@@ -71,24 +87,8 @@ do_test malloc5-1.3 {
   # in the cache belonging to db2.
   #
   set ::pgalloc [sqlite3_release_memory]
-} {0}
-
-# The sizes of memory allocations from system malloc() might vary,
-# depending on the memory allocator algorithms used.  The following
-# routine is designed to support answers that fall within a range
-# of values while also supplying easy-to-understand "expected" values
-# when errors occur.
-#
-proc value_in_range {target x args} {
-  set v [lindex $args 0]
-  if {$v!=""} {
-    if {$v<$target*$x} {return $v}
-    if {$v>$target/$x} {return $v}
-  }
-  return "number between [expr {int($target*$x)}] and [expr {int($target/$x)}]"
-}
-set mrange 0.98   ;#  plus or minus 2%
-
+  value_in_range 1288 0.75
+} [value_in_range 1288 0.75]
 
 do_test malloc5-1.4 {
   # Commit the transaction and open a new one. Read 1 page into the cache.
@@ -117,12 +117,12 @@ do_test malloc5-1.6 {
   db2 close
   execsql {
     BEGIN;
-    SELECT * FROM abc;
     CREATE TABLE def(d, e, f);
+    SELECT * FROM abc;
   }
+  breakpoint
   value_in_range $::pgalloc $::mrange [sqlite3_release_memory 500]
 } [value_in_range $::pgalloc $::mrange]
-
 do_test malloc5-1.7 {
   # Database should not be locked this time. 
   sqlite3 db2 test.db
@@ -236,12 +236,12 @@ do_test malloc5-4.2 {
   db eval {PRAGMA cache_size=1}
   db cache flush
   sqlite3_release_memory
-  sqlite3_soft_heap_limit 100000
+  sqlite3_soft_heap_limit 200000
   sqlite3_memory_highwater 1
   execsql {SELECT * FROM abc}
   set nMaxBytes [sqlite3_memory_highwater 1]
   puts -nonewline " (Highwater mark: $nMaxBytes) "
-  expr $nMaxBytes <= 110000
+  expr $nMaxBytes <= 210000
 } {1}
 do_test malloc5-4.3 {
   # Check that the content of table abc is at least roughly as expected.
@@ -346,7 +346,7 @@ do_test malloc5-6.2.2 {
   # If we now try to reclaim some memory, it should come from the db2 cache.
   sqlite3_release_memory 3000
   expr [nPage db] + [nPage db2]
-} {4}
+} {1}
 do_test malloc5-6.2.3 {
   # Access the db2 cache again, so that all the db2 pages have been used
   # more recently than all the db pages. Then try to reclaim 3000 bytes.
@@ -354,7 +354,7 @@ do_test malloc5-6.2.3 {
   execsql { SELECT * FROM abc } db2
   sqlite3_release_memory 3000
   expr [nPage db] + [nPage db2]
-} {4}
+} {0}
 
 do_test malloc5-6.3.1 {
   # Now open a transaction and update 2 pages in the db2 cache. Then
@@ -364,6 +364,8 @@ do_test malloc5-6.3.1 {
   # sync() to free up the dirty db2 pages. The only page that cannot be
   # freed is page1 of db2. Because there is an open transaction, the
   # btree layer holds a reference to page 1 in the db2 cache.
+  #
+  # UPDATE: No longer. As release_memory() does not cause a sync()
   execsql {
     BEGIN;
     UPDATE abc SET c = randstr(100,100) 
@@ -377,13 +379,13 @@ do_test malloc5-6.3.2 {
   # non-dirty pages held by db2.
   sqlite3_release_memory [expr 7*1132]
   list [nPage db] [nPage db2]
-} {1 3}
+} {0 3}
 do_test malloc5-6.3.3 {
   # Try to release another 1000 bytes. This should come fromt the db
   # cache, since all three pages held by db2 are either in-use or diry.
   sqlite3_release_memory 1000
   list [nPage db] [nPage db2]
-} {1 3}
+} {0 3}
 do_test malloc5-6.3.4 {
   # Now release 9900 more (about 9 pages worth). This should expunge
   # the rest of the db cache. But the db2 cache remains intact, because
@@ -394,20 +396,20 @@ do_test malloc5-6.3.4 {
     sqlite3_release_memory 9900
   }
   list [nPage db] [nPage db2]
-} {1 3}
+} {0 3}
 do_test malloc5-6.3.5 {
   # But if we are really insistent, SQLite will consent to call sync()
   # if there is no other option. UPDATE: As of 3.6.2, SQLite will not
   # call sync() in this scenario. So no further memory can be reclaimed.
   sqlite3_release_memory 1000
   list [nPage db] [nPage db2]
-} {1 3}
+} {0 3}
 do_test malloc5-6.3.6 {
   # The referenced page (page 1 of the db2 cache) will not be freed no
   # matter how much memory we ask for:
   sqlite3_release_memory 31459
   list [nPage db] [nPage db2]
-} {1 3}
+} {0 3}
 
 db2 close
 
diff --git a/test/mallocI.test b/test/mallocI.test
index 1229455505..ee4a603dee 100644
--- a/test/mallocI.test
+++ b/test/mallocI.test
@@ -61,4 +61,15 @@ do_malloc_test mallocI-4 -tclprep {
 }
 catch { db2 close }
 
+do_faultsim_test mallocI-5 -faults oom* -prep {
+  catch { db close }
+  sqlite3 db test.db
+  sqlite3_db_config_lookaside db 0 0 0
+} -body {
+  db eval { Select CAST(1 AS blob) }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+
 finish_test
diff --git a/test/mallocM.test b/test/mallocM.test
new file mode 100644
index 0000000000..4da3a9e112
--- /dev/null
+++ b/test/mallocM.test
@@ -0,0 +1,48 @@
+# 2017 March 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Further OOM tests.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix mallocM
+
+sqlite3_db_config_lookaside db 0 0 0
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x);
+}
+do_faultsim_test 1 -faults oom* -body {
+  execsql {
+    SELECT 'abc' FROM ( SELECT 'xyz' FROM t1 WHERE (SELECT 1) )
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_execsql_test 2.0.1 { SELECT instr(x'', x'') }         {1}
+do_execsql_test 2.0.2 { SELECT instr(x'12345678', x'') } {1}
+do_execsql_test 2.0.3 { SELECT instr(x'', x'1234') }     {0}
+
+do_faultsim_test 2.1 -faults oom* -body {
+  execsql { SELECT instr (x'00', zeroblob(1)) }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+do_faultsim_test 2.2 -faults oom* -body {
+  execsql { SELECT instr (zeroblob(1), x'00') }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+finish_test
diff --git a/test/memsubsys1.test b/test/memsubsys1.test
index e9a4cf08a8..36427f9bae 100644
--- a/test/memsubsys1.test
+++ b/test/memsubsys1.test
@@ -100,10 +100,12 @@ reset_highwater_marks
 build_test_db memsubsys1-2 {PRAGMA page_size=1024; PRAGMA mmap_size=0}
 #show_memstats
 set MEMORY_MANAGEMENT $sqlite_options(memorymanage)
-ifcapable !malloc_usable_size {
- do_test memsubsys1-2.3 {
-    set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
-  } [expr ($TEMP_STORE>1 || $MEMORY_MANAGEMENT==0)*1024]
+ifcapable pagecache_overflow_stats {
+  ifcapable !malloc_usable_size {
+    do_test memsubsys1-2.3 {
+      set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2]
+    } [expr ($TEMP_STORE>1 || $MEMORY_MANAGEMENT==0)*1024]
+  }
 }
 do_test memsubsys1-2.4 {
   set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2]
@@ -255,7 +257,7 @@ do_test memsubsys1-7.4 {
 } 0
 do_test memsubsys1-7.5 {
   set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2]
-  expr {$maxreq<4100 + 4200*[nonzero_reserved_bytes]}
+  expr {$maxreq<(4100 + 8200*[nonzero_reserved_bytes])}
 } 1
 do_test memsubsys1-7.6 {
   set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2]
diff --git a/test/minmax.test b/test/minmax.test
index fb9bbb383a..04a3f06dff 100644
--- a/test/minmax.test
+++ b/test/minmax.test
@@ -628,5 +628,19 @@ do_test_13_noopt 13.7 {
   SELECT min(c), count(c) FROM t1 WHERE a='a';
 } {1 5}
 
+# 2016-07-26.  https://www.sqlite.org/src/info/a0bac8b3c3d1bb75
+# Incorrect result on a min() query after a CREATE INDEX.
+#
+do_execsql_test 14.1 {
+  CREATE TABLE t14(a INTEGER, b INTEGER);
+  INSERT INTO t14(a,b) VALUES(100,2),(200,2),(300,2),(400,1),(500,2);
+  SELECT min(a) FROM t14 WHERE b='2' AND a>'50';
+} {100}
+do_execsql_test 14.2 {
+  CREATE INDEX t14ba ON t14(b,a);
+  SELECT min(a) FROM t14 WHERE b='2' AND a>'50';
+} {100}
+
+
 
 finish_test
diff --git a/test/misc5.test b/test/misc5.test
index 30176b8082..0e7e34dd16 100644
--- a/test/misc5.test
+++ b/test/misc5.test
@@ -571,6 +571,8 @@ ifcapable subquery&&compound {
 # Overflow the lemon parser stack by providing an overly complex
 # expression.  Make sure that the overflow is detected and reported.
 #
+# This test fails when building with -DYYSTACKDEPTH=0
+#
 do_test misc5-7.1 {
   execsql {CREATE TABLE t1(x)}
   set sql "INSERT INTO t1 VALUES("
diff --git a/test/misc8.test b/test/misc8.test
index 3817cadf94..b639dd1463 100644
--- a/test/misc8.test
+++ b/test/misc8.test
@@ -108,4 +108,30 @@ do_execsql_test misc8-3.0 {
    ORDER BY 1;
 } {0 1 6 0 1 7}
 
+# The SQLITE_DBCONFIG_MAINDBNAME interface
+#
+db close
+forcedelete test.db test2.db
+sqlite3 db test.db
+do_execsql_test misc8-4.0 {
+  CREATE TABLE t1(a,b,c);
+  INSERT INTO t1 VALUES(1,2,3);
+  ATTACH 'test2.db' AS aux2;
+  CREATE TABLE aux2.t2(c,d,e);
+  INSERT INTO t2 VALUES(4,5,6);
+  SELECT * FROM t1, t2;
+} {1 2 3 4 5 6}
+do_execsql_test misc8-4.1 {
+  PRAGMA database_list;
+} {/0 main .* 2 aux2/}
+dbconfig_maindbname_icecube db
+do_execsql_test misc8-4.2 {
+  SELECT name FROM icecube.sqlite_master;
+} {t1}
+do_execsql_test misc8-4.3 {
+  PRAGMA database_list;
+} {/0 icecube .* 2 aux2/}
+
+
+
 finish_test
diff --git a/test/multiplex.test b/test/multiplex.test
index 5db56f264a..6ea328906e 100644
--- a/test/multiplex.test
+++ b/test/multiplex.test
@@ -195,15 +195,16 @@ do_test multiplex-2.3.1 {
 
 
 unset -nocomplain ::log
-do_test multiplex-2.4.1 {
-  sqlite3_multiplex_shutdown
-} {SQLITE_MISUSE}
+#do_test multiplex-2.4.1 {
+#  sqlite3_multiplex_shutdown
+#} {SQLITE_MISUSE}
 do_test multiplex-2.4.2 {
   execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) }
 } {}
-do_test multiplex-2.4.3 {
-  set ::log
-} {SQLITE_MISUSE {sqlite3_multiplex_shutdown() called while database connections are still open}}
+#do_test multiplex-2.4.3 {
+#  set ::log
+#} {SQLITE_MISUSE {sqlite3_multiplex_shutdown() called while database connections are still open}}
+
 do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168}
 do_test multiplex-2.4.5 {
   db close
@@ -445,11 +446,7 @@ multiplex_set db main 32768 16
 
 # Return a list of all currently defined multiplexs.
 proc multiplex_list {} {
-  set allq {}
-  foreach q [sqlite3_multiplex_dump] {
-    lappend allq [lindex $q 0]
-  }
-  return [lsort $allq]
+  glob -nocomplain test2.db*
 }
 
 do_test multiplex-4.1.6 {
@@ -494,7 +491,7 @@ do_test multiplex-4.1.11 {
 do_test multiplex-4.1.12 {
   db close
   multiplex_list
-} {}
+} {test2.db}
 
 
 #-------------------------------------------------------------------------
diff --git a/test/mutex1.test b/test/mutex1.test
index 340e271175..f567a0d930 100644
--- a/test/mutex1.test
+++ b/test/mutex1.test
@@ -97,7 +97,7 @@ do_test mutex1-1.9 {
 #   * Multi-threaded mode,
 #   * Single-threaded mode.
 #
-ifcapable threadsafe&&shared_cache {
+ifcapable threadsafe1&&shared_cache {
   set enable_shared_cache [sqlite3_enable_shared_cache 1]
   foreach {mode mutexes} {
     singlethread {}
diff --git a/test/nockpt.test b/test/nockpt.test
new file mode 100644
index 0000000000..bd3953f1ee
--- /dev/null
+++ b/test/nockpt.test
@@ -0,0 +1,66 @@
+# 2016 October 31
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
+# option.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+source $testdir/wal_common.tcl
+ifcapable !wal {finish_test ; return }
+if {[permutation]=="journaltest" || [permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
+set testprefix nockpt
+
+do_execsql_test 1.0 {
+  PRAGMA auto_vacuum=OFF;
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  CREATE TABLE c1(x, y, z);
+  INSERT INTO c1 VALUES(1, 2, 3);
+} {wal}
+
+do_test 1.1 { file exists test.db-wal } 1
+do_test 1.2 { file size test.db-wal } [wal_file_size 3 1024]
+do_test 1.3 { db close } {}
+do_test 1.4 { file exists test.db-wal } 0
+
+sqlite3 db test.db
+do_execsql_test 1.5 {
+  INSERT INTO c1 VALUES(4, 5, 6);
+  INSERT INTO c1 VALUES(7, 8, 9);
+}
+do_test 1.6 { file exists test.db-wal } 1
+do_test 1.7 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1}
+do_test 1.8 { file size test.db-wal } [wal_file_size 2 1024]
+do_test 1.9 { db close } {}
+do_test 1.10 { file exists test.db-wal } 1
+do_test 1.11 { file size test.db-wal } [wal_file_size 2 1024]
+
+sqlite3 db test.db
+do_execsql_test 1.12 {
+  SELECT * FROM c1
+} {1 2 3 4 5 6 7 8 9}
+
+do_execsql_test 1.13 { PRAGMA main.journal_mode } {wal}
+do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1}
+do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete}
+do_test 1.15 { file exists test.db-wal } {0}
+
+
+
+finish_test
diff --git a/test/nolock.test b/test/nolock.test
index fdaef6ef07..e732dcf13b 100644
--- a/test/nolock.test
+++ b/test/nolock.test
@@ -183,35 +183,38 @@ db2 close
 db close
 tvfs delete
 
-# 2016-03-11:  Make sure all works when transitioning to WAL mode under nolock.
-#
-do_test nolock-4.1 {
-  forcedelete test.db
-  sqlite3 db file:test.db?nolock=1 -uri 1
-  db eval {
-     PRAGMA journal_mode=WAL;
-     CREATE TABLE t1(x);
-     INSERT INTO t1 VALUES('youngling');
-     SELECT * FROM t1;
-  }
-} {delete youngling}
-db close
-
-do_test nolock-4.2 {
-  forcedelete test.db
-  sqlite3 db test.db
-  db eval {
-    PRAGMA journal_mode=WAL;
-    CREATE TABLE t1(x);
-    INSERT INTO t1 VALUES('catbird');
-    SELECT * FROM t1;
-  }
-} {wal catbird}
-do_test nolock-4.3 {
+if {[permutation]!="inmemory_journal"} {
+  # 2016-03-11:  Make sure all works when transitioning to WAL mode
+  # under nolock.
+  #
+  do_test nolock-4.1 {
+    forcedelete test.db
+    sqlite3 db file:test.db?nolock=1 -uri 1
+    db eval {
+       PRAGMA journal_mode=WAL;
+       CREATE TABLE t1(x);
+       INSERT INTO t1 VALUES('youngling');
+       SELECT * FROM t1;
+    }
+  } {delete youngling}
   db close
-  sqlite3 db file:test.db?nolock=1 -uri 1
-  set rc [catch {db eval {SELECT * FROM t1}} msg]
-  lappend rc $msg
-} {1 {unable to open database file}}
+  
+  do_test nolock-4.2 {
+    forcedelete test.db
+    sqlite3 db test.db
+    db eval {
+      PRAGMA journal_mode=WAL;
+      CREATE TABLE t1(x);
+      INSERT INTO t1 VALUES('catbird');
+      SELECT * FROM t1;
+    }
+  } {wal catbird}
+  do_test nolock-4.3 {
+    db close
+    sqlite3 db file:test.db?nolock=1 -uri 1
+    set rc [catch {db eval {SELECT * FROM t1}} msg]
+    lappend rc $msg
+  } {1 {unable to open database file}}
+}
 
 finish_test
diff --git a/test/ossfuzz.c b/test/ossfuzz.c
new file mode 100644
index 0000000000..51983548b9
--- /dev/null
+++ b/test/ossfuzz.c
@@ -0,0 +1,156 @@
+/*
+** This module interfaces SQLite to the Google OSS-Fuzz, fuzzer as a service.
+** (https://github.com/google/oss-fuzz)
+*/
+#include 
+#include 
+#include 
+#include 
+#include "sqlite3.h"
+
+/* Global debugging settings.  OSS-Fuzz will have all debugging turned
+** off.  But if LLVMFuzzerTestOneInput() is called interactively from
+** the ossshell utility program, then these flags might be set.
+*/
+static unsigned mDebug = 0;
+#define FUZZ_SQL_TRACE       0x0001   /* Set an sqlite3_trace() callback */
+#define FUZZ_SHOW_MAX_DELAY  0x0002   /* Show maximum progress callback delay */
+#define FUZZ_SHOW_ERRORS     0x0004   /* Print error messages from SQLite */
+
+/* The ossshell utility program invokes this interface to see the
+** debugging flags.  Unused by OSS-Fuzz.
+*/
+void ossfuzz_set_debug_flags(unsigned x){
+  mDebug = x;
+}
+
+/* Return the current real-world time in milliseconds since the
+** Julian epoch (-4714-11-24).
+*/
+static sqlite3_int64 timeOfDay(void){
+  static sqlite3_vfs *clockVfs = 0;
+  sqlite3_int64 t;
+  if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
+  if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
+    clockVfs->xCurrentTimeInt64(clockVfs, &t);
+  }else{
+    double r;
+    clockVfs->xCurrentTime(clockVfs, &r);
+    t = (sqlite3_int64)(r*86400000.0);
+  }
+  return t;
+}
+
+/* An instance of the following object is passed by pointer as the
+** client data to various callbacks.
+*/
+typedef struct FuzzCtx {
+  sqlite3 *db;               /* The database connection */
+  sqlite3_int64 iCutoffTime; /* Stop processing at this time. */
+  sqlite3_int64 iLastCb;     /* Time recorded for previous progress callback */
+  sqlite3_int64 mxInterval;  /* Longest interval between two progress calls */
+  unsigned nCb;              /* Number of progress callbacks */
+} FuzzCtx;
+
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+/*
+** Progress handler callback.
+**
+** The argument is the cutoff-time after which all processing should
+** stop.  So return non-zero if the cut-off time is exceeded.
+*/
+static int progress_handler(void *pClientData) {
+  FuzzCtx *p = (FuzzCtx*)pClientData;
+  sqlite3_int64 iNow = timeOfDay();
+  int rc = iNow>=p->iCutoffTime;
+  sqlite3_int64 iDiff = iNow - p->iLastCb;
+  if( iDiff > p->mxInterval ) p->mxInterval = iDiff;
+  p->nCb++;
+  return rc;
+}
+#endif
+
+/*
+** Callback for sqlite3_exec().
+*/
+static int exec_handler(void *pCnt, int argc, char **argv, char **namev){
+  int i;
+  if( argv ){
+    for(i=0; i>= 1;
+
+  /* Remaining bits of the selector determine a limit on the number of
+  ** output rows */
+  execCnt = uSelector + 1;
+
+  /* Run the SQL.  The sqlite_exec() interface expects a zero-terminated
+  ** string, so make a copy. */
+  zSql = sqlite3_mprintf("%.*s", (int)size, data);
+  sqlite3_exec(cx.db, zSql, exec_handler, (void*)&execCnt, &zErrMsg);
+
+  /* Show any errors */
+  if( (mDebug & FUZZ_SHOW_ERRORS)!=0 && zErrMsg ){
+    printf("Error: %s\n", zErrMsg);
+  }
+
+  /* Cleanup and return */
+  sqlite3_free(zErrMsg);
+  sqlite3_free(zSql);
+  sqlite3_exec(cx.db, "PRAGMA temp_store_directory=''", 0, 0, 0);
+  sqlite3_close(cx.db);
+
+  if( mDebug & FUZZ_SHOW_MAX_DELAY ){
+    printf("Progress callback count....... %d\n", cx.nCb);
+    printf("Max time between callbacks.... %d ms\n", (int)cx.mxInterval);
+  }
+  return 0;
+}
diff --git a/test/ossshell.c b/test/ossshell.c
new file mode 100644
index 0000000000..00cc3391c8
--- /dev/null
+++ b/test/ossshell.c
@@ -0,0 +1,94 @@
+/*
+** This is a test interface for the ossfuzz.c module.  The ossfuzz.c module
+** is an adaptor for OSS-FUZZ.  (https://github.com/google/oss-fuzz)
+**
+** This program links against ossfuzz.c.  It reads files named on the
+** command line and passes them one by one into ossfuzz.c.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "sqlite3.h"
+
+/*
+** The entry point in ossfuzz.c that this routine will be calling
+*/
+int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
+
+/* Must match equivalent #defines in ossfuzz.c */
+#define FUZZ_SQL_TRACE       0x0001   /* Set an sqlite3_trace() callback */
+#define FUZZ_SHOW_MAX_DELAY  0x0002   /* Show maximum progress callback delay */
+#define FUZZ_SHOW_ERRORS     0x0004   /* Show SQL errors */
+extern void ossfuzz_set_debug_flags(unsigned);
+
+
+
+/*
+** Read files named on the command-line and invoke the fuzzer for
+** each one.
+*/
+int main(int argc, char **argv){
+  FILE *in;
+  int i;
+  int nErr = 0;
+  uint8_t *zBuf = 0;
+  size_t sz;
+  unsigned mDebug = 0;
+
+  for(i=1; i $v2" $gt   \
+    2.$tn.le "$v1 <= $v2" $le   \
+    2.$tn.ge "$v1 >= $v2" $ge   \
+  ] {
+    do_execsql_test $tn2 "SELECT $expr" [list $res]
+
+    set map(0) [list]
+    set map() [list]
+    set map(1) [list 1]
+    do_execsql_test $tn2.where1 "SELECT * FROM one WHERE $expr" $map($res)
+
+    set map(0) [list 1]
+    set map() [list]
+    set map(1) [list]
+    do_execsql_test $tn2.where2 "SELECT * FROM one WHERE NOT $expr" $map($res)
+  }
+}
+
+do_execsql_test 3.0 {
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(1, 1);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(2, 3);
+  INSERT INTO t1 VALUES(2, 4);
+  INSERT INTO t1 VALUES(3, 5);
+  INSERT INTO t1 VALUES(3, 6);
+}
+
+foreach {tn r order} {
+  1 "(1, 1)"           "ORDER BY y"
+  2 "(1, 1)"           "ORDER BY x, y"
+  3 "(1, 2)"           "ORDER BY x, y DESC"
+  4 "(3, 6)"           "ORDER BY x DESC, y DESC"
+  5 "((3, 5))"         "ORDER BY x DESC, y"
+  6 "(SELECT 3, 5)"    "ORDER BY x DESC, y"
+} {
+  do_execsql_test 3.$tn.1 "SELECT $r == (SELECT x,y FROM t1 $order)" 1
+  do_execsql_test 3.$tn.2 "SELECT $r == (SELECT * FROM t1 $order)" 1
+
+  do_execsql_test 3.$tn.3 "
+    SELECT (SELECT * FROM t1 $order) == (SELECT * FROM t1 $order)
+  " 1
+  do_execsql_test 3.$tn.4 "
+    SELECT (SELECT 0, 0) == (SELECT * FROM t1 $order)
+  " 0
+}
+
+foreach {tn expr res} {
+  1 {(2, 2) BETWEEN (2, 2) AND (3, 3)} 1
+  2 {(2, 2) BETWEEN (2, NULL) AND (3, 3)} {}
+  3 {(2, 2) BETWEEN (3, NULL) AND (3, 3)} 0
+} {
+  do_execsql_test 4.$tn "SELECT $expr" [list $res]
+}
+
+foreach {tn expr res} {
+  1 {(2, 4) IN (SELECT * FROM t1)} 1
+  2 {(3, 4) IN (SELECT * FROM t1)} 0
+
+  3 {(NULL, 4) IN (SELECT * FROM t1)} {}
+  4 {(NULL, 0) IN (SELECT * FROM t1)} 0
+
+  5 {(NULL, 4) NOT IN (SELECT * FROM t1)} {}
+  6 {(NULL, 0) NOT IN (SELECT * FROM t1)} 1
+} {
+  do_execsql_test 5.$tn "SELECT $expr" [list $res]
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE hh(a, b, c);
+  INSERT INTO hh VALUES('abc', 1, 'i');
+  INSERT INTO hh VALUES('ABC', 1, 'ii');
+  INSERT INTO hh VALUES('def', 2, 'iii');
+  INSERT INTO hh VALUES('DEF', 2, 'iv');
+  INSERT INTO hh VALUES('GHI', 3, 'v');
+  INSERT INTO hh VALUES('ghi', 3, 'vi');
+
+  CREATE INDEX hh_ab ON hh(a, b); 
+}
+
+do_execsql_test 6.1 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc', 1);
+} {i}
+do_execsql_test 6.2 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc' COLLATE nocase, 1);
+} {i}
+do_execsql_test 6.3 {
+  SELECT c FROM hh WHERE a = (SELECT 'abc' COLLATE nocase) AND b = (SELECT 1);
+} {i}
+do_execsql_test 6.4 {
+  SELECT c FROM hh WHERE +a = (SELECT 'abc' COLLATE nocase) AND b = (SELECT 1);
+} {i}
+do_execsql_test 6.5 {
+  SELECT c FROM hh WHERE a = (SELECT 'abc') COLLATE nocase AND b = (SELECT 1);
+} {i ii}
+do_catchsql_test 6.6 {
+  SELECT c FROM hh WHERE (a, b) = (SELECT 'abc', 1) COLLATE nocase;
+} {1 {row value misused}}
+do_catchsql_test 6.7 {
+  SELECT c FROM hh WHERE (a, b) = 1;
+} {1 {row value misused}}
+do_execsql_test 6.8 {
+  SELECT c FROM hh WHERE (a COLLATE nocase, b) = (SELECT 'def', 2);
+} {iii iv}
+do_execsql_test 6.9 {
+  SELECT c FROM hh WHERE (a COLLATE nocase, b) IS NOT (SELECT 'def', 2);
+} {i ii v vi}
+do_execsql_test 6.10 {
+  SELECT c FROM hh WHERE (b, a) = (SELECT 2, 'def');
+} {iii}
+
+do_execsql_test 7.0 {
+  CREATE TABLE xy(i INTEGER PRIMARY KEY, j, k);
+  INSERT INTO xy VALUES(1, 1, 1);
+  INSERT INTO xy VALUES(2, 2, 2);
+  INSERT INTO xy VALUES(3, 3, 3);
+  INSERT INTO xy VALUES(4, 4, 4);
+}
+
+
+foreach {tn sql res eqp} {
+  1 "SELECT * FROM xy WHERE (i, j) IS (2, 2)" {2 2 2} 
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid=?)}"
+
+  2 "SELECT * FROM xy WHERE (k, j) < (2, 3)" {1 1 1 2 2 2}
+    "0 0 0 {SCAN TABLE xy}"
+
+  3 "SELECT * FROM xy WHERE (i, j) < (2, 3)" {1 1 1 2 2 2}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid (2, 1)" {2 2 2 3 3 3 4 4 4}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid>?)}"
+
+  5 "SELECT * FROM xy WHERE (i, j) > ('2', 1)" {2 2 2 3 3 3 4 4 4}
+    "0 0 0 {SEARCH TABLE xy USING INTEGER PRIMARY KEY (rowid>?)}"
+
+} {
+  do_eqp_test 7.$tn.1 $sql $eqp
+  do_execsql_test 7.$tn.2 $sql $res
+}
+
+do_execsql_test 8.0 {
+  CREATE TABLE j1(a);
+}
+do_execsql_test 8.1 {
+  SELECT * FROM j1 WHERE (select min(a) FROM j1) IN (?, ?, ?)
+}
+
+do_execsql_test 9.0 {
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c);
+  INSERT INTO t2 VALUES(1, 1, 1);
+  INSERT INTO t2 VALUES(2, 2, 2);
+  INSERT INTO t2 VALUES(3, 3, 3);
+  INSERT INTO t2 VALUES(4, 4, 4);
+  INSERT INTO t2 VALUES(5, 5, 5);
+}
+
+foreach {tn q res} {
+  1 "(a, b) > (2, 1)" {2 3 4 5}
+  2 "(a, b) > (2, 2)" {3 4 5}
+  3 "(a, b) < (4, 5)" {1 2 3 4}
+  4 "(a, b) < (4, 3)" {1 2 3}
+} {
+  do_execsql_test 9.$tn "SELECT c FROM t2 WHERE $q" $res
+} 
+
+do_execsql_test 10.0 {
+  CREATE TABLE dual(dummy); INSERT INTO dual(dummy) VALUES('X');
+  CREATE TABLE t3(a TEXT,b TEXT,c TEXT,d TEXT,e TEXT,f TEXT);
+  CREATE INDEX t3x ON t3(b,c,d,e,f);
+
+  SELECT a FROM t3
+    WHERE (c,d) IN (SELECT 'c','d' FROM dual)
+    AND (a,b,e) IN (SELECT 'a','b','d' FROM dual);
+}
+
+do_catchsql_test 11.1 {
+  CREATE TABLE t11(a);
+  SELECT * FROM t11 WHERE (a,a)<=1;
+} {1 {row value misused}}
+do_catchsql_test 11.2 {
+  SELECT * FROM t11 WHERE (a,a)<1;
+} {1 {row value misused}}
+do_catchsql_test 11.3 {
+  SELECT * FROM t11 WHERE (a,a)>=1;
+} {1 {row value misused}}
+do_catchsql_test 11.4 {
+  SELECT * FROM t11 WHERE (a,a)>1;
+} {1 {row value misused}}
+do_catchsql_test 11.5 {
+  SELECT * FROM t11 WHERE (a,a)==1;
+} {1 {row value misused}}
+do_catchsql_test 11.6 {
+  SELECT * FROM t11 WHERE (a,a)<>1;
+} {1 {row value misused}}
+do_catchsql_test 11.7 {
+  SELECT * FROM t11 WHERE (a,a) IS 1;
+} {1 {row value misused}}
+do_catchsql_test 11.8 {
+  SELECT * FROM t11 WHERE (a,a) IS NOT 1;
+} {1 {row value misused}}
+
+# 2016-10-27: https://www.sqlite.org/src/tktview/fef4bb4bd9185ec8f
+# Incorrect result from a LEFT JOIN with a row-value constraint
+#
+do_execsql_test 12.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b); INSERT INTO t1 VALUES(1,2);
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(x,y); INSERT INTO t2 VALUES(3,4);
+  SELECT *,'x' FROM t1 LEFT JOIN t2 ON (a,b)=(x,y);
+} {1 2 {} {} x}
+
+
+foreach {tn sql} {
+  0 "SELECT (1,2) AS x WHERE x=3"
+  1 "SELECT (1,2) BETWEEN 1 AND 2"
+  2 "SELECT 1 BETWEEN (1,2) AND 2"
+  3 "SELECT 2 BETWEEN 1 AND (1,2)"
+  4 "SELECT (1,2) FROM (SELECT 1) ORDER BY 1"
+  5 "SELECT (1,2) FROM (SELECT 1) GROUP BY 1"
+} {
+  do_catchsql_test 13.$tn $sql {1 {row value misused}}
+}
+
+do_execsql_test 14.0 {
+  CREATE TABLE t12(x);
+  INSERT INTO t12 VALUES(2), (4);
+}
+do_execsql_test 14.1 "SELECT 1 WHERE (2,2) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.2 "SELECT CASE (2,2) WHEN (1, 1) THEN 2 ELSE 1 END" 1
+do_execsql_test 14.3 "SELECT CASE (SELECT 2,2) WHEN (1, 1) THEN 2 ELSE 1 END" 1
+do_execsql_test 14.4 "SELECT 1 WHERE (SELECT 2,2) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.5 "SELECT 1 FROM t12 WHERE (x,1) BETWEEN (1,1) AND (3,3)" 1
+do_execsql_test 14.6 {
+  SELECT 1 FROM t12 WHERE (1,x) BETWEEN (1,1) AND (3,3)
+} {1 1}
+
+#-------------------------------------------------------------------------
+# Test that errors are not concealed by the SELECT flattening or
+# WHERE-clause push-down optimizations.
+do_execsql_test 14.1 {
+  CREATE TABLE x1(a PRIMARY KEY, b);
+  CREATE TABLE x2(a INTEGER PRIMARY KEY, b);
+}
+
+foreach {tn n sql} {
+  1 0 "SELECT * FROM (SELECT (1, 1) AS c FROM x1) WHERE c=1"
+  2 2 "SELECT * FROM (SELECT 1 AS x, (SELECT 8,9) AS y) WHERE y<1"
+  3 3 "SELECT * FROM (SELECT 1 AS x, (SELECT 8,9,10) AS y) WHERE y<1"
+  4 0 "SELECT * FROM (SELECT (a, b) AS c FROM x1), x2 WHERE c=a"
+  5 0 "SELECT * FROM (SELECT a AS c, (1, 2, 3) FROM x1), x2 WHERE c=a"
+  6 0 "SELECT * FROM (SELECT 1 AS c, (1, 2, 3) FROM x1) WHERE c=1"
+} {
+  if {$n==0} {
+    set err "row value misused"
+  } else {
+    set err "sub-select returns $n columns - expected 1"
+  }
+  do_catchsql_test 14.2.$tn $sql [list 1 $err]
+}
+
+#--------------------------------------------------------------------------
+# Test for vector size mismatches concealed by unexpanded subqueries.
+#
+do_catchsql_test 15.1 {
+  DETACH (SELECT * FROM (SELECT 1,2))<3;
+} {1 {row value misused}}
+do_catchsql_test 15.2 {
+  UPDATE x1 SET a=(SELECT * FROM (SELECT b,2))<3;
+} {1 {row value misused}}
+do_catchsql_test 15.3 {
+  UPDATE x1 SET a=NULL WHERE  a<(SELECT * FROM (SELECT b,2));
+} {1 {sub-select returns 2 columns - expected 1}}
+do_catchsql_test 15.4 {
+  DELETE FROM x1 WHERE  a<(SELECT * FROM (SELECT b,2));
+} {1 {sub-select returns 2 columns - expected 1}}
+do_catchsql_test 15.5 {
+  INSERT INTO x1(a,b) VALUES(1,(SELECT * FROM (SELECT 1,2))<3);
+} {1 {row value misused}}
+
+#-------------------------------------------------------------------------
+# Row-values used in UPDATE statements within TRIGGERs
+#
+# Ticket https://www.sqlite.org/src/info/8c9458e703666e1a
+#
+do_execsql_test 16.1 {
+  CREATE TABLE t16a(a,b,c);
+  INSERT INTO t16a VALUES(1,2,3);
+  CREATE TABLE t16b(x);
+  INSERT INTO t16b(x) VALUES(1);
+  CREATE TRIGGER t16r AFTER UPDATE ON t16b BEGIN
+     UPDATE t16a SET (a,b,c)=(SELECT new.x,new.x+1,new.x+2);
+  END;
+  UPDATE t16b SET x=7;
+  SELECT * FROM t16a;
+} {7 8 9}
+do_execsql_test 16.2 {
+  UPDATE t16b SET x=97;
+  SELECT * FROM t16a;
+} {97 98 99}
+
+do_execsql_test 16.3 {
+  CREATE TABLE t16c(a, b, c, d, e);
+  INSERT INTO t16c VALUES(1, 'a', 'b', 'c', 'd');
+  CREATE TRIGGER t16c1 AFTER INSERT ON t16c BEGIN
+    UPDATE t16c SET (c, d) = (SELECT 'A', 'B'), (e, b) = (SELECT 'C', 'D')
+      WHERE a = new.a-1;
+  END;
+
+  SELECT * FROM t16c;
+} {1 a b c d}
+
+do_execsql_test 16.4 {
+  INSERT INTO t16c VALUES(2, 'w', 'x', 'y', 'z');
+  SELECT * FROM t16c;
+} {
+  1 D A B C 
+  2 w x y z
+}
+
+do_execsql_test 16.5 {
+  DROP TRIGGER t16c1;
+  PRAGMA recursive_triggers = 1;
+  INSERT INTO t16c VALUES(3, 'i', 'ii', 'iii', 'iv');
+  CREATE TRIGGER t16c1 AFTER UPDATE ON t16c WHEN new.a>1 BEGIN
+    UPDATE t16c SET (e, d) = (
+      SELECT b, c FROM t16c WHERE a = new.a-1
+    ), (c, b) = (
+      SELECT d, e FROM t16c WHERE a = new.a-1
+    ) WHERE a = new.a-1;
+  END;
+
+  UPDATE t16c SET a=a WHERE a=3;
+  SELECT * FROM t16c;
+} {
+  1 C B A D
+  2 z y x w
+  3 i ii iii iv
+}
+
+finish_test
diff --git a/test/rowvalue2.test b/test/rowvalue2.test
new file mode 100644
index 0000000000..1502e31a54
--- /dev/null
+++ b/test/rowvalue2.test
@@ -0,0 +1,279 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SELECT statement.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue2
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES(0, 0, 0);
+  INSERT INTO t1 VALUES(0, 1, 1);
+  INSERT INTO t1 VALUES(1, 0, 2);
+  INSERT INTO t1 VALUES(1, 1, 3);
+
+  CREATE INDEX i1 ON t1(a, b);
+}
+
+do_execsql_test 1.1.1 { SELECT c FROM t1 WHERE (a, b) >= (1, 0) } {2 3} 
+do_execsql_test 1.1.2 { SELECT c FROM t1 WHERE (a, b) > (1, 0)  } {3}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0.1 {
+  CREATE TABLE t2(a INTEGER, b INTEGER, c INTEGER, d INTEGER);
+  CREATE INDEX i2 ON t2(a, b, c);
+}
+do_test 2.0.2 {
+  foreach a {0 1 2 3} {
+  foreach b {0 1 2 3} {
+  foreach c {0 1 2 3} {
+    execsql { INSERT INTO t2 VALUES($a, $b, $c, $c + $b*4 + $a*16); }
+  }}}
+} {}
+
+do_execsql_test 2.1 {
+  SELECT d FROM t2 WHERE (a, b) > (2, 2);
+} [db eval { SELECT d FROM t2 WHERE a>2 OR (a=2 AND b>2) }]
+
+do_execsql_test 2.2 {
+  SELECT d FROM t2 WHERE (a, b) >= (2, 2);
+} [db eval { SELECT d FROM t2 WHERE a>2 OR (a=2 AND b>=2) }]
+
+do_execsql_test 2.3 {
+  SELECT d FROM t2 WHERE a=1 AND (b, c) >= (1, 2);
+} [db eval { SELECT d FROM t2 WHERE +a=1 AND (b>1 OR (b==1 AND c>=2)) }]
+
+do_execsql_test 2.4 {
+  SELECT d FROM t2 WHERE a=1 AND (b, c) > (1, 2);
+} [db eval { SELECT d FROM t2 WHERE +a=1 AND (b>1 OR (b==1 AND c>2)) }]
+
+#-------------------------------------------------------------------------
+
+set words {
+airfare airfield airfields airflow airfoil
+airfoils airframe airframes airily airing
+airings airless airlift airlifts airline
+airliner airlines airlock airlocks airmail
+airmails airman airmen airplane airplanes
+
+arraignment arraignments arraigns arrange arranged
+arrangement arrangements arranger arrangers arranges
+arranging arrant array arrayed arrays
+arrears arrest arrested arrester arresters
+arresting arrestingly arrestor arrestors arrests
+
+edifices edit edited editing edition
+editions editor editorial editorially editorials
+editors edits educable educate educated
+educates educating education educational educationally
+educations educator educators eel eelgrass
+}
+
+do_test 3.0 {
+  execsql { CREATE TABLE t3(a, b, c, w); }
+  foreach w $words {
+    set a [string range $w 0 2]
+    set b [string range $w 3 5]
+    set c [string range $w 6 end]
+    execsql { INSERT INTO t3 VALUES($a, $b, $c, $w) }
+  }
+} {}
+
+
+foreach {tn idx} {
+  IDX1 {}
+  IDX2 { CREATE INDEX i3 ON t3(a, b, c); }
+  IDX3 { CREATE INDEX i3 ON t3(a, b); }
+  IDX4 { CREATE INDEX i3 ON t3(a); }
+} {
+  execsql { DROP INDEX IF EXISTS i3 }
+  execsql $idx
+
+  foreach w $words {
+    set a [string range $w 0 2]
+    set b [string range $w 3 5]
+    set c [string range $w 6 end]
+
+    foreach op [list > >= < <= == IS] {
+      do_execsql_test 3.1.$tn.$w.$op [subst -novar {
+        SELECT rowid FROM t3 WHERE (a, b, c) [set op] ($a, $b, $c) 
+        ORDER BY +rowid
+      }] [db eval [subst -novar {
+        SELECT rowid FROM t3 WHERE w [set op] $w ORDER BY +rowid
+      }]]
+
+      do_execsql_test 3.1.$tn.$w.$op.subselect [subst -novar {
+        SELECT rowid FROM t3 WHERE (a, b, c) [set op] (
+          SELECT a, b, c FROM t3 WHERE w = $w
+        )
+        ORDER BY +rowid
+      }] [db eval [subst -novar {
+        SELECT rowid FROM t3 WHERE w [set op] $w ORDER BY +rowid
+      }]]
+    }
+
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 4.0 {
+  CREATE TABLE t4(a, b, c);
+  INSERT INTO t4 VALUES(NULL, NULL, NULL);
+  INSERT INTO t4 VALUES(NULL, NULL, 0);
+  INSERT INTO t4 VALUES(NULL, NULL, 1);
+  INSERT INTO t4 VALUES(NULL,    0, NULL);
+  INSERT INTO t4 VALUES(NULL,    0, 0);
+  INSERT INTO t4 VALUES(NULL,    0, 1);
+  INSERT INTO t4 VALUES(NULL,    1, NULL);
+  INSERT INTO t4 VALUES(NULL,    1, 0);
+  INSERT INTO t4 VALUES(NULL,    1, 1);
+
+  INSERT INTO t4 VALUES(   0, NULL, NULL);
+  INSERT INTO t4 VALUES(   0, NULL, 0);
+  INSERT INTO t4 VALUES(   0, NULL, 1);
+  INSERT INTO t4 VALUES(   0,    0, NULL);
+  INSERT INTO t4 VALUES(   0,    0, 0);
+  INSERT INTO t4 VALUES(   0,    0, 1);
+  INSERT INTO t4 VALUES(   0,    1, NULL);
+  INSERT INTO t4 VALUES(   0,    1, 0);
+  INSERT INTO t4 VALUES(   0,    1, 1);
+
+  INSERT INTO t4 VALUES(   1, NULL, NULL);
+  INSERT INTO t4 VALUES(   1, NULL, 0);
+  INSERT INTO t4 VALUES(   1, NULL, 1);
+  INSERT INTO t4 VALUES(   1,    0, NULL);
+  INSERT INTO t4 VALUES(   1,    0, 0);
+  INSERT INTO t4 VALUES(   1,    0, 1);
+  INSERT INTO t4 VALUES(   1,    1, NULL);
+  INSERT INTO t4 VALUES(   1,    1, 0);
+  INSERT INTO t4 VALUES(   1,    1, 1);
+}
+
+proc make_expr1 {cList vList op} {
+  return "([join $cList ,]) $op ([join $vList ,])"
+}
+
+proc make_expr3 {cList vList op} {
+  set n [llength $cList]
+
+  set aList [list]
+  foreach c [lrange $cList 0 end-1] v [lrange $vList 0 end-1] {
+    lappend aList "$c == $v"
+  }
+  lappend aList "[lindex $cList end] $op [lindex $vList end]"
+
+  return "([join $aList { AND }])"
+}
+
+proc make_expr2 {cList vList op} {
+  set ret ""
+
+  switch -- $op {
+    == - IS {
+      set aList [list]
+      foreach c $cList v $vList { lappend aList "($c $op $v)" }
+      set ret [join $aList " AND "]
+    }
+
+    < - > {
+      set oList [list]
+      for {set i 0} {$i < [llength $cList]} {incr i} {
+        lappend oList [make_expr3 [lrange $cList 0 $i] [lrange $vList 0 $i] $op]
+      }
+      set ret [join $oList " OR "]
+    }
+
+    <= - >= {
+      set o2 [string range $op 0 0]
+      set oList [list]
+      for {set i 0} {$i < [llength $cList]-1} {incr i} {
+        lappend oList [make_expr3 [lrange $cList 0 $i] [lrange $vList 0 $i] $o2]
+      }
+      lappend oList [make_expr3 $cList $vList $op]
+      set ret [join $oList " OR "]
+    }
+
+
+    default {
+      error "Unknown op: $op"
+    }
+  }
+
+  set ret
+}
+
+foreach {tn idx} {
+  IDX1 {}
+  IDX2 { CREATE INDEX i4 ON t4(a, b, c); }
+  IDX3 { CREATE INDEX i4 ON t4(a, b); }
+  IDX4 { CREATE INDEX i4 ON t4(a); }
+} {
+  execsql { DROP INDEX IF EXISTS i4 }
+  execsql $idx
+
+  foreach {tn2 vector} {
+    1 {0 0 0}
+    2 {1 1 1}
+    3 {0 0 NULL}
+    4 {0 NULL 0}
+    5 {NULL 0 0}
+    6 {1 1 NULL}
+    7 {1 NULL 1}
+    8 {NULL 1 1}
+  } {
+    foreach op { IS == < <= > >= } {
+      set e1 [make_expr1 {a b c} $vector $op]
+      set e2 [make_expr2 {a b c} $vector $op]
+
+      do_execsql_test 4.$tn.$tn2.$op \
+          "SELECT rowid FROM t4 WHERE $e2 ORDER BY +rowid" [
+          db eval "SELECT rowid FROM t4 WHERE $e1 ORDER BY +rowid"
+      ]
+    }
+  }
+}
+
+do_execsql_test 5.0 {
+  CREATE TABLE r1(a TEXT, iB TEXT);
+  CREATE TABLE r2(x TEXT, zY INTEGER);
+  CREATE INDEX r1ab ON r1(a, iB);
+
+  INSERT INTO r1 VALUES(35, 35);
+  INSERT INTO r2 VALUES(35, 36);
+  INSERT INTO r2 VALUES(35, 4);
+  INSERT INTO r2 VALUES(35, 35);
+} {}
+
+foreach {tn lhs rhs} {
+  1 {x +zY} {a iB}
+  2 {x  zY} {a iB}
+  3 {x  zY} {a +iB}
+  4 {+x  zY} {a iB}
+  5 {x  zY} {+a iB}
+} {
+  foreach op { IS == < <= > >= } {
+    set e1 [make_expr1 $lhs $rhs $op]
+    set e2 [make_expr2 $lhs $rhs $op]
+    do_execsql_test 5.$tn.$op \
+      "SELECT * FROM r1, r2 WHERE $e2 ORDER BY iB" [db eval \
+      "SELECT * FROM r1, r2 WHERE $e1 ORDER BY iB"
+    ]
+  }
+}
+
+
+finish_test
diff --git a/test/rowvalue3.test b/test/rowvalue3.test
new file mode 100644
index 0000000000..988eea9008
--- /dev/null
+++ b/test/rowvalue3.test
@@ -0,0 +1,220 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing "(...) IN (SELECT ...)" expressions
+# where the SELECT statement returns more than one column.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue3
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX i1 ON t1(a, b);
+  INSERT INTO t1 VALUES(1, 2, 3);
+  INSERT INTO t1 VALUES(4, 5, 6);
+  INSERT INTO t1 VALUES(7, 8, 9);
+}
+
+foreach {tn sql res} {
+  1  "SELECT 1 WHERE (4, 5) IN (SELECT a, b FROM t1)"  1
+  2  "SELECT 1 WHERE (5, 5) IN (SELECT a, b FROM t1)"  {}
+  3  "SELECT 1 WHERE (5, 4) IN (SELECT a, b FROM t1)"  {}
+  4  "SELECT 1 WHERE (5, 4) IN (SELECT b, a FROM t1)"  1
+  5  "SELECT 1 WHERE (SELECT a, b FROM t1 WHERE c=6) IN (SELECT a, b FROM t1)" 1
+  6  "SELECT (5, 4) IN (SELECT a, b FROM t1)" 0
+  7  "SELECT 1 WHERE (5, 4) IN (SELECT +b, +a FROM t1)"  1
+  8  "SELECT (5, 4) IN (SELECT +b, +a FROM t1)"  1
+  9  "SELECT (1, 2) IN (SELECT rowid, b FROM t1)"  1
+  10 "SELECT 1 WHERE (1, 2) IN (SELECT rowid, b FROM t1)"  1
+  11 "SELECT 1 WHERE (1, NULL) IN (SELECT rowid, b FROM t1)"  {}
+  12 "SELECT 1 FROM t1 WHERE (a, b) = (SELECT +a, +b FROM t1)" {1}
+} {
+  do_execsql_test 1.$tn $sql $res
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 2.0 {
+  CREATE TABLE z1(x, y, z);
+  CREATE TABLE kk(a, b);
+
+  INSERT INTO z1 VALUES('a', 'b', 'c');
+  INSERT INTO z1 VALUES('d', 'e', 'f');
+  INSERT INTO z1 VALUES('g', 'h', 'i');
+
+  -- INSERT INTO kk VALUES('y', 'y');
+  INSERT INTO kk VALUES('d', 'e');
+  -- INSERT INTO kk VALUES('x', 'x');
+
+}
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX z1idx ON z1(x, y) }
+  3 { CREATE UNIQUE INDEX z1idx ON z1(x, y) }
+  4 { CREATE INDEX z1idx ON kk(a, b) }
+} {
+  execsql "DROP INDEX IF EXISTS z1idx"
+  execsql $idx
+
+  do_execsql_test 2.$tn.1 {
+    SELECT * FROM z1 WHERE x IN (SELECT a FROM kk)
+  } {d e f}
+
+  do_execsql_test 2.$tn.2 {
+    SELECT * FROM z1 WHERE (x,y) IN (SELECT a, b FROM kk)
+  } {d e f}
+
+  do_execsql_test 2.$tn.3 {
+    SELECT * FROM z1 WHERE (x, +y) IN (SELECT a, b FROM kk)
+  } {d e f}
+  
+  do_execsql_test 2.$tn.4 {
+    SELECT * FROM z1 WHERE (x, +y) IN (SELECT a, b||'x' FROM kk)
+  } {}
+
+  do_execsql_test 2.$tn.5 {
+    SELECT * FROM z1 WHERE (+x, y) IN (SELECT a, b FROM kk)
+  } {d e f}
+}
+
+#-------------------------------------------------------------------------
+#
+
+do_execsql_test 3.0 {
+  CREATE TABLE c1(a, b, c, d);
+  INSERT INTO c1(rowid, a, b) VALUES(1,   NULL, 1);
+  INSERT INTO c1(rowid, a, b) VALUES(2,   2, NULL);
+  INSERT INTO c1(rowid, a, b) VALUES(3,   2, 2);
+  INSERT INTO c1(rowid, a, b) VALUES(4,   3, 3);
+
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(101, 'a', 'b', 1, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(102, 'a', 'b', 1, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(103, 'a', 'b', 1, 3);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(104, 'a', 'b', 2, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(105, 'a', 'b', 2, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(106, 'a', 'b', 2, 3);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(107, 'a', 'b', 3, 1);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(108, 'a', 'b', 3, 2);
+  INSERT INTO c1(rowid, a, b, c, d) VALUES(109, 'a', 'b', 3, 3);
+}
+
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX c1ab ON c1(a, b); }
+  3 { CREATE INDEX c1ba ON c1(b, a); }
+
+  4 { CREATE INDEX c1cd ON c1(c, d); }
+  5 { CREATE INDEX c1dc ON c1(d, c); }
+} {
+  drop_all_indexes
+
+  foreach {tn2 sql res} {
+    1 "SELECT (1, 2) IN (SELECT a, b FROM c1)" {0}
+    2 "SELECT (1, 1) IN (SELECT a, b FROM c1)" {{}}
+    3 "SELECT (2, 1) IN (SELECT a, b FROM c1)" {{}}
+    4 "SELECT (2, 2) IN (SELECT a, b FROM c1)" {1}
+    5 "SELECT c, d FROM c1 WHERE (c, d) IN (SELECT d, c FROM c1)"
+      { 1 1 1 2 1 3   2 1 2 2 2 3   3 1 3 2 3 3 }
+
+    6 "SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) ORDER BY c DESC"
+      { 3 1 3 2 3 3   2 1 2 2 2 3   1 1 1 2 1 3 }
+
+    7 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c DESC, d ASC
+      } { 3 1 3 2 3 3   2 1 2 2 2 3   1 1 1 2 1 3 }
+
+    8 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c ASC, d DESC
+      } { 1 3 1 2 1 1   2 3 2 2 2 1   3 3 3 2 3 1 }
+
+    9 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c ASC, d ASC
+      } { 1 1 1 2 1 3   2 1 2 2 2 3   3 1 3 2 3 3 }
+    10 {
+        SELECT c, d FROM c1 WHERE (c,d) IN (SELECT d, c FROM c1) 
+        ORDER BY c DESC, d DESC
+      } { 3 3 3 2 3 1   2 3 2 2 2 1   1 3 1 2 1 1 }
+
+  } {
+    do_execsql_test 3.$tn.$tn2 $sql $res
+  }
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 4.0 {
+  CREATE TABLE hh(a, b, c);
+
+  INSERT INTO hh VALUES('a', 'a', 1);
+  INSERT INTO hh VALUES('a', 'b', 2);
+  INSERT INTO hh VALUES('b', 'a', 3);
+  INSERT INTO hh VALUES('b', 'b', 4);
+
+  CREATE TABLE k1(x, y);
+  INSERT INTO k1 VALUES('a', 'a');
+  INSERT INTO k1 VALUES('b', 'b');
+  INSERT INTO k1 VALUES('a', 'b');
+  INSERT INTO k1 VALUES('b', 'a');
+}
+
+foreach {tn idx} {
+  1 { }
+  2 { CREATE INDEX h1 ON hh(a, b); }
+  3 { CREATE UNIQUE INDEX k1idx ON k1(x, y) }
+  4 { CREATE UNIQUE INDEX k1idx ON k1(x, y DESC) }
+  5 { 
+    CREATE INDEX h1 ON hh(a, b);
+    CREATE UNIQUE INDEX k1idx ON k1(x, y); 
+  }
+  6 { 
+    CREATE INDEX h1 ON hh(a, b);
+    CREATE UNIQUE INDEX k1idx ON k1(x, y DESC); 
+  }
+} {
+  drop_all_indexes
+  execsql $idx
+  foreach {tn2 orderby res} {
+    1 "a ASC, b ASC"  {1 2 3 4}
+    2 "a ASC, b DESC" {2 1 4 3}
+    3 "a DESC, b ASC" {3 4 1 2}
+    4 "a DESC, b DESC" {4 3 2 1}
+  } {
+    do_execsql_test 4.$tn.$tn2 "
+      SELECT c FROM hh WHERE (a, b) in (SELECT x, y FROM k1) ORDER BY $orderby
+    " $res
+  }
+}
+
+#-------------------------------------------------------------------------
+
+# 2016-11-17.  Query flattening in a vector SELECT on the RHS of an IN
+# operator.  Ticket https://www.sqlite.org/src/info/da7841375186386c
+#
+do_execsql_test 5.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE T1(a TEXT);
+  INSERT INTO T1(a) VALUES ('aaa');
+  CREATE TABLE T2(a TEXT PRIMARY KEY,n INT);
+  INSERT INTO T2(a, n) VALUES('aaa',0);
+  SELECT * FROM T2
+  WHERE (a,n) IN (SELECT T1.a, V.n FROM T1, (SELECT * FROM (SELECT 0 n)) V);
+} {aaa 0}
+
+
+finish_test
diff --git a/test/rowvalue4.test b/test/rowvalue4.test
new file mode 100644
index 0000000000..2e6a28c230
--- /dev/null
+++ b/test/rowvalue4.test
@@ -0,0 +1,313 @@
+# 2016 July 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is syntax errors involving row-value constructors
+# and sub-selects that return multiple arguments.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue4
+
+#-------------------------------------------------------------------------
+# Test some error conditions:
+# 
+#   * row values used where they are not supported,
+#   * row values or sub-selects that contain/return the wrong number
+#     of elements.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1bac ON t1(b, a, c);
+}
+
+foreach {tn e} {
+  1 "(1, 2, 3)"
+  2 "1 + (1, 2)"
+  3 "(1,2,3) == (1, 2)"
+} {
+  do_catchsql_test 1.$tn "SELECT $e" {1 {row value misused}}
+}
+
+foreach {tn s error} {
+  1 "SELECT * FROM t1 WHERE a = (1, 2)"       {row value misused}
+  2 "SELECT * FROM t1 WHERE b = (1, 2)"       {row value misused}
+  3 "SELECT * FROM t1 WHERE NOT (b = (1, 2))" {row value misused}
+  4 "SELECT * FROM t1 LIMIT (1, 2)"           {row value misused}
+  5 "SELECT (a, b) IN (SELECT * FROM t1) FROM t1" 
+                             {sub-select returns 3 columns - expected 2}
+
+  6 "SELECT * FROM t1 WHERE (a, b) IN (SELECT * FROM t1)" 
+                             {sub-select returns 3 columns - expected 2}
+  7 "SELECT * FROM t1 WHERE (c, c) <= 1" {row value misused}
+  8 "SELECT * FROM t1 WHERE (b, b) <= 1" {row value misused}
+} {
+  do_catchsql_test 2.$tn "$s" [list 1 $error]
+}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE TABLE t2(a, b, c, d);
+  INSERT INTO t2 VALUES(1, 1, 1,   1);
+  INSERT INTO t2 VALUES(1, 1, 2,   2);
+  INSERT INTO t2 VALUES(1, 1, 3,   3);
+  INSERT INTO t2 VALUES(1, 2, 1,   4);
+  INSERT INTO t2 VALUES(1, 2, 2,   5);
+  INSERT INTO t2 VALUES(1, 2, 3,   6);
+  INSERT INTO t2 VALUES(1, 3, 1,   7);
+  INSERT INTO t2 VALUES(1, 3, 2,   8);
+  INSERT INTO t2 VALUES(1, 3, 3,   9);
+
+  INSERT INTO t2 VALUES(2, 1, 1,   10);
+  INSERT INTO t2 VALUES(2, 1, 2,   11);
+  INSERT INTO t2 VALUES(2, 1, 3,   12);
+  INSERT INTO t2 VALUES(2, 2, 1,   13);
+  INSERT INTO t2 VALUES(2, 2, 2,   14);
+  INSERT INTO t2 VALUES(2, 2, 3,   15);
+  INSERT INTO t2 VALUES(2, 3, 1,   16);
+  INSERT INTO t2 VALUES(2, 3, 2,   17);
+  INSERT INTO t2 VALUES(2, 3, 3,   18);
+
+  INSERT INTO t2 VALUES(3, 1, 1,   19);
+  INSERT INTO t2 VALUES(3, 1, 2,   20);
+  INSERT INTO t2 VALUES(3, 1, 3,   21);
+  INSERT INTO t2 VALUES(3, 2, 1,   22);
+  INSERT INTO t2 VALUES(3, 2, 2,   23);
+  INSERT INTO t2 VALUES(3, 2, 3,   24);
+  INSERT INTO t2 VALUES(3, 3, 1,   25);
+  INSERT INTO t2 VALUES(3, 3, 2,   26);
+  INSERT INTO t2 VALUES(3, 3, 3,   27);
+}
+
+foreach {nm idx} {
+  idx1 {}
+  idx2 { CREATE INDEX t2abc ON t2(a, b, c); }
+  idx3 { CREATE INDEX t2abc ON t2(a, b DESC, c); }
+  idx4 { CREATE INDEX t2abc ON t2(a DESC, b DESC, c DESC); }
+  idx5 { CREATE INDEX t2abc ON t2(a ASC, b ASC, c ASC); }
+  idx6 { CREATE INDEX t2abc ON t2(a DESC, b, c); }
+  idx7 { CREATE INDEX t2abc ON t2(a DESC, b DESC) }
+  idx8 { CREATE INDEX t2abc ON t2(c, b, a); }
+  idx9 { CREATE INDEX t2d ON t2(d); }
+  idx10 { CREATE INDEX t2abc ON t2(a DESC, b, c DESC); }
+} {
+  drop_all_indexes
+  execsql $idx
+
+  foreach {tn where res} {
+    1 "(a, b, c) < (2, 2, 2)"  {1 2 3 4 5 6 7 8 9 10 11 12 13}
+    2 "(a, b, c) <= (2, 2, 2)" {1 2 3 4 5 6 7 8 9 10 11 12 13 14}
+    3 "(a, b, c) > (2, 2, 2)"  {15 16 17 18 19 20 21 22 23 24 25 26 27}
+    4 "(a, b, c) >= (2, 2, 2)" {14 15 16 17 18 19 20 21 22 23 24 25 26 27}
+    5 "(a, b, c) >= (2, 2, NULL)" {16 17 18 19 20 21 22 23 24 25 26 27}
+    6 "(a, b, c) <= (2, 2, NULL)" {1 2 3 4 5 6 7 8 9 10 11 12}
+    7 "(a, b, c) >= (2, NULL, NULL)" {19 20 21 22 23 24 25 26 27}
+    8 "(a, b, c) <= (2, NULL, NULL)" {1 2 3 4 5 6 7 8 9}
+
+    9 "(a, b, c) < (SELECT a, b, c FROM t2 WHERE d=14)"  
+      {1 2 3 4 5 6 7 8 9 10 11 12 13}
+
+    10 "(a, b, c) = (SELECT a, b, c FROM t2 WHERE d=14)" 14
+
+    11 "a = 2 AND (b, c) > (2, 2)" {15 16 17 18}
+    12 "a = 2 AND (b, c) < (3, 3) AND (b, c) > (1, 1)" {11 12 13 14 15 16 17}
+  } {
+    set result [db eval "SELECT d FROM t2 WHERE $where"]
+    do_test 2.1.$nm.$tn { lsort -integer $result } $res
+  }
+
+  foreach {tn e res} {
+    1 "(2, 1) IN (SELECT a, b FROM t2)" 1
+    2 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d)" 1
+    3 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 9)" 0
+    4 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 10)" 1
+
+    5 "(3, 3) = (SELECT a, b FROM t2 ORDER BY d DESC LIMIT 1)" 1
+    6 "(3, 3) = (SELECT a, b FROM t2 ORDER BY d ASC LIMIT 1)" 0
+    7 "(1, NULL) = (SELECT a, b FROM t2 ORDER BY d ASC LIMIT 1)" {{}}
+
+    8 "(3, 1) = (SELECT b, c FROM t2 ORDER BY d DESC LIMIT 1 OFFSET 2)" 1
+    9 "(3, 1) = (SELECT b, c FROM t2 ORDER BY d ASC LIMIT 1 OFFSET 2)" 0
+    10 "(1, NULL) = (SELECT b, c FROM t2 ORDER BY d ASC LIMIT 1 OFFSET 2)" {{}}
+
+    11 "(3, 3) = (SELECT max(a), max(b) FROM t2)" 1
+    12 "(3, 1) = (SELECT max(a), min(b) FROM t2)" 1
+    13 "(NULL, NULL) = (SELECT max(a), min(b) FROM t2)" {{}}
+
+    14 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 5 OFFSET 11)" 1
+    15 "(2, 1) IN (SELECT a, b FROM t2 ORDER BY d LIMIT 5 OFFSET 12)" 0
+  } {
+    do_execsql_test 2.2.$nm.$tn "SELECT $e" $res
+  }
+}
+
+ifcapable stat4 {
+  do_execsql_test 3.0 {
+    CREATE TABLE c1(a, b, c, d);
+    INSERT INTO c1(a, b) VALUES(1, 'a');
+    INSERT INTO c1(a, b) VALUES(1, 'b');
+    INSERT INTO c1(a, b) VALUES(1, 'c');
+    INSERT INTO c1(a, b) VALUES(1, 'd');
+    INSERT INTO c1(a, b) VALUES(1, 'e');
+    INSERT INTO c1(a, b) VALUES(1, 'f');
+    INSERT INTO c1(a, b) VALUES(1, 'g');
+    INSERT INTO c1(a, b) VALUES(1, 'h');
+    INSERT INTO c1(a, b) VALUES(1, 'i');
+    INSERT INTO c1(a, b) VALUES(1, 'j');
+    INSERT INTO c1(a, b) VALUES(1, 'k');
+    INSERT INTO c1(a, b) VALUES(1, 'l');
+    INSERT INTO c1(a, b) VALUES(1, 'm');
+    INSERT INTO c1(a, b) VALUES(1, 'n');
+    INSERT INTO c1(a, b) VALUES(1, 'o');
+    INSERT INTO c1(a, b) VALUES(1, 'p');
+    INSERT INTO c1(a, b) VALUES(2, 'a');
+    INSERT INTO c1(a, b) VALUES(2, 'b');
+    INSERT INTO c1(a, b) VALUES(2, 'c');
+    INSERT INTO c1(a, b) VALUES(2, 'd');
+    INSERT INTO c1(a, b) VALUES(2, 'e');
+    INSERT INTO c1(a, b) VALUES(2, 'f');
+    INSERT INTO c1(a, b) VALUES(2, 'g');
+    INSERT INTO c1(a, b) VALUES(2, 'h');
+
+    INSERT INTO c1(c, d) SELECT a, b FROM c1;
+
+    CREATE INDEX c1ab ON c1(a, b);
+    CREATE INDEX c1cd ON c1(c, d);
+    ANALYZE;
+  }
+
+  do_eqp_test 3.1.1 { SELECT * FROM c1 WHERE a=1 AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c=?)}
+  }
+  do_eqp_test 3.1.2 { SELECT * FROM c1 WHERE a=1 AND b>'d' AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c=?)}
+  }
+  do_eqp_test 3.1.3 { SELECT * FROM c1 WHERE a=1 AND b>'l' AND c=2 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=? AND b>?)}
+  }
+
+  do_eqp_test 3.2.1 { SELECT * FROM c1 WHERE a=1 AND c>1 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd (c>?)}
+  }
+  do_eqp_test 3.2.2 { SELECT * FROM c1 WHERE a=1 AND c>0 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.3 { SELECT * FROM c1 WHERE a=1 AND c>=1 } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.4 { SELECT * FROM c1 WHERE a=1 AND (c, d)>(1, 'c') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+  do_eqp_test 3.2.5 { SELECT * FROM c1 WHERE a=1 AND (c, d)>(1, 'o') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1cd ((c,d)>(?,?))}
+  }
+  do_eqp_test 3.2.6 { SELECT * FROM c1 WHERE a=1 AND (c, +b)>(1, 'c') } {
+    0 0 0 {SEARCH TABLE c1 USING INDEX c1ab (a=?)}
+  }
+}
+
+#------------------------------------------------------------------------
+
+do_execsql_test 5.0 {
+  CREATE TABLE d1(x, y);
+  CREATE TABLE d2(a, b, c);
+  CREATE INDEX d2ab ON d2(a, b);
+  CREATE INDEX d2c ON d2(c);
+
+  WITH i(i) AS (
+    VALUES(1) UNION ALL SELECT i+1 FROM i WHERE i<1000
+  )
+  INSERT INTO d2 SELECT i/3, i%3, i/3 FROM i;
+  ANALYZE;
+}
+
+do_eqp_test 5.1 {
+  SELECT * FROM d2 WHERE 
+    (a, b) IN (SELECT x, y FROM d1) AND
+    (c) IN (SELECT y FROM d1)
+} {
+  0 0 0 {SEARCH TABLE d2 USING INDEX d2ab (a=? AND b=?)}
+  0 0 0 {EXECUTE LIST SUBQUERY 1} 
+  1 0 0 {SCAN TABLE d1}
+  0 0 0 {EXECUTE LIST SUBQUERY 2} 
+  2 0 0 {SCAN TABLE d1}
+}
+
+do_execsql_test 6.0 {
+  CREATE TABLE e1(a, b, c, d, e);
+  CREATE INDEX e1ab ON e1(a, b);
+  CREATE INDEX e1cde ON e1(c, d, e);
+}
+
+do_eqp_test 6.1 {
+  SELECT * FROM e1 WHERE (a, b) > (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1ab ((a,b)>(?,?))}
+}
+do_eqp_test 6.2 {
+  SELECT * FROM e1 WHERE (a, b) < (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1ab ((a,b)<(?,?))}
+}
+do_eqp_test 6.3 {
+  SELECT * FROM e1 WHERE c = ? AND (d, e) > (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)>(?,?))}
+}
+do_eqp_test 6.4 {
+  SELECT * FROM e1 WHERE c = ? AND (d, e) < (?, ?)
+} {
+  0 0 0 {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)<(?,?))}
+}
+
+do_eqp_test 6.5 {
+  SELECT * FROM e1 WHERE (d, e) BETWEEN (?, ?) AND (?, ?) AND c = ?
+} {
+  0 0 0 
+  {SEARCH TABLE e1 USING INDEX e1cde (c=? AND (d,e)>(?,?) AND (d,e)<(?,?))}
+}
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 7.1 {
+  CREATE TABLE f1(a, b, c);
+  CREATE INDEX f1ab ON f1(a, b);
+}
+
+do_catchsql_test 7.2 {
+  SELECT (a COLLATE nocase, b) IN (SELECT a, b FROM f1) FROM f1;
+} {0 {}}
+
+do_catchsql_test 7.3 {
+  SELECT (a COLLATE nose, b) IN (SELECT a, b FROM f1) FROM f1;
+} {1 {no such collation sequence: nose}}
+
+do_catchsql_test 7.4 {
+  SELECT * FROM f1 WHERE (?, ? COLLATE nose) > (a, b);
+} {1 {no such collation sequence: nose}}
+
+#-------------------------------------------------------------------------
+drop_all_tables
+do_execsql_test 8.1 {
+  CREATE TABLE c1(x, y);
+  CREATE TABLE c2(a, b, c);
+  CREATE INDEX c2ab ON c2(a, b);
+  CREATE INDEX c2c ON c2(c);
+
+  CREATE TABLE c3(d);
+}
+do_catchsql_test 8.2 {
+  SELECT * FROM c2 CROSS JOIN c3 WHERE 
+    ( (a, b) == (SELECT x, y FROM c1) AND c3.d = c ) OR
+    ( c == (SELECT x, y FROM c1) AND c3.d = c )
+} {1 {row value misused}}
+
+finish_test
diff --git a/test/rowvalue5.test b/test/rowvalue5.test
new file mode 100644
index 0000000000..58b39f2793
--- /dev/null
+++ b/test/rowvalue5.test
@@ -0,0 +1,115 @@
+# 2016 July 29
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is syntax errors involving row-values and
+# virtual tables.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue5
+
+ifcapable !vtab {
+  finish_test
+  return
+}
+
+proc vtab_command {method args} {
+  switch -- $method {
+    xConnect {
+      return "CREATE TABLE t1(a, b, c, d, expr)"
+    }
+
+    xBestIndex {
+      set COL(0) a
+      set COL(1) b
+      set COL(2) c
+      set COL(3) d
+      set COL(4) expr
+
+      set OP(eq) =
+      set OP(ne) !=
+      set OP(gt) >
+      set OP(le) <=
+      set OP(lt) <
+      set OP(ge) >=
+      set OP(match) MATCH
+      set OP(like) LIKE
+      set OP(glob) GLOB
+      set OP(regexp) REGEXP
+
+      set clist [lindex $args 0]
+      set ret [list]
+      set elist [list]
+      set i 0
+      foreach c $clist {
+        array set C $c
+        if {$C(usable)} {
+          lappend ret omit $i
+          lappend elist "$COL($C(column)) $OP($C(op)) %$i%"
+        }
+        incr i
+      }
+
+      lappend ret idxstr [join $elist " AND "]
+      #puts "xBestIndex: $ret"
+      return $ret
+    }
+
+    xFilter {
+      foreach {idxnum idxstr arglist} $args {}
+      set i 0
+      set ee $idxstr
+      foreach a $arglist {
+        if {[string is double $a]==0} {
+          set a "'[string map {' ''} $a]'"
+        }
+        set ee [string map [list "%$i%" $a] $ee]
+        incr i
+      }
+      set ee [string map [list "'" "''"] $ee]
+
+      set ret [list sql "SELECT 1, 'a', 'b', 'c', 'd', '$ee'"]
+      #puts "xFilter: $ret"
+      return $ret
+    }
+  }
+
+  return {}
+}
+
+register_tcl_module db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING tcl(vtab_command);
+} {}
+
+
+foreach {tn where res} {
+  1 "1"                              {{}}
+  2 "a=1"                            {{a = 1}}
+  3 "a=1 AND 4 = b"                  {{a = 1 AND b = 4}}
+  4 "c>'hello'"                      {{c > 'hello'}}
+  5 "c<='hel''lo'"                   {{c <= 'hel''lo'}}
+  6 "(a, b) = (SELECT 9, 10)"        {{a = 9 AND b = 10}}
+  7 "(+a, b) = (SELECT 'a', 'b')"    {{b = 'b'}}
+  8 "(a, +b) = (SELECT 'a', 'b')"    {{a = 'a'}}
+  11 "(+a, b) IN (SELECT 'a', 'b')"  {{b = 'b'}}
+  12 "(a, +b) IN (SELECT 'a', 'b')"  {{a = 'a'}}
+
+  13 "(a, b) < ('d', 'e')"           {{a <= 'd'}}
+  14 "(a, b) < ('a', 'c')"           {{a <= 'a'}}
+  15 "(a, b) <= ('a', 'b')"          {{a <= 'a'}}
+  16 "(a, b) < ('a', 'b')"           {}
+} {
+  do_execsql_test 1.$tn "SELECT expr FROM x1 WHERE $where" $res
+}
+
+finish_test
diff --git a/test/rowvalue6.test b/test/rowvalue6.test
new file mode 100644
index 0000000000..d90e61c811
--- /dev/null
+++ b/test/rowvalue6.test
@@ -0,0 +1,36 @@
+# 2016-08-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# The focus of this file is handling of NULL values in row-value IN
+# expressions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue6
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a,b,c);
+  CREATE INDEX t1x1 ON t1(a,b);
+  INSERT INTO t1 VALUES(1,NULL,200);
+
+  CREATE TABLE t2(x,y,z);
+  INSERT INTO t2 VALUES(1,NULL,55);
+
+  SELECT c FROM t1 WHERE (a,b) IN (SELECT x,y FROM t2 WHERE z==55);
+} {}
+do_execsql_test 1.2 {
+  INSERT INTO t1 VALUES(2,3,400);
+  INSERT INTO t2 VALUES(2,3,55);  
+
+  SELECT c FROM t1 WHERE (a,b) IN (SELECT x,y FROM t2 WHERE z==55);
+} {400}
+
+finish_test
diff --git a/test/rowvalue7.test b/test/rowvalue7.test
new file mode 100644
index 0000000000..f6764f301e
--- /dev/null
+++ b/test/rowvalue7.test
@@ -0,0 +1,58 @@
+# 2016-08-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# The focus of this file is vector assignments in the SET clause of
+# an UPDATE statement.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue7
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a,b,c,d);
+  CREATE INDEX t1x ON t1(a,b);
+  INSERT INTO t1(a,b,c,d) VALUES(1,2,0,0),(3,4,0,0),(5,6,0,0);
+  CREATE TABLE t2(w,x,y,z);
+  CREATE INDEX t2x ON t2(w,x);
+  INSERT INTO t2(w,x,y,z) VALUES(1,2,11,22),(8,9,88,99),(3,5,33,55),(5,6,55,66);
+
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 0 0 | 3 4 0 0 | 5 6 0 0 |}
+
+do_execsql_test 1.2 {
+  UPDATE t1 SET (c,d) = (SELECT y,z FROM t2 WHERE (w,x)=(a,b));
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 {} {} | 5 6 55 66 |}
+
+do_execsql_test 1.3 {
+  UPDATE t1 SET (c,d) = (SELECT y,z FROM t2 WHERE w=a);
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 33 55 | 5 6 55 66 |}
+
+do_execsql_test 1.4 {
+  UPDATE t1 SET (c) = 99 WHERE a=3;
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 4 99 55 | 5 6 55 66 |}
+
+do_execsql_test 1.5 {
+  UPDATE t1 SET b = 8, (c,d) = (SELECT 123,456) WHERE a=3;
+  SELECT *,'|' FROM t1 ORDER BY a;
+} {1 2 11 22 | 3 8 123 456 | 5 6 55 66 |}
+
+do_catchsql_test 2.1 {
+  UPDATE t1 SET (c,d) = (SELECT x,y,z FROM t2 WHERE w=a);
+} {1 {2 columns assigned 3 values}}
+
+do_catchsql_test 2.2 {
+  UPDATE t1 SET (b,c,d) = (SELECT x,y FROM t2 WHERE w=a);
+} {1 {3 columns assigned 2 values}}
+
+finish_test
diff --git a/test/rowvalue8.test b/test/rowvalue8.test
new file mode 100644
index 0000000000..432dad1278
--- /dev/null
+++ b/test/rowvalue8.test
@@ -0,0 +1,59 @@
+# 2016-08-22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# Use of row values in CASE statements.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue8
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);
+  INSERT INTO t1(a,b,c,d) VALUES
+     (1,1,2,3),
+     (2,2,3,4),
+     (3,1,2,4),
+     (4,2,3,5),
+     (5,3,4,6),
+     (6,4,5,9);
+  SELECT a, CASE (b,c) WHEN (1,2) THEN 'aleph'
+                       WHEN (2,3) THEN 'bet'
+                       WHEN (3,4) THEN 'gimel'
+                       ELSE '-' END,
+         '|'
+    FROM t1
+   ORDER BY a;
+} {1 aleph | 2 bet | 3 aleph | 4 bet | 5 gimel | 6 - |}
+do_execsql_test 1.2 {
+  SELECT a, CASE (b,c,d) WHEN (1,2,3) THEN 'aleph'
+                         WHEN (2,3,4) THEN 'bet'
+                         WHEN (3,4,6) THEN 'gimel'
+                         ELSE '-' END,
+         '|'
+    FROM t1
+   ORDER BY a;
+} {1 aleph | 2 bet | 3 - | 4 - | 5 gimel | 6 - |}
+
+do_execsql_test 2.1 {
+  CREATE TABLE t2(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t2(x,y) VALUES(1,6),(2,5),(3,4),(4,3),(5,2),(6,1);
+  SELECT x, CASE (SELECT b,c FROM t1 WHERE a=y)
+            WHEN (1,2) THEN 'aleph'
+            WHEN (2,3) THEN 'bet'
+            WHEN (3,4) THEN 'gimel'
+            ELSE '-' END,
+         '|'
+    FROM t2
+   ORDER BY +x;
+} {1 - | 2 gimel | 3 bet | 4 aleph | 5 bet | 6 aleph |}
+
+
+finish_test
diff --git a/test/rowvalue9.test b/test/rowvalue9.test
new file mode 100644
index 0000000000..0fa652f97d
--- /dev/null
+++ b/test/rowvalue9.test
@@ -0,0 +1,301 @@
+# 2016 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing SQL statements that use row value
+# constructors.
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix rowvalue9
+
+# Tests:
+#
+#  1.*: Test that affinities are handled correctly by various row-value
+#       operations without indexes.
+#
+#  2.*: Test an affinity bug that came up during testing.
+#
+#  3.*: Test a row-value version of the bug tested by 2.*.
+#
+#  4.*: Test that affinities are handled correctly by various row-value
+#       operations with assorted indexes.
+#
+
+do_execsql_test 1.0.1 {
+  CREATE TABLE a1(c, b INTEGER, a TEXT, PRIMARY KEY(a, b));
+ 
+  INSERT INTO a1 (rowid, c, b, a) VALUES(3,  '0x03', 1, 1);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(14, '0x0E', 2, 2);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(15, '0x0F', 3, 3);
+  INSERT INTO a1 (rowid, c, b, a) VALUES(92, '0x5C', 4, 4);
+
+  CREATE TABLE a2(x BLOB, y BLOB);
+  INSERT INTO a2(x, y) VALUES(1, 1);
+  INSERT INTO a2(x, y) VALUES(2, '2');
+  INSERT INTO a2(x, y) VALUES('3', 3);
+  INSERT INTO a2(x, y) VALUES('4', '4');
+}
+
+do_execsql_test 1.0.2 { 
+  SELECT x, typeof(x), y, typeof(y) FROM a2 ORDER BY rowid
+} {
+  1 integer 1 integer 
+  2 integer 2 text 
+  3 text    3 integer 
+  4 text    4 text
+}
+
+do_execsql_test 1.1.1 {
+  SELECT (SELECT rowid FROM a1 WHERE a=x AND b=y) FROM a2
+} {{} {} 15 92}
+do_execsql_test 1.1.2 {
+  SELECT (SELECT rowid FROM a1 WHERE (a, b) = (x, y)) FROM a2
+} {{} {} 15 92}
+
+do_execsql_test 1.2.3 {
+  SELECT a1.rowid FROM a1, a2 WHERE a=x AND b=y;
+} {15 92}
+do_execsql_test 1.2.4 {
+  SELECT a1.rowid FROM a1, a2 WHERE (a, b) = (x, y)
+} {15 92}
+
+
+do_execsql_test 1.3.1 {
+  SELECT a1.rowid FROM a1, a2 WHERE coalesce(NULL,x)=a AND coalesce(NULL,y)=b
+} {3 14 15 92}
+do_execsql_test 1.3.2 {
+  SELECT a1.rowid FROM a1, a2 
+  WHERE (coalesce(NULL,x), coalesce(NULL,y)) = (a, b)
+} {3 14 15 92}
+
+do_execsql_test 1.4.1 {
+  SELECT a1.rowid FROM a1, a2 WHERE +x=a AND +y=b
+} {3 14 15 92}
+do_execsql_test 1.4.2 {
+  SELECT a1.rowid FROM a1, a2 WHERE (+x, +y) = (a, b)
+} {3 14 15 92}
+
+do_execsql_test 1.5.1 {
+  SELECT (SELECT rowid FROM a1 WHERE a=+x AND b=+y) FROM a2
+} {3 14 15 92}
+do_execsql_test 1.5.2 {
+  SELECT (SELECT rowid FROM a1 WHERE (a, b) = (+x, +y)) FROM a2
+} {3 14 15 92}
+do_execsql_test 1.5.3 {
+  SELECT (SELECT rowid FROM a1 WHERE (+x, +y) = (a, b)) FROM a2
+} {3 14 15 92}
+
+do_execsql_test 1.6.1 {
+  SELECT a1.rowid FROM a1 WHERE (a, b) IN (SELECT x, y FROM a2)
+} {15 92}
+do_execsql_test 1.6.2 {
+  SELECT a1.rowid FROM a1, a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE a=x AND b=y
+  )
+} {3 14 15 92 3 14 15 92}
+
+# Test that [199df416] is fixed.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE b1(a TEXT);
+  CREATE TABLE b2(x BLOB);
+  INSERT INTO b1 VALUES(1);
+  INSERT INTO b2 VALUES(1);
+}
+do_execsql_test 2.2 { SELECT * FROM b1, b2 WHERE a=x; } {}
+do_execsql_test 2.3 { SELECT * FROM b1 WHERE a IN (SELECT x FROM b2) } {}
+do_execsql_test 2.4 { CREATE UNIQUE INDEX b1a ON b1(a); }
+do_execsql_test 2.5 { SELECT * FROM b1 WHERE a IN (SELECT x FROM b2) } {}
+
+# Test that a multi-column version of the query that revealed problem 
+# [199df416] also works.
+#
+do_execsql_test 3.1 {
+  CREATE TABLE c1(a INTEGER, b TEXT);
+  INSERT INTO c1 VALUES(1, 1);
+  CREATE TABLE c2(x BLOB, y BLOB);
+  INSERT INTO c2 VALUES(1, 1);
+}
+do_execsql_test 3.2 {
+  SELECT * FROM c1 WHERE (a, b) IN (SELECT x, y FROM c2)
+} {}
+do_execsql_test 3.3 {
+  CREATE UNIQUE INDEX c1ab ON c1(a, b);
+  SELECT * FROM c1 WHERE (a, b) IN (SELECT x, y FROM c2)
+} {}
+do_execsql_test 3.4 {
+  SELECT * FROM c1 WHERE (a, +b) IN (SELECT x, y FROM c2)
+} {}
+
+do_execsql_test 3.5 {
+  SELECT c1.rowid FROM c1 WHERE b = (SELECT y FROM c2);
+} {}
+do_execsql_test 3.6 {
+  SELECT c1.rowid FROM c1 WHERE (a, b) = (SELECT x, y FROM c2);
+} {}
+
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 4.0 {
+  CREATE TABLE d1(a TEXT, b INTEGER, c NUMERIC);
+  CREATE TABLE d2(x BLOB, y BLOB);
+
+  INSERT INTO d1 VALUES(1, 1, 1);
+  INSERT INTO d1 VALUES(2, 2, 2);
+  INSERT INTO d1 VALUES(3, 3, 3);
+  INSERT INTO d1 VALUES(4, 4, 4);
+
+  INSERT INTO d2 VALUES (1, 1);
+  INSERT INTO d2 VALUES (2, '2');
+  INSERT INTO d2 VALUES ('3', 3);
+  INSERT INTO d2 VALUES ('4', '4');
+}
+
+foreach {tn idx} {
+  1 {}
+  2 { CREATE INDEX idx ON d1(a) }
+  3 { CREATE INDEX idx ON d1(a, c) }
+  4 { CREATE INDEX idx ON d1(c) }
+  5 { CREATE INDEX idx ON d1(c, a) }
+
+  6 { 
+    CREATE INDEX idx ON d1(c, a) ;
+    CREATE INDEX idx1 ON d2(x, y);
+  }
+
+  7 { 
+    CREATE INDEX idx ON d1(c, a) ;
+    CREATE UNIQUE INDEX idx2 ON d2(x, y) ;
+  }
+
+  8 { 
+    CREATE INDEX idx ON d1(c) ;
+    CREATE UNIQUE INDEX idx2 ON d2(x);
+  }
+
+} {
+  execsql { DROP INDEX IF EXISTS idx } 
+  execsql { DROP INDEX IF EXISTS idx2 } 
+  execsql { DROP INDEX IF EXISTS idx3 } 
+  execsql $idx
+
+  do_execsql_test 4.$tn.1 {
+    SELECT rowid FROM d1 WHERE (a, c) IN (SELECT x, y FROM d2);
+  } {3 4}
+
+  do_execsql_test 4.$tn.2 {
+    SELECT rowid FROM d1 WHERE (c, a) IN (SELECT x, y FROM d2);
+  } {2 4}
+
+  do_execsql_test 4.$tn.3 {
+    SELECT rowid FROM d1 WHERE (+c, a) IN (SELECT x, y FROM d2);
+  } {2}
+
+  do_execsql_test 4.$tn.4 {
+    SELECT rowid FROM d1 WHERE (c, a) = (
+      SELECT x, y FROM d2 WHERE d2.rowid=d1.rowid
+    );
+  } {2 4}
+
+  do_execsql_test 4.$tn.5 {
+    SELECT d1.rowid FROM d1, d2 WHERE a = y;
+  } {2 4}
+
+  do_execsql_test 4.$tn.6 {
+    SELECT d1.rowid FROM d1 WHERE a = (
+      SELECT y FROM d2 where d2.rowid=d1.rowid
+    );
+  } {2 4}
+}
+
+do_execsql_test 5.0 {
+  CREATE TABLE e1(a TEXT, c NUMERIC);
+  CREATE TABLE e2(x BLOB, y BLOB);
+
+  INSERT INTO e1 VALUES(2, 2);
+
+  INSERT INTO e2 VALUES ('2', 2);
+  INSERT INTO e2 VALUES ('2', '2');
+  INSERT INTO e2 VALUES ('2', '2.0');
+
+  CREATE INDEX e1c ON e1(c);
+}
+
+do_execsql_test 5.1 {
+  SELECT rowid FROM e1 WHERE (a, c) IN (SELECT x, y FROM e2);
+} {1}
+do_execsql_test 5.2 {
+  SELECT rowid FROM e2 WHERE rowid IN (SELECT +c FROM e1);
+} {2}
+do_execsql_test 5.3 {
+  SELECT rowid FROM e2 WHERE rowid IN (SELECT 0+c FROM e1);
+} {2}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 6.0 {
+  CREATE TABLE f1(a, b);
+  CREATE TABLE f2(c, d);
+  CREATE TABLE f3(e, f);
+}
+
+do_execsql_test 6.1 {
+  SELECT * FROM f3 WHERE (e, f) IN (
+    SELECT a, b FROM f1 UNION ALL SELECT c, d FROM f2
+  );
+}
+do_execsql_test 6.2 {
+  CREATE INDEX f3e ON f3(e);
+  SELECT * FROM f3 WHERE (e, f) IN (
+    SELECT a, b FROM f1 UNION ALL SELECT c, d FROM f2
+  );
+}
+
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+  CREATE TABLE g1(a, b); 
+  INSERT INTO g1 VALUES
+      (1, 1), (1, 2), (1, 3), (1, 'i'), (1, 'j'),
+      (1, 6), (1, 7), (1, 8), (1, 9), (1, 10),
+      (1, 4), (1, 5);
+
+  CREATE TABLE g2(x, y);
+  CREATE INDEX g2x ON g2(x);
+
+  INSERT INTO g2 VALUES(1, 4);
+  INSERT INTO g2 VALUES(1, 5);
+}
+
+do_execsql_test 7.1 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY +a, +b LIMIT 10
+  );
+} { 1 4 1 5 }
+
+do_execsql_test 7.2 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY a, b LIMIT 10
+  );
+} { 1 4 1 5 }
+
+do_execsql_test 7.3 {
+  SELECT * FROM g2 WHERE (x, y) IN (
+    SELECT a, b FROM g1 ORDER BY 1, 2 LIMIT 10
+  );
+} { 1 4 1 5 }
+
+
+finish_test
diff --git a/test/rowvaluefault.test b/test/rowvaluefault.test
new file mode 100644
index 0000000000..ac1b236ba4
--- /dev/null
+++ b/test/rowvaluefault.test
@@ -0,0 +1,71 @@
+# 2016 June 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix rowvaluefault
+
+do_execsql_test 1.0 {
+  CREATE TABLE xyz(one, two, thr, fou);
+  INSERT INTO xyz VALUES('A', 'A', 'A',  1);
+  INSERT INTO xyz VALUES('B', 'B', 'B',  2);
+  INSERT INTO xyz VALUES('C', 'C', 'C',  3);
+  INSERT INTO xyz VALUES('D', 'D', 'D',  4);
+
+  CREATE UNIQUE INDEX xyz_one_two ON xyz(one, two);
+}
+
+do_faultsim_test 1 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two, thr) = ('B', 'B', 'B') }
+} -test {
+  faultsim_test_result {0 2} 
+}
+
+do_faultsim_test 2 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (two, thr) IS ('C', 'C') }
+} -test {
+  faultsim_test_result {0 3} 
+}
+
+do_faultsim_test 3 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two, thr) > ('B', 'B', 'B') }
+} -test {
+  faultsim_test_result {0 {3 4}} 
+}
+
+do_faultsim_test 4 -faults oom* -body {
+  execsql { SELECT fou FROM xyz WHERE (one, two) IN (SELECT one, two FROM xyz) }
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+}
+
+do_faultsim_test 5 -faults oom* -body {
+  execsql { 
+    SELECT fou FROM xyz 
+    WHERE (one, two, thr) IN (SELECT one, two, thr FROM xyz) 
+  }
+} -test {
+  faultsim_test_result {0 {1 2 3 4}} 
+}
+
+do_faultsim_test 6 -faults oom* -body {
+  execsql { 
+    SELECT fou FROM xyz 
+    WHERE (one, two, thr) BETWEEN ('B', 'B', 'B') AND ('C', 'C', 'C') }
+} -test {
+  faultsim_test_result {0 {2 3}} 
+}
+
+finish_test
diff --git a/test/savepoint7.test b/test/savepoint7.test
index d8a02f1f80..59c3cd6cdc 100644
--- a/test/savepoint7.test
+++ b/test/savepoint7.test
@@ -95,4 +95,38 @@ do_test savepoint7-2.2 {
   list $rc $msg [db eval {SELECT * FROM t2}]
 } {1 {abort due to ROLLBACK} {}}
 
+# Ticket: https://www.sqlite.org/src/tktview/7f7f8026eda387d544b
+# Segfault in the in-memory journal logic triggered by a tricky
+# combination of SAVEPOINT operations.
+#
+unset -nocomplain i
+for {set i 248} {$i<=253} {incr i} {
+  do_test savepoint7-3.$i {
+    db close
+    forcedelete test.db
+    sqlite3 db test.db
+    db eval {
+      PRAGMA page_size=1024;
+      PRAGMA temp_store=MEMORY;
+      BEGIN;
+      CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT);
+      WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<$::i)
+      INSERT INTO t1(x,y) SELECT x*10, printf('%04d%.800c',x,'*') FROM c;
+      SAVEPOINT one;
+        SELECT count(*) FROM t1;
+        WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<$::i)
+        INSERT INTO t1(x,y) SELECT x*10+1, printf('%04d%.800c',x,'*') FROM c;
+      ROLLBACK TO one;
+        SELECT count(*) FROM t1;
+        SAVEPOINT twoB;
+          WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<10)
+          INSERT INTO t1(x,y) SELECT x*10+2, printf('%04d%.800c',x,'*') FROM c;
+        ROLLBACK TO twoB;
+      RELEASE one;
+      COMMIT;
+    }
+  } [list $i $i]
+}
+
+
 finish_test
diff --git a/test/schema4.test b/test/schema4.test
index 6618d75388..9b794ee555 100644
--- a/test/schema4.test
+++ b/test/schema4.test
@@ -149,7 +149,7 @@ ifcapable altertable {
   } {}
 
   do_execsql_test schema4-2.8 {
-    select sql from sqlite_temp_master WHERE type='table';
+    select sql from temp.sqlite_master WHERE type='table';
   } {{CREATE TABLE x1(x)}}
 
   do_execsql_test schema4-2.7 { ALTER TABLE tbl RENAME TO tbl2 } {}
diff --git a/test/select4.test b/test/select4.test
index be8d0e0ab1..51a1b1c4c5 100644
--- a/test/select4.test
+++ b/test/select4.test
@@ -12,7 +12,6 @@
 # focus of this file is testing UNION, INTERSECT and EXCEPT operators
 # in SELECT statements.
 #
-# $Id: select4.test,v 1.30 2009/04/16 00:24:24 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -936,4 +935,76 @@ do_execsql_test select4-15.1 {
    ORDER BY 1;
 } {1 33 456 2 33 789}
 
+# Enhancement (2016-03-15):  Use a co-routine for subqueries if the
+# subquery is guaranteed to be the outer-most query
+#
+do_execsql_test select4-16.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,
+  PRIMARY KEY(a,b DESC)) WITHOUT ROWID;
+
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b,c,d)
+    SELECT x%10, x/10, x, printf('xyz%dabc',x) FROM c;
+
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+do_execsql_test select4-16.2 {
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    CROSS JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+do_execsql_test select4-16.3 {
+  SELECT t3.c FROM 
+    (SELECT a,max(b) AS m FROM t1 WHERE a>=5 GROUP BY a) AS t2
+    LEFT JOIN t1 AS t3
+  WHERE t2.a=t3.a AND t2.m=t3.b
+  ORDER BY t3.a;
+} {95 96 97 98 99}
+
+# Ticket https://www.sqlite.org/src/tktview/f7f8c97e975978d45  on 2016-04-25
+#
+# The where push-down optimization from 2015-06-02 is suppose to disable
+# on aggregate subqueries.  But if the subquery is a compound where the
+# last SELECT is non-aggregate but some other SELECT is an aggregate, the
+# test is incomplete and the optimization is not properly disabled.
+# 
+# The following test cases verify that the fix works.
+#
+do_execsql_test select4-17.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a int, b int);
+  INSERT INTO t1 VALUES(1,2),(1,18),(2,19);
+  SELECT x, y FROM (
+    SELECT 98 AS x, 99 AS y
+    UNION
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 20 98 99}
+do_execsql_test select4-17.2 {
+  SELECT x, y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a
+    UNION
+    SELECT 98 AS x, 99 AS y
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 20 98 99}
+do_catchsql_test select4-17.3 {
+  SELECT x, y FROM (
+    SELECT a AS x, sum(b) AS y FROM t1 GROUP BY a LIMIT 3
+    UNION
+    SELECT 98 AS x, 99 AS y
+  ) AS w WHERE y>=20
+  ORDER BY +x;
+} {1 {LIMIT clause should come after UNION not before}}
+
+
+
 finish_test
diff --git a/test/select7.test b/test/select7.test
index 0df84e13d9..d705ebfaf4 100644
--- a/test/select7.test
+++ b/test/select7.test
@@ -114,26 +114,22 @@ ifcapable {subquery && compound} {
       CREATE TABLE t2(a,b);
       SELECT 5 IN (SELECT a,b FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.2 {
     catchsql {
       SELECT 5 IN (SELECT * FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.3 {
     catchsql {
       SELECT 5 IN (SELECT a,b FROM t2 UNION SELECT b,a FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
   do_test select7-5.4 {
     catchsql {
       SELECT 5 IN (SELECT * FROM t2 UNION SELECT * FROM t2);
     }
-  } [list 1 \
-     {only a single result allowed for a SELECT that is part of an expression}]
+  } {1 {sub-select returns 2 columns - expected 1}}
 }
 
 # Verify that an error occurs if you have too many terms on a
diff --git a/test/selectA.test b/test/selectA.test
index 0338338902..78d04be15d 100644
--- a/test/selectA.test
+++ b/test/selectA.test
@@ -1434,5 +1434,14 @@ do_catchsql_test 5.4 {
   SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE
 } {1 {1st ORDER BY term does not match any column in the result set}}
 
+do_execsql_test 6.1 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1(a INTEGER);
+  CREATE TABLE t2(b TEXT);
+  INSERT INTO t2(b) VALUES('12345');
+  SELECT * FROM (SELECT a FROM t1 UNION SELECT b FROM t2) WHERE a=a;
+} {12345}
+
 
 finish_test
diff --git a/test/selectC.test b/test/selectC.test
index dedac41fc4..4d79963007 100644
--- a/test/selectC.test
+++ b/test/selectC.test
@@ -14,6 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix selectC
 
 # Ticket #
 do_test selectC-1.1 {
@@ -233,4 +234,37 @@ do_execsql_test selectC-4.3 {
   select a, udf() from (select distinct a, b from t_distinct_bug)
 } {1 1 1 2 1 3}
 
+#-------------------------------------------------------------------------
+# Test that the problem in ticket #190c2507 has been fixed.
+#
+do_execsql_test 5.0 {
+  CREATE TABLE x1(a);
+  CREATE TABLE x2(b);
+  CREATE TABLE x3(c);
+  CREATE VIEW vvv AS SELECT b FROM x2 ORDER BY 1;
+
+  INSERT INTO x1 VALUES('a'), ('b');
+  INSERT INTO x2 VALUES(22), (23), (25), (24), (21);
+  INSERT INTO x3 VALUES(302), (303), (301);
+}
+
+do_execsql_test 5.1 {
+  CREATE TABLE x4 AS SELECT b FROM vvv UNION ALL SELECT c from x3;
+  SELECT * FROM x4;
+} {21 22 23 24 25 302 303 301}
+
+do_execsql_test 5.2 {
+  SELECT * FROM x1, x4
+} {
+  a 21 a 22 a 23 a 24 a 25 a 302 a 303 a 301
+  b 21 b 22 b 23 b 24 b 25 b 302 b 303 b 301
+}
+
+do_execsql_test 5.3 {
+  SELECT * FROM x1, (SELECT b FROM vvv UNION ALL SELECT c from x3);
+} {
+  a 21 a 22 a 23 a 24 a 25 a 302 a 303 a 301
+  b 21 b 22 b 23 b 24 b 25 b 302 b 303 b 301
+}
+
 finish_test
diff --git a/test/session.test b/test/session.test
new file mode 100644
index 0000000000..4fd1ec0184
--- /dev/null
+++ b/test/session.test
@@ -0,0 +1,22 @@
+# 2008 June 23
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file runs all rtree related tests.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/permutations.test
+
+ifcapable session {
+  # First run tests with sqlite3_extended_error_codes() set, then
+  # again with it clear.
+  run_test_suite session_eec
+  run_test_suite session
+  run_test_suite session_strm
+}
+
+finish_test
diff --git a/test/shell1.test b/test/shell1.test
index 1b9ea1d383..ac3403f810 100644
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -21,15 +21,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -53,7 +45,7 @@ do_test shell1-1.1.1b {
 } {1 1}
 # error on extra options
 do_test shell1-1.1.2 {
-  catchcmd "test.db \"select 3\" \"select 4\"" ""
+  catchcmd "test.db \"select+3\" \"select+4\"" ""
 } {0 {3
 4}}
 # error on extra options
@@ -84,20 +76,20 @@ do_test shell1-1.3.3 {
 
 # -echo                print commands before execution
 do_test shell1-1.4.1 {
-  catchcmd "-echo test.db" "" 
+  catchcmd "-echo test.db" ""
 } {0 {}}
 
 # -[no]header          turn headers on or off
 do_test shell1-1.5.1 {
-  catchcmd "-header test.db" "" 
+  catchcmd "-header test.db" ""
 } {0 {}}
 do_test shell1-1.5.2 {
-  catchcmd "-noheader test.db" "" 
+  catchcmd "-noheader test.db" ""
 } {0 {}}
 
 # -bail                stop after hitting an error
 do_test shell1-1.6.1 {
-  catchcmd "-bail test.db" "" 
+  catchcmd "-bail test.db" ""
 } {0 {}}
 
 # -interactive         force interactive I/O
@@ -111,40 +103,40 @@ do_test shell1-1.7.1 {
 
 # -batch               force batch I/O
 do_test shell1-1.8.1 {
-  catchcmd "-batch test.db" "" 
+  catchcmd "-batch test.db" ""
 } {0 {}}
 
 # -column              set output mode to 'column'
 do_test shell1-1.9.1 {
-  catchcmd "-column test.db" "" 
+  catchcmd "-column test.db" ""
 } {0 {}}
 
 # -csv                 set output mode to 'csv'
 do_test shell1-1.10.1 {
-  catchcmd "-csv test.db" "" 
+  catchcmd "-csv test.db" ""
 } {0 {}}
 
 # -html                set output mode to HTML
 do_test shell1-1.11.1 {
-  catchcmd "-html test.db" "" 
+  catchcmd "-html test.db" ""
 } {0 {}}
 
 # -line                set output mode to 'line'
 do_test shell1-1.12.1 {
-  catchcmd "-line test.db" "" 
+  catchcmd "-line test.db" ""
 } {0 {}}
 
 # -list                set output mode to 'list'
 do_test shell1-1.13.1 {
-  catchcmd "-list test.db" "" 
+  catchcmd "-list test.db" ""
 } {0 {}}
 
 # -separator 'x'       set output field separator (|)
 do_test shell1-1.14.1 {
-  catchcmd "-separator 'x' test.db" "" 
+  catchcmd "-separator 'x' test.db" ""
 } {0 {}}
 do_test shell1-1.14.2 {
-  catchcmd "-separator x test.db" "" 
+  catchcmd "-separator x test.db" ""
 } {0 {}}
 do_test shell1-1.14.3 {
   set res [catchcmd "-separator" ""]
@@ -155,7 +147,7 @@ do_test shell1-1.14.3 {
 
 # -stats               print memory stats before each finalize
 do_test shell1-1.14b.1 {
-  catchcmd "-stats test.db" "" 
+  catchcmd "-stats test.db" ""
 } {0 {}}
 
 # -nullvalue 'text'    set text string for NULL values
@@ -183,7 +175,7 @@ do_test shell1-1.16.1 {
 
 # check first token handling
 do_test shell1-2.1.1 {
-  catchcmd "test.db" ".foo" 
+  catchcmd "test.db" ".foo"
 } {1 {Error: unknown command or invalid arguments:  "foo". Enter ".help" for help}}
 do_test shell1-2.1.2 {
   catchcmd "test.db" ".\"foo OFF\""
@@ -207,10 +199,10 @@ do_test shell1-2.2.4 {
 } {0 {}}
 do_test shell1-2.2.5 {
   catchcmd "test.db" ".mode \"insert FOO"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-2.2.6 {
   catchcmd "test.db" ".mode \'insert FOO"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 
 # check multiple tokens, and quoted tokens
 do_test shell1-2.3.1 {
@@ -238,7 +230,7 @@ do_test shell1-2.3.7 {
 # check quoted args are unquoted
 do_test shell1-2.4.1 {
   catchcmd "test.db" ".mode FOO"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-2.4.2 {
   catchcmd "test.db" ".mode csv"
 } {0 {}}
@@ -281,14 +273,16 @@ do_test shell1-3.2.4 {
   catchcmd "test.db" ".bail OFF BAD"
 } {1 {Usage: .bail on|off}}
 
+ifcapable vtab {
 # .databases             List names and files of attached databases
 do_test shell1-3.3.1 {
   catchcmd "-csv test.db" ".databases"
-} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/"
+} "/0.+main.+[string map {/ ".{1,2}"} [string range [get_pwd] 0 10]].*/"
 do_test shell1-3.3.2 {
   # extra arguments ignored
   catchcmd "test.db" ".databases BAD"
-} "/0 +.*main +[string map {/ .} [string range [get_pwd] 0 10]].*/"
+} "/0.+main.+[string map {/ ".{1,2}"} [string range [get_pwd] 0 10]].*/"
+}
 
 # .dump ?TABLE? ...      Dump the database in an SQL text format
 #                          If TABLE specified, only dump tables matching
@@ -306,7 +300,7 @@ do_test shell1-3.4.2 {
 do_test shell1-3.4.3 {
   # too many arguments
   catchcmd "test.db" ".dump FOO BAD"
-} {1 {Usage: .dump ?LIKE-PATTERN?}}
+} {1 {Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?}}
 
 # .echo ON|OFF           Turn command echo on or off
 do_test shell1-3.5.1 {
@@ -435,10 +429,10 @@ do_test shell1-3.12.3 {
 #                          tcl      TCL list elements
 do_test shell1-3.13.1 {
   catchcmd "test.db" ".mode"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-3.13.2 {
   catchcmd "test.db" ".mode FOO"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-3.13.3 {
   catchcmd "test.db" ".mode csv"
 } {0 {}}
@@ -471,10 +465,10 @@ do_test shell1-3.13.11 {
 # don't allow partial mode type matches
 do_test shell1-3.13.12 {
   catchcmd "test.db" ".mode l"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-3.13.13 {
   catchcmd "test.db" ".mode li"
-} {1 {Error: mode should be one of: ascii column csv html insert line list tabs tcl}}
+} {1 {Error: mode should be one of: ascii column csv html insert line list quote tabs tcl}}
 do_test shell1-3.13.14 {
   catchcmd "test.db" ".mode lin"
 } {0 {}}
@@ -576,7 +570,7 @@ do_test shell1-3.21.2 {
 do_test shell1-3.21.3 {
   # too many arguments
   catchcmd "test.db" ".schema FOO BAD"
-} {1 {Usage: .schema ?LIKE-PATTERN?}}
+} {1 {Usage: .schema ?--indent? ?LIKE-PATTERN?}}
 
 do_test shell1-3.21.4 {
   catchcmd "test.db" {
@@ -746,21 +740,154 @@ do_test shell1-4.1 {
 } {0 {PRAGMA foreign_keys=OFF;
 BEGIN TRANSACTION;
 CREATE TABLE t1(x);
-INSERT INTO "t1" VALUES(NULL);
-INSERT INTO "t1" VALUES('');
-INSERT INTO "t1" VALUES(1);
-INSERT INTO "t1" VALUES(2.25);
-INSERT INTO "t1" VALUES('hello');
-INSERT INTO "t1" VALUES(X'807F');
+INSERT INTO t1 VALUES(NULL);
+INSERT INTO t1 VALUES('');
+INSERT INTO t1 VALUES(1);
+INSERT INTO t1 VALUES(2.25);
+INSERT INTO t1 VALUES('hello');
+INSERT INTO t1 VALUES(X'807f');
 CREATE TABLE t3(x,y);
-INSERT INTO "t3" VALUES(1,NULL);
-INSERT INTO "t3" VALUES(2,'');
-INSERT INTO "t3" VALUES(3,1);
-INSERT INTO "t3" VALUES(4,2.25);
-INSERT INTO "t3" VALUES(5,'hello');
-INSERT INTO "t3" VALUES(6,X'807F');
+INSERT INTO t3 VALUES(1,NULL);
+INSERT INTO t3 VALUES(2,'');
+INSERT INTO t3 VALUES(3,1);
+INSERT INTO t3 VALUES(4,2.25);
+INSERT INTO t3 VALUES(5,'hello');
+INSERT INTO t3 VALUES(6,X'807f');
 COMMIT;}}
 
+
+ifcapable vtab {
+
+# The --preserve-rowids option to .dump
+#
+do_test shell1-4.1.1 {
+  catchcmd test.db {.dump --preserve-rowids}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t1(x);
+INSERT INTO t1(rowid,x) VALUES(1,NULL);
+INSERT INTO t1(rowid,x) VALUES(2,'');
+INSERT INTO t1(rowid,x) VALUES(3,1);
+INSERT INTO t1(rowid,x) VALUES(4,2.25);
+INSERT INTO t1(rowid,x) VALUES(5,'hello');
+INSERT INTO t1(rowid,x) VALUES(6,X'807f');
+CREATE TABLE t3(x,y);
+INSERT INTO t3(rowid,x,y) VALUES(1,1,NULL);
+INSERT INTO t3(rowid,x,y) VALUES(2,2,'');
+INSERT INTO t3(rowid,x,y) VALUES(3,3,1);
+INSERT INTO t3(rowid,x,y) VALUES(4,4,2.25);
+INSERT INTO t3(rowid,x,y) VALUES(5,5,'hello');
+INSERT INTO t3(rowid,x,y) VALUES(6,6,X'807f');
+COMMIT;}}
+
+# If the table contains an INTEGER PRIMARY KEY, do not record a separate
+# rowid column in the output.
+#
+do_test shell1-4.1.2 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+    INSERT INTO t1 VALUES(1,null), (2,''), (3,1),
+                         (4,2.25), (5,'hello'), (6,x'807f');
+  }
+  catchcmd test2.db {.dump --preserve-rowids}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+INSERT INTO t1 VALUES(1,NULL);
+INSERT INTO t1 VALUES(2,'');
+INSERT INTO t1 VALUES(3,1);
+INSERT INTO t1 VALUES(4,2.25);
+INSERT INTO t1 VALUES(5,'hello');
+INSERT INTO t1 VALUES(6,X'807f');
+COMMIT;}}
+
+# Verify that the table named [table] is correctly quoted and that
+# an INTEGER PRIMARY KEY DESC is not an alias for the rowid.
+#
+do_test shell1-4.1.3 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE [table](x INTEGER PRIMARY KEY DESC, y);
+    INSERT INTO [table] VALUES(1,null), (12,''), (23,1),
+                         (34,2.25), (45,'hello'), (56,x'807f');
+  }
+  catchcmd test2.db {.dump --preserve-rowids}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE [table](x INTEGER PRIMARY KEY DESC, y);
+INSERT INTO "table"(rowid,x,y) VALUES(1,1,NULL);
+INSERT INTO "table"(rowid,x,y) VALUES(2,12,'');
+INSERT INTO "table"(rowid,x,y) VALUES(3,23,1);
+INSERT INTO "table"(rowid,x,y) VALUES(4,34,2.25);
+INSERT INTO "table"(rowid,x,y) VALUES(5,45,'hello');
+INSERT INTO "table"(rowid,x,y) VALUES(6,56,X'807f');
+COMMIT;}}
+
+# Do not record rowids for a WITHOUT ROWID table.  Also check correct quoting
+# of table names that contain odd characters.
+#
+do_test shell1-4.1.4 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID;
+    INSERT INTO [ta<>ble] VALUES(1,null), (12,''), (23,1),
+                         (34,2.25), (45,'hello'), (56,x'807f');
+  }
+  catchcmd test2.db {.dump --preserve-rowids}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID;
+INSERT INTO "ta<>ble" VALUES(1,NULL);
+INSERT INTO "ta<>ble" VALUES(12,'');
+INSERT INTO "ta<>ble" VALUES(23,1);
+INSERT INTO "ta<>ble" VALUES(34,2.25);
+INSERT INTO "ta<>ble" VALUES(45,'hello');
+INSERT INTO "ta<>ble" VALUES(56,X'807f');
+COMMIT;}}
+
+# Do not record rowids if the rowid is inaccessible
+#
+do_test shell1-4.1.5 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(_ROWID_,rowid,oid);
+    INSERT INTO t1 VALUES(1,null,'alpha'), (12,'',99), (23,1,x'b0b1b2');
+  }
+  catchcmd test2.db {.dump --preserve-rowids}
+} {0 {PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE t1(_ROWID_,rowid,oid);
+INSERT INTO t1 VALUES(1,NULL,'alpha');
+INSERT INTO t1 VALUES(12,'',99);
+INSERT INTO t1 VALUES(23,1,X'b0b1b2');
+COMMIT;}}
+
+} else {
+
+do_test shell1-4.1.6 {
+  db close
+  forcedelete test2.db
+  sqlite3 db test2.db
+  db eval {
+    CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+    INSERT INTO t1 VALUES(1,null), (2,''), (3,1),
+                         (4,2.25), (5,'hello'), (6,x'807f');
+  }
+  catchcmd test2.db {.dump --preserve-rowids}
+} {1 {The --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE}}
+
+}
+
+
 # Test the output of ".mode insert"
 #
 do_test shell1-4.2.1 {
@@ -878,6 +1005,8 @@ do_test shell1-5.0 {
     if {$i==0x0D || ($tcl_platform(platform)=="windows" && $i==0x1A)} {
       continue
     }
+    if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"}  continue
+    if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"}  continue
     set hex [format %02X $i]
     set char [subst \\x$hex]; set oldChar $char
     set escapes [list]
@@ -913,9 +1042,55 @@ do_test shell1-5.0 {
       error "failed with error: $res"
     }
     if {$res ne "$oldChar\n"} {
-      error "failed with byte $hex mismatch"
+      if {[llength $res] > 0} {
+        set got [format %02X [scan $res %c]]
+      } else {
+        set got 
+      }
+      error "failed with byte $hex mismatch, got $got"
     }
   }
 } {}
 
+# These test cases do not work on MinGW
+if 0 {
+
+# The string used here is the word "test" in Chinese.
+# In UTF-8, it is encoded as: \xE6\xB5\x8B\xE8\xAF\x95
+set test \u6D4B\u8BD5
+
+do_test shell1-6.0 {
+  set fileName $test; append fileName .db
+  catch {forcedelete $fileName}
+  set x [catchcmdex $fileName "CREATE TABLE t1(x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE t1(x);"} {
+    error "failed with mismatch: $res"
+  }
+  if {![file exists $fileName]} {
+    error "file \"$fileName\" (Unicode) does not exist"
+  }
+  forcedelete $fileName
+} {}
+
+do_test shell1-6.1 {
+  catch {forcedelete test3.db}
+  set x [catchcmdex test3.db \
+      "CREATE TABLE [encoding convertto utf-8 $test](x);\n.schema\n"]
+  set code [lindex $x 0]
+  set res [string trim [lindex $x 1]]
+  if {$code ne "0"} {
+    error "failed with error: $res"
+  }
+  if {$res ne "CREATE TABLE ${test}(x);"} {
+    error "failed with mismatch: $res"
+  }
+  forcedelete test3.db
+} {}
+}
+
 finish_test
diff --git a/test/shell2.test b/test/shell2.test
index 616610bd4b..2de6bf7514 100644
--- a/test/shell2.test
+++ b/test/shell2.test
@@ -20,15 +20,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -51,7 +43,7 @@ do_test shell2-1.1.1 {
 # Shell silently ignores extra parameters.
 # Ticket [f5cb008a65].
 do_test shell2-1.2.1 {
-  set rc [catch { eval exec $CLI \":memory:\" \"select 3\" \"select 4\" } msg]
+  set rc [catch { eval exec $CLI \":memory:\" \"select+3\" \"select+4\" } msg]
   list $rc $msg
 } {0 {3
 4}}
diff --git a/test/shell3.test b/test/shell3.test
index ce1fd4ecb5..bb2524c1cc 100644
--- a/test/shell3.test
+++ b/test/shell3.test
@@ -21,19 +21,23 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
 
+# There are inconsistencies in command-line argument quoting on Windows.
+# In particular, individual applications are responsible for command-line
+# parsing in Windows, not the shell.  Depending on whether the sqlite3.exe
+# program is compiled with MinGW or MSVC, the command-line parsing is
+# different.  This causes problems for the tests below.  To avoid
+# issues, these tests are disabled for windows.
+#
+if {$::tcl_platform(platform)=="windows"} {
+  finish_test
+  return
+}
+
 #----------------------------------------------------------------------------
 #   shell3-1.*: Basic tests for running SQL statments from command line.
 #
@@ -92,6 +96,6 @@ do_test shell3-2.6 {
 } {0 {}}
 do_test shell3-2.7 {
   catchcmd "foo.db" "CREATE TABLE"
-} {1 {Error: incomplete SQL: CREATE TABLE}}
+} {1 {Error: near line 1: near "TABLE": syntax error}}
 
 finish_test
diff --git a/test/shell4.test b/test/shell4.test
index 18e7d7fdf6..88e5e69a28 100644
--- a/test/shell4.test
+++ b/test/shell4.test
@@ -18,18 +18,11 @@
 #
 #   shell4-1.*: Basic tests specific to the "stats" command.
 #   shell4-2.*: Basic tests for ".trace"
+#   shell4-3.*: The ".read" command takes the shell out of interactive mode
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 sqlite3 db test.db
@@ -132,5 +125,17 @@ do_test shell4-2.5 {
 } {0 {SELECT * FROM t1;}}
 }
 
+do_test shell4-3.1 {
+  set fd [open t1.txt wb]
+  puts $fd "SELECT 'squirrel';"
+  close $fd
+  exec $::CLI :memory: --interactive ".read t1.txt"
+} {squirrel}
+do_test shell4-3.2 {
+  set fd [open t1.txt wb]
+  puts $fd "SELECT 'pound: \302\243';"
+  close $fd
+  exec $::CLI :memory: --interactive ".read t1.txt"
+} {pound: £}
 
 finish_test
diff --git a/test/shell5.test b/test/shell5.test
index b921accca5..bb2511a525 100644
--- a/test/shell5.test
+++ b/test/shell5.test
@@ -21,15 +21,7 @@
 #
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-if {$tcl_platform(platform)=="windows"} {
-  set CLI "sqlite3.exe"
-} else {
-  set CLI "./sqlite3"
-}
-if {![file executable $CLI]} {
-  finish_test
-  return
-}
+set CLI [test_find_cli]
 db close
 forcedelete test.db test.db-journal test.db-wal
 
diff --git a/test/shell7.test b/test/shell7.test
new file mode 100644
index 0000000000..a062885140
--- /dev/null
+++ b/test/shell7.test
@@ -0,0 +1,54 @@
+# 2016 December 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test the readfile() function built into the shell tool. Specifically,
+# that it does not truncate the blob read at the first embedded 0x00
+# byte.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix shell7
+set CLI [test_find_cli]
+
+
+do_execsql_test 1.0 { 
+  CREATE TABLE f1(tn INTEGER PRIMARY KEY, x BLOB); 
+  CREATE TABLE f2(tn INTEGER PRIMARY KEY, x BLOB); 
+
+  INSERT INTO f1 VALUES(1, X'01020304');
+  INSERT INTO f1 VALUES(2, X'01000304');
+  INSERT INTO f1 VALUES(3, randomblob(200));
+}
+
+foreach {tn l x} [db eval { SELECT tn, length(x) AS l, x FROM f1 }] {
+  forcedelete shell7_test.bin
+  set fd [open shell7_test.bin w]
+  fconfigure $fd -encoding binary
+  fconfigure $fd -translation binary
+  puts -nonewline $fd $x
+  close $fd
+
+  do_test 1.$tn.1 { file size shell7_test.bin } $l
+  do_test 1.$tn.2 { 
+    catchcmd test.db "INSERT INTO f2 VALUES($tn, readfile('shell7_test.bin'));"
+  } {0 {}}
+
+  do_execsql_test 1.$tn.3 { 
+    SELECT (SELECT x FROM f1 WHERE tn=1)==(SELECT x FROM f2 WHERE tn=1)
+  } {1}
+}
+
+
+
+finish_test
+
+
diff --git a/test/skipscan1.test b/test/skipscan1.test
index ac26711603..6cdeed5439 100644
--- a/test/skipscan1.test
+++ b/test/skipscan1.test
@@ -43,23 +43,23 @@ do_execsql_test skipscan1-1.2 {
 } {abc 345 7 8 | def 345 9 10 |}
 do_execsql_test skipscan1-1.2eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a;
 } {/* USING INDEX t1abc (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-1.2sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a;
 } {~/*ORDER BY*/}
 
 do_execsql_test skipscan1-1.3 {
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a DESC;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {def 345 9 10 | abc 345 7 8 |}
 do_execsql_test skipscan1-1.3eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {/* USING INDEX t1abc (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-1.3sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t1 WHERE d<>99 AND b=345 ORDER BY a DESC;
 } {~/*ORDER BY*/}
 
 do_execsql_test skipscan1-1.4 {
@@ -150,15 +150,15 @@ do_execsql_test skipscan1-2.1 {
 } {}
 
 do_execsql_test skipscan1-2.2 {
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {abc 345 7 8 | def 345 9 10 |}
 do_execsql_test skipscan1-2.2eqp {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {/* USING INDEX sqlite_autoindex_t2_1 (ANY(a) AND b=?)*/}
 do_execsql_test skipscan1-2.2sort {
   EXPLAIN QUERY PLAN
-  SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a;
+  SELECT a,b,c,d,'|' FROM t2 WHERE d<>99 AND b=345 ORDER BY a;
 } {~/*ORDER BY*/}
 
 
@@ -322,4 +322,19 @@ do_execsql_test skipscan1-8.2 {
   ORDER BY +x;
 } {1 AB 2 CD}
 
+# Segfault reported on the mailing list by Keith Medcalf on 2016-09-18.
+# A skip-scan with a "column IN (SELECT ...)" on the second term of the
+# index.
+#
+do_execsql_test skipscan1-9.2 {
+  CREATE TABLE t9a(a,b,c);
+  CREATE INDEX t9a_ab ON t9a(a,b);
+  CREATE TABLE t9b(x,y);
+  ANALYZE sqlite_master;
+  INSERT INTO sqlite_stat1 VALUES('t9a','t9a_ab','1000000 250000 1');
+  ANALYZE sqlite_master;
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t9a WHERE b IN (SELECT x FROM t9b WHERE y!=5);
+} {/USING INDEX t9a_ab .ANY.a. AND b=./}
+
 finish_test
diff --git a/test/snapshot.test b/test/snapshot.test
index ba3a666796..99d3ed47c3 100644
--- a/test/snapshot.test
+++ b/test/snapshot.test
@@ -17,343 +17,466 @@ source $testdir/tester.tcl
 ifcapable !snapshot {finish_test; return}
 set testprefix snapshot
 
-#-------------------------------------------------------------------------
-# Check some error conditions in snapshot_get(). It is an error if:
-#
-#  1) snapshot_get() is called on a non-WAL database, or
-#  2) there is an open write transaction on the database.
-#
-do_execsql_test 1.0 {
-  CREATE TABLE t1(a, b);
-  INSERT INTO t1 VALUES(1, 2);
-  INSERT INTO t1 VALUES(3, 4);
+# This test does not work with the inmemory_journal permutation. The reason
+# is that each connection opened as part of this permutation executes
+# "PRAGMA journal_mode=memory", which fails if the database is in wal mode
+# and there are one or more existing connections.
+if {[permutation]=="inmemory_journal"} {
+  finish_test
+  return
 }
 
-do_test 1.1.1 {
-  execsql { BEGIN; SELECT * FROM t1; }
-  list [catch { sqlite3_snapshot_get db main } msg] $msg
-} {1 SQLITE_ERROR}
-do_execsql_test 1.1.2 COMMIT
-
-do_test 1.2.1 {
-  execsql {
-    PRAGMA journal_mode = WAL;
-    BEGIN;
-      INSERT INTO t1 VALUES(5, 6);
-      INSERT INTO t1 VALUES(7, 8);
+foreach {tn tcl} {
+  1 {
+    proc snapshot_get {DB DBNAME} {
+      uplevel [list sqlite3_snapshot_get $DB $DBNAME]
+    }
+    proc snapshot_open {DB DBNAME SNAPSHOT} {
+      uplevel [list sqlite3_snapshot_open $DB $DBNAME $SNAPSHOT]
+    }
+    proc snapshot_free {SNAPSHOT} {
+      uplevel [list sqlite3_snapshot_free $SNAPSHOT]
+    }
+    proc snapshot_cmp {SNAPSHOT1 SNAPSHOT2} {
+      uplevel [list sqlite3_snapshot_cmp $SNAPSHOT1 $SNAPSHOT2]
+    }
   }
-  list [catch { sqlite3_snapshot_get db main } msg] $msg
-} {1 SQLITE_ERROR}
-do_execsql_test 1.3.2 COMMIT
 
-#-------------------------------------------------------------------------
-# Check that a simple case works. Reuse the database created by the
-# block of tests above.
-#
-do_execsql_test 2.1.0 {
-  BEGIN;
-    SELECT * FROM t1;
-} {1 2 3 4 5 6 7 8}
-
-do_test 2.1.1 {
-  set snapshot [sqlite3_snapshot_get db main]
-  execsql {
-    COMMIT;
-    INSERT INTO t1 VALUES(9, 10);
-    SELECT * FROM t1;
+  2 {
+    proc snapshot_get {DB DBNAME} {
+      uplevel [list sqlite3_snapshot_get_blob $DB $DBNAME]
+    }
+    proc snapshot_open {DB DBNAME SNAPSHOT} {
+      uplevel [list sqlite3_snapshot_open_blob $DB $DBNAME $SNAPSHOT]
+    }
+    proc snapshot_free {SNAPSHOT} {
+    }
+    proc snapshot_cmp {SNAPSHOT1 SNAPSHOT2} {
+      uplevel [list sqlite3_snapshot_cmp_blob $SNAPSHOT1 $SNAPSHOT2]
+    }
   }
-} {1 2 3 4 5 6 7 8 9 10}
+} {
 
-do_test 2.1.2 {
-  execsql BEGIN
-  sqlite3_snapshot_open db main $snapshot
-  execsql {
-    SELECT * FROM t1;
+  reset_db
+  eval $tcl
+
+  #-------------------------------------------------------------------------
+  # Check some error conditions in snapshot_get(). It is an error if:
+  #
+  #  1) snapshot_get() is called on a non-WAL database, or
+  #  2) there is an open write transaction on the database.
+  #  3) the database handle is in auto-commit mode
+  #
+  do_execsql_test $tn.1.0 {
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
   }
-} {1 2 3 4 5 6 7 8}
 
-do_test 2.1.3 {
-  sqlite3_snapshot_free $snapshot
-  execsql COMMIT
-} {}
+  do_test $tn.1.1.1 {
+    execsql { BEGIN; SELECT * FROM t1; }
+    list [catch { snapshot_get db main } msg] $msg
+  } {1 SQLITE_ERROR}
+  do_execsql_test $tn.1.1.2 COMMIT
 
-do_test 2.2.0 {
-  sqlite3 db2 test.db
-  execsql {
+  do_test $tn.1.2.1 {
+    execsql {
+      PRAGMA journal_mode = WAL;
+      BEGIN;
+        INSERT INTO t1 VALUES(5, 6);
+        INSERT INTO t1 VALUES(7, 8);
+    }
+    list [catch { snapshot_get db main } msg] $msg
+  } {1 SQLITE_ERROR}
+  do_execsql_test $tn.1.2.2 COMMIT
+
+  do_test $tn.1.3.1 {
+    list [catch { snapshot_get db main } msg] $msg
+  } {1 SQLITE_ERROR}
+  do_test $tn.1.3.2 {
+    db trans { set snap [snapshot_get db main] }
+    snapshot_free $snap
+  } {}
+
+  #-------------------------------------------------------------------------
+  # Check that a simple case works. Reuse the database created by the
+  # block of tests above.
+  #
+  do_execsql_test $tn.2.1.0 {
     BEGIN;
       SELECT * FROM t1;
-  } db2
-} {1 2 3 4 5 6 7 8 9 10}
+  } {1 2 3 4 5 6 7 8}
 
-do_test 2.2.1 {
-  set snapshot [sqlite3_snapshot_get db2 main]
-  execsql {
-    INSERT INTO t1 VALUES(11, 12);
-    SELECT * FROM t1;
-  }
-} {1 2 3 4 5 6 7 8 9 10 11 12}
+  do_test $tn.2.1.1 {
+    set snapshot [snapshot_get db main]
+    execsql {
+      COMMIT;
+      INSERT INTO t1 VALUES(9, 10);
+      SELECT * FROM t1;
+    }
+  } {1 2 3 4 5 6 7 8 9 10}
 
-do_test 2.2.2 {
-  execsql BEGIN
-  sqlite3_snapshot_open db main $snapshot
-  execsql {
-    SELECT * FROM t1;
-  }
-} {1 2 3 4 5 6 7 8 9 10}
+  do_test $tn.2.1.2 {
+    execsql BEGIN
+    snapshot_open db main $snapshot
+    execsql {
+      SELECT * FROM t1;
+    }
+  } {1 2 3 4 5 6 7 8}
 
-do_test 2.2.3 {
-  sqlite3_snapshot_free $snapshot
-  execsql COMMIT
-  execsql COMMIT db2
-  db2 close
-} {}
+  do_test $tn.2.1.3 {
+    snapshot_free $snapshot
+    execsql COMMIT
+  } {}
 
-do_test 2.3.1 {
-  execsql { DELETE FROM t1 WHERE a>6 }
-  set snapshot [sqlite3_snapshot_get db main]
-  execsql {
-    INSERT INTO t1 VALUES('a', 'b');
-    INSERT INTO t1 VALUES('c', 'd');
-    SELECT * FROM t1;
-  }
-} {1 2 3 4 5 6 a b c d}
-do_test 2.3.2 {
-  execsql BEGIN
-  sqlite3_snapshot_open db main $snapshot
-  execsql { SELECT * FROM t1 }
-} {1 2 3 4 5 6}
+  do_test $tn.2.2.0 {
+    sqlite3 db2 test.db
+    execsql {
+      BEGIN;
+        SELECT * FROM t1;
+    } db2
+  } {1 2 3 4 5 6 7 8 9 10}
 
-do_test 2.3.3 {
-  catchsql {
-    INSERT INTO t1 VALUES('x','y')
-  }
-} {1 {database is locked}}
-do_test 2.3.4 {
-  execsql COMMIT
-  sqlite3_snapshot_free $snapshot
-} {}
+  do_test $tn.2.2.1 {
+    set snapshot [snapshot_get db2 main]
+    execsql {
+      INSERT INTO t1 VALUES(11, 12);
+      SELECT * FROM t1;
+    }
+  } {1 2 3 4 5 6 7 8 9 10 11 12}
 
-#-------------------------------------------------------------------------
-# Check some errors in sqlite3_snapshot_open(). It is an error if:
-#
-#   1) the db is in auto-commit mode,
-#   2) the db has an open (read or write) transaction,
-#   3) the db is not a wal database,
-#
-# Reuse the database created by earlier tests.
-#
-do_execsql_test 3.0.0 {
-  CREATE TABLE t2(x, y);
-  INSERT INTO t2 VALUES('a', 'b');
-  INSERT INTO t2 VALUES('c', 'd');
-  BEGIN;
-    SELECT * FROM t2;
-} {a b c d}
-do_test 3.0.1 {
-  set snapshot [sqlite3_snapshot_get db main]
-  execsql { COMMIT }
-  execsql { INSERT INTO t2 VALUES('e', 'f'); }
-} {}
+  do_test $tn.2.2.2 {
+    execsql BEGIN
+    snapshot_open db main $snapshot
+    execsql {
+      SELECT * FROM t1;
+    }
+  } {1 2 3 4 5 6 7 8 9 10}
 
-do_test 3.1 {
-  list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg
-} {1 SQLITE_ERROR}
+  do_test $tn.2.2.3 {
+    snapshot_free $snapshot
+    execsql COMMIT
+    execsql COMMIT db2
+    db2 close
+  } {}
 
-do_test 3.2.1 {
-  execsql {
+  do_test $tn.2.3.1 {
+    execsql { DELETE FROM t1 WHERE a>6 }
+    db trans { set snapshot [snapshot_get db main] }
+    execsql {
+      INSERT INTO t1 VALUES('a', 'b');
+      INSERT INTO t1 VALUES('c', 'd');
+      SELECT * FROM t1;
+    }
+  } {1 2 3 4 5 6 a b c d}
+  do_test $tn.2.3.2 {
+    execsql BEGIN
+    snapshot_open db main $snapshot
+    execsql { SELECT * FROM t1 }
+  } {1 2 3 4 5 6}
+
+  do_test $tn.2.3.3 {
+    catchsql {
+      INSERT INTO t1 VALUES('x','y')
+    }
+  } {1 {database is locked}}
+  do_test $tn.2.3.4 {
+    execsql COMMIT
+    snapshot_free $snapshot
+  } {}
+
+  #-------------------------------------------------------------------------
+  # Check some errors in snapshot_open(). It is an error if:
+  #
+  #   1) the db is in auto-commit mode,
+  #   2) the db has an open (read or write) transaction,
+  #   3) the db is not a wal database,
+  #
+  # Reuse the database created by earlier tests.
+  #
+  do_execsql_test $tn.3.0.0 {
+    CREATE TABLE t2(x, y);
+    INSERT INTO t2 VALUES('a', 'b');
+    INSERT INTO t2 VALUES('c', 'd');
     BEGIN;
       SELECT * FROM t2;
-  }
-} {a b c d e f}
-do_test 3.2.2 {
-  list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg
-} {1 SQLITE_ERROR}
+  } {a b c d}
+  do_test $tn.3.0.1 {
+    set snapshot [snapshot_get db main]
+    execsql { COMMIT }
+    execsql { INSERT INTO t2 VALUES('e', 'f'); }
+  } {}
 
-do_test 3.2.3 {
-  execsql {
-    COMMIT;
-    BEGIN;
-      INSERT INTO t2 VALUES('g', 'h');
-  }
-  list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg
-} {1 SQLITE_ERROR}
-do_execsql_test 3.2.4 COMMIT
+  do_test $tn.3.1 {
+    list [catch {snapshot_open db main $snapshot } msg] $msg
+  } {1 SQLITE_ERROR}
 
-do_test 3.3.1 {
-  execsql { PRAGMA journal_mode = DELETE }
-  execsql { BEGIN }
-  list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg
-} {1 SQLITE_ERROR}
+  do_test $tn.3.2.1 {
+    execsql {
+      BEGIN;
+        SELECT * FROM t2;
+    }
+  } {a b c d e f}
+  do_test $tn.3.2.2 {
+    list [catch {snapshot_open db main $snapshot } msg] $msg
+  } {1 SQLITE_ERROR}
 
-do_test 3.3.2 {
-  sqlite3_snapshot_free $snapshot
-  execsql COMMIT
-} {}
+  do_test $tn.3.2.3 {
+    execsql {
+      COMMIT;
+      BEGIN;
+        INSERT INTO t2 VALUES('g', 'h');
+    }
+    list [catch {snapshot_open db main $snapshot } msg] $msg
+  } {1 SQLITE_ERROR}
+  do_execsql_test $tn.3.2.4 COMMIT
 
-#-------------------------------------------------------------------------
-# Check that SQLITE_BUSY_SNAPSHOT is returned if the specified snapshot
-# no longer exists because the wal file has been checkpointed.
-#
-#   1. Reading a snapshot from the middle of a wal file is not possible
-#      after the wal file has been checkpointed.
-#
-#   2. That a snapshot from the end of a wal file can not be read once
-#      the wal file has been wrapped.
-#
-do_execsql_test 4.1.0 {
-  PRAGMA journal_mode = wal;
-  CREATE TABLE t3(i, j);
-  INSERT INTO t3 VALUES('o', 't');
-  INSERT INTO t3 VALUES('t', 'f');
-  BEGIN;
-    SELECT * FROM t3;
-} {wal o t t f}
+  do_test $tn.3.3.1 {
+    execsql { PRAGMA journal_mode = DELETE }
+    execsql { BEGIN }
+    list [catch {snapshot_open db main $snapshot } msg] $msg
+  } {1 SQLITE_ERROR}
 
-do_test 4.1.1 {
-  set snapshot [sqlite3_snapshot_get db main]
-  execsql COMMIT
-} {}
-do_test 4.1.2 {
-  execsql { 
-    INSERT INTO t3 VALUES('f', 's'); 
-    BEGIN;
-  }
-  sqlite3_snapshot_open db main $snapshot
-  execsql { SELECT * FROM t3 }
-} {o t t f}
+  do_test $tn.$tn.3.3.2 {
+    snapshot_free $snapshot
+    execsql COMMIT
+  } {}
 
-do_test 4.1.3 {
-  execsql { 
-    COMMIT;
-    PRAGMA wal_checkpoint;
-    BEGIN;
-  }
-  list [catch {sqlite3_snapshot_open db main $snapshot} msg] $msg
-} {1 SQLITE_BUSY_SNAPSHOT}
-do_test 4.1.4 {
-  sqlite3_snapshot_free $snapshot
-  execsql COMMIT
-} {}
-
-do_test 4.2.1 {
-  execsql {
-    INSERT INTO t3 VALUES('s', 'e');
-    INSERT INTO t3 VALUES('n', 't');
+  #-------------------------------------------------------------------------
+  # Check that SQLITE_BUSY_SNAPSHOT is returned if the specified snapshot
+  # no longer exists because the wal file has been checkpointed.
+  #
+  #   1. Reading a snapshot from the middle of a wal file is not possible
+  #      after the wal file has been checkpointed.
+  #
+  #   2. That a snapshot from the end of a wal file can not be read once
+  #      the wal file has been wrapped.
+  #
+  do_execsql_test $tn.4.1.0 {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t3(i, j);
+    INSERT INTO t3 VALUES('o', 't');
+    INSERT INTO t3 VALUES('t', 'f');
     BEGIN;
       SELECT * FROM t3;
-  }
-} {o t t f f s s e n t}
-do_test 4.2.2 {
-  set snapshot [sqlite3_snapshot_get db main]
-  execsql {
-    COMMIT;
-    PRAGMA wal_checkpoint;
+  } {wal o t t f}
+
+  do_test $tn.4.1.1 {
+    set snapshot [snapshot_get db main]
+    execsql COMMIT
+  } {}
+  do_test $tn.4.1.2 {
+    execsql { 
+      INSERT INTO t3 VALUES('f', 's'); 
+      BEGIN;
+    }
+    snapshot_open db main $snapshot
+    execsql { SELECT * FROM t3 }
+  } {o t t f}
+
+  do_test $tn.4.1.3 {
+    execsql { 
+      COMMIT;
+      PRAGMA wal_checkpoint;
+      BEGIN;
+    }
+    list [catch {snapshot_open db main $snapshot} msg] $msg
+  } {1 SQLITE_BUSY_SNAPSHOT}
+  do_test $tn.4.1.4 {
+    snapshot_free $snapshot
+    execsql COMMIT
+  } {}
+
+  do_test $tn.4.2.1 {
+    execsql {
+      INSERT INTO t3 VALUES('s', 'e');
+      INSERT INTO t3 VALUES('n', 't');
+      BEGIN;
+        SELECT * FROM t3;
+    }
+  } {o t t f f s s e n t}
+  do_test $tn.4.2.2 {
+    set snapshot [snapshot_get db main]
+    execsql {
+      COMMIT;
+      PRAGMA wal_checkpoint;
+      BEGIN;
+    }
+    snapshot_open db main $snapshot
+    execsql { SELECT * FROM t3 }
+  } {o t t f f s s e n t}
+  do_test $tn.4.2.3 {
+    execsql {
+      COMMIT;
+      INSERT INTO t3 VALUES('e', 't');
+      BEGIN;
+    }
+    list [catch {snapshot_open db main $snapshot} msg] $msg
+  } {1 SQLITE_BUSY_SNAPSHOT}
+  do_test $tn.4.2.4 {
+    snapshot_free $snapshot
+  } {}
+
+  #-------------------------------------------------------------------------
+  # Check that SQLITE_BUSY is returned if a checkpoint is running when
+  # sqlite3_snapshot_open() is called.
+  #
+  reset_db
+  db close
+  testvfs tvfs
+  sqlite3 db test.db -vfs tvfs
+
+  do_execsql_test $tn.5.1 {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE x1(x, xx, xxx);
+    INSERT INTO x1 VALUES('z', 'zz', 'zzz');
     BEGIN;
+      SELECT * FROM x1;
+  } {wal z zz zzz}
+
+  do_test $tn.5.2 {
+    set ::snapshot [snapshot_get db main]
+    sqlite3 db2 test.db -vfs tvfs
+    execsql {
+      INSERT INTO x1 VALUES('a', 'aa', 'aaa');
+      COMMIT;
+    }
+  } {}
+
+  set t53 0
+  proc write_callback {args} {
+    do_test $tn.5.3.[incr ::t53] {
+      execsql BEGIN
+      list [catch { snapshot_open db main $::snapshot } msg] $msg
+    } {1 SQLITE_BUSY}
+    catchsql COMMIT
   }
-  sqlite3_snapshot_open db main $snapshot
-  execsql { SELECT * FROM t3 }
-} {o t t f f s s e n t}
-do_test 4.2.3 {
-  execsql {
-    COMMIT;
-    INSERT INTO t3 VALUES('e', 't');
+
+  tvfs filter xWrite
+  tvfs script write_callback
+  db2 eval { PRAGMA wal_checkpoint }
+  db close
+  db2 close
+  tvfs delete
+  snapshot_free $snapshot
+
+  #-------------------------------------------------------------------------
+  # Test that sqlite3_snapshot_get() may be called immediately after
+  # "BEGIN; PRAGMA user_version;". And that sqlite3_snapshot_open() may
+  # be called after opening the db handle and running the script
+  # "PRAGMA user_version; BEGIN".
+  reset_db
+  do_execsql_test $tn.6.1 {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE x1(x, xx, xxx);
+    INSERT INTO x1 VALUES('z', 'zz', 'zzz');
     BEGIN;
+      PRAGMA user_version;
+  } {wal 0}
+  do_test $tn.6.2 {
+    set ::snapshot [snapshot_get db main]
+    execsql {
+      INSERT INTO x1 VALUES('a', 'aa', 'aaa');
+      COMMIT;
+    }
+  } {}
+  do_test $tn.6.3 {
+    sqlite3 db2 test.db 
+    db2 eval "PRAGMA user_version ; BEGIN"
+    snapshot_open db2 main $::snapshot
+    db2 eval { SELECT * FROM x1 }
+  } {z zz zzz}
+  do_test $tn.6.4 {
+    db2 close
+    sqlite3 db2 test.db 
+    db2 eval "PRAGMA application_id"
+    db2 eval "BEGIN"
+    snapshot_open db2 main $::snapshot
+    db2 eval { SELECT * FROM x1 }
+  } {z zz zzz}
+
+  do_test $tn.6.5 {
+    db2 close
+    sqlite3 db2 test.db 
+    db2 eval "BEGIN"
+    list [catch {snapshot_open db2 main $::snapshot} msg] $msg
+  } {1 SQLITE_ERROR}
+
+  snapshot_free $snapshot
+
+  #-------------------------------------------------------------------------
+  # The following tests investigate the sqlite3_snapshot_cmp() API.
+  #
+
+  # Compare snapshots $p1 and $p2, checking that the result is $r.
+  #
+  proc do_snapshot_cmp_test {tn p1 p2 r} {
+    uplevel [list do_test $tn.1 [list snapshot_cmp $p1 $p2] $r]
+    uplevel [list do_test $tn.2 [list snapshot_cmp $p2 $p1] [expr $r*-1]]
+    uplevel [list do_test $tn.3 [list snapshot_cmp $p1 $p1] 0]
+    uplevel [list do_test $tn.4 [list snapshot_cmp $p2 $p2] 0]
   }
-  list [catch {sqlite3_snapshot_open db main $snapshot} msg] $msg
-} {1 SQLITE_BUSY_SNAPSHOT}
-do_test 4.2.4 {
-  sqlite3_snapshot_free $snapshot
-} {}
 
-#-------------------------------------------------------------------------
-# Check that SQLITE_BUSY is returned if a checkpoint is running when
-# sqlite3_snapshot_open() is called.
-#
-reset_db
-db close
-testvfs tvfs
-sqlite3 db test.db -vfs tvfs
+  catch { db2 close }
+  reset_db
 
-do_execsql_test 5.1 {
-  PRAGMA journal_mode = wal;
-  CREATE TABLE x1(x, xx, xxx);
-  INSERT INTO x1 VALUES('z', 'zz', 'zzz');
-  BEGIN;
-    SELECT * FROM x1;
-} {wal z zz zzz}
+  do_execsql_test $tn.7.1 {
+    PRAGMA journal_mode = wal;
+    CREATE TABLE t1(x);
+  } wal
 
-do_test 5.2 {
-  set ::snapshot [sqlite3_snapshot_get db main]
-  sqlite3 db2 test.db -vfs tvfs
-  execsql {
-    INSERT INTO x1 VALUES('a', 'aa', 'aaa');
-    COMMIT;
-  }
-} {}
+  do_test $tn.7.1.2 {
+    execsql { BEGIN ; PRAGMA application_id }
+    set p1 [snapshot_get db main]
+    execsql {
+      INSERT INTO t1 VALUES(10);
+      COMMIT;
+    }
+    execsql { BEGIN ; PRAGMA application_id }
+    set p2 [snapshot_get db main]
+    execsql COMMIT
+  } {}
 
-set t53 0
-proc write_callback {args} {
-  do_test 5.3.[incr ::t53] {
-    execsql BEGIN
-    list [catch { sqlite3_snapshot_open db main $::snapshot } msg] $msg
-  } {1 SQLITE_BUSY}
-  catchsql COMMIT
+  do_snapshot_cmp_test $tn.7.1.3 $p1 $p2 -1
+  snapshot_free $p1
+  snapshot_free $p2
+
+  do_execsql_test $tn.7.2.1 {
+    INSERT INTO t1 VALUES(11);
+    INSERT INTO t1 VALUES(12);
+    INSERT INTO t1 VALUES(13);
+    BEGIN; 
+      PRAGMA application_id;
+  } {0}
+  do_test $tn.7.2.2 {
+    set p1 [snapshot_get db main]
+    execsql {
+      COMMIT;
+      INSERT INTO t1 VALUES(14);
+      PRAGMA wal_checkpoint;
+      BEGIN;
+        PRAGMA application_id;
+    }
+    set p2 [snapshot_get db main]
+    execsql COMMIT
+  } {}
+
+  do_snapshot_cmp_test $tn.7.2.3 $p1 $p2 -1
+  snapshot_free $p2
+
+  do_test $tn.7.3.1 {
+    execsql {
+      INSERT INTO t1 VALUES(14);
+      BEGIN;
+        PRAGMA application_id;
+    }
+    set p2 [snapshot_get db main]
+    execsql COMMIT
+  } {}
+
+  do_snapshot_cmp_test $tn.7.3.2 $p1 $p2 -1
+  snapshot_free $p1
+  snapshot_free $p2
 }
 
-tvfs filter xWrite
-tvfs script write_callback
-db2 eval { PRAGMA wal_checkpoint }
-db close
-db2 close
-tvfs delete
-sqlite3_snapshot_free $snapshot
-
-#-------------------------------------------------------------------------
-# Test that sqlite3_snapshot_get() may be called immediately after
-# "BEGIN; PRAGMA user_version;". And that sqlite3_snapshot_open() may
-# be called after opening the db handle and running the script
-# "PRAGMA user_version; BEGIN".
-reset_db
-do_execsql_test 6.1 {
-  PRAGMA journal_mode = wal;
-  CREATE TABLE x1(x, xx, xxx);
-  INSERT INTO x1 VALUES('z', 'zz', 'zzz');
-  BEGIN;
-    PRAGMA user_version;
-} {wal 0}
-do_test 6.2 {
-  set ::snapshot [sqlite3_snapshot_get db main]
-  execsql {
-    INSERT INTO x1 VALUES('a', 'aa', 'aaa');
-    COMMIT;
-  }
-} {}
-do_test 6.3 {
-  sqlite3 db2 test.db 
-  db2 eval "PRAGMA user_version ; BEGIN"
-  sqlite3_snapshot_open db2 main $::snapshot
-  db2 eval { SELECT * FROM x1 }
-} {z zz zzz}
-do_test 6.4 {
-  db2 close
-  sqlite3 db2 test.db 
-  db2 eval "PRAGMA application_id"
-  db2 eval "BEGIN"
-  sqlite3_snapshot_open db2 main $::snapshot
-  db2 eval { SELECT * FROM x1 }
-} {z zz zzz}
-
-# EVIDENCE-OF: R-55491-50411 A snapshot will fail to open if the
-# database connection D has not previously completed at least one read
-# operation against the database file.
-#
-do_test 6.5 {
-  db2 close
-  sqlite3 db2 test.db 
-  db2 eval "BEGIN"
-  list [catch {sqlite3_snapshot_open db2 main $::snapshot} msg] $msg
-} {1 SQLITE_ERROR}
-
-sqlite3_snapshot_free $snapshot
-
 finish_test
diff --git a/test/snapshot2.test b/test/snapshot2.test
new file mode 100644
index 0000000000..41e555258b
--- /dev/null
+++ b/test/snapshot2.test
@@ -0,0 +1,200 @@
+# 2016 November 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The focus
+# of this file is the sqlite3_snapshot_xxx() APIs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+ifcapable !snapshot {finish_test; return}
+set testprefix snapshot2
+
+# This test does not work with the inmemory_journal permutation. The reason
+# is that each connection opened as part of this permutation executes
+# "PRAGMA journal_mode=memory", which fails if the database is in wal mode
+# and there are one or more existing connections.
+if {[permutation]=="inmemory_journal"} {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Check that it is not possible to obtain a snapshot immediately after
+# a wal mode database with an empty wal file is opened. But it is after
+# the file has been written, even by some other connection.
+#
+do_execsql_test 1.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES(1, 2, 3);
+  INSERT INTO t1 VALUES(4, 5, 6);
+} {wal}
+
+db close
+do_test 1.1.1 { list [file exists test.db] [file exists test.db-wal] } {1 0}
+
+sqlite3 db test.db
+do_execsql_test 1.1.2 { SELECT * FROM t1 } {1 2 3 4 5 6}
+
+do_test 1.1.3 {
+  execsql BEGIN
+  list [catch { sqlite3_snapshot_get_blob db main } msg] $msg
+} {1 SQLITE_ERROR}
+execsql COMMIT
+
+do_test 1.1.4 {
+  execsql { INSERT INTO t1 VALUES(7, 8, 9) }
+  execsql BEGIN
+  string length [sqlite3_snapshot_get_blob db main]
+} 48
+execsql COMMIT
+
+db close
+do_test 1.2.1 { list [file exists test.db] [file exists test.db-wal] } {1 0}
+
+sqlite3 db test.db
+do_execsql_test 1.2.2 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8 9}
+
+do_test 1.2.3 {
+  execsql BEGIN
+  list [catch { sqlite3_snapshot_get_blob db main } msg] $msg
+} {1 SQLITE_ERROR}
+execsql COMMIT
+
+do_test 1.2.4 {
+  sqlite3 db2 test.db
+  execsql { INSERT INTO t1 VALUES(10, 11, 12) } db2
+  execsql BEGIN
+  string length [sqlite3_snapshot_get_blob db main]
+} 48
+execsql COMMIT
+db2 close
+
+#-------------------------------------------------------------------------
+# Simple tests for sqlite3_snapshot_recover().
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x);
+  PRAGMA journal_mode = wal;
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+} {wal}
+
+do_test 2.1 {
+  db trans { set snap [sqlite3_snapshot_get_blob db main] }
+  sqlite3_db_config db NO_CKPT_ON_CLOSE 1
+  db close
+  sqlite3 db test.db
+
+  execsql {SELECT * FROM sqlite_master}
+  execsql BEGIN
+  sqlite3_snapshot_open_blob db main $snap
+  execsql COMMIT;
+  execsql { INSERT INTO t1 VALUES(3); }
+} {}
+
+do_test 2.2 {
+  sqlite3_db_config db NO_CKPT_ON_CLOSE 1
+  db close
+  sqlite3 db test.db
+
+  execsql {SELECT * FROM sqlite_master}
+  execsql BEGIN
+  list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg
+} {1 SQLITE_BUSY_SNAPSHOT}
+
+do_test 2.3 {
+  execsql COMMIT
+  sqlite3_snapshot_recover db main
+  execsql BEGIN
+  sqlite3_snapshot_open_blob db main $snap
+  execsql { SELECT * FROM t1 }
+} {1 2}
+
+do_test 2.4 {
+  execsql COMMIT
+  execsql { SELECT * FROM t1 }
+} {1 2 3}
+
+do_test 2.5 {
+  execsql { PRAGMA wal_checkpoint }
+  sqlite3_db_config db NO_CKPT_ON_CLOSE 1
+  db close
+  sqlite3 db test.db
+
+  sqlite3_snapshot_recover db main
+  execsql BEGIN
+  list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg
+} {1 SQLITE_BUSY_SNAPSHOT}
+
+#-------------------------------------------------------------------------
+# Check that calling sqlite3_snapshot_recover() does not confuse the
+# pager cache.
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES('a', 'b');
+  INSERT INTO t1 VALUES('c', 'd');
+} {wal}
+do_test 3.1 {
+  sqlite3 db2 test.db
+  execsql { INSERT INTO t1 VALUES('e', 'f') } db2
+  db2 close
+  sqlite3_snapshot_recover db main
+} {}
+do_execsql_test 3.2 {
+  SELECT * FROM t1;
+} {a b c d e f}
+
+#-------------------------------------------------------------------------
+# Check that sqlite3_snapshot_recover() returns an error if it is called
+# with an open read-transaction. Or on a database that does not exist. Or
+# on the temp database. Or on a db that is not in wal mode.
+#
+do_test 4.1 {
+  sqlite3_snapshot_recover db main
+} {}
+do_test 4.2 {
+  execsql {
+    BEGIN;
+      SELECT * FROM sqlite_master;
+  }
+  list [catch { sqlite3_snapshot_recover db main } msg] $msg
+} {1 SQLITE_ERROR}
+do_test 4.3 {
+  execsql COMMIT
+  sqlite3_snapshot_recover db main
+} {}
+do_test 4.4 {
+  list [catch { sqlite3_snapshot_recover db aux } msg] $msg
+} {1 SQLITE_ERROR}
+do_test 4.5 {
+  forcedelete test.db2
+  execsql {
+    ATTACH 'test.db2' AS aux;
+    PRAGMA aux.journal_mode = wal;
+    CREATE TABLE aux.t2(x, y);
+  }
+  list [catch { sqlite3_snapshot_recover db aux } msg] $msg
+} {0 {}}
+do_test 4.6 {
+  list [catch { sqlite3_snapshot_recover db temp } msg] $msg
+} {1 SQLITE_ERROR}
+do_test 4.7 {
+  execsql {
+    PRAGMA aux.journal_mode = delete;
+  }
+  list [catch { sqlite3_snapshot_recover db aux } msg] $msg
+} {1 SQLITE_ERROR}
+
+finish_test
diff --git a/test/snapshot_fault.test b/test/snapshot_fault.test
index 3ac13daefd..5c6cf40a6f 100644
--- a/test/snapshot_fault.test
+++ b/test/snapshot_fault.test
@@ -146,7 +146,7 @@ do_faultsim_test 3.0 -prep {
     error $msg
   }
 } -test {
-  faultsim_test_result {0 {}} {1 SQLITE_IOERR} \
+  faultsim_test_result {0 {}} {1 SQLITE_IOERR} {1 SQLITE_NOMEM} \
                               {1 SQLITE_IOERR_NOMEM} {1 SQLITE_IOERR_READ}
   if {$testrc==0} {
     set res [db eval { 
@@ -159,6 +159,68 @@ do_faultsim_test 3.0 -prep {
   sqlite3_snapshot_free $::snapshot
 }
 
+#-------------------------------------------------------------------------
+# Test the handling of faults that occur within sqlite3_snapshot_recover().
+#
+reset_db
+do_execsql_test 4.0 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(zzz);
+  INSERT INTO t1 VALUES('abc');
+  INSERT INTO t1 VALUES('def');
+} {wal}
+faultsim_save_and_close
+
+do_test 4.0.1 {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM sqlite_master } 
+  sqlite3_snapshot_recover db main
+} {}
+db close
+
+do_faultsim_test 4.0 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM sqlite_master } 
+} -body {
+  sqlite3_snapshot_recover db main
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM}
+}
+
+# The following test cases contrive to call sqlite3_snapshot_recover()
+# before all pages of the *-shm file have been mapped. This tests an
+# extra branch of error handling logic in snapshot_recover().
+#
+reset_db
+do_execsql_test 4.1.0 {
+  PRAGMA page_size = 512;
+  PRAGMA journal_mode = wal;
+  PRAGMA wal_autocheckpoint = 0;
+  CREATE TABLE t1(zzz);
+  INSERT INTO t1 VALUES(randomblob( 500 * 9500 ));
+  PRAGMA user_version = 211;
+} {wal 0}
+
+do_test 4.1.1 {
+  list [file size test.db-shm] [file size test.db]
+} {98304 512}
+
+faultsim_save_and_close
+do_faultsim_test 4.1 -faults shm* -prep {
+  catch { db2 close } 
+  catch { db close } 
+  faultsim_restore_and_reopen
+  sqlite3 db2 test.db
+  db2 eval { SELECT * FROM sqlite_master } 
+  db eval BEGIN
+  sqlite3_snapshot_get_blob db main
+  db eval COMMIT
+} -body {
+  sqlite3_snapshot_recover db main
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_IOERR}
+}
+
 
 
 finish_test
diff --git a/test/sort5.test b/test/sort5.test
index 5b1292bb48..6ada81c9cc 100644
--- a/test/sort5.test
+++ b/test/sort5.test
@@ -41,4 +41,78 @@ do_execsql_test 1.2 {
 
 db close
 tvfs delete
+
+#-------------------------------------------------------------------------
+# Test that the PMA size is determined correctly. The PMA size should be
+# roughly the same amount of memory allocated to the main pager cache, or
+# 250 pages if this is larger.
+#
+testvfs tvfs
+tvfs script tv_callback
+tvfs filter {xOpen xWrite}
+
+proc tv_callback {method args} {
+  global iTemp
+  global F
+  switch $method {
+    xOpen {
+      if {[lindex $args 0]==""} { return "temp[incr iTemp]" }
+      return "SQLITE_OK"
+    }
+
+    xWrite {
+      foreach {filename id off amt} $args {}
+      if {[info exists F($id)]==0 || $F($id)<($off + $amt)} {
+        set F($id) [expr $off+$amt]
+      }
+    }
+  }
+}
+
+catch { db close }
+forcedelete test.db
+sqlite3 db test.db -vfs tvfs
+execsql { CREATE TABLE t1(x) }
+
+# Each iteration of the following loop attempts to sort 10001 records
+# each a bit over 100 bytes in size. In total a little more than 1MiB 
+# of data.
+#
+foreach {tn pgsz cachesz bTemp} {
+  1 4096   1000  0
+  2 1024   1000  1
+
+  3 4096  -1000  1
+  4 1024  -1000  1
+
+  5 4096  -9000  0
+  6 1024  -9000  0
+} {
+  do_execsql_test 2.$tn.0 "
+    PRAGMA page_size = $pgsz;
+    VACUUM;
+    PRAGMA cache_size = $cachesz;
+  "
+
+  if {[db one {PRAGMA page_size}]!=$pgsz} {
+    # SEE is not able to change page sizes and that messes up the
+    # results that follow.
+    continue
+  }
+
+  do_test 2.$tn.1 {
+    set ::iTemp 0
+    catch { array unset F }
+    execsql {
+      WITH x(i, j) AS (
+        SELECT 1, randomblob(100)
+        UNION ALL
+        SELECT i+1, randomblob(100) FROM x WHERE i<10000
+      )
+      SELECT * FROM x ORDER BY j;
+    }
+    expr {[array names F]!=""}
+  } $bTemp
+}
+
 finish_test
diff --git a/test/speed3.test b/test/speed3.test
index 1beaeb74f3..4aa90943ba 100644
--- a/test/speed3.test
+++ b/test/speed3.test
@@ -105,7 +105,7 @@ proc io_log {db} {
   puts "Normal    : Read $stats2(read), wrote $stats2(write)"
 }
 
-proc reset_db {} {
+proc speed3_reset_db {} {
   db close
   sqlite3 db test.db
   db eval { 
@@ -117,7 +117,7 @@ proc reset_db {} {
 }
 
 forcedelete test2.db test2.db-journal
-reset_db
+speed3_reset_db
 
 # Set up a database in auto-vacuum mode and create a database schema.
 #
@@ -154,7 +154,7 @@ do_test speed3-0.4 {
 # Delete all content in a table, one row at a time.
 #
 #io_log db
-reset_db
+speed3_reset_db
 speed_trial speed3-1.incrvacuum $::NROW row {DELETE FROM main.t1 WHERE 1}
 speed_trial speed3-1.normal     $::NROW row {DELETE FROM aux.t1 WHERE 1}
 io_log db
@@ -164,7 +164,7 @@ io_log db
 #
 #db eval {PRAGMA incremental_vacuum(500000)}
 populate_t1 db
-reset_db
+speed3_reset_db
 speed_trial speed3-2.incrvacuum $::NROW row {SELECT c FROM main.t1}
 speed_trial speed3-2.normal     $::NROW row {SELECT c FROM aux.t1}
 io_log db
diff --git a/test/speedtest1.c b/test/speedtest1.c
index d7f2af47b8..db3a558a38 100644
--- a/test/speedtest1.c
+++ b/test/speedtest1.c
@@ -15,6 +15,7 @@ static const char zHelp[] =
   "  --journal M         Set the journal_mode to M\n"
   "  --key KEY           Set the encryption key to KEY\n"
   "  --lookaside N SZ    Configure lookaside for N slots of SZ bytes each\n"
+  "  --mmap SZ           MMAP the first SZ bytes of the database file\n"
   "  --multithread       Set multithreaded mode\n"
   "  --nomemstat         Disable memory statistics\n"
   "  --nosync            Set PRAGMA synchronous=OFF\n"
@@ -22,6 +23,7 @@ static const char zHelp[] =
   "  --pagesize N        Set the page size to N\n"
   "  --pcache N SZ       Configure N pages of pagecache each of size SZ bytes\n"
   "  --primarykey        Use PRIMARY KEY instead of UNIQUE where appropriate\n"
+  "  --repeat N          Repeat each SELECT N times (default: 1)\n"
   "  --reprepare         Reprepare each statement upon every invocation\n"
   "  --scratch N SZ      Configure scratch memory for N slots of SZ bytes each\n"
   "  --serialized        Set serialized threading mode\n"
@@ -30,7 +32,8 @@ static const char zHelp[] =
   "  --shrink-memory     Invoke sqlite3_db_release_memory() frequently.\n"
   "  --size N            Relative test size.  Default=100\n"
   "  --stats             Show statistics at the end\n"
-  "  --testset T         Run test-set T\n"
+  "  --temp N            N from 0 to 9.  0: no temp table. 9: all temp tables\n"
+  "  --testset T         Run test-set T (main, cte, rtree, orm, debug)\n"
   "  --trace             Turn on SQL tracing\n"
   "  --threads N         Use up to N threads for sorting\n"
   "  --utf16be           Set text encoding to UTF-16BE\n"
@@ -47,15 +50,17 @@ static const char zHelp[] =
 #include 
 #include 
 #include 
+#ifndef _WIN32
+# include 
+#else
+# include 
+#endif
 #define ISSPACE(X) isspace((unsigned char)(X))
 #define ISDIGIT(X) isdigit((unsigned char)(X))
 
 #if SQLITE_VERSION_NUMBER<3005000
 # define sqlite3_int64 sqlite_int64
 #endif
-#ifdef SQLITE_ENABLE_RBU
-# include "sqlite3rbu.h"
-#endif
 
 /* All global state is held in this structure */
 static struct Global {
@@ -69,7 +74,9 @@ static struct Global {
   int bExplain;              /* Print SQL with EXPLAIN prefix */
   int bVerify;               /* Try to verify that results are correct */
   int bMemShrink;            /* Call sqlite3_db_release_memory() often */
+  int eTemp;                 /* 0: no TEMP.  9: always TEMP. */
   int szTest;                /* Scale factor for test iterations */
+  int nRepeat;               /* Repeat selects this many times */
   const char *zWR;           /* Might be WITHOUT ROWID */
   const char *zNN;           /* Might be NOT NULL */
   const char *zPK;           /* Might be UNIQUE or PRIMARY KEY */
@@ -78,6 +85,12 @@ static struct Global {
   char zResult[3000];        /* Text of the current result */
 } g;
 
+/* Return " TEMP" or "", as appropriate for creating a table.
+*/
+static const char *isTemp(int N){
+  return g.eTemp>=N ? " TEMP" : "";
+}
+
 
 /* Print an error message and exit */
 static void fatal_error(const char *zMsg, ...){
@@ -413,12 +426,14 @@ void speedtest1_run(void){
   speedtest1_shrink_memory();
 }
 
+#ifndef SQLITE_OMIT_DEPRECATED
 /* The sqlite3_trace() callback function */
 static void traceCallback(void *NotUsed, const char *zSql){
   int n = (int)strlen(zSql);
   while( n>0 && (zSql[n-1]==';' || ISSPACE(zSql[n-1])) ) n--;
   fprintf(stderr,"%.*s;\n", n, zSql);
 }
+#endif /* SQLITE_OMIT_DEPRECATED */
 
 /* Substitute random() function that gives the same random
 ** sequence on each run, for repeatability. */
@@ -443,6 +458,68 @@ static int est_square_root(int x){
   return y0;
 }
 
+
+#if SQLITE_VERSION_NUMBER<3005004
+/*
+** An implementation of group_concat().  Used only when testing older
+** versions of SQLite that lack the built-in group_concat().
+*/
+struct groupConcat {
+  char *z;
+  int nAlloc;
+  int nUsed;
+};
+static void groupAppend(struct groupConcat *p, const char *z, int n){
+  if( p->nUsed+n >= p->nAlloc ){
+    int n2 = (p->nAlloc+n+1)*2;
+    char *z2 = sqlite3_realloc(p->z, n2);
+    if( z2==0 ) return;
+    p->z = z2;
+    p->nAlloc = n2;
+  }
+  memcpy(p->z+p->nUsed, z, n);
+  p->nUsed += n;
+}
+static void groupStep(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zVal;
+  struct groupConcat *p;
+  const char *zSep;
+  int nVal, nSep;
+  assert( argc==1 || argc==2 );
+  if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
+  p= (struct groupConcat*)sqlite3_aggregate_context(context, sizeof(*p));
+
+  if( p ){
+    int firstTerm = p->nUsed==0;
+    if( !firstTerm ){
+      if( argc==2 ){
+        zSep = (char*)sqlite3_value_text(argv[1]);
+        nSep = sqlite3_value_bytes(argv[1]);
+      }else{
+        zSep = ",";
+        nSep = 1;
+      }
+      if( nSep ) groupAppend(p, zSep, nSep);
+    }
+    zVal = (char*)sqlite3_value_text(argv[0]);
+    nVal = sqlite3_value_bytes(argv[0]);
+    if( zVal ) groupAppend(p, zVal, nVal);
+  }
+}
+static void groupFinal(sqlite3_context *context){
+  struct groupConcat *p;
+  p = sqlite3_aggregate_context(context, 0);
+  if( p && p->z ){
+    p->z[p->nUsed] = 0;
+    sqlite3_result_text(context, p->z, p->nUsed, sqlite3_free);
+  }
+}
+#endif
+
 /*
 ** The main and default testset
 */
@@ -451,16 +528,17 @@ void testset_main(void){
   int n;                        /* iteration count */
   int sz;                       /* Size of the tables */
   int maxb;                     /* Maximum swizzled value */
-  unsigned x1, x2;              /* Parameters */
-  int len;                      /* Length of the zNum[] string */
+  unsigned x1 = 0, x2 = 0;      /* Parameters */
+  int len = 0;                  /* Length of the zNum[] string */
   char zNum[2000];              /* A number name */
 
   sz = n = g.szTest*500;
+  zNum[0] = 0;
   maxb = roundup_allones(sz);
   speedtest1_begin_test(100, "%d INSERTs into table with no index", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t1(a INTEGER %s, b INTEGER %s, c TEXT %s);",
-                  g.zNN, g.zNN, g.zNN);
+  speedtest1_exec("CREATE%s TABLE t1(a INTEGER %s, b INTEGER %s, c TEXT %s);",
+                  isTemp(9), g.zNN, g.zNN, g.zNN);
   speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2,?3); --  %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -477,8 +555,9 @@ void testset_main(void){
   n = sz;
   speedtest1_begin_test(110, "%d ordered INSERTS with one index/PK", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
-                   g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+  speedtest1_exec(
+     "CREATE%s TABLE t2(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
+     isTemp(5), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_prepare("INSERT INTO t2 VALUES(?1,?2,?3); -- %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -495,8 +574,9 @@ void testset_main(void){
   n = sz;
   speedtest1_begin_test(120, "%d unordered INSERTS with one index/PK", n);
   speedtest1_exec("BEGIN");
-  speedtest1_exec("CREATE TABLE t3(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
-                   g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+  speedtest1_exec(
+      "CREATE%s TABLE t3(a INTEGER %s %s, b INTEGER %s, c TEXT %s) %s",
+      isTemp(3), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_prepare("INSERT INTO t3 VALUES(?1,?2,?3); -- %d times", n);
   for(i=1; i<=n; i++){
     x1 = swizzle(i,maxb);
@@ -509,17 +589,23 @@ void testset_main(void){
   speedtest1_exec("COMMIT");
   speedtest1_end_test();
 
+#if SQLITE_VERSION_NUMBER<3005004
+  sqlite3_create_function(g.db, "group_concat", 1, SQLITE_UTF8, 0,
+                          0, groupStep, groupFinal);
+#endif
 
   n = 25;
   speedtest1_begin_test(130, "%d SELECTS, numeric BETWEEN, unindexed", n);
   speedtest1_exec("BEGIN");
   speedtest1_prepare(
-    "SELECT count(*), avg(b), sum(length(c)) FROM t1\n"
+    "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM t1\n"
     " WHERE b BETWEEN ?1 AND ?2; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    x2 = speedtest1_random()%10 + sz/5000 + x1;
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      x2 = speedtest1_random()%10 + sz/5000 + x1;
+    }
     sqlite3_bind_int(g.pStmt, 1, x1);
     sqlite3_bind_int(g.pStmt, 2, x2);
     speedtest1_run();
@@ -532,16 +618,18 @@ void testset_main(void){
   speedtest1_begin_test(140, "%d SELECTS, LIKE, unindexed", n);
   speedtest1_exec("BEGIN");
   speedtest1_prepare(
-    "SELECT count(*), avg(b), sum(length(c)) FROM t1\n"
+    "SELECT count(*), avg(b), sum(length(c)), group_concat(c) FROM t1\n"
     " WHERE c LIKE ?1; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    zNum[0] = '%';
-    len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
-    zNum[len] = '%';
-    zNum[len+1] = 0;
-    sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC);
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      zNum[0] = '%';
+      len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
+      zNum[len] = '%';
+      zNum[len+1] = 0;
+    }
+    sqlite3_bind_text(g.pStmt, 1, zNum, len+1, SQLITE_STATIC);
     speedtest1_run();
   }
   speedtest1_exec("COMMIT");
@@ -556,12 +644,14 @@ void testset_main(void){
     " ORDER BY a; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    zNum[0] = '%';
-    len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
-    zNum[len] = '%';
-    zNum[len+1] = 0;
-    sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC);
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      zNum[0] = '%';
+      len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
+      zNum[len] = '%';
+      zNum[len+1] = 0;
+    }
+    sqlite3_bind_text(g.pStmt, 1, zNum, len+1, SQLITE_STATIC);
     speedtest1_run();
   }
   speedtest1_exec("COMMIT");
@@ -575,12 +665,14 @@ void testset_main(void){
     " ORDER BY a LIMIT 10; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    zNum[0] = '%';
-    len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
-    zNum[len] = '%';
-    zNum[len+1] = 0;
-    sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC);
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      zNum[0] = '%';
+      len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2);
+      zNum[len] = '%';
+      zNum[len+1] = 0;
+    }
+    sqlite3_bind_text(g.pStmt, 1, zNum, len+1, SQLITE_STATIC);
     speedtest1_run();
   }
   speedtest1_exec("COMMIT");
@@ -602,12 +694,14 @@ void testset_main(void){
   speedtest1_begin_test(160, "%d SELECTS, numeric BETWEEN, indexed", n);
   speedtest1_exec("BEGIN");
   speedtest1_prepare(
-    "SELECT count(*), avg(b), sum(length(c)) FROM t1\n"
+    "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t1\n"
     " WHERE b BETWEEN ?1 AND ?2; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    x2 = speedtest1_random()%10 + sz/5000 + x1;
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      x2 = speedtest1_random()%10 + sz/5000 + x1;
+    }
     sqlite3_bind_int(g.pStmt, 1, x1);
     sqlite3_bind_int(g.pStmt, 2, x2);
     speedtest1_run();
@@ -620,12 +714,14 @@ void testset_main(void){
   speedtest1_begin_test(161, "%d SELECTS, numeric BETWEEN, PK", n);
   speedtest1_exec("BEGIN");
   speedtest1_prepare(
-    "SELECT count(*), avg(b), sum(length(c)) FROM t2\n"
+    "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t2\n"
     " WHERE a BETWEEN ?1 AND ?2; -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = speedtest1_random()%maxb;
-    x2 = speedtest1_random()%10 + sz/5000 + x1;
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = speedtest1_random()%maxb;
+      x2 = speedtest1_random()%10 + sz/5000 + x1;
+    }
     sqlite3_bind_int(g.pStmt, 1, x1);
     sqlite3_bind_int(g.pStmt, 2, x2);
     speedtest1_run();
@@ -638,12 +734,14 @@ void testset_main(void){
   speedtest1_begin_test(170, "%d SELECTS, text BETWEEN, indexed", n);
   speedtest1_exec("BEGIN");
   speedtest1_prepare(
-    "SELECT count(*), avg(b), sum(length(c)) FROM t1\n"
+    "SELECT count(*), avg(b), sum(length(c)), group_concat(a) FROM t1\n"
     " WHERE c BETWEEN ?1 AND (?1||'~'); -- %d times", n
   );
   for(i=1; i<=n; i++){
-    x1 = swizzle(i, maxb);
-    len = speedtest1_numbername(x1, zNum, sizeof(zNum)-1);
+    if( (i-1)%g.nRepeat==0 ){
+      x1 = swizzle(i, maxb);
+      len = speedtest1_numbername(x1, zNum, sizeof(zNum)-1);
+    }
     sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC);
     speedtest1_run();
   }
@@ -654,12 +752,12 @@ void testset_main(void){
   speedtest1_begin_test(180, "%d INSERTS with three indexes", n);
   speedtest1_exec("BEGIN");
   speedtest1_exec(
-    "CREATE TABLE t4(\n"
+    "CREATE%s TABLE t4(\n"
     "  a INTEGER %s %s,\n"
     "  b INTEGER %s,\n"
     "  c TEXT %s\n"
     ") %s",
-    g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
+    isTemp(1), g.zNN, g.zPK, g.zNN, g.zNN, g.zWR);
   speedtest1_exec("CREATE INDEX t4b ON t4(b)");
   speedtest1_exec("CREATE INDEX t4c ON t4(c)");
   speedtest1_exec("INSERT INTO t4 SELECT * FROM t1");
@@ -802,6 +900,65 @@ void testset_main(void){
   speedtest1_run();
   speedtest1_end_test();
 
+  sz = n = g.szTest*700;
+  zNum[0] = 0;
+  maxb = roundup_allones(sz/3);
+  speedtest1_begin_test(400, "%d REPLACE ops on an IPK", n);
+  speedtest1_exec("BEGIN");
+  speedtest1_exec("CREATE%s TABLE t5(a INTEGER PRIMARY KEY, b %s);",
+                  isTemp(9), g.zNN);
+  speedtest1_prepare("REPLACE INTO t5 VALUES(?1,?2); --  %d times",n);
+  for(i=1; i<=n; i++){
+    x1 = swizzle(i,maxb);
+    speedtest1_numbername(i, zNum, sizeof(zNum));
+    sqlite3_bind_int(g.pStmt, 1, (sqlite3_int64)x1);
+    sqlite3_bind_text(g.pStmt, 2, zNum, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_exec("COMMIT");
+  speedtest1_end_test();
+  speedtest1_begin_test(410, "%d SELECTS on an IPK", n);
+  speedtest1_prepare("SELECT b FROM t5 WHERE a=?1; --  %d times",n);
+  for(i=1; i<=n; i++){
+    x1 = swizzle(i,maxb);
+    sqlite3_bind_int(g.pStmt, 1, (sqlite3_int64)x1);
+    speedtest1_run();
+  }
+  speedtest1_end_test();
+
+  sz = n = g.szTest*700;
+  zNum[0] = 0;
+  maxb = roundup_allones(sz/3);
+  speedtest1_begin_test(500, "%d REPLACE on TEXT PK", n);
+  speedtest1_exec("BEGIN");
+  speedtest1_exec("CREATE%s TABLE t6(a TEXT PRIMARY KEY, b %s)%s;",
+                  isTemp(9), g.zNN,
+                  sqlite3_libversion_number()>=3008002 ? "WITHOUT ROWID" : "");
+  speedtest1_prepare("REPLACE INTO t6 VALUES(?1,?2); --  %d times",n);
+  for(i=1; i<=n; i++){
+    x1 = swizzle(i,maxb);
+    speedtest1_numbername(x1, zNum, sizeof(zNum));
+    sqlite3_bind_int(g.pStmt, 2, i);
+    sqlite3_bind_text(g.pStmt, 1, zNum, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_exec("COMMIT");
+  speedtest1_end_test();
+  speedtest1_begin_test(510, "%d SELECTS on a TEXT PK", n);
+  speedtest1_prepare("SELECT b FROM t6 WHERE a=?1; --  %d times",n);
+  for(i=1; i<=n; i++){
+    x1 = swizzle(i,maxb);
+    speedtest1_numbername(x1, zNum, sizeof(zNum));
+    sqlite3_bind_text(g.pStmt, 1, zNum, -1, SQLITE_STATIC);
+    speedtest1_run();
+  }
+  speedtest1_end_test();
+  speedtest1_begin_test(520, "%d SELECT DISTINCT", n);
+  speedtest1_exec("SELECT DISTINCT b FROM t5;");
+  speedtest1_exec("SELECT DISTINCT b FROM t6;");
+  speedtest1_end_test();
+
+
   speedtest1_begin_test(980, "PRAGMA integrity_check");
   speedtest1_exec("PRAGMA integrity_check");
   speedtest1_end_test();
@@ -1021,10 +1178,10 @@ void testset_rtree(int p1, int p2){
   unsigned mxCoord;
   unsigned x0, x1, y0, y1, z0, z1;
   unsigned iStep;
-  int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 );
+  int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*500 );
 
   mxCoord = 15000;
-  n = g.szTest*100;
+  n = g.szTest*500;
   speedtest1_begin_test(100, "%d INSERTs into an r-tree", n);
   speedtest1_exec("BEGIN");
   speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)");
@@ -1051,7 +1208,7 @@ void testset_rtree(int p1, int p2){
   speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1");
   speedtest1_end_test();
 
-  n = g.szTest*20;
+  n = g.szTest*100;
   speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n);
   speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2");
   iStep = mxCoord/n;
@@ -1064,7 +1221,7 @@ void testset_rtree(int p1, int p2){
   speedtest1_end_test();
 
   if( g.bVerify ){
-    n = g.szTest*20;
+    n = g.szTest*100;
     speedtest1_begin_test(111, "Verify result from 1-D intersect slice queries");
     speedtest1_prepare("SELECT count(*) FROM t1 WHERE x0>=?1 AND x1<=?2");
     iStep = mxCoord/n;
@@ -1080,7 +1237,7 @@ void testset_rtree(int p1, int p2){
     speedtest1_end_test();
   }
   
-  n = g.szTest*20;
+  n = g.szTest*100;
   speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n);
   speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2");
   iStep = mxCoord/n;
@@ -1093,7 +1250,7 @@ void testset_rtree(int p1, int p2){
   speedtest1_end_test();
 
   if( g.bVerify ){
-    n = g.szTest*20;
+    n = g.szTest*100;
     speedtest1_begin_test(121, "Verify result from 1-D overlap slice queries");
     speedtest1_prepare("SELECT count(*) FROM t1 WHERE y1>=?1 AND y0<=?2");
     iStep = mxCoord/n;
@@ -1110,7 +1267,7 @@ void testset_rtree(int p1, int p2){
   }
   
 
-  n = g.szTest*20;
+  n = g.szTest*100;
   speedtest1_begin_test(125, "%d custom geometry callback queries", n);
   sqlite3_rtree_geometry_callback(g.db, "xslice", xsliceGeometryCallback, 0);
   speedtest1_prepare("SELECT count(*) FROM rt1 WHERE id MATCH xslice(?1,?2)");
@@ -1126,7 +1283,7 @@ void testset_rtree(int p1, int p2){
   }
   speedtest1_end_test();
 
-  n = g.szTest*80;
+  n = g.szTest*400;
   speedtest1_begin_test(130, "%d three-dimensional intersect box queries", n);
   speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x1>=?1 AND x0<=?2"
                      " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2");
@@ -1139,7 +1296,7 @@ void testset_rtree(int p1, int p2){
   }
   speedtest1_end_test();
 
-  n = g.szTest*100;
+  n = g.szTest*500;
   speedtest1_begin_test(140, "%d rowid queries", n);
   speedtest1_prepare("SELECT * FROM rt1 WHERE id=?1");
   for(i=1; i<=n; i++){
@@ -1150,6 +1307,275 @@ void testset_rtree(int p1, int p2){
 }
 #endif /* SQLITE_ENABLE_RTREE */
 
+/*
+** A testset that does key/value storage on tables with many columns.
+** This is the kind of workload generated by ORMs such as CoreData.
+*/
+void testset_orm(void){
+  unsigned i, j, n;
+  unsigned nRow;
+  unsigned x1, len;
+  char zNum[2000];              /* A number name */
+  static const char zType[] =   /* Types for all non-PK columns, in order */
+    "IBBIIITIVVITBTBFBFITTFBTBVBVIFTBBFITFFVBIFIVBVVVBTVTIBBFFIVIBTB"
+    "TVTTFTVTVFFIITIFBITFTTFFFVBIIBTTITFTFFVVVFIIITVBBVFFTVVB";
+
+  nRow = n = g.szTest*250;
+  speedtest1_begin_test(100, "Fill %d rows", n);
+  speedtest1_exec(
+    "BEGIN;"
+    "CREATE TABLE ZLOOKSLIKECOREDATA ("
+    "  ZPK INTEGER PRIMARY KEY,"
+    "  ZTERMFITTINGHOUSINGCOMMAND INTEGER,"
+    "  ZBRIEFGOBYDODGERHEIGHT BLOB,"
+    "  ZCAPABLETRIPDOORALMOND BLOB,"
+    "  ZDEPOSITPAIRCOLLEGECOMET INTEGER,"
+    "  ZFRAMEENTERSIMPLEMOUTH INTEGER,"
+    "  ZHOPEFULGATEHOLECHALK INTEGER,"
+    "  ZSLEEPYUSERGRANDBOWL TIMESTAMP,"
+    "  ZDEWPEACHCAREERCELERY INTEGER,"
+    "  ZHANGERLITHIUMDINNERMEET VARCHAR,"
+    "  ZCLUBRELEASELIZARDADVICE VARCHAR,"
+    "  ZCHARGECLICKHUMANEHIRE INTEGER,"
+    "  ZFINGERDUEPIZZAOPTION TIMESTAMP,"
+    "  ZFLYINGDOCTORTABLEMELODY BLOB,"
+    "  ZLONGFINLEAVEIMAGEOIL TIMESTAMP,"
+    "  ZFAMILYVISUALOWNERMATTER BLOB,"
+    "  ZGOLDYOUNGINITIALNOSE FLOAT,"
+    "  ZCAUSESALAMITERMCYAN BLOB,"
+    "  ZSPREADMOTORBISCUITBACON FLOAT,"
+    "  ZGIFTICEFISHGLUEHAIR INTEGER,"
+    "  ZNOTICEPEARPOLICYJUICE TIMESTAMP,"
+    "  ZBANKBUFFALORECOVERORBIT TIMESTAMP,"
+    "  ZLONGDIETESSAYNATURE FLOAT,"
+    "  ZACTIONRANGEELEGANTNEUTRON BLOB,"
+    "  ZCADETBRIGHTPLANETBANK TIMESTAMP,"
+    "  ZAIRFORGIVEHEADFROG BLOB,"
+    "  ZSHARKJUSTFRUITMOVIE VARCHAR,"
+    "  ZFARMERMORNINGMIRRORCONCERN BLOB,"
+    "  ZWOODPOETRYCOBBLERBENCH VARCHAR,"
+    "  ZHAFNIUMSCRIPTSALADMOTOR INTEGER,"
+    "  ZPROBLEMCLUBPOPOVERJELLY FLOAT,"
+    "  ZEIGHTLEADERWORKERMOST TIMESTAMP,"
+    "  ZGLASSRESERVEBARIUMMEAL BLOB,"
+    "  ZCLAMBITARUGULAFAJITA BLOB,"
+    "  ZDECADEJOYOUSWAVEHABIT FLOAT,"
+    "  ZCOMPANYSUMMERFIBERELF INTEGER,"
+    "  ZTREATTESTQUILLCHARGE TIMESTAMP,"
+    "  ZBROWBALANCEKEYCHOWDER FLOAT,"
+    "  ZPEACHCOPPERDINNERLAKE FLOAT,"
+    "  ZDRYWALLBEYONDBROWNBOWL VARCHAR,"
+    "  ZBELLYCRASHITEMLACK BLOB,"
+    "  ZTENNISCYCLEBILLOFFICER INTEGER,"
+    "  ZMALLEQUIPTHANKSGLUE FLOAT,"
+    "  ZMISSREPLYHUMANLIVING INTEGER,"
+    "  ZKIWIVISUALPRIDEAPPLE VARCHAR,"
+    "  ZWISHHITSKINMOTOR BLOB,"
+    "  ZCALMRACCOONPROGRAMDEBIT VARCHAR,"
+    "  ZSHINYASSISTLIVINGCRAB VARCHAR,"
+    "  ZRESOLVEWRISTWRAPAPPLE VARCHAR,"
+    "  ZAPPEALSIMPLESECONDHOUSING BLOB,"
+    "  ZCORNERANCHORTAPEDIVER TIMESTAMP,"
+    "  ZMEMORYREQUESTSOURCEBIG VARCHAR,"
+    "  ZTRYFACTKEEPMILK TIMESTAMP,"
+    "  ZDIVERPAINTLEATHEREASY INTEGER,"
+    "  ZSORTMISTYQUOTECABBAGE BLOB,"
+    "  ZTUNEGASBUFFALOCAPITAL BLOB,"
+    "  ZFILLSTOPLAWJOYFUL FLOAT,"
+    "  ZSTEELCAREFULPLATENUMBER FLOAT,"
+    "  ZGIVEVIVIDDIVINEMEANING INTEGER,"
+    "  ZTREATPACKFUTURECONVERT VARCHAR,"
+    "  ZCALMLYGEMFINISHEFFECT INTEGER,"
+    "  ZCABBAGESOCKEASEMINUTE BLOB,"
+    "  ZPLANETFAMILYPUREMEMORY TIMESTAMP,"
+    "  ZMERRYCRACKTRAINLEADER BLOB,"
+    "  ZMINORWAYPAPERCLASSY TIMESTAMP,"
+    "  ZEAGLELINEMINEMAIL VARCHAR,"
+    "  ZRESORTYARDGREENLET TIMESTAMP,"
+    "  ZYARDOREGANOVIVIDJEWEL TIMESTAMP,"
+    "  ZPURECAKEVIVIDNEATLY FLOAT,"
+    "  ZASKCONTACTMONITORFUN TIMESTAMP,"
+    "  ZMOVEWHOGAMMAINCH VARCHAR,"
+    "  ZLETTUCEBIRDMEETDEBATE TIMESTAMP,"
+    "  ZGENENATURALHEARINGKITE VARCHAR,"
+    "  ZMUFFINDRYERDRAWFORTUNE FLOAT,"
+    "  ZGRAYSURVEYWIRELOVE FLOAT,"
+    "  ZPLIERSPRINTASKOREGANO INTEGER,"
+    "  ZTRAVELDRIVERCONTESTLILY INTEGER,"
+    "  ZHUMORSPICESANDKIDNEY TIMESTAMP,"
+    "  ZARSENICSAMPLEWAITMUON INTEGER,"
+    "  ZLACEADDRESSGROUNDCAREFUL FLOAT,"
+    "  ZBAMBOOMESSWASABIEVENING BLOB,"
+    "  ZONERELEASEAVERAGENURSE INTEGER,"
+    "  ZRADIANTWHENTRYCARD TIMESTAMP,"
+    "  ZREWARDINSIDEMANGOINTENSE FLOAT,"
+    "  ZNEATSTEWPARTIRON TIMESTAMP,"
+    "  ZOUTSIDEPEAHENCOUNTICE TIMESTAMP,"
+    "  ZCREAMEVENINGLIPBRANCH FLOAT,"
+    "  ZWHALEMATHAVOCADOCOPPER FLOAT,"
+    "  ZLIFEUSELEAFYBELL FLOAT,"
+    "  ZWEALTHLINENGLEEFULDAY VARCHAR,"
+    "  ZFACEINVITETALKGOLD BLOB,"
+    "  ZWESTAMOUNTAFFECTHEARING INTEGER,"
+    "  ZDELAYOUTCOMEHORNAGENCY INTEGER,"
+    "  ZBIGTHINKCONVERTECONOMY BLOB,"
+    "  ZBASEGOUDAREGULARFORGIVE TIMESTAMP,"
+    "  ZPATTERNCLORINEGRANDCOLBY TIMESTAMP,"
+    "  ZCYANBASEFEEDADROIT INTEGER,"
+    "  ZCARRYFLOORMINNOWDRAGON TIMESTAMP,"
+    "  ZIMAGEPENCILOTHERBOTTOM FLOAT,"
+    "  ZXENONFLIGHTPALEAPPLE TIMESTAMP,"
+    "  ZHERRINGJOKEFEATUREHOPEFUL FLOAT,"
+    "  ZCAPYEARLYRIVETBRUSH FLOAT,"
+    "  ZAGEREEDFROGBASKET VARCHAR,"
+    "  ZUSUALBODYHALIBUTDIAMOND VARCHAR,"
+    "  ZFOOTTAPWORDENTRY VARCHAR,"
+    "  ZDISHKEEPBLESTMONITOR FLOAT,"
+    "  ZBROADABLESOLIDCASUAL INTEGER,"
+    "  ZSQUAREGLEEFULCHILDLIGHT INTEGER,"
+    "  ZHOLIDAYHEADPONYDETAIL INTEGER,"
+    "  ZGENERALRESORTSKYOPEN TIMESTAMP,"
+    "  ZGLADSPRAYKIDNEYGUPPY VARCHAR,"
+    "  ZSWIMHEAVYMENTIONKIND BLOB,"
+    "  ZMESSYSULFURDREAMFESTIVE BLOB,"
+    "  ZSKYSKYCLASSICBRIEF VARCHAR,"
+    "  ZDILLASKHOKILEMON FLOAT,"
+    "  ZJUNIORSHOWPRESSNOVA FLOAT,"
+    "  ZSIZETOEAWARDFRESH TIMESTAMP,"
+    "  ZKEYFAILAPRICOTMETAL VARCHAR,"
+    "  ZHANDYREPAIRPROTONAIRPORT VARCHAR,"
+    "  ZPOSTPROTEINHANDLEACTOR BLOB"
+    ");"
+  );
+  speedtest1_prepare(
+    "INSERT INTO ZLOOKSLIKECOREDATA(ZPK,ZAIRFORGIVEHEADFROG,"
+    "ZGIFTICEFISHGLUEHAIR,ZDELAYOUTCOMEHORNAGENCY,ZSLEEPYUSERGRANDBOWL,"
+    "ZGLASSRESERVEBARIUMMEAL,ZBRIEFGOBYDODGERHEIGHT,"
+    "ZBAMBOOMESSWASABIEVENING,ZFARMERMORNINGMIRRORCONCERN,"
+    "ZTREATPACKFUTURECONVERT,ZCAUSESALAMITERMCYAN,ZCALMRACCOONPROGRAMDEBIT,"
+    "ZHOLIDAYHEADPONYDETAIL,ZWOODPOETRYCOBBLERBENCH,ZHAFNIUMSCRIPTSALADMOTOR,"
+    "ZUSUALBODYHALIBUTDIAMOND,ZOUTSIDEPEAHENCOUNTICE,ZDIVERPAINTLEATHEREASY,"
+    "ZWESTAMOUNTAFFECTHEARING,ZSIZETOEAWARDFRESH,ZDEWPEACHCAREERCELERY,"
+    "ZSTEELCAREFULPLATENUMBER,ZCYANBASEFEEDADROIT,ZCALMLYGEMFINISHEFFECT,"
+    "ZHANDYREPAIRPROTONAIRPORT,ZGENENATURALHEARINGKITE,ZBROADABLESOLIDCASUAL,"
+    "ZPOSTPROTEINHANDLEACTOR,ZLACEADDRESSGROUNDCAREFUL,ZIMAGEPENCILOTHERBOTTOM,"
+    "ZPROBLEMCLUBPOPOVERJELLY,ZPATTERNCLORINEGRANDCOLBY,ZNEATSTEWPARTIRON,"
+    "ZAPPEALSIMPLESECONDHOUSING,ZMOVEWHOGAMMAINCH,ZTENNISCYCLEBILLOFFICER,"
+    "ZSHARKJUSTFRUITMOVIE,ZKEYFAILAPRICOTMETAL,ZCOMPANYSUMMERFIBERELF,"
+    "ZTERMFITTINGHOUSINGCOMMAND,ZRESORTYARDGREENLET,ZCABBAGESOCKEASEMINUTE,"
+    "ZSQUAREGLEEFULCHILDLIGHT,ZONERELEASEAVERAGENURSE,ZBIGTHINKCONVERTECONOMY,"
+    "ZPLIERSPRINTASKOREGANO,ZDECADEJOYOUSWAVEHABIT,ZDRYWALLBEYONDBROWNBOWL,"
+    "ZCLUBRELEASELIZARDADVICE,ZWHALEMATHAVOCADOCOPPER,ZBELLYCRASHITEMLACK,"
+    "ZLETTUCEBIRDMEETDEBATE,ZCAPABLETRIPDOORALMOND,ZRADIANTWHENTRYCARD,"
+    "ZCAPYEARLYRIVETBRUSH,ZAGEREEDFROGBASKET,ZSWIMHEAVYMENTIONKIND,"
+    "ZTRAVELDRIVERCONTESTLILY,ZGLADSPRAYKIDNEYGUPPY,ZBANKBUFFALORECOVERORBIT,"
+    "ZFINGERDUEPIZZAOPTION,ZCLAMBITARUGULAFAJITA,ZLONGFINLEAVEIMAGEOIL,"
+    "ZLONGDIETESSAYNATURE,ZJUNIORSHOWPRESSNOVA,ZHOPEFULGATEHOLECHALK,"
+    "ZDEPOSITPAIRCOLLEGECOMET,ZWEALTHLINENGLEEFULDAY,ZFILLSTOPLAWJOYFUL,"
+    "ZTUNEGASBUFFALOCAPITAL,ZGRAYSURVEYWIRELOVE,ZCORNERANCHORTAPEDIVER,"
+    "ZREWARDINSIDEMANGOINTENSE,ZCADETBRIGHTPLANETBANK,ZPLANETFAMILYPUREMEMORY,"
+    "ZTREATTESTQUILLCHARGE,ZCREAMEVENINGLIPBRANCH,ZSKYSKYCLASSICBRIEF,"
+    "ZARSENICSAMPLEWAITMUON,ZBROWBALANCEKEYCHOWDER,ZFLYINGDOCTORTABLEMELODY,"
+    "ZHANGERLITHIUMDINNERMEET,ZNOTICEPEARPOLICYJUICE,ZSHINYASSISTLIVINGCRAB,"
+    "ZLIFEUSELEAFYBELL,ZFACEINVITETALKGOLD,ZGENERALRESORTSKYOPEN,"
+    "ZPURECAKEVIVIDNEATLY,ZKIWIVISUALPRIDEAPPLE,ZMESSYSULFURDREAMFESTIVE,"
+    "ZCHARGECLICKHUMANEHIRE,ZHERRINGJOKEFEATUREHOPEFUL,ZYARDOREGANOVIVIDJEWEL,"
+    "ZFOOTTAPWORDENTRY,ZWISHHITSKINMOTOR,ZBASEGOUDAREGULARFORGIVE,"
+    "ZMUFFINDRYERDRAWFORTUNE,ZACTIONRANGEELEGANTNEUTRON,ZTRYFACTKEEPMILK,"
+    "ZPEACHCOPPERDINNERLAKE,ZFRAMEENTERSIMPLEMOUTH,ZMERRYCRACKTRAINLEADER,"
+    "ZMEMORYREQUESTSOURCEBIG,ZCARRYFLOORMINNOWDRAGON,ZMINORWAYPAPERCLASSY,"
+    "ZDILLASKHOKILEMON,ZRESOLVEWRISTWRAPAPPLE,ZASKCONTACTMONITORFUN,"
+    "ZGIVEVIVIDDIVINEMEANING,ZEIGHTLEADERWORKERMOST,ZMISSREPLYHUMANLIVING,"
+    "ZXENONFLIGHTPALEAPPLE,ZSORTMISTYQUOTECABBAGE,ZEAGLELINEMINEMAIL,"
+    "ZFAMILYVISUALOWNERMATTER,ZSPREADMOTORBISCUITBACON,ZDISHKEEPBLESTMONITOR,"
+    "ZMALLEQUIPTHANKSGLUE,ZGOLDYOUNGINITIALNOSE,ZHUMORSPICESANDKIDNEY)"
+    "VALUES(?1,?26,?20,?93,?8,?33,?3,?81,?28,?60,?18,?47,?109,?29,?30,?104,?86,"
+    "?54,?92,?117,?9,?58,?97,?61,?119,?73,?107,?120,?80,?99,?31,?96,?85,?50,?71,"
+    "?42,?27,?118,?36,?2,?67,?62,?108,?82,?94,?76,?35,?40,?11,?88,?41,?72,?4,"
+    "?83,?102,?103,?112,?77,?111,?22,?13,?34,?15,?23,?116,?7,?5,?90,?57,?56,"
+    "?75,?51,?84,?25,?63,?37,?87,?114,?79,?38,?14,?10,?21,?48,?89,?91,?110,"
+    "?69,?45,?113,?12,?101,?68,?105,?46,?95,?74,?24,?53,?39,?6,?64,?52,?98,"
+    "?65,?115,?49,?70,?59,?32,?44,?100,?55,?66,?16,?19,?106,?43,?17,?78);"
+  );
+  for(i=0; i=3006000
       }else if( strcmp(z,"multithread")==0 ){
         sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
       }else if( strcmp(z,"nomemstat")==0 ){
         sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
+#endif
+#if SQLITE_VERSION_NUMBER>=3007017
+      }else if( strcmp(z, "mmap")==0 ){
+        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
+        mmapSize = integerValue(argv[++i]);
+ #endif
       }else if( strcmp(z,"nosync")==0 ){
         noSync = 1;
       }else if( strcmp(z,"notnull")==0 ){
         g.zNN = "NOT NULL";
-#ifdef SQLITE_ENABLE_RBU
-      }else if( strcmp(z,"rbu")==0 ){
-        sqlite3ota_create_vfs("rbu", 0);
-        sqlite3_vfs_register(sqlite3_vfs_find("rbu"), 1);
-#endif
       }else if( strcmp(z,"pagesize")==0 ){
         if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
         pageSize = integerValue(argv[++i]);
@@ -1300,6 +1735,10 @@ int main(int argc, char **argv){
         i += 2;
       }else if( strcmp(z,"primarykey")==0 ){
         g.zPK = "PRIMARY KEY";
+      }else if( strcmp(z,"repeat")==0 ){
+        if( i>=argc-1 ) fatal_error("missing arguments on %s\n", argv[i]);
+        g.nRepeat = integerValue(argv[i+1]);
+        i += 1;
       }else if( strcmp(z,"reprepare")==0 ){
         g.bReprepare = 1;
       }else if( strcmp(z,"scratch")==0 ){
@@ -1307,10 +1746,12 @@ int main(int argc, char **argv){
         nScratch = integerValue(argv[i+1]);
         szScratch = integerValue(argv[i+2]);
         i += 2;
+#if SQLITE_VERSION_NUMBER>=3006000
       }else if( strcmp(z,"serialized")==0 ){
         sqlite3_config(SQLITE_CONFIG_SERIALIZED);
       }else if( strcmp(z,"singlethread")==0 ){
         sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+#endif
       }else if( strcmp(z,"sqlonly")==0 ){
         g.bSqlOnly = 1;
       }else if( strcmp(z,"shrink-memory")==0 ){
@@ -1320,6 +1761,13 @@ int main(int argc, char **argv){
         g.szTest = integerValue(argv[++i]);
       }else if( strcmp(z,"stats")==0 ){
         showStats = 1;
+      }else if( strcmp(z,"temp")==0 ){
+        if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
+        i++;
+        if( argv[i][0]<'0' || argv[i][0]>'9' || argv[i][1]!=0 ){
+          fatal_error("argument to --temp should be integer between 0 and 9");
+        }
+        g.eTemp = argv[i][0] - '0';
       }else if( strcmp(z,"testset")==0 ){
         if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]);
         zTSet = argv[++i];
@@ -1351,11 +1799,7 @@ int main(int argc, char **argv){
                   argv[i], argv[0]);
     }
   }
-#if 0
-  if( zDbName==0 ){
-    fatal_error(zHelp, argv[0]);
-  }
-#endif
+  if( zDbName!=0 ) unlink(zDbName);
 #if SQLITE_VERSION_NUMBER>=3006001
   if( nHeap>0 ){
     pHeap = malloc( nHeap );
@@ -1379,7 +1823,7 @@ int main(int argc, char **argv){
     rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch);
     if( rc ) fatal_error("scratch configuration failed: %d\n", rc);
   }
-  if( nLook>0 ){
+  if( nLook>=0 ){
     sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0);
   }
 #endif
@@ -1398,7 +1842,12 @@ int main(int argc, char **argv){
 
   /* Set database connection options */
   sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0);
+#ifndef SQLITE_OMIT_DEPRECATED
   if( doTrace ) sqlite3_trace(g.db, traceCallback, 0);
+#endif
+  if( mmapSize>0 ){
+    speedtest1_exec("PRAGMA mmap_size=%d", mmapSize);
+  }
   speedtest1_exec("PRAGMA threads=%d", nThread);
   if( zKey ){
     speedtest1_exec("PRAGMA key('%s')", zKey);
@@ -1430,6 +1879,8 @@ int main(int argc, char **argv){
     testset_main();
   }else if( strcmp(zTSet,"debug1")==0 ){
     testset_debug1();
+  }else if( strcmp(zTSet,"orm")==0 ){
+    testset_orm();
   }else if( strcmp(zTSet,"cte")==0 ){
     testset_cte();
   }else if( strcmp(zTSet,"rtree")==0 ){
diff --git a/test/sqldiff1.test b/test/sqldiff1.test
index 3201fb3654..ea4e1f9993 100644
--- a/test/sqldiff1.test
+++ b/test/sqldiff1.test
@@ -14,16 +14,8 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-if {$tcl_platform(platform)=="windows"} {
-  set PROG "sqldiff.exe"
-} else {
-  set PROG "./sqldiff"
-}
-if {![file exe $PROG]} {
-  puts "sqldiff cannot run because $PROG is not available"
-  finish_test
-  return
-}
+set PROG [test_find_sqldiff]
+
 db close
 forcedelete test.db test2.db
 sqlite3 db test.db
diff --git a/test/stat.test b/test/stat.test
index 95586e94bc..66ca5e2f2b 100644
--- a/test/stat.test
+++ b/test/stat.test
@@ -32,12 +32,28 @@ db func a_string a_string
 
 register_dbstat_vtab db
 do_execsql_test stat-0.0 {
+  PRAGMA table_info(dbstat);
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+
+# Attempts to drop an eponymous virtual table are a no-op.
+do_execsql_test stat-0.1 {
+  DROP TABLE dbstat;
+  PRAGMA table_info=dbstat;
+} {/0 name TEXT .* 1 path TEXT .* 9 pgsize INTEGER/}
+
+db close
+forcedelete test.db
+sqlite3 db test.db
+db func a_string a_string
+register_dbstat_vtab db
+do_execsql_test stat-0.2 {
   PRAGMA auto_vacuum = OFF;
   CREATE VIRTUAL TABLE temp.stat USING dbstat;
   SELECT * FROM stat;
 } {}
 
-ifcapable wal {
+
+if {[wal_is_capable]} {
   do_execsql_test stat-0.1 {
     PRAGMA journal_mode = WAL;
     PRAGMA journal_mode = delete;
diff --git a/test/subselect.test b/test/subselect.test
index 247f68ee80..4d0efde47a 100644
--- a/test/subselect.test
+++ b/test/subselect.test
@@ -40,7 +40,7 @@ do_test subselect-1.1 {
 do_test subselect-1.2 {
   set v [catch {execsql {SELECT * FROM t1 WHERE a = (SELECT * FROM t1)}} msg]
   lappend v $msg
-} {1 {only a single result allowed for a SELECT that is part of an expression}}
+} {1 {row value misused}}
 
 # A subselect without an aggregate.
 #
diff --git a/test/sync.test b/test/sync.test
index 2ee636a266..210039acb5 100644
--- a/test/sync.test
+++ b/test/sync.test
@@ -81,6 +81,7 @@ do_test sync-1.3 {
   set sqlite_sync_count
 } 11
 ifcapable pager_pragmas {
+if {[permutation]!="journaltest"} {
   do_test sync-1.4 {
     set sqlite_sync_count 0
     execsql {
@@ -94,6 +95,7 @@ ifcapable pager_pragmas {
     set sqlite_sync_count
   } 0
 }
+}
 
 
 finish_test
diff --git a/test/sync2.test b/test/sync2.test
new file mode 100644
index 0000000000..4f6ed7754c
--- /dev/null
+++ b/test/sync2.test
@@ -0,0 +1,160 @@
+# 2017 March 16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+# Specificly, it tests that "PRAGMA synchronous" appears to work.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix sync2
+
+#
+# These tests are only applicable when pager pragma are
+# enabled. Also, since every test uses an ATTACHed database, they
+# are only run when ATTACH is enabled.
+#
+ifcapable !pager_pragmas||!attach {
+  finish_test
+  return
+}
+if {$::tcl_platform(platform)!="unix" 
+  || [permutation] == "journaltest"
+  || [permutation] == "inmemory_journal"
+} {
+  finish_test
+  return
+}
+
+proc execsql_sync {sql} {
+  set s $::sqlite_sync_count
+  set res [execsql $sql]
+  concat [expr $::sqlite_sync_count-$s] $res
+}
+
+proc do_execsql_sync_test {tn sql res} {
+  uplevel [list do_test $tn [list execsql_sync $sql] [list {*}$res]]
+}
+
+#-----------------------------------------------------------------------
+# Tests for journal mode.
+#
+sqlite3 db test.db
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+do_execsql_sync_test 1.1 { INSERT INTO t1 VALUES(3, 4) } 4
+
+# synchronous=normal. So, 1 sync on the directory, 1 on the journal, 1 
+# on the db file. 3 in total.
+do_execsql_test      1.2.1 { PRAGMA main.synchronous = NORMAL }
+do_execsql_test      1.2.2 { PRAGMA main.synchronous }     1
+do_execsql_sync_test 1.2.3 { INSERT INTO t1 VALUES(5, 6) } 3
+
+# synchronous=off. No syncs.
+do_execsql_test      1.3.1 { PRAGMA main.synchronous = OFF }
+do_execsql_test      1.3.2 { PRAGMA main.synchronous }     0
+do_execsql_sync_test 1.3.3 { INSERT INTO t1 VALUES(7, 8) } 0
+ 
+# synchronous=full, journal_mode=delete. So, 1 sync on the directory,
+# 2 on the journal, 1 on the db file. 4 in total.
+do_execsql_test      1.4.1 { PRAGMA main.synchronous = FULL }
+do_execsql_test      1.4.2 { PRAGMA main.synchronous }      2
+do_execsql_sync_test 1.4.3 { INSERT INTO t1 VALUES(9, 10) } 4
+
+#-----------------------------------------------------------------------
+# Tests for wal mode.
+#
+do_execsql_test      1.5 { PRAGMA journal_mode = wal } {wal}
+
+# sync=full, journal_mode=wal. One sync on the directory, two on the 
+# wal file.
+do_execsql_sync_test 1.6 { INSERT INTO t1 VALUES(11, 12) } 3
+
+# One sync on the wal file.
+do_execsql_sync_test 1.7 { INSERT INTO t1 VALUES(13, 14) } 1
+
+# No syncs.
+do_execsql_test      1.8.1 { PRAGMA main.synchronous = NORMAL }
+do_execsql_test      1.8.2 { PRAGMA main.synchronous }          1
+do_execsql_sync_test 1.8.3 { INSERT INTO t1 VALUES(15, 16) }    0
+
+# One sync on wal file, one on the db file.
+do_execsql_sync_test 1.9   { PRAGMA wal_checkpoint }  {2 0 3 3}
+
+# No syncs.
+do_execsql_test      1.10.1 { PRAGMA main.synchronous = OFF }
+do_execsql_test      1.10.2 { PRAGMA main.synchronous }          0
+do_execsql_sync_test 1.10.3 { INSERT INTO t1 VALUES(17, 18) }    0
+
+#-----------------------------------------------------------------------
+# Tests for the compile time settings SQLITE_DEFAULT_SYNCHRONOUS and
+# SQLITE_DEFAULT_WAL_SYNCHRONOUS. These tests only run if the former
+# is set to "2" and the latter to "1". This is not the default, but
+# it is currently the recommended configuration.
+#
+#   https://sqlite.org/compile.html#recommended_compile_time_options
+#
+if {$SQLITE_DEFAULT_SYNCHRONOUS==2 && $SQLITE_DEFAULT_WAL_SYNCHRONOUS==1} {
+
+  db close
+  sqlite3 db test.db
+
+  # Wal mode, sync=normal. The first transaction does one sync on directory,
+  # one on the wal file. The second does no syncs.
+  do_execsql_sync_test 1.11.1 { INSERT INTO t1 VALUES(19, 20) } 2
+  do_execsql_sync_test 1.11.2 { INSERT INTO t1 VALUES(21, 22) } 0
+  do_execsql_test 1.11.3      { PRAGMA main.synchronous }       1
+
+  # One sync on wal file, one on the db file.
+  do_execsql_sync_test 1.12   { PRAGMA wal_checkpoint }  {2 0 2 2}
+
+  # First transaction syncs the wal file once, the second not at all.
+  # one on the wal file. The second does no syncs.
+  do_execsql_sync_test 1.13.1 { INSERT INTO t1 VALUES(22, 23) } 1
+  do_execsql_sync_test 1.13.2 { INSERT INTO t1 VALUES(24, 25) } 0
+
+  do_execsql_test 1.14 { PRAGMA journal_mode = delete } {delete}
+  
+  # Delete mode, sync=full. The first transaction does one sync on 
+  # directory, two on the journal file, one on the db. The second does 
+  # the same.
+  do_execsql_sync_test 1.15.1 { INSERT INTO t1 VALUES(26, 27) } 4
+  do_execsql_sync_test 1.15.2 { INSERT INTO t1 VALUES(28, 29) } 4
+  do_execsql_test 1.15.3      { PRAGMA main.synchronous }       2
+
+  # Switch back to wal mode.
+  do_execsql_test 1.16 { PRAGMA journal_mode = wal } {wal}
+
+  do_execsql_sync_test 1.17.1 { INSERT INTO t1 VALUES(30, 31) } 2
+  do_execsql_sync_test 1.17.2 { INSERT INTO t1 VALUES(32, 33) } 0
+  do_execsql_test 1.17.3      { PRAGMA main.synchronous }       1
+
+  # Now set synchronous=off, then switch back to delete mode. Check
+  # that the db handle is still using synchronous=off.
+  do_execsql_test 1.18.3      { PRAGMA main.synchronous=off }
+  do_execsql_test 1.18 { PRAGMA journal_mode = delete } {delete}
+
+  do_execsql_sync_test 1.19.1 { INSERT INTO t1 VALUES(34, 35) } 0
+  do_execsql_sync_test 1.19.2 { INSERT INTO t1 VALUES(36, 37) } 0
+  do_execsql_test 1.19.3      { PRAGMA main.synchronous }       0
+
+  # Close and reopen the db. Back to synchronous=normal.
+  db close
+  sqlite3 db test.db
+  do_execsql_sync_test 1.20.1 { INSERT INTO t1 VALUES(38, 39) } 4
+  do_execsql_sync_test 1.20.2 { INSERT INTO t1 VALUES(40, 41) } 4
+  do_execsql_test 1.20.3      { PRAGMA main.synchronous }       2
+}
+
+finish_test
diff --git a/test/tabfunc01.test b/test/tabfunc01.test
index 879a045b8e..dcaafa420c 100644
--- a/test/tabfunc01.test
+++ b/test/tabfunc01.test
@@ -22,6 +22,8 @@ ifcapable !vtab {
   return
 }
 load_static_extension db series
+load_static_extension db carray
+load_static_extension db remember
 
 do_execsql_test tabfunc01-1.1 {
   SELECT *, '|' FROM generate_series WHERE start=1 AND stop=9 AND step=2;
@@ -135,4 +137,82 @@ do_execsql_test tabfunc01-500 {
   ORDER BY +1;
 } {1 7 11 17}
 
+# Table-valued functions on the RHS of an IN operator
+#
+do_execsql_test tabfunc01-600 {
+  CREATE TABLE t600(a INTEGER PRIMARY KEY, b TEXT);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+    INSERT INTO t600(a,b) SELECT x, printf('(%03d)',x) FROM c;
+  SELECT b FROM t600 WHERE a IN generate_series(2,52,10);
+} {(002) (012) (022) (032) (042) (052)}
+
+
+do_test tabfunc01-700 {
+  set PTR1 [intarray_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR1,5) WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-701 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int32');
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-702 {
+  db eval {
+    SELECT b FROM t600 WHERE a IN carray($PTR1,4,'int32');
+  }
+} {(005) (007) (013) (017)}
+do_catchsql_test tabfunc01-710 {
+  SELECT b FROM t600 WHERE a IN carray($PTR1,5,'int33');
+} {1 {unknown datatype: 'int33'}}
+
+do_test tabfunc01-720 {
+  set PTR2 [int64array_addr 5 7 13 17 23]
+  db eval {
+    SELECT b FROM t600, carray($PTR2,5,'int64') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+do_test tabfunc01-721 {
+  db eval {
+    SELECT remember(123,$PTR2);
+    SELECT value FROM carray($PTR2,5,'int64');
+  }
+} {123 123 7 13 17 23}
+do_test tabfunc01-722 {
+  set PTR3 [expr {$PTR2+16}]
+  db eval {
+    SELECT remember(987,$PTR3);
+    SELECT value FROM carray($PTR2,5,'int64');
+  }
+} {987 123 7 987 17 23}
+
+do_test tabfunc01-730 {
+  set PTR4 [doublearray_addr 5.0 7.0 13.0 17.0 23.0]
+  db eval {
+    SELECT b FROM t600, carray($PTR4,5,'double') WHERE a=value;
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-740 {
+  set PTR5 [textarray_addr x5 x7 x13 x17 x23]
+  db eval {
+    SELECT b FROM t600, carray($PTR5,5,'char*') WHERE a=trim(value,'x');
+  }
+} {(005) (007) (013) (017) (023)}
+
+do_test tabfunc01-750 {
+  db eval {
+    SELECT aa.value, bb.value, '|'
+      FROM carray($PTR4,5,'double') AS aa
+      JOIN carray($PTR5,5,'char*') AS bb ON aa.rowid=bb.rowid;
+  }
+} {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |}
+
+# Free up memory allocations
+intarray_addr
+int64array_addr
+doublearray_addr
+textarray_addr
+
 finish_test
diff --git a/test/tclsqlite.test b/test/tclsqlite.test
index d5c078de15..bdb0fc56d1 100644
--- a/test/tclsqlite.test
+++ b/test/tclsqlite.test
@@ -34,7 +34,7 @@ do_test tcl-1.1 {
 do_test tcl-1.2 {
   set v [catch {db bogus} msg]
   lappend v $msg
-} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, version, or wal_hook}}
+} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}}
 do_test tcl-1.2.1 {
   set v [catch {db cache bogus} msg]
   lappend v $msg
@@ -635,6 +635,39 @@ do_test tcl-14.2 {
   db one {SELECT x FROM t6 WHERE xCall()!='value'}
 } {}
 
+# Verify that the "exists" and "onecolumn" methods work when
+# a "profile" is registered.
+#
+catch {db close}
+sqlite3 db :memory:
+proc noop-profile {args} {
+  return
+}
+do_test tcl-15.0 {
+  db eval {CREATE TABLE t1(a); INSERT INTO t1 VALUES(1),(2),(3);}
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.1 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.2 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+db profile noop-profile
+do_test tcl-15.3 {
+  db onecolumn {SELECT a FROM t1 WHERE a>2}
+} {3}
+do_test tcl-15.4 {
+  db exists {SELECT a FROM t1 WHERE a>2}
+} {1}
+do_test tcl-15.5 {
+  db exists {SELECT a FROM t1 WHERE a>3}
+} {0}
+
+
+
+
+
 
 
 finish_test
diff --git a/test/tempdb2.test b/test/tempdb2.test
new file mode 100644
index 0000000000..5f019bd117
--- /dev/null
+++ b/test/tempdb2.test
@@ -0,0 +1,76 @@
+# 2016 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix tempdb2
+
+db close
+sqlite3 db ""
+
+proc int2str {i} { string range [string repeat "$i." 450] 0 899 }
+db func int2str int2str
+
+#-------------------------------------------------------------------------
+#
+#  1.1: Write a big transaction to the db. One so large that it forces
+#       the file to be created and the cache flushed to disk on COMMIT.
+#
+#  1.2: Write a small transaction - one small enough that it remains in
+#       memory on COMMIT. All the pages of table [t1] are now dirty.
+#
+#  1.3: Delete the contents of [t1]. This moves all of its leaves to the
+#       free-list and causes the btree layer to call PagerDontWrite() on
+#       each of them.
+#
+#       Then do a big update on table [t2]. So big that the former leaves
+#       of [t1] are forced out of the cache. Then roll back the transaction.
+#       If the PagerDontWrite() calls are honoured and the data is not written
+#       to disk, the update made in test 1.2 will be lost at this point. Or, if
+#       they are ignored (as they should be for temp databases), the update
+#       will be safely written out to disk before the cache entries are
+#       discarded.
+#
+do_execsql_test 1.1 {
+  PRAGMA page_size=1024;
+  PRAGMA cache_size=50;
+
+  BEGIN;
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+    INSERT INTO t1 VALUES(1, int2str(1));
+    INSERT INTO t1 VALUES(2, int2str(1));
+    INSERT INTO t1 VALUES(3, int2str(1));
+
+    CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+    WITH c(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100 ) 
+    INSERT INTO t2 SELECT x, int2str(x) FROM c;
+  COMMIT;
+
+  PRAGMA lock_status;
+} {main unlocked temp closed}
+
+do_execsql_test 1.2 {
+  UPDATE t1 SET b=int2str(2);
+  SELECT b=int2str(2) FROM t1
+} {1 1 1}
+
+do_execsql_test 1.3 {
+  BEGIN;
+    DELETE FROM t1;
+    UPDATE t2 SET b=int2str(a+1);
+  ROLLBACK;
+}
+
+do_execsql_test 1.4 {
+  SELECT b=int2str(2) FROM t1
+} {1 1 1}
+
+finish_test
diff --git a/test/tempfault.test b/test/tempfault.test
new file mode 100644
index 0000000000..4eae116a72
--- /dev/null
+++ b/test/tempfault.test
@@ -0,0 +1,134 @@
+# 2016 April 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for fault-injection when SQLite is used with
+# a temp file database.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set testprefix tempfault
+
+# sqlite3_memdebug_vfs_oom_test 0
+
+do_faultsim_test 1 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    CREATE TABLE t1(a, b);
+    INSERT INTO t1 VALUES(1, 2);
+    INSERT INTO t1 VALUES(3, 4);
+  }
+} -body {
+  execsql { INSERT INTO t1 VALUES(5, 6) }
+} -test {
+  faultsim_test_result {0 {}}
+  set rc [catch { execsql { SELECT * FROM t1 } } msg]
+  if {$rc==0 && $msg != "1 2 3 4 5 6" && $msg != "1 2 3 4"} {
+    error "data mismatch 1: $msg"
+  }
+  if {$testrc==0 && $msg != "1 2 3 4 5 6"} {
+    error "data mismatch 2: $msg"
+  }
+  faultsim_integrity_check
+}
+
+do_faultsim_test 2 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { UPDATE t1 SET a = randomblob(99) }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+catch { db close }
+do_faultsim_test 2.1 -faults * -prep {
+  if {[info commands db]==""} {
+    sqlite3 db ""
+    execsql {
+      PRAGMA page_size = 1024;
+      PRAGMA cache_size = 10;
+      CREATE TABLE t1(a, b);
+      CREATE INDEX i1 ON t1(b, a);
+      WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100)
+          INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+    }
+  } 
+} -body {
+  execsql { UPDATE t1 SET a = randomblob(99) }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+do_faultsim_test 3 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { 
+    BEGIN;
+      UPDATE t1 SET a = randomblob(99);
+      SAVEPOINT abc;
+        UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0;
+      ROLLBACK TO abc;
+        UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0;
+      ROLLBACK TO abc;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+  faultsim_integrity_check db
+}
+
+do_faultsim_test 4 -faults * -prep {
+  sqlite3 db ""
+  db eval {
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(b, a);
+    WITH x(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<50)
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  }
+} -body {
+  execsql { 
+    BEGIN;
+      UPDATE t1 SET a = randomblob(99);
+      SAVEPOINT abc;
+        UPDATE t1 SET a = randomblob(98) WHERE (rowid%10)==0;
+      ROLLBACK TO abc;
+        UPDATE t1 SET a = randomblob(97) WHERE (rowid%5)==0;
+      ROLLBACK TO abc;
+    COMMIT;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+sqlite3_memdebug_vfs_oom_test 1
+finish_test
diff --git a/test/temptable2.test b/test/temptable2.test
new file mode 100644
index 0000000000..ffa69b6150
--- /dev/null
+++ b/test/temptable2.test
@@ -0,0 +1,357 @@
+# 2016 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix temptable2
+
+do_execsql_test 1.1 {
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+}
+
+do_execsql_test 1.2 {
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100000 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM X;
+} {}
+
+do_execsql_test 1.3 {
+  PRAGMA temp.integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.1 {
+  CREATE TEMP TABLE t2(a, b);
+  INSERT INTO t2 VALUES(1, 2);
+} {}
+
+do_execsql_test 2.2 {
+  BEGIN;
+    INSERT INTO t2 VALUES(3, 4);
+    SELECT * FROM t2;
+} {1 2 3 4}
+
+do_execsql_test 2.3 {
+  ROLLBACK;
+  SELECT * FROM t2;
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t1;
+} {1000}
+do_execsql_test 3.1.2 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0;
+  ROLLBACK;
+}
+do_execsql_test 3.1.3 {
+  SELECT count(*) FROM t1;
+} {1000}
+do_execsql_test 3.1.4 { PRAGMA temp.integrity_check } {ok}
+
+do_execsql_test 3.2.1 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==0;
+    SAVEPOINT abc;
+      UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==1;
+    ROLLBACK TO abc;
+    UPDATE t1 SET b=randomblob(100) WHERE (rowid%10)==2;
+  COMMIT;
+}
+do_execsql_test 3.2.2 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<10 )
+  INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t1;
+  PRAGMA temp.page_count;
+} {10 9}
+
+do_execsql_test 4.1.2 {
+  BEGIN;
+    UPDATE t1 SET b=randomblob(100);
+  ROLLBACK;
+}
+
+do_execsql_test 4.1.3 {
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+
+  SELECT count(*) FROM t2;
+  SELECT count(*) FROM t1;
+} {500 10}
+
+do_test 4.1.4 {
+  set n [db one { PRAGMA temp.page_count }]
+  expr ($n >280 && $n < 300) 
+} 1
+
+do_execsql_test 4.1.4 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 5.1.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+
+  CREATE TEMP TABLE t1(a, b);
+  CREATE INDEX i1 ON t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+}
+
+# Test that the temp database is now much bigger than the configured
+# cache size (10 pages).
+do_test 5.1.2 {
+  set n [db one { PRAGMA temp.page_count }]
+  expr ($n > 270 && $n < 290)
+} {1}
+
+do_execsql_test 5.1.3 {
+  BEGIN;
+    UPDATE t1 SET a=2;
+    UPDATE t2 SET a=randomblob(100);
+    SELECT count(*) FROM t1;
+  ROLLBACK;
+} {1}
+
+do_execsql_test 5.1.4 {
+  UPDATE t2 SET a=randomblob(100);
+
+  SELECT * FROM t1;
+} {1 2}
+
+do_execsql_test 5.1.5 { PRAGMA temp.integrity_check } {ok}
+
+#-------------------------------------------------------------------------
+# Test this:
+#
+#   1. Page is DIRTY at the start of a transaction.
+#   2. Page is written out as part of the transaction.
+#   3. Page is then read back in.
+#   4. Transaction is rolled back. Is the page now clean or dirty?
+#
+# This actually does work. Step 4 marks the page as clean. But it also
+# writes to the database file itself. So marking it clean is correct - 
+# the page does match the contents of the db file.
+#
+reset_db
+
+do_execsql_test 6.1 {
+  PRAGMA main.cache_size = 10;
+  PRAGMA temp.cache_size = 10;
+
+  CREATE TEMP TABLE t1(x);
+  INSERT INTO t1 VALUES('one');
+
+  CREATE TEMP TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x;
+}
+
+do_execsql_test 6.2 {
+  UPDATE t1 SET x='two';             -- step 1
+  BEGIN;
+    UPDATE t2 SET a=randomblob(100); -- step 2
+    SELECT * FROM t1;                -- step 3
+  ROLLBACK;                          -- step 4
+
+  SELECT count(*) FROM t2;
+  SELECT * FROM t1;
+} {two 500 two}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3 db ""
+do_execsql_test 7.1 {
+  PRAGMA auto_vacuum=INCREMENTAL;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(zeroblob(900));
+  INSERT INTO t1 VALUES(zeroblob(900));
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1 SELECT x FROM t1;
+  BEGIN;
+  DELETE FROM t1 WHERE rowid%2;
+  PRAGMA incremental_vacuum(4);
+  ROLLBACK;
+  PRAGMA integrity_check;
+} {ok}
+
+#-------------------------------------------------------------------------
+# Try changing the page size using a backup operation when pages are
+# stored in main-memory only.
+#
+reset_db
+do_execsql_test 8.1 {
+  PRAGMA auto_vacuum = OFF;
+  CREATE TABLE t2(a, b);
+  CREATE INDEX i2 ON t2(a, b);
+  WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<20 )
+  INSERT INTO t2 SELECT randomblob(100), randomblob(100) FROM x ORDER BY 1, 2;
+  PRAGMA page_count;
+} {13}
+
+do_test 8.2 {
+  sqlite3 tmp ""
+  execsql {
+    PRAGMA auto_vacuum = OFF;
+    PRAGMA page_size = 8192;
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(a, b);
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<100 )
+    INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x ORDER BY 1, 2;
+    PRAGMA page_count;
+  } tmp
+} {10}
+
+do_test 8.3 {
+  sqlite3_backup B tmp main db main
+  B step 5
+  B finish
+} {SQLITE_READONLY}
+
+do_test 8.4 {
+  execsql {
+    SELECT count(*) FROM t1;
+    PRAGMA integrity_check;
+    PRAGMA page_size;
+  } tmp
+} {100 ok 8192}
+
+do_test 8.5 { 
+  tmp eval { UPDATE t1 SET a=randomblob(100) }
+} {}
+
+do_test 8.6 {
+  sqlite3_backup B tmp main db main
+  B step 1000
+  B finish
+} {SQLITE_READONLY}
+
+tmp close
+
+#-------------------------------------------------------------------------
+# Try inserts and deletes with a large db in auto-vacuum mode. Check
+#
+foreach {tn mode} {
+  1 delete
+  2 wal
+} {
+  reset_db
+  sqlite3 db ""
+  do_execsql_test 9.$tn.1.1 {
+    PRAGMA cache_size = 15;
+    PRAGMA auto_vacuum = 1;
+  }
+  execsql "PRAGMA journal_mode = $mode"
+
+  do_execsql_test 9.$tn.1.2 {
+    CREATE TABLE tx(a, b);
+    CREATE INDEX i1 ON tx(a);
+    CREATE INDEX i2 ON tx(b);
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000 )
+      INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+  }
+
+  for {set i 2} {$i<20} {incr i} {
+    do_execsql_test 9.$tn.$i.1 { DELETE FROM tx WHERE (random()%3)==0 }
+
+    do_execsql_test 9.$tn.$i.2 { PRAGMA integrity_check } ok
+
+      do_execsql_test 9.$tn.$i.3 { 
+        WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<400 )
+          INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+      }
+
+    do_execsql_test 9.$tn.$i.4 { PRAGMA integrity_check } ok
+
+    do_execsql_test 9.$tn.$i.5 { 
+      BEGIN;
+      DELETE FROM tx WHERE (random()%3)==0;
+      WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+        INSERT INTO tx SELECT randomblob(100), randomblob(100) FROM x;
+      COMMIT;
+    }
+
+    do_execsql_test 9.$tn.$i.6 { PRAGMA integrity_check } ok
+  }
+}
+
+#-------------------------------------------------------------------------
+# When using mmap mode with a temp file, SQLite must search the cache 
+# before using a mapped page even when there is no write transaction
+# open. For a temp file, the on-disk version may not be up to date.
+#
+sqlite3 db ""
+do_execsql_test 10.0 {
+  PRAGMA cache_size = 50;
+  PRAGMA page_size = 1024;
+  CREATE TABLE t1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  CREATE INDEX i1 ON t1(a);
+  CREATE TABLE t2(x, y);
+  INSERT INTO t2 VALUES(1, 2);
+}
+
+do_execsql_test 10.1 {
+  BEGIN;
+    WITH x(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<500 )
+      INSERT INTO t1 SELECT randomblob(100), randomblob(100) FROM x;
+  COMMIT;
+  INSERT INTO t2 VALUES(3, 4);
+}
+
+ifcapable mmap {
+  if {[permutation]!="journaltest"} {
+    # The journaltest permutation does not support mmap, so this part of
+    # the test is omitted.
+    do_execsql_test 10.2 { PRAGMA mmap_size = 512000 } 512000
+  }
+}
+
+do_execsql_test 10.3 { SELECT * FROM t2 } {1 2 3 4}
+do_execsql_test 10.4 { PRAGMA integrity_check } ok
+
+finish_test
diff --git a/test/temptable3.test b/test/temptable3.test
new file mode 100644
index 0000000000..16f51d7832
--- /dev/null
+++ b/test/temptable3.test
@@ -0,0 +1,41 @@
+# 2016-05-10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix temptable3
+
+db close
+sqlite3 db {}
+do_execsql_test 1.1 {
+  PRAGMA cache_size = 1;
+  PRAGMA page_size = 1024;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES( randomblob(800) );
+  INSERT INTO t1 VALUES( randomblob(800) );
+  CREATE TABLE t2(x);
+  PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db {}
+do_execsql_test 1.2 {
+  PRAGMA cache_size = 1;
+  PRAGMA auto_vacuum = 2;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(x UNIQUE);
+  INSERT INTO t2 VALUES(1), (2), (3);
+  DROP TABLE t1;
+  PRAGMA integrity_check;
+} {ok}
+
+finish_test
diff --git a/test/temptrigger.test b/test/temptrigger.test
index 551c620479..e4277adf65 100644
--- a/test/temptrigger.test
+++ b/test/temptrigger.test
@@ -236,7 +236,7 @@ do_test 5.1 {
 
 do_execsql_test 5.2 {
   SELECT * FROM sqlite_master;
-  SELECT * FROM sqlite_temp_master;
+  SELECT * FROM temp.sqlite_master;
 } {
   trigger tr1 t1 0 
   {CREATE TRIGGER tr1 BEFORE INSERT ON t1 BEGIN SELECT 1,2,3; END}
diff --git a/test/tester.tcl b/test/tester.tcl
index 3a881138e2..dc6547d033 100644
--- a/test/tester.tcl
+++ b/test/tester.tcl
@@ -25,6 +25,7 @@
 #      copy_file              FROM TO
 #      delete_file            FILENAME
 #      drop_all_tables        ?DB?
+#      drop_all_indexes       ?DB?
 #      forcecopy              FROM TO
 #      forcedelete            FILENAME
 #
@@ -373,6 +374,7 @@ proc do_not_use_codec {} {
   set ::do_not_use_codec 1
   reset_db
 }
+unset -nocomplain do_not_use_codec
 
 # Return true if the "reserved_bytes" integer on database files is non-zero.
 #
@@ -459,7 +461,7 @@ if {[info exists cmdlinearg]==0} {
       }
       {^-+backtrace=.+$} {
         foreach {dummy cmdlinearg(backtrace)} [split $a =] break
-        sqlite3_memdebug_backtrace $value
+        sqlite3_memdebug_backtrace $cmdlinearg(backtrace)
       }
       {^-+binarylog=.+$} {
         foreach {dummy cmdlinearg(binarylog)} [split $a =] break
@@ -521,7 +523,11 @@ if {[info exists cmdlinearg]==0} {
       }
 
       default {
-        lappend leftover [file normalize $a]
+        if {[file tail $a]==$a} {
+          lappend leftover $a
+        } else {
+          lappend leftover [file normalize $a]
+        }
       }
     }
   }
@@ -716,6 +722,17 @@ proc puts {args} { uplevel puts_override $args }
 
 # Invoke the do_test procedure to run a single test
 #
+# The $expected parameter is the expected result.  The result is the return
+# value from the last TCL command in $cmd.
+#
+# Normally, $expected must match exactly.  But if $expected is of the form
+# "/regexp/" then regular expression matching is used.  If $expected is
+# "~/regexp/" then the regular expression must NOT match.  If $expected is
+# of the form "#/value-list/" then each term in value-list must be numeric
+# and must approximately match the corresponding numeric term in $result.
+# Values must match within 10%.  Or if the $expected term is A..B then the
+# $result term must be in between A and B.
+#
 proc do_test {name cmd expected} {
   global argv cmdlinearg
 
@@ -749,7 +766,7 @@ proc do_test {name cmd expected} {
       output2 "\nError: $result"
       fail_test $name
     } else {
-      if {[regexp {^~?/.*/$} $expected]} {
+      if {[regexp {^[~#]?/.*/$} $expected]} {
         # "expected" is of the form "/PATTERN/" then the result if correct if
         # regular expression PATTERN matches the result.  "~/PATTERN/" means
         # the regular expression must not match.
@@ -763,6 +780,21 @@ proc do_test {name cmd expected} {
             set ok [regexp $re $result]
           }
           set ok [expr {!$ok}]
+        } elseif {[string index $expected 0]=="#"} {
+          # Numeric range value comparison.  Each term of the $result is matched
+          # against one term of $expect.  Both $result and $expected terms must be
+          # numeric.  The values must match within 10%.  Or if $expected is of the
+          # form A..B then the $result term must be between A and B.
+          set e2 [string range $expected 2 end-1]
+          foreach i $result j $e2 {
+            if {[regexp {^(-?\d+)\.\.(-?\d)$} $j all A B]} {
+              set ok [expr {$i+0>=$A && $i+0<=$B}]
+            } else {
+              set ok [expr {$i+0>=0.9*$j && $i+0<=1.1*$j}]
+            }
+            if {!$ok} break
+          }
+          if {$ok && [llength $result]!=[llength $e2]} {set ok 0}
         } else {
           set re [string range $expected 1 end-1]
           if {[string index $re 0]=="*"} {
@@ -881,10 +913,43 @@ proc fix_testname {varname} {
   }
 }
 
-proc do_execsql_test {testname sql {result {}}} {
-  fix_testname testname
-  uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
+proc normalize_list {L} {
+  set L2 [list]
+  foreach l $L {lappend L2 $l}
+  set L2
 }
+
+# Either:
+#
+#   do_execsql_test TESTNAME SQL ?RES?
+#   do_execsql_test -db DB TESTNAME SQL ?RES?
+#
+proc do_execsql_test {args} {
+  set db db
+  if {[lindex $args 0]=="-db"} {
+    set db [lindex $args 1]
+    set args [lrange $args 2 end]
+  }
+
+  if {[llength $args]==2} {
+    foreach {testname sql} $args {}
+    set result ""
+  } elseif {[llength $args]==3} {
+    foreach {testname sql result} $args {}
+  } else {
+    error [string trim {
+      wrong # args: should be "do_execsql_test ?-db DB? testname sql ?result?"
+    }]
+  }
+
+  fix_testname testname
+
+  uplevel do_test                 \
+      [list $testname]            \
+      [list "execsql {$sql} $db"] \
+      [list [list {*}$result]]
+}
+
 proc do_catchsql_test {testname sql result} {
   fix_testname testname
   uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result]
@@ -1252,9 +1317,9 @@ proc explain_i {sql {db db}} {
     set D ""
   }
   foreach opcode {
-      Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind
+      Seek SeekGE SeekGT SeekLE SeekLT NotFound Last Rewind
       NoConflict Next Prev VNext VPrev VFilter
-      SorterSort SorterNext
+      SorterSort SorterNext NextIfOpen
   } {
     set color($opcode) $B
   }
@@ -1275,9 +1340,15 @@ proc explain_i {sql {db db}} {
       set bSeenGoto 1
     }
 
+    if {$opcode=="Once"} {
+      for {set i $addr} {$i<$p2} {incr i} {
+        set star($i) $addr
+      }
+    }
+
     if {$opcode=="Next"  || $opcode=="Prev" 
      || $opcode=="VNext" || $opcode=="VPrev"
-     || $opcode=="SorterNext"
+     || $opcode=="SorterNext" || $opcode=="NextIfOpen"
     } {
       for {set i $p2} {$i<$addr} {incr i} {
         incr x($i) 2
@@ -1301,6 +1372,12 @@ proc explain_i {sql {db db}} {
     }
     set I [string repeat " " $x($addr)]
 
+    if {[info exists star($addr)]} {
+      set ii [expr $x($star($addr))]
+      append I "  "
+      set I [string replace $I $ii $ii *]
+    }
+
     set col ""
     catch { set col $color($opcode) }
 
@@ -1456,6 +1533,7 @@ proc crashsql {args} {
   set tclbody {}
   set crashfile ""
   set dc ""
+  set dfltvfs 0
   set sql [lindex $args end]
 
   for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} {
@@ -1469,7 +1547,8 @@ proc crashsql {args} {
     elseif {$n>1 && [string first $z -file]==0}      {set crashfile $z2}  \
     elseif {$n>1 && [string first $z -tclbody]==0}   {set tclbody $z2}  \
     elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \
-    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \
+    elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" }\
+    elseif {$n>1 && [string first $z -dfltvfs]==0} {set dfltvfs $z2 }\
     else   { error "Unrecognized option: $z" }
   }
 
@@ -1483,7 +1562,7 @@ proc crashsql {args} {
   set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]]
 
   set f [open crash.tcl w]
-  puts $f "sqlite3_crash_enable 1"
+  puts $f "sqlite3_crash_enable 1 $dfltvfs"
   puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile"
   puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte"
 
@@ -1902,6 +1981,16 @@ proc drop_all_tables {{db db}} {
   }
 }
 
+# Drop all auxiliary indexes from the main database opened by handle [db].
+#
+proc drop_all_indexes {{db db}} {
+  set L [$db eval {
+    SELECT name FROM sqlite_master WHERE type='index' AND sql LIKE 'create%'
+  }]
+  foreach idx $L { $db eval "DROP INDEX $idx" }
+}
+
+
 #-------------------------------------------------------------------------
 # If a test script is executed with global variable $::G(perm:name) set to
 # "wal", then the tests are run in WAL mode. Otherwise, they should be run
@@ -1938,6 +2027,12 @@ proc wal_check_journal_mode {testname {db db}} {
   }
 }
 
+proc wal_is_capable {} {
+  ifcapable !wal { return 0 }
+  if {[permutation]=="journaltest"} { return 0 }
+  return 1
+}
+
 proc permutation {} {
   set perm ""
   catch {set perm $::G(perm:name)}
@@ -2136,6 +2231,41 @@ proc test_restore_config_pagecache {} {
   sqlite3 db test.db
 }
 
+proc test_find_binary {nm} {
+  if {$::tcl_platform(platform)=="windows"} {
+    set ret "$nm.exe"
+  } else {
+    set ret $nm
+  }
+  set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]]
+  if {![file executable $ret]} {
+    finish_test
+    return ""
+  }
+  return $ret
+}
+
+# Find the name of the 'shell' executable (e.g. "sqlite3.exe") to use for
+# the tests in shell[1-5].test. If no such executable can be found, invoke
+# [finish_test ; return] in the callers context.
+#
+proc test_find_cli {} {
+  set prog [test_find_binary sqlite3]
+  if {$prog==""} { return -code return }
+  return $prog
+}
+
+# Find the name of the 'sqldiff' executable (e.g. "sqlite3.exe") to use for
+# the tests in sqldiff tests. If no such executable can be found, invoke
+# [finish_test ; return] in the callers context.
+#
+proc test_find_sqldiff {} {
+  set prog [test_find_binary sqldiff]
+  if {$prog==""} { return -code return }
+  return $prog
+}
+
+
 # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set
 # to non-zero, then set the global variable $AUTOVACUUM to 1.
 set AUTOVACUUM $sqlite_options(default_autovacuum)
diff --git a/test/time-wordcount.sh b/test/time-wordcount.sh
new file mode 100644
index 0000000000..df9edc6f2c
--- /dev/null
+++ b/test/time-wordcount.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# This script runs the wordcount program in different ways and generates
+# an output useful for performance comparisons.
+#
+
+# Select the source text to be analyzed.
+#
+if test "x$1" = "x";
+then echo "Usage: $0 FILENAME [ARGS...]"; exit 1;
+fi
+
+# Do test runs
+#
+rm -f wcdb1.db
+./wordcount --tag A: --timer --summary wcdb1.db $* --insert
+rm -f wcdb2.db
+./wordcount --tag B: --timer --summary wcdb2.db $* --insert --without-rowid
+rm -f wcdb1.db
+./wordcount --tag C: --timer --summary wcdb1.db $* --replace
+rm -f wcdb2.db
+./wordcount --tag D: --timer --summary wcdb2.db $* --replace --without-rowid
+rm -f wcdb1.db
+./wordcount --tag E: --timer --summary wcdb1.db $* --select
+rm -f wcdb2.db
+./wordcount --tag F: --timer --summary wcdb2.db $* --select --without-rowid
+./wordcount --tag G: --timer --summary wcdb1.db $* --query
+./wordcount --tag H: --timer --summary wcdb1.db $* --query --without-rowid
+./wordcount --tag I: --timer --summary wcdb1.db $* --delete
+./wordcount --tag J: --timer --summary wcdb2.db $* --delete --without-rowid
+
+# Clean up temporary files created.
+#
+rm -f wcdb1.db wcdb2.db
diff --git a/test/tkt-2d1a5c67d.test b/test/tkt-2d1a5c67d.test
index 3fef187ecd..0d12a6ec43 100644
--- a/test/tkt-2d1a5c67d.test
+++ b/test/tkt-2d1a5c67d.test
@@ -19,7 +19,8 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 set testprefix tkt-2d1a5c67d
 
-ifcapable {!wal || !vtab} {finish_test; return}
+ifcapable {!vtab} {finish_test; return}
+if {[wal_is_capable]==0} {finish_test; return}
 
 for {set ii 1} {$ii<=10} {incr ii} {
   do_test tkt-2d1a5c67d.1.$ii {
diff --git a/test/tkt-313723c356.test b/test/tkt-313723c356.test
index 8c08c34976..5325fd2292 100644
--- a/test/tkt-313723c356.test
+++ b/test/tkt-313723c356.test
@@ -18,7 +18,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 
-ifcapable !wal { finish_test ; return }
+if {![wal_is_capable]} { finish_test ; return }
 
 do_execsql_test tkt-313723c356.1 {
   PRAGMA page_size = 1024;
diff --git a/test/tkt-5d863f876e.test b/test/tkt-5d863f876e.test
index 86024e300b..9a2fa3f470 100644
--- a/test/tkt-5d863f876e.test
+++ b/test/tkt-5d863f876e.test
@@ -18,7 +18,7 @@ set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 set ::testprefix tkt-5d863f876e
-ifcapable !wal {finish_test ; return }
+if {![wal_is_capable]} {finish_test ; return }
 
 do_multiclient_test tn {
   do_test $tn.1 {
diff --git a/test/tkt-78e04e52ea.test b/test/tkt-78e04e52ea.test
index 975e5b3d1c..963ecf18d1 100644
--- a/test/tkt-78e04e52ea.test
+++ b/test/tkt-78e04e52ea.test
@@ -42,7 +42,7 @@ do_test tkt-78e04-1.3 {
 } {}
 do_test tkt-78e04-1.4 {
   execsql {
-    EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE 'abc%';
+    EXPLAIN QUERY PLAN SELECT "" FROM "" WHERE "" LIKE '1abc%';
   }
 } {0 0 0 {SCAN TABLE  USING COVERING INDEX i1}}
 do_test tkt-78e04-1.5 {
diff --git a/test/tkt-80e031a00f.test b/test/tkt-80e031a00f.test
index af1d636eed..4ad0a772a5 100644
--- a/test/tkt-80e031a00f.test
+++ b/test/tkt-80e031a00f.test
@@ -20,10 +20,6 @@ source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 source $testdir/malloc_common.tcl
 
-# EVIDENCE-OF: R-58875-56087 The IN and NOT IN operators take a single
-# scalar operand on the left and a vector operand on the right formed by
-# an explicit list of zero or more scalars or by a single subquery.
-#
 # EVIDENCE-OF: R-52275-55503 When the right operand is an empty set, the
 # result of IN is false and the result of NOT IN is true, regardless of
 # the left operand and even if the left operand is NULL.
diff --git a/test/tkt-9d68c883.test b/test/tkt-9d68c883.test
index 18dc6ccd67..ba91b39e39 100644
--- a/test/tkt-9d68c883.test
+++ b/test/tkt-9d68c883.test
@@ -50,4 +50,6 @@ for {set i 0} {$i < 100} {incr i} {
   } {ok}
 }
 
+catch { db close } 
+unregister_devsim
 finish_test
diff --git a/test/tkt3630.test b/test/tkt3630.test
index b329c48dcf..1fe7ce70fa 100644
--- a/test/tkt3630.test
+++ b/test/tkt3630.test
@@ -23,7 +23,7 @@ source $testdir/tester.tcl
 do_test tkt3630-1 {
   db eval {
     CREATE TEMP TABLE temp1(a,b,c);
-    SELECT * FROM sqlite_temp_master WHERE sql GLOB '*TEMP*';
+    SELECT * FROM temp.sqlite_master WHERE sql GLOB '*TEMP*';
   }
 } {}
 do_test tkt3630-2 {
@@ -39,7 +39,7 @@ ifcapable altertable {
     db eval {
       ALTER TABLE temp2 ADD COLUMN d;
       ALTER TABLE temp2 RENAME TO temp2rn;
-      SELECT name FROM sqlite_temp_master WHERE name LIKE 'temp2%';
+      SELECT name FROM temp.sqlite_master WHERE name LIKE 'temp2%';
     }
   } {temp2rn}
 }
diff --git a/test/tkt3810.test b/test/tkt3810.test
index 7e1b2ec5c9..f15eb7d63a 100644
--- a/test/tkt3810.test
+++ b/test/tkt3810.test
@@ -62,7 +62,7 @@ do_test tkt3810-3 {
 # an orphan.
 #
 do_test tkt3810-4 {
-  execsql {SELECT name FROM sqlite_temp_master ORDER BY name}
+  execsql {SELECT name FROM temp.sqlite_master ORDER BY name}
 } {r1}
 
 # Because it is an orphan, it cannot be dropped.
diff --git a/test/trace3.test b/test/trace3.test
new file mode 100644
index 0000000000..271009a3fc
--- /dev/null
+++ b/test/trace3.test
@@ -0,0 +1,233 @@
+# 2016 July 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The focus of
+# this test file is the "sqlite3_trace_v2()" and "sqlite3_expanded_sql()"
+# APIs.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+ifcapable !trace { finish_test ; return }
+set ::testprefix trace3
+
+proc trace_v2_error { args } {
+  lappend ::stmtlist(error) [string trim $args]
+  error "trace error"; # this will be ignored.
+}
+proc trace_v2_record { args } {
+  lappend ::stmtlist(record) [string trim $args]
+}
+proc trace_v2_nop { args } {}; # do nothing.
+
+do_test trace3-1.0 {
+  execsql {
+    CREATE TABLE t1(a,b);
+    INSERT INTO t1 VALUES(1,NULL);
+    INSERT INTO t1 VALUES(2,-1);
+    INSERT INTO t1 VALUES(3,0);
+    INSERT INTO t1 VALUES(4,1);
+    INSERT INTO t1 VALUES(5,-2147483648);
+    INSERT INTO t1 VALUES(6,2147483647);
+    INSERT INTO t1 VALUES(7,-9223372036854775808);
+    INSERT INTO t1 VALUES(8,9223372036854775807);
+    INSERT INTO t1 VALUES(9,-1.0);
+    INSERT INTO t1 VALUES(10,0.0);
+    INSERT INTO t1 VALUES(11,1.0);
+    INSERT INTO t1 VALUES(12,'');
+    INSERT INTO t1 VALUES(13,'1');
+    INSERT INTO t1 VALUES(14,'one');
+    INSERT INTO t1 VALUES(15,x'abcd0123');
+    INSERT INTO t1 VALUES(16,x'4567cdef');
+  }
+} {}
+
+do_test trace3-1.1 {
+  set rc [catch {db trace_v2 1 2 3} msg]
+  lappend rc $msg
+} {1 {wrong # args: should be "db trace_v2 ?CALLBACK? ?MASK?"}}
+do_test trace3-1.2 {
+  set rc [catch {db trace_v2 1 bad} msg]
+  lappend rc $msg
+} {1 {bad trace type "bad": must be statement, profile, row, or close}}
+
+do_test trace3-2.1 {
+  db trace_v2 trace_v2_nop
+  db trace_v2
+} {trace_v2_nop}
+
+do_test trace3-3.1 {
+  unset -nocomplain ::stmtlist
+  db trace_v2 trace_v2_nop
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  array get ::stmtlist
+} {}
+do_test trace3-3.2 {
+  set ::stmtlist(error) {}
+  db trace_v2 trace_v2_error
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(error)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.3 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.4 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record statement
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+do_test trace3-3.5 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 1
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ \{SELECT a, b FROM t1 ORDER BY a;\}\}$/}
+
+do_test trace3-4.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record profile
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ -?\d+\}$/}
+do_test trace3-4.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 2
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} {/^\{-?\d+ -?\d+\}$/}
+
+do_test trace3-5.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record row
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]]\$/"
+do_test trace3-5.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 4
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]]\$/"
+
+do_test trace3-6.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record {profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^[string trim [string repeat {-?\d+ } 16]] \\\{-?\\d+ -?\\d+\\\}\$/"
+do_test trace3-6.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record {statement profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  set ::stmtlist(record)
+} "/^\\\{-?\\d+ \\\{SELECT a, b FROM t1 ORDER BY a;\\\}\\\} [string trim \
+[string repeat {-?\d+ } 16]] \\\{-?\\d+ -?\\d+\\\}\$/"
+
+do_test trace3-7.1 {
+  set DB [sqlite3_connection_pointer db]
+
+  set STMT [sqlite3_prepare_v2 $DB \
+      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 TAIL]
+} {/^[0-9A-Fa-f]+$/}
+
+do_test trace3-8.1 {
+  list [sqlite3_bind_null $STMT 1] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = NULL ORDER BY a;}}
+do_test trace3-8.2 {
+  list [sqlite3_bind_int $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123 ORDER BY a;}}
+do_test trace3-8.3 {
+  list [sqlite3_bind_int64 $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123 ORDER BY a;}}
+do_test trace3-8.4 {
+  list [sqlite3_bind_text $STMT 1 "some string" 11] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'some string' ORDER BY a;}}
+do_test trace3-8.5 {
+  list [sqlite3_bind_text $STMT 1 "some 'bad' string" 17] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'some ''bad'' string' ORDER BY a;}}
+do_test trace3-8.6 {
+  list [sqlite3_bind_double $STMT 1 123] [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 123.0 ORDER BY a;}}
+do_test trace3-8.7 {
+  list [sqlite3_bind_text16 $STMT 1 \
+      [encoding convertto unicode hi\000yall\000] 16] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = 'hi' ORDER BY a;}}
+do_test trace3-8.8 {
+  list [sqlite3_bind_blob $STMT 1 "\x12\x34\x56" 3] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = x'123456' ORDER BY a;}}
+do_test trace3-8.9 {
+  list [sqlite3_bind_blob $STMT 1 "\xAB\xCD\xEF" 3] \
+      [sqlite3_expanded_sql $STMT]
+} {{} {SELECT a, b FROM t1 WHERE b = x'abcdef' ORDER BY a;}}
+
+do_test trace3-9.1 {
+  sqlite3_finalize $STMT
+} {SQLITE_OK}
+
+do_test trace3-10.1 {
+  db trace_v2 ""
+  db trace_v2
+} {}
+do_test trace3-10.2 {
+  unset -nocomplain ::stmtlist
+  db trace_v2 "" {statement profile row}
+  execsql {
+    SELECT a, b FROM t1 ORDER BY a;
+  }
+  array get ::stmtlist
+} {}
+
+do_test trace3-11.1 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record close
+  db close
+  set ::stmtlist(record)
+} {/^-?\d+$/}
+
+reset_db
+
+do_test trace3-11.2 {
+  set ::stmtlist(record) {}
+  db trace_v2 trace_v2_record 8
+  db close
+  set ::stmtlist(record)
+} {/^-?\d+$/}
+
+finish_test
diff --git a/test/triggerF.test b/test/triggerF.test
new file mode 100644
index 0000000000..d5e3565edd
--- /dev/null
+++ b/test/triggerF.test
@@ -0,0 +1,72 @@
+# 2017 January 4
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice', here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix triggerF
+ifcapable {!trigger} {
+  finish_test
+  return
+}
+
+
+foreach {tn sql log} {
+  1 { } { }
+
+  2 { 
+    CREATE TRIGGER trd AFTER DELETE ON t1 BEGIN
+      INSERT INTO log VALUES(old.a || old.b || (SELECT count(*) FROM t1));
+    END;
+  } {1one2 2two1 3three1}
+
+  3 { 
+    CREATE TRIGGER trd BEFORE DELETE ON t1 BEGIN
+      INSERT INTO log VALUES(old.a || old.b || (SELECT count(*) FROM t1));
+    END;
+  } {1one3 2two2 3three2}
+
+  4 { 
+    CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
+      INSERT INTO log VALUES(old.a || old.b || (SELECT count(*) FROM t1));
+    END;
+    CREATE TRIGGER tr2 BEFORE DELETE ON t1 BEGIN
+      INSERT INTO log VALUES(old.a || old.b || (SELECT count(*) FROM t1));
+    END;
+  } {1one3 1one2 2two2 2two1 3three2 3three1}
+
+} {
+  reset_db
+  do_execsql_test 1.$tn.0 {
+    PRAGMA recursive_triggers = on;
+    CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID;
+    CREATE TABLE log(t);
+  }
+  
+  execsql $sql
+
+  do_execsql_test 1.$tn.1 {
+    INSERT INTO t1 VALUES(1, 'one');
+    INSERT INTO t1 VALUES(2, 'two');
+    INSERT INTO t1 VALUES(3, 'three');
+
+    DELETE FROM t1 WHERE a=1;
+    INSERT OR REPLACE INTO t1 VALUES(2, 'three');
+    UPDATE OR REPLACE t1 SET a=3 WHERE a=2;
+  }
+
+  do_execsql_test 1.$tn.2 {
+    SELECT * FROM log ORDER BY rowid;
+  } $log
+}
+
+finish_test
+
diff --git a/test/triggerG.test b/test/triggerG.test
new file mode 100644
index 0000000000..c770b5bd76
--- /dev/null
+++ b/test/triggerG.test
@@ -0,0 +1,65 @@
+# 2017-03-24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice', here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix triggerG
+ifcapable {!trigger} {
+  finish_test
+  return
+}
+
+# Test cases for ticket 
+# https://www.sqlite.org/src/tktview/06796225f59c057cd120f
+#
+# The OP_Once opcode was not working correctly for recursive triggers.
+#
+do_execsql_test 100 {
+  PRAGMA recursive_triggers = 1;
+  
+  CREATE TABLE t1(a);
+  CREATE INDEX i1 ON t1(a);
+  INSERT INTO t1(a) VALUES(0),(2),(3),(8),(9);
+  CREATE TABLE t2(b);
+  CREATE TABLE t3(c);
+  
+  CREATE TRIGGER tr AFTER INSERT ON t3 BEGIN
+    INSERT INTO t3 SELECT new.c+1 WHERE new.c<5;
+    INSERT INTO t2 SELECT new.c*100+a FROM t1 WHERE a IN (1, 2, 3, 4);
+  END;
+  
+  INSERT INTO t3 VALUES(2);
+  SELECT c FROM t3 ORDER BY c;;
+} {2 3 4 5}
+do_execsql_test 110 {
+  SELECT b FROM t2 ORDER BY b;
+} {202 203 302 303 402 403 502 503}
+
+do_execsql_test 200 {
+  DELETE FROM t1;
+  INSERT INTO t1(a) VALUES(0),(2),(3),(8),(9);
+  DELETE FROM t2;
+  DELETE FROM t3;
+  DROP TRIGGER tr;
+  CREATE TRIGGER tr AFTER INSERT ON t3 BEGIN
+    INSERT INTO t3 SELECT new.c+1 WHERE new.c<5;
+    INSERT INTO t2 SELECT new.c*10000+xx.a*100+yy.a
+                     FROM t1 AS xx, t1 AS yy
+                    WHERE xx.a IN (1,2,3,4)
+                      AND yy.a IN (2,3,4,5);
+  END;
+
+  INSERT INTO t3 VALUES(2);
+  SELECT b FROM t2 ORDER BY b;
+} {20202 20203 20302 20303 30202 30203 30302 30303 40202 40203 40302 40303 50202 50203 50302 50303}
+
+finish_test
diff --git a/test/types2.test b/test/types2.test
index 4a70aa5fe2..d6d84e65b5 100644
--- a/test/types2.test
+++ b/test/types2.test
@@ -333,7 +333,7 @@ ifcapable subquery {
   test_boolset types2-8.8 {o IN (SELECT t FROM t4)} {7}
   test_boolset types2-8.9 {i IN (SELECT o FROM t4)} {9 10 11 12}
   test_boolset types2-8.6 {n IN (SELECT o FROM t4)} {9 10 11 12}
-  test_boolset types2-8.7 {t IN (SELECT o FROM t4)} {9 11}
+  test_boolset types2-8.7 {t IN (SELECT o FROM t4)} {}
   test_boolset types2-8.8 {o IN (SELECT o FROM t4)} {9 10}
 }
 
diff --git a/test/update2.test b/test/update2.test
new file mode 100644
index 0000000000..6a359d11a9
--- /dev/null
+++ b/test/update2.test
@@ -0,0 +1,205 @@
+# 2017 January 9
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix update2
+
+db func repeat [list string repeat]
+
+#-------------------------------------------------------------------------
+# 1.1.* A one-pass UPDATE that does balance() operations on the IPK index
+#       that it is scanning.
+#
+# 1.2.* Same again, but with a WITHOUT ROWID table.
+#
+set nrow [expr 10]
+do_execsql_test 1.1.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
+  WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow )
+  INSERT INTO t1(b) SELECT char((i % 26) + 65) FROM s;
+  INSERT INTO t2 SELECT * FROM t1;
+}
+
+do_execsql_test 1.1.1 {
+  UPDATE t1 SET b = repeat(b, 100)
+}
+
+do_execsql_test 1.1.2 {
+  SELECT * FROM t1;
+} [db eval { SELECT a, repeat(b, 100) FROM t2 }]
+
+do_execsql_test 1.2.0 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID;
+  WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE i<$nrow )
+  INSERT INTO t1(a, b) SELECT i+1, char((i % 26) + 65) FROM s;
+}
+
+#explain_i { UPDATE t1 SET b = repeat(b, 100) }
+do_execsql_test 1.2.1 {
+  UPDATE t1 SET b = repeat(b, 100)
+}
+
+do_execsql_test 1.2.2 {
+  SELECT * FROM t1;
+} [db eval { SELECT a, repeat(b, 100) FROM t2 }]
+
+
+#-------------------------------------------------------------------------
+# A one-pass UPDATE that does balance() operations on the IPK index
+# that it is scanning.
+#
+do_execsql_test 2.1 {
+  CREATE TABLE t3(a PRIMARY KEY, b, c);
+  CREATE INDEX t3i ON t3(b);
+} {}
+do_execsql_test 2.2 { UPDATE t3 SET c=1 WHERE b=?      } {}
+do_execsql_test 2.3 { UPDATE t3 SET c=1 WHERE rowid=?  } {}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.0 {
+  CREATE TABLE t4(a PRIMARY KEY, b, c) WITHOUT ROWID;
+  CREATE INDEX t4c ON t4(c);
+  INSERT INTO t4 VALUES(1, 2, 3);
+  INSERT INTO t4 VALUES(2, 3, 4);
+}
+
+do_execsql_test 3.1 {
+  UPDATE t4 SET c=c+2 WHERE c>2;
+  SELECT a, c FROM t4 ORDER BY a;
+} {1 5 2 6}
+
+#-------------------------------------------------------------------------
+#
+foreach {tn sql} {
+  1 { 
+    CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c);
+    CREATE TABLE c1(a INTEGER PRIMARY KEY, b, c, d)
+  }
+  2 { 
+    CREATE TABLE b1(a INT PRIMARY KEY, b, c) WITHOUT ROWID;
+    CREATE TABLE c1(a INT PRIMARY KEY, b, c, d) WITHOUT ROWID;
+  }
+} {
+  execsql { DROP TABLE IF EXISTS b1; DROP TABLE IF EXISTS c1; }
+  execsql $sql
+
+  do_execsql_test 4.$tn.0 {
+    CREATE UNIQUE INDEX b1c ON b1(c);
+    INSERT INTO b1 VALUES(1, 'a', 1);
+    INSERT INTO b1 VALUES(2, 'b', 15);
+    INSERT INTO b1 VALUES(3, 'c', 3);
+    INSERT INTO b1 VALUES(4, 'd', 4);
+    INSERT INTO b1 VALUES(5, 'e', 5);
+    INSERT INTO b1 VALUES(6, 'f', 6);
+    INSERT INTO b1 VALUES(7, 'g', 7);
+  }
+
+  do_execsql_test 4.$tn.1 {
+    UPDATE OR REPLACE b1 SET c=c+10 WHERE a BETWEEN 4 AND 7;
+    SELECT * FROM b1 ORDER BY a;
+  } {
+    1 a 1
+    3 c 3
+    4 d 14
+    5 e 15
+    6 f 16
+    7 g 17
+  }
+
+  do_execsql_test 4.$tn.2 {
+    CREATE INDEX c1d ON c1(d, b);
+    CREATE UNIQUE INDEX c1c ON c1(c, b);
+
+    INSERT INTO c1 VALUES(1, 'a', 1,  1);
+    INSERT INTO c1 VALUES(2, 'a', 15, 2);
+    INSERT INTO c1 VALUES(3, 'a', 3,  3);
+    INSERT INTO c1 VALUES(4, 'a', 4,  4);
+    INSERT INTO c1 VALUES(5, 'a', 5,  5);
+    INSERT INTO c1 VALUES(6, 'a', 6,  6);
+    INSERT INTO c1 VALUES(7, 'a', 7,  7);
+  }
+
+  do_execsql_test 4.$tn.3 {
+    UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7;
+    SELECT * FROM c1 ORDER BY a;
+  } {
+    1 a 1 1
+    3 a 3 3
+    4 a 14 4
+    5 a 15 5
+    6 a 16 6
+    7 a 17 7
+  }
+
+  do_execsql_test 4.$tn.4 { PRAGMA integrity_check } ok
+
+  do_execsql_test 4.$tn.5 {
+    DROP INDEX c1d;
+    DROP INDEX c1c;
+    DELETE FROM c1;
+
+    INSERT INTO c1 VALUES(1, 'a', 1,  1);
+    INSERT INTO c1 VALUES(2, 'a', 15, 2);
+    INSERT INTO c1 VALUES(3, 'a', 3,  3);
+    INSERT INTO c1 VALUES(4, 'a', 4,  4);
+    INSERT INTO c1 VALUES(5, 'a', 5,  5);
+    INSERT INTO c1 VALUES(6, 'a', 6,  6);
+    INSERT INTO c1 VALUES(7, 'a', 7,  7);
+
+    CREATE INDEX c1d ON c1(d);
+    CREATE UNIQUE INDEX c1c ON c1(c);
+  }
+
+  do_execsql_test 4.$tn.6 {
+    UPDATE OR REPLACE c1 SET c=c+10 WHERE d BETWEEN 4 AND 7;
+    SELECT * FROM c1 ORDER BY a;
+  } {
+    1 a 1 1
+    3 a 3 3
+    4 a 14 4
+    5 a 15 5
+    6 a 16 6
+    7 a 17 7
+  }
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 5.0 {
+  CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c);
+  CREATE INDEX x1c ON x1(b, c);
+  INSERT INTO x1 VALUES(1, 'a', 1);
+  INSERT INTO x1 VALUES(2, 'a', 2);
+  INSERT INTO x1 VALUES(3, 'a', 3);
+}
+
+do_execsql_test 5.1.1 {
+  UPDATE x1 SET c=c+1 WHERE b='a';
+}
+
+do_execsql_test 5.1.2 {
+  SELECT * FROM x1;
+} {1 a 2 2 a 3 3 a 4}
+
+do_test 5.2 {
+  catch { array unset A }
+  db eval { EXPLAIN UPDATE x1 SET c=c+1 WHERE b='a' } { incr A($opcode) }
+  set A(NotExists)
+} {1}
+
+
+finish_test
+
diff --git a/test/uri.test b/test/uri.test
index a0a88d2263..0931a25e51 100644
--- a/test/uri.test
+++ b/test/uri.test
@@ -55,6 +55,10 @@ foreach {tn uri file} {
 
   ifcapable !curdir { if {$tn==3} break }
 
+  ifcapable uri_00_error {
+    if {[string first %00 $uri]>=0} continue
+  }
+
   if {$tcl_platform(platform)=="windows"} {
     #
     # NOTE: Due to limits on legal characters for file names imposed by
@@ -124,6 +128,10 @@ foreach {tn uri kvlist} {
   14     http:test.db?hello&world                     {}
 } {
 
+  ifcapable uri_00_error {
+    if {[string first %00 $uri]>=0} continue
+  }
+
   if {$tcl_platform(platform) == "windows" && $tn>12} {
     continue
   }
diff --git a/test/uri2.test b/test/uri2.test
new file mode 100644
index 0000000000..52feb7b50f
--- /dev/null
+++ b/test/uri2.test
@@ -0,0 +1,63 @@
+# 2016 October 26
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests for SQLITE_ENABLE_URI_00_ERROR builds.
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !uri_00_error {
+  finish_test
+  return
+}
+
+set testprefix uri2
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+foreach {tn uri} {
+  1 file:test.db%00trailing
+  2 file:test.db?%00trailing=1
+  3 file:test.db?trailing=1%00
+  4 file:test.db?trailing=1&abc%00def
+  5 file:test.db?trailing=1&abc%00def
+} {
+  do_test 1.$tn.1 {
+    set rc [catch { sqlite3 db $uri } msg]
+    list $rc $msg
+  } {1 {unexpected %00 in uri}}
+
+  do_test 1.$tn.2 {
+    set DB2 [sqlite3_open $uri]
+    sqlite3_errcode $DB2
+  } {SQLITE_ERROR}
+
+  catch { sqlite3_close $DB2 }
+
+  do_test 1.$tn.2 {
+    sqlite3 db ""
+    catchsql { ATTACH $uri AS aux }
+  } {1 {unexpected %00 in uri}}
+
+  do_test 1.$tn.3 {
+    sqlite3_errcode db
+  } {SQLITE_ERROR}
+
+  catch { db close }
+}
+
+reset_db
+do_test 2.0 {
+  expr {[lsearch [execsql {PRAGMA compile_options}] ENABLE_URI_00_ERROR] >= 0}
+} 1
+
+finish_test
diff --git a/test/vacuum5.test b/test/vacuum5.test
new file mode 100644
index 0000000000..8e76a9393c
--- /dev/null
+++ b/test/vacuum5.test
@@ -0,0 +1,153 @@
+# 2016-08-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file implements a test for VACUUM on attached databases.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuum5
+
+# If the VACUUM statement is disabled in the current build, skip all
+# the tests in this file.
+#
+ifcapable !vacuum {
+  finish_test
+  return
+}
+
+forcedelete test2.db test3.db
+do_execsql_test vacuum5-1.1 {
+  PRAGMA auto_vacuum = 0;
+  CREATE TABLE main.t1(a,b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000)
+    INSERT INTO t1(a,b) SELECT x, randomblob(1000) FROM c;
+  CREATE TEMP TABLE ttemp(x,y);
+  INSERT INTO ttemp SELECT * FROM t1;
+  ATTACH 'test2.db' AS x2;
+  ATTACH 'test3.db' AS x3;
+  CREATE TABLE x2.t2(c,d);
+  INSERT INTO t2 SELECT * FROM t1;
+  CREATE TABLE x3.t3(e,f);
+  INSERT INTO t3 SELECT * FROM t1;
+  DELETE FROM t1 WHERE (rowid%3)!=0;
+  DELETE FROM t2 WHERE (rowid%4)!=0;
+  DELETE FROM t3 WHERE (rowid%5)!=0;
+  PRAGMA main.integrity_check;
+  PRAGMA x2.integrity_check;
+  PRAGMA x3.integrity_check;
+} {ok ok ok}
+set size1 [file size test.db]
+set size2 [file size test2.db]
+set size3 [file size test3.db]
+
+do_execsql_test vacuum5-1.2.1 {
+  VACUUM main;
+} {}
+do_test vacuum5-1.2.2 {
+  expr {[file size test.db]<$size1}
+} {1}
+do_test vacuum5-1.2.3 {
+  file size test2.db
+} $size2
+do_test vacuum5-1.2.4 {
+  file size test3.db
+} $size3
+set size1 [file size test.db]
+do_execsql_test vacuum-1.2.5 {
+  DELETE FROM t1;
+  PRAGMA main.integrity_check;
+} {ok}
+
+do_execsql_test vacuum5-1.3.1 {
+  VACUUM x2;
+} {}
+do_test vacuum5-1.3.2 {
+  file size test.db
+} $size1
+do_test vacuum5-1.3.3 {
+  expr {[file size test2.db]<$size2}
+} 1
+do_test vacuum5-1.3.4 {
+  file size test3.db
+} $size3
+set size2 [file size test2.db]
+do_execsql_test vacuum-1.3.5 {
+  DELETE FROM t2;
+  PRAGMA x2.integrity_check;
+} {ok}
+
+do_execsql_test vacuum5-1.4.1 {
+  VACUUM x3;
+} {}
+do_test vacuum5-1.3.2 {
+  file size test.db
+} $size1
+do_test vacuum5-1.3.3 {
+  file size test2.db
+} $size2
+do_test vacuum5-1.3.4 {
+  expr {[file size test3.db]<$size3}
+} 1
+
+# VACUUM is a no-op on the TEMP table
+#
+set sizeTemp [db one {PRAGMA temp.page_count}]
+do_execsql_test vacuum5-1.4.1 {
+  VACUUM temp;
+} {}
+do_execsql_test vacuum5-1.4.2 {
+  PRAGMA temp.page_count;
+} $sizeTemp
+
+do_catchsql_test vacuum5-2.0 {
+  VACUUM olaf;
+} {1 {unknown database olaf}}
+
+#-------------------------------------------------------------------------
+# Test that a temp file is opened as part of VACUUM.
+#
+if {$::TEMP_STORE<3 && [permutation]!="inmemory_journal"} {
+  db close
+  testvfs tvfs 
+  tvfs filter xOpen
+  tvfs script open_cb
+  forcedelete test.db
+
+  set ::openfiles [list]
+  proc open_cb {method args} {
+    lappend ::openfiles [file tail [lindex $args 0]]
+  }
+  sqlite3 db test.db -vfs tvfs
+
+  do_execsql_test 3.0 {
+    PRAGMA temp_store = file;
+    PRAGMA page_size = 1024;
+    PRAGMA cache_size = 50;
+    CREATE TABLE t1(i INTEGER PRIMARY KEY, j UNIQUE);
+    WITH s(i) AS (
+      VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000
+    )
+    INSERT INTO t1 SELECT NULL, randomblob(100) FROM s;
+  }
+
+  do_execsql_test 3.1 { VACUUM }
+
+  db close
+  tvfs delete
+  do_test 3.2 {
+    lrange $::openfiles 0 4
+  } {test.db test.db-journal test.db-journal {} test.db-journal}
+} 
+
+
+
+finish_test
diff --git a/test/vacuummem.test b/test/vacuummem.test
new file mode 100644
index 0000000000..0f77c1b400
--- /dev/null
+++ b/test/vacuummem.test
@@ -0,0 +1,73 @@
+# 2005 February 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing that the VACUUM statement correctly
+# frees any memory used for a temporary cache.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuummem
+
+if {[permutation]=="memsubsys1"} {
+  finish_test
+  return
+}
+
+# If ENABLE_MEMORY_MANAGEMENT is defined, when VACUUM is run the temp db
+# is able to borrow space from the main db (and it does, because the
+# temp db is configure with a very small cache). When the VACUUM is
+# finished and the temp db closed, all the page-cache memory currently 
+# assigned to the temp db is freed. If ENABLE_MEMORY_MANAGEMENT is defined
+# this causes the total memory usage to drop much more than expected,
+# causing tests in this file to fail.
+#
+ifcapable memorymanage {
+  finish_test
+  return
+}
+
+
+proc memory_used {} { 
+  set stat [sqlite3_status SQLITE_STATUS_MEMORY_USED 1]  
+  lindex $stat 1
+}
+
+do_execsql_test 1.0 {
+  PRAGMA cache_size = -2000;
+  CREATE TABLE t1(a, b, c);
+
+  WITH r(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM r WHERE i<100000
+  )
+  INSERT INTO t1 SELECT randomblob(100),randomblob(100),randomblob(100) FROM r;
+
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+}
+set ans "#/[memory_used]/"
+
+do_test 1.1 { memory_used } $ans
+
+do_execsql_test 1.2 VACUUM
+
+do_test 1.3 { memory_used } $ans
+
+do_execsql_test 1.4 {
+  SELECT count(*) FROM t1 WHERE +a IS NOT NULL
+} {100000}
+
+do_test 1.5 { memory_used } $ans
+
+
+
+finish_test
diff --git a/test/view.test b/test/view.test
index 235f8605ac..694b3c6abd 100644
--- a/test/view.test
+++ b/test/view.test
@@ -92,6 +92,26 @@ do_test view-1.8 {
   }
 } {2 3 5 6 8 9}
 
+do_execsql_test view-1.10 {
+  CREATE TABLE t9(x INTEGER);
+  CREATE VIEW v9a AS SELECT x FROM t9;
+  CREATE VIEW v9b AS SELECT * FROM t9;
+  CREATE VIEW v9c(x) AS SELECT x FROM t9;
+  CREATE VIEW v9d(x) AS SELECT * FROM t9;
+} {}
+do_execsql_test view-1.11 {
+  PRAGMA table_info(v9a);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.12 {
+  PRAGMA table_info(v9b);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.13 {
+  PRAGMA table_info(v9c);
+} {0 x INTEGER 0 {} 0}
+do_execsql_test view-1.14 {
+  PRAGMA table_info(v9d);
+} {0 x INTEGER 0 {} 0}
+
 do_test view-2.1 {
   execsql {
     CREATE VIEW v2 AS SELECT * FROM t1 WHERE a>5
diff --git a/test/vtab1.test b/test/vtab1.test
index 6b6a0e2683..59ab959746 100644
--- a/test/vtab1.test
+++ b/test/vtab1.test
@@ -1295,25 +1295,25 @@ do_execsql_test 18.1.0 {
   CREATE INDEX i6 ON t6(b, a);
   INSERT INTO t6 VALUES(1, 'Peter');
   INSERT INTO t6 VALUES(2, 'Andrew');
-  INSERT INTO t6 VALUES(3, 'James');
-  INSERT INTO t6 VALUES(4, 'John');
+  INSERT INTO t6 VALUES(3, '8James');
+  INSERT INTO t6 VALUES(4, '8John');
   INSERT INTO t6 VALUES(5, 'Phillip');
   INSERT INTO t6 VALUES(6, 'Bartholomew');
   CREATE VIRTUAL TABLE e6 USING echo(t6);
 }
 
 foreach {tn sql res filter} {
-  1.1 "SELECT a FROM e6 WHERE b>'James'" {4 1 5}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} James}
+  1.1 "SELECT a FROM e6 WHERE b>'8James'" {4 2 6 1 5}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b > ?} 8James}
 
-  1.2 "SELECT a FROM e6 WHERE b>='J' AND b<'K'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} J K}
+  1.2 "SELECT a FROM e6 WHERE b>='8' AND b<'9'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b >= ? AND b < ?} 8 9}
 
-  1.3 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%}
+  1.3 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%}
 
-  1.4 "SELECT a FROM e6 WHERE b LIKE 'j%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%}
+  1.4 "SELECT a FROM e6 WHERE b LIKE '8j%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%}
 } {
   set echo_module {}
   do_execsql_test 18.$tn.1 $sql $res
@@ -1322,11 +1322,11 @@ foreach {tn sql res filter} {
 
 do_execsql_test 18.2.0 {  PRAGMA case_sensitive_like = ON }
 foreach {tn sql res filter} {
-  2.1 "SELECT a FROM e6 WHERE b LIKE 'J%'" {3 4}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} J%}
+  2.1 "SELECT a FROM e6 WHERE b LIKE '8J%'" {3 4}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8J%}
 
-  2.2 "SELECT a FROM e6 WHERE b LIKE 'j%'" {}
-    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} j%}
+  2.2 "SELECT a FROM e6 WHERE b LIKE '8j%'" {}
+    {xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8j%}
 } {
   set echo_module {}
   do_execsql_test 18.$tn.1 $sql $res
diff --git a/test/vtabF.test b/test/vtabF.test
index b9341ea597..7b11cb8c71 100644
--- a/test/vtabF.test
+++ b/test/vtabF.test
@@ -21,7 +21,7 @@ ifcapable !vtab||!schema_pragmas { finish_test ; return }
 # Register the echo module
 register_echo_module [sqlite3_connection_pointer db]
 
-do_test vtabE-1.1 {
+do_test vtabF-1.1 {
   execsql {
     CREATE TABLE t1(a, b);
     CREATE INDEX i1 ON t1(a);
@@ -37,7 +37,7 @@ do_test vtabE-1.1 {
     SELECT b FROM t1 WHERE a IS NOT NULL;
   }
 } {110 111 112 113}
-do_test vtabE-1.2 {
+do_test vtabF-1.2 {
   execsql {SELECT b FROM tv1 WHERE a IS NOT NULL}
 } {110 111 112 113}
 
diff --git a/test/vtabH.test b/test/vtabH.test
index d16db13674..c5684ff516 100644
--- a/test/vtabH.test
+++ b/test/vtabH.test
@@ -31,14 +31,14 @@ do_execsql_test 1.0 {
 }
 
 foreach {tn sql expect} {
-  1 "SELECT * FROM e6 WHERE b LIKE 'abc'" {
+  1 "SELECT * FROM e6 WHERE b LIKE '8abc'" {
     xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b like ?}
-    xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} abc
+    xFilter {SELECT rowid, a, b FROM 't6' WHERE b like ?} 8abc
   }
 
-  2 "SELECT * FROM e6 WHERE b GLOB 'abc'" {
+  2 "SELECT * FROM e6 WHERE b GLOB '8abc'" {
     xBestIndex {SELECT rowid, a, b FROM 't6' WHERE b glob ?}
-    xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} abc
+    xFilter {SELECT rowid, a, b FROM 't6' WHERE b glob ?} 8abc
   }
 } {
   do_test 1.$tn {
@@ -108,7 +108,12 @@ foreach ::tclvar_set_omit {0 1} {
 
 #-------------------------------------------------------------------------
 #
-if {1} {
+if {$tcl_platform(platform)=="windows"} {
+  set drive [string range [pwd] 0 1]
+  set ::env(fstreeDrive) $drive
+}
+if {$tcl_platform(platform)!="windows" || \
+    [regexp -nocase -- {^[A-Z]:} $drive]} {
   reset_db
   register_fs_module db
   do_execsql_test 3.0 {
@@ -116,32 +121,66 @@ if {1} {
     SELECT name FROM fsdir WHERE dir = '.' AND name = '.'
   } {test.db .}
 
+  proc sort_files { names {nocase false} } {
+    if {$nocase && $::tcl_platform(platform) eq "windows"} {
+      return [lsort -nocase $names]
+    } else {
+      return [lsort $names]
+    }
+  }
+
   proc list_root_files {} {
     if {$::tcl_platform(platform) eq "windows"} {
-      set res [list]
-      foreach name [glob -directory $::env(SystemDrive)/ -- *] {
+      set res [list]; set dir $::env(fstreeDrive)/; set names [list]
+      eval lappend names [glob -nocomplain -directory $dir -- *]
+      foreach name $names {
         if {[string index [file tail $name] 0] eq "."} continue
+        if {[file attributes $name -hidden]} continue
+        if {[file attributes $name -system]} continue
         lappend res $name
       }
-      return $res
+      return [sort_files $res true]
     } else {
-      return [string map {/ {}} [glob /*]]
+      return [sort_files [string map {/ {}} [glob -nocomplain -- /*]]]
     }
   }
 
   proc list_files { pattern } {
     if {$::tcl_platform(platform) eq "windows"} {
-      set res [list]
-      foreach name [glob -nocomplain $pattern] {
+      set res [list]; set names [list]
+      eval lappend names [glob -nocomplain -- $pattern]
+      foreach name $names {
         if {[string index [file tail $name] 0] eq "."} continue
+        if {[file attributes $name -hidden]} continue
+        if {[file attributes $name -system]} continue
         lappend res $name
       }
-      return $res
+      return [sort_files $res]
     } else {
-      return [glob -nocomplain $pattern]
+      return [sort_files [glob -nocomplain -- $pattern]]
     }
   }
 
+  # Read the first 5 entries from the root directory.  Except, ignore
+  # files that contain the "$" character in their names as these are
+  # special files on some Windows platforms.
+  #
+  set res [list]
+  set root_files [list_root_files]
+  foreach p $root_files {
+    if {$::tcl_platform(platform) eq "windows"} {
+      if {![regexp {\$} $p]} {lappend res $p}
+    } else {
+      lappend res "/$p"
+    }
+  }
+  set num_root_files [llength $root_files]
+  do_test 3.1 {
+    sort_files [execsql {
+      SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files;
+    }] true
+  } [sort_files $res true]
+
   # Read all entries in the current directory.
   #
   proc contents {pattern} {
@@ -158,7 +197,7 @@ if {1} {
   set res [contents $pwd]
   do_execsql_test 3.2 {
     SELECT path FROM fstree WHERE path GLOB $pwd ORDER BY 1
-  } [lsort $res]
+  } [sort_files $res]
 
   # Add some sub-directories and files to the current directory.
   #
diff --git a/test/wal2.test b/test/wal2.test
index 4b9bbf315f..0b15b15461 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -1194,6 +1194,9 @@ foreach {tn sql reslist} {
   2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2}
   3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0}
 } {
+  ifcapable default_ckptfullfsync {
+    if {[string trim $sql]==""} continue
+  }
   faultsim_delete_and_reopen
 
   execsql {PRAGMA auto_vacuum = 0; PRAGMA synchronous = FULL;}
diff --git a/test/wal3.test b/test/wal3.test
index da3d318773..56f40ab539 100644
--- a/test/wal3.test
+++ b/test/wal3.test
@@ -220,6 +220,7 @@ foreach {tn syncmode synccount} {
     sqlite3 db test.db -vfs T
   
     execsql "PRAGMA synchronous = $syncmode"
+    execsql "PRAGMA checkpoint_fullfsync = 0"
     execsql { PRAGMA journal_mode = WAL }
     execsql { CREATE TABLE filler(a,b,c); }
 
diff --git a/test/wal5.test b/test/wal5.test
index 0f6dcfa6a7..50c517286a 100644
--- a/test/wal5.test
+++ b/test/wal5.test
@@ -141,13 +141,13 @@ foreach {testprefix do_wal_checkpoint} {
     do_test 1.$tn.7 {
       reopen_all
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {7 0 0}
+    } [expr {[nonzero_reserved_bytes]?"/# # 0/":"7 0 0"}]
 
     do_test 1.$tn.8  { sql2 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5}
     do_test 1.$tn.9  {
       sql1 { INSERT INTO t1 VALUES(6, zeroblob(1200)) }
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {7 5 0}
+    } [expr {[nonzero_reserved_bytes]?"/# # #/":"7 5 0"}]
     do_test 1.$tn.10 { sql3 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5 6}
 
     set ::busy_handler_script { 
@@ -158,7 +158,7 @@ foreach {testprefix do_wal_checkpoint} {
     do_test 1.$tn.11 {
       code1 { do_wal_checkpoint db -mode restart }
       list [db_page_count] [wal_page_count] $::nBusyHandler
-    } {10 5 8}
+    } [expr {[nonzero_reserved_bytes]?"/# # #/":"10 5 8"}]
     do_test 1.$tn.12 { set ::db_file_size } 10
   }
 
diff --git a/test/wal6.test b/test/wal6.test
index 2574a64a84..9bbc58409d 100644
--- a/test/wal6.test
+++ b/test/wal6.test
@@ -234,5 +234,36 @@ do_test 4.4.2 {
   catchsql { SELECT * FROM t2 } db2 
 } {1 {database disk image is malformed}}
 
+#-------------------------------------------------------------------------
+# Confirm that it is possible to get an SQLITE_BUSY_SNAPSHOT error from
+# "BEGIN EXCLUSIVE" if the connection already has an open read-transaction.
+#
+db close
+db2 close
+reset_db
+sqlite3 db2 test.db
+do_execsql_test 5.1 {
+  PRAGMA journal_mode = wal;
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES(3, 4);
+} {wal}
+do_test 5.2 {
+  set res [list]
+  db eval {
+    SELECT * FROM t1
+  } {
+    if {$x==1} {
+      db2 eval { INSERT INTO t1 VALUES(5, 6) }
+    } 
+    if {$x==3} {
+      set res [catchsql {BEGIN EXCLUSIVE}]
+      lappend res [sqlite3_extended_errcode db]
+    } 
+  }
+  set res
+} {1 {database is locked} SQLITE_BUSY_SNAPSHOT}
+
+
 
 finish_test
diff --git a/test/walcrash4.test b/test/walcrash4.test
new file mode 100644
index 0000000000..80839b39e5
--- /dev/null
+++ b/test/walcrash4.test
@@ -0,0 +1,85 @@
+# 2010 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+ifcapable !wal {finish_test ; return }
+set testprefix walcrash4
+do_not_use_codec
+
+#-------------------------------------------------------------------------
+# At one point, if "PRAGMA synchronous=full" is set and the platform
+# does not support POWERSAFE_OVERWRITE, and the last frame written to 
+# the wal file in a transaction is aligned with a sector boundary, the
+# xSync() call was omitted. 
+#
+# The following test verifies that this has been fixed.
+#
+do_execsql_test 1.0 {
+  PRAGMA autovacuum = 0;
+  PRAGMA page_size = 1024;
+  PRAGMA journal_mode = wal;
+  PRAGMA main.synchronous = full;
+} {wal}
+
+faultsim_save_and_close
+
+# The error message is different on unix and windows
+#
+if {$::tcl_platform(platform)=="windows"} {
+ set msg "child killed: unknown signal"
+} else {
+ set msg "child process exited abnormally"
+}
+
+for {set nExtra 0} {$nExtra < 10} {incr nExtra} {
+  for {set i 0} {$i < 10} {incr i} {
+    do_test 1.nExtra=$nExtra.i=$i.1 {
+      faultsim_restore_and_reopen
+    
+      set fd [open crash.tcl w]
+      puts $fd [subst -nocommands {
+        sqlite3_crash_enable 1
+        sqlite3_test_control_pending_byte $::sqlite_pending_byte
+        sqlite3 db test.db -vfs crash
+        db eval {
+          PRAGMA main.synchronous=FULL;
+          BEGIN;
+          CREATE TABLE t1(x UNIQUE);
+        }
+        for {set e 2} {[set e] < ($nExtra+2)} {incr e} {
+          db eval "CREATE TABLE t[set e] (x)"
+        }
+        db eval {
+          INSERT INTO t1 VALUES( randomblob(170000) );
+          COMMIT;
+        }
+        sqlite3_crash_now
+      }]
+      close $fd
+    
+      set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg]
+      list $r $msg
+    } "1 {$msg}"
+  
+    do_execsql_test 1.nExtra=$nExtra.i=$i.2 { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    } {1 ok}
+  } 
+}
+
+
+finish_test
diff --git a/test/walprotocol.test b/test/walprotocol.test
index d658de9d7c..ee8d0b72a5 100644
--- a/test/walprotocol.test
+++ b/test/walprotocol.test
@@ -69,7 +69,7 @@ proc lock_callback {method filename handle lock} {
   if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY }
   return SQLITE_OK
 }
-puts "# Warning: This next test case causes SQLite to call xSlee(1) 100 times."
+puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
 puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite"
 puts "# is built on unix without HAVE_USLEEP defined, it may be much longer."
 do_test 1.3 {
diff --git a/test/where3.test b/test/where3.test
index c2804b5579..6edbe2bcad 100644
--- a/test/where3.test
+++ b/test/where3.test
@@ -45,7 +45,7 @@ do_test where3-1.1 {
   }
 } {222 two 2 222 {} {}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.1.1 {
      explain_no_trace {SELECT * FROM t1, t2 LEFT JOIN t3 ON q=x
                         WHERE p=2 AND a=q}
@@ -86,7 +86,7 @@ do_test where3-1.2 {
   }
 } {1 {Value for C1.1} {Value for C2.1} 2 {} {Value for C2.2} 3 {Value for C1.3} {Value for C2.3}}
 
-ifcapable explain {
+ifcapable explain&&!cursorhints {
   do_test where3-1.2.1 {
      explain_no_trace {
        SELECT parent1.parent1key, child1.value, child2.value
diff --git a/test/whereD.test b/test/whereD.test
index de0f9618d2..8ced0ffc72 100644
--- a/test/whereD.test
+++ b/test/whereD.test
@@ -337,4 +337,81 @@ do_searchcount_test 6.6.4 {
   SELECT c FROM x1 WHERE b=6 OR c=11 OR a=1
 } {7 11 3 search 7}
 
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+  CREATE TABLE y1(a, b);
+  CREATE TABLE y2(x, y);
+  CREATE INDEX y2xy ON y2(x, y);
+  INSERT INTO y1 VALUES(1, 1);
+  INSERT INTO y2 VALUES(3, 3);
+}
+
+do_execsql_test 7.1 {
+  SELECT * FROM y1 LEFT JOIN y2 ON ((x=1 AND y=b) OR (x=2 AND y=b))
+} {1 1 {} {}}
+
+do_execsql_test 7.3 {
+  CREATE TABLE foo (Id INTEGER PRIMARY KEY, fa INTEGER, fb INTEGER); 
+  CREATE TABLE bar (Id INTEGER PRIMARY KEY, ba INTEGER, bb INTEGER);
+
+  INSERT INTO foo VALUES(1, 1, 1);
+  INSERT INTO foo VALUES(2, 1, 2);
+  INSERT INTO foo VALUES(3, 1, 3);
+  INSERT INTO foo VALUES(4, 1, 4);
+  INSERT INTO foo VALUES(5, 1, 5);
+  INSERT INTO foo VALUES(6, 1, 6);
+  INSERT INTO foo VALUES(7, 1, 7);
+  INSERT INTO foo VALUES(8, 1, 8);
+  INSERT INTO foo VALUES(9, 1, 9);
+
+  INSERT INTO bar VALUES(NULL, 1, 1);
+  INSERT INTO bar VALUES(NULL, 2, 2);
+  INSERT INTO bar VALUES(NULL, 3, 3);
+  INSERT INTO bar VALUES(NULL, 1, 4);
+  INSERT INTO bar VALUES(NULL, 2, 5);
+  INSERT INTO bar VALUES(NULL, 3, 6);
+  INSERT INTO bar VALUES(NULL, 1, 7);
+  INSERT INTO bar VALUES(NULL, 2, 8);
+  INSERT INTO bar VALUES(NULL, 3, 9);
+}
+
+do_execsql_test 7.4 {
+  SELECT 
+    bar.Id, bar.ba, bar.bb, foo.fb
+    FROM foo LEFT JOIN bar
+           ON (bar.ba = 1 AND bar.bb = foo.fb)
+           OR (bar.ba = 5 AND bar.bb = foo.fb);
+} {
+  1 1 1 1 
+  {} {} {} 2 
+  {} {} {} 3 
+  4 1 4 4 
+  {} {} {} 5 
+  {} {} {} 6 
+  7 1 7 7 
+  {} {} {} 8 
+  {} {} {} 9
+}
+
+do_execsql_test 7.5 {
+  CREATE INDEX idx_bar ON bar(ba, bb);
+  SELECT 
+    bar.Id, bar.ba, bar.bb, foo.fb
+    FROM foo LEFT JOIN bar
+           ON (bar.ba = 1 AND bar.bb = foo.fb)
+           OR (bar.ba = 5 AND bar.bb = foo.fb);
+} {
+  1 1 1 1 
+  {} {} {} 2 
+  {} {} {} 3 
+  4 1 4 4 
+  {} {} {} 5 
+  {} {} {} 6 
+  7 1 7 7 
+  {} {} {} 8 
+  {} {} {} 9
+}
+
+
 finish_test
diff --git a/test/win32nolock.test b/test/win32nolock.test
new file mode 100644
index 0000000000..8128860392
--- /dev/null
+++ b/test/win32nolock.test
@@ -0,0 +1,126 @@
+# 2016 July 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+if {$tcl_platform(platform)!="windows"} return
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix win32nolock
+
+do_test win32nolock-1.0 {
+  sqlite3 db test.db
+  execsql {
+    CREATE TABLE t1(a, b);
+    BEGIN;
+    INSERT INTO t1 VALUES(1, 2);
+  }
+} {}
+
+do_test win32nolock-1.1 {
+  execsql COMMIT
+  catchsql { SELECT * FROM t1 }
+} {0 {1 2}}
+
+db close
+
+do_test win32nolock-1.2 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db -vfs win32-none
+  execsql { PRAGMA mmap_size = 0 } db2
+  execsql {
+    BEGIN;
+    INSERT INTO t1 VALUES(3, 4);
+  }
+} {}
+
+do_test win32nolock-1.3 {
+  execsql { SELECT * FROM t1 }
+} {1 2 3 4}
+
+do_test win32nolock-1.4 {
+  execsql { SELECT * FROM t1; } db2
+} {1 2}
+
+do_test win32nolock-1.5 {
+  execsql {
+    BEGIN;
+    SELECT * FROM t1;
+  } db2
+} {1 2}
+
+do_test win32nolock-1.6 {
+  execsql COMMIT
+  execsql {SELECT * FROM t1} db2
+} {1 2}
+
+ifcapable memorymanage {
+  do_test win32nolock-1.7 {
+    sqlite3_release_memory 1000000
+    execsql {SELECT * FROM t1} db2
+  } {1 2 3 4}
+}
+
+do_test win32nolock-1.8 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.9.1 {
+  sqlite3 db test.db
+  sqlite3 db2 test.db
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {1 {database is locked}}}
+
+do_test win32nolock-1.9.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.10.1 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.10.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.11.1 {
+  sqlite3 db test.db
+  sqlite3 db2 test.db -vfs win32-none
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.11.2 {
+  db close
+  db2 close
+} {}
+
+do_test win32nolock-1.12.1 {
+  sqlite3 db test.db -vfs win32-none
+  sqlite3 db2 test.db -vfs win32-none
+  list [catchsql { BEGIN EXCLUSIVE; } db] \
+      [catchsql { BEGIN EXCLUSIVE; } db2]
+} {{0 {}} {0 {}}}
+
+do_test win32nolock-1.12.2 {
+  db close
+  db2 close
+} {}
+
+finish_test
diff --git a/test/with3.test b/test/with3.test
index 62e88441ab..5540a7fca6 100644
--- a/test/with3.test
+++ b/test/with3.test
@@ -61,4 +61,60 @@ do_execsql_test 2.1 {
   SELECT * FROM t1;
 } {200}
 
+#-------------------------------------------------------------------------
+# Test that the planner notices LIMIT clauses on recursive WITH queries.
+#
+
+ifcapable analyze {
+  do_execsql_test 3.1.1 {
+    CREATE TABLE y1(a, b);
+    CREATE INDEX y1a ON y1(a);
+
+    WITH cnt(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM cnt LIMIT 1000)
+      INSERT INTO y1 SELECT i%10, i FROM cnt;
+    ANALYZE;
+
+  }
+
+  do_eqp_test 3.1.2 {
+    WITH cnt(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM cnt LIMIT 1)
+    SELECT * FROM cnt, y1 WHERE i=a
+  } {
+    3 0 0 {SCAN TABLE cnt} 
+    1 0 0 {COMPOUND SUBQUERIES 0 AND 0 (UNION ALL)}
+    0 0 0 {SCAN SUBQUERY 1} 
+    0 1 1 {SEARCH TABLE y1 USING INDEX y1a (a=?)}
+  }
+
+  do_eqp_test 3.1.3 {
+    WITH cnt(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM cnt LIMIT 1000000)
+    SELECT * FROM cnt, y1 WHERE i=a
+  } {
+    3 0 0 {SCAN TABLE cnt} 
+    1 0 0 {COMPOUND SUBQUERIES 0 AND 0 (UNION ALL)}
+    0 0 1 {SCAN TABLE y1} 
+    0 1 0 {SEARCH SUBQUERY 1 USING AUTOMATIC COVERING INDEX (i=?)}
+  }
+}
+
+do_execsql_test 3.2.1 {
+  CREATE TABLE w1(pk INTEGER PRIMARY KEY, x INTEGER);
+  CREATE TABLE w2(pk INTEGER PRIMARY KEY);
+}
+
+do_eqp_test 3.2.2 {
+  WITH RECURSIVE c(w,id) AS (SELECT 0, (SELECT pk FROM w2 LIMIT 1)
+     UNION ALL SELECT c.w + 1, x FROM w1, c LIMIT 1)
+     SELECT * FROM c, w2, w1
+     WHERE c.id=w2.pk AND c.id=w1.pk;
+} {
+  2 0 0 {EXECUTE SCALAR SUBQUERY 3} 
+  3 0 0 {SCAN TABLE w2} 
+  4 0 0 {SCAN TABLE w1}
+  4 1 1 {SCAN TABLE c} 
+  1 0 0 {COMPOUND SUBQUERIES 0 AND 0 (UNION ALL)} 0 0 0 {SCAN SUBQUERY 1}
+  0 1 1 {SEARCH TABLE w2 USING INTEGER PRIMARY KEY (rowid=?)} 
+  0 2 2 {SEARCH TABLE w1 USING INTEGER PRIMARY KEY (rowid=?)}
+}
+
 finish_test
diff --git a/test/without_rowid3.test b/test/without_rowid3.test
index 2af43a9191..387a213b99 100644
--- a/test/without_rowid3.test
+++ b/test/without_rowid3.test
@@ -1031,7 +1031,7 @@ ifcapable altertable {
       PRAGMA foreign_keys = off;
       ALTER TABLE t2 ADD COLUMN h DEFAULT 'text' REFERENCES t1;
       PRAGMA foreign_keys = on;
-      SELECT sql FROM sqlite_temp_master WHERE name='t2';
+      SELECT sql FROM temp.sqlite_master WHERE name='t2';
     }
   } {{CREATE TABLE t2(a, b, c REFERENCES t1, d DEFAULT NULL REFERENCES t1, e REFERENCES t1 DEFAULT NULL, h DEFAULT 'text' REFERENCES t1)}}
 
@@ -1064,7 +1064,7 @@ ifcapable altertable {
   ]
   do_test without_rowid3-14.2tmp.2.2 {
     execsql { ALTER TABLE t1 RENAME TO t4 }
-    execsql { SELECT sql FROM sqlite_temp_master WHERE type = 'table'}
+    execsql { SELECT sql FROM temp.sqlite_master WHERE type = 'table'}
   } [list \
     {CREATE TABLE "t4"(a PRIMARY KEY, b REFERENCES "t4") WITHOUT rowid}      \
     {CREATE TABLE t2(a PRIMARY KEY, b REFERENCES "t4", c REFERENCES t2)
diff --git a/test/wordcount.c b/test/wordcount.c
index 72aa6b2f0b..bc1d499f15 100644
--- a/test/wordcount.c
+++ b/test/wordcount.c
@@ -13,23 +13,6 @@
 **
 ** Option:
 **
-**     --without-rowid      Use a WITHOUT ROWID table to store the words.
-**     --insert             Use INSERT mode (the default)
-**     --replace            Use REPLACE mode
-**     --select             Use SELECT mode
-**     --update             Use UPDATE mode
-**     --delete             Use DELETE mode
-**     --query              Use QUERY mode
-**     --nocase             Add the NOCASE collating sequence to the words.
-**     --trace              Enable sqlite3_trace() output.
-**     --summary            Show summary information on the collected data.
-**     --stats              Show sqlite3_status() results at the end.
-**     --pagesize NNN       Use a page size of NNN
-**     --cachesize NNN      Use a cache size of NNN
-**     --commit NNN         Commit after every NNN operations
-**     --nosync             Use PRAGMA synchronous=OFF
-**     --journal MMMM       Use PRAGMA journal_mode=MMMM
-**     --timer              Time the operation of this program
 **
 ** Modes:
 **
@@ -80,8 +63,39 @@
 #include 
 #include 
 #include "sqlite3.h"
+#ifndef _WIN32
+# include 
+#else
+# include 
+#endif
 #define ISALPHA(X) isalpha((unsigned char)(X))
 
+const char zHelp[] = 
+"Usage: wordcount [OPTIONS] DATABASE [INPUT]\n"
+" --all                Repeat the test for all test modes\n"
+" --cachesize NNN      Use a cache size of NNN\n"
+" --commit NNN         Commit after every NNN operations\n"
+" --delete             Use DELETE mode\n"
+" --insert             Use INSERT mode (the default)\n"
+" --journal MMMM       Use PRAGMA journal_mode=MMMM\n"
+" --nocase             Add the NOCASE collating sequence to the words.\n"
+" --nosync             Use PRAGMA synchronous=OFF\n"
+" --pagesize NNN       Use a page size of NNN\n"
+" --query              Use QUERY mode\n"
+" --replace            Use REPLACE mode\n"
+" --select             Use SELECT mode\n"
+" --stats              Show sqlite3_status() results at the end.\n"
+" --summary            Show summary information on the collected data.\n"
+" --tag NAME           Tag all output using NAME.  Use only stdout.\n"
+" --timer              Time the operation of this program\n"
+" --trace              Enable sqlite3_trace() output.\n"
+" --update             Use UPDATE mode\n"
+" --without-rowid      Use a WITHOUT ROWID table to store the words.\n"
+;
+
+/* Output tag */
+char *zTag = "--";
+
 /* Return the current wall-clock time */
 static sqlite3_int64 realTime(void){
   static sqlite3_vfs *clockVfs = 0;
@@ -106,6 +120,12 @@ static void fatal_error(const char *zMsg, ...){
   exit(1);
 }
 
+/* Print a usage message and quit */
+static void usage(void){
+  printf("%s",zHelp);
+  exit(0);
+}
+
 /* The sqlite3_trace() callback function */
 static void traceCallback(void *NotUsed, const char *zSql){
   printf("%s;\n", zSql);
@@ -115,7 +135,7 @@ static void traceCallback(void *NotUsed, const char *zSql){
 ** each column separated by a single space. */
 static int printResult(void *NotUsed, int nArg, char **azArg, char **azNm){
   int i;
-  printf("--");
+  printf("%s", zTag);
   for(i=0; i=MODE_COUNT*2 ) return 0;
+  i = (*piLoopCnt)++;
+  *pUseWithoutRowid = i&1;
+  *piMode2 = i>>1;
+  return 1;
+}
 
 int main(int argc, char **argv){
   const char *zFileToRead = 0;  /* Input file.  NULL for stdin */
   const char *zDbName = 0;      /* Name of the database file to create */
   int useWithoutRowid = 0;      /* True for --without-rowid */
   int iMode = MODE_INSERT;      /* One of MODE_xxxxx */
+  int iMode2;                   /* Mode to use for current --all iteration */
+  int iLoopCnt = 0;             /* Which iteration when running --all */
   int useNocase = 0;            /* True for --nocase */
   int doTrace = 0;              /* True for --trace */
   int showStats = 0;            /* True for --stats */
@@ -220,8 +277,10 @@ int main(int argc, char **argv){
   FILE *in;                     /* The open input file */
   int rc;                       /* Return code from an SQLite interface */
   int iCur, iHiwtr;             /* Statistics values, current and "highwater" */
+  FILE *pTimer = stderr;        /* Output channel for the timer */
   sqlite3_int64 sumCnt = 0;     /* Sum in QUERY mode */
-  sqlite3_int64 startTime;
+  sqlite3_int64 startTime;      /* Time of start */
+  sqlite3_int64 totalTime = 0;  /* Total time */
   char zInput[2000];            /* A single line of input */
 
   /* Process command-line arguments */
@@ -243,6 +302,9 @@ int main(int argc, char **argv){
         iMode = MODE_DELETE;
       }else if( strcmp(z,"query")==0 ){
         iMode = MODE_QUERY;
+      }else if( strcmp(z,"all")==0 ){
+        iMode = MODE_ALL;
+        showTimer = -99;
       }else if( strcmp(z,"nocase")==0 ){
         useNocase = 1;
       }else if( strcmp(z,"trace")==0 ){
@@ -266,23 +328,33 @@ int main(int argc, char **argv){
         commitInterval = atoi(argv[i]);
       }else if( strcmp(z,"journal")==0 && i0 && (nOp%commitInterval)==0 ){
+          sqlite3_exec(db, "COMMIT; BEGIN IMMEDIATE", 0, 0, 0);
         }
       }
-      i = j-1;
-
-      /* Increment the operation counter.  Do a COMMIT if it is time. */
-      nOp++;
-      if( commitInterval>0 && (nOp%commitInterval)==0 ){
-        sqlite3_exec(db, "COMMIT; BEGIN IMMEDIATE", 0, 0, 0);
+    }
+    sqlite3_exec(db, "COMMIT", 0, 0, 0);
+    sqlite3_finalize(pInsert);  pInsert = 0;
+    sqlite3_finalize(pUpdate);  pUpdate = 0;
+    sqlite3_finalize(pSelect);  pSelect = 0;
+    sqlite3_finalize(pDelete);  pDelete = 0;
+  
+    if( iMode2==MODE_QUERY && iMode!=MODE_ALL ){
+      printf("%s sum of cnt: %lld\n", zTag, sumCnt);
+      rc = sqlite3_prepare_v2(db,"SELECT sum(cnt*cnt) FROM wordcount", -1,
+                              &pSelect, 0);
+      if( rc==SQLITE_OK && sqlite3_step(pSelect)==SQLITE_ROW ){
+        printf("%s double-check: %lld\n", zTag,sqlite3_column_int64(pSelect,0));
+      }
+      sqlite3_finalize(pSelect);
+    }
+  
+  
+    if( showTimer ){
+      sqlite3_int64 elapseTime = realTime() - startTime;
+      totalTime += elapseTime;
+      fprintf(pTimer, "%3d.%03d wordcount", (int)(elapseTime/1000),
+                                   (int)(elapseTime%1000));
+      if( iMode==MODE_ALL ){
+        fprintf(pTimer, " %s%s\n", azMode[iMode2],
+                useWithoutRowid? " --without-rowid" : "");
+      }else{
+        for(i=1; i NUL
   GOTO :EOF
diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl
index 9b34cf2e03..a6c58f1a25 100644
--- a/tool/addopcodes.tcl
+++ b/tool/addopcodes.tcl
@@ -37,6 +37,8 @@ set extras {
   UMINUS
   UPLUS
   REGISTER
+  VECTOR
+  SELECT_COLUMN
   ASTERISK
   SPAN
   SPACE
diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat
index ae8b76af5a..aaeb67bdfb 100755
--- a/tool/build-all-msvc.bat
+++ b/tool/build-all-msvc.bat
@@ -268,7 +268,7 @@ REM
 IF DEFINED TCLSH_CMD (
   SET TCLSH_FILE=%TCLSH_CMD%
 ) ELSE (
-  SET TCLSH_FILE=tclsh85.exe
+  SET TCLSH_FILE=tclsh.exe
 )
 
 FOR %%T IN (%TCLSH_FILE%) DO (
@@ -460,7 +460,7 @@ FOR %%P IN (%PLATFORMS%) DO (
     REM
     REM NOTE: Reset the PATH here to the absolute bare minimum required.
     REM
-    SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot%
+    CALL :fn_ResetPath
 
     REM
     REM NOTE: This is the inner loop.  There are normally two iterations, one
@@ -818,6 +818,10 @@ GOTO no_errors
   CALL :fn_ResetErrorLevel
   GOTO :EOF
 
+:fn_ResetPath
+  SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot%
+  GOTO :EOF
+
 :fn_AppendVariable
   SET __ECHO_CMD=ECHO %%%1%%
   IF DEFINED %1 (
diff --git a/tool/cg_anno.tcl b/tool/cg_anno.tcl
index f806c5a5af..43d2b74fcd 100755
--- a/tool/cg_anno.tcl
+++ b/tool/cg_anno.tcl
@@ -2,23 +2,76 @@
 #
 # A wrapper around cg_annotate that sets appropriate command-line options
 # and rearranges the output so that annotated files occur in a consistent
-# sorted order.  Used by the run-speed-test.tcl script.
+# sorted order.  Used by the speed-check.tcl script.
 #
 
 set in [open "|cg_annotate --show=Ir --auto=yes --context=40 $argv" r]
 set dest !
 set out(!) {}
+set linenum 0
+set cntlines 0      ;# true to remember cycle counts on each line
+set seenSqlite3 0   ;# true if we have seen the sqlite3.c file
 while {![eof $in]} {
   set line [string map {\t {        }} [gets $in]]
   if {[regexp {^-- Auto-annotated source: (.*)} $line all name]} {
     set dest $name
-  } elseif {[regexp {^-- line \d+ ------} $line]} {
+    if {[string match */sqlite3.c $dest]} {
+      set cntlines 1
+      set seenSqlite3 1
+    } else {
+      set cntlines 0
+    }
+  } elseif {[regexp {^-- line (\d+) ------} $line all ln]} {
     set line [lreplace $line 2 2 {#}]
+    set linenum [expr {$ln-1}]
   } elseif {[regexp {^The following files chosen for } $line]} {
     set dest !
   }
   append out($dest) $line\n
+  if {$cntlines} {
+    incr linenum
+    if {[regexp {^ *([0-9,]+) } $line all x]} {
+      set x [string map {, {}} $x]
+      set cycles($linenum) $x
+    }
+  }
 }
 foreach x [lsort [array names out]] {
   puts $out($x)
 }
+
+# If the sqlite3.c file has been seen, then output a summary of the
+# cycle counts for each file that went into making up sqlite3.c
+#
+if {$seenSqlite3} {
+  close $in
+  set in [open sqlite3.c]
+  set linenum 0
+  set fn sqlite3.c
+  set pattern1 {^/\*+ Begin file ([^ ]+) \*}
+  set pattern2 {^/\*+ Continuing where we left off in ([^ ]+) \*}
+  while {![eof $in]} {
+    set line [gets $in]
+    incr linenum
+    if {[regexp $pattern1 $line all newfn]} {
+      set fn $newfn
+    } elseif {[regexp $pattern2 $line all newfn]} {
+      set fn $newfn
+    } elseif {[info exists cycles($linenum)]} {
+      incr fcycles($fn) $cycles($linenum)
+    }
+  }
+  close $in
+  puts {**********************************************************************}
+  set lx {}
+  set sum 0
+  foreach {fn cnt} [array get fcycles] {
+    lappend lx [list $cnt $fn]
+    incr sum $cnt
+  }
+  puts [format {%20s %14d  %8.3f%%} TOTAL $sum 100]
+  foreach entry [lsort -index 0 -integer -decreasing $lx] {
+    foreach {cnt fn} $entry break
+    puts [format {%20s %14d  %8.3f%%} $fn $cnt [expr {$cnt*100.0/$sum}]]
+  }
+}
diff --git a/tool/dbhash.c b/tool/dbhash.c
new file mode 100644
index 0000000000..b1c72b3e36
--- /dev/null
+++ b/tool/dbhash.c
@@ -0,0 +1,506 @@
+/*
+** 2016-06-07
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This is a utility program that computes an SHA1 hash on the content
+** of an SQLite database.
+**
+** The hash is computed over just the content of the database.  Free
+** space inside of the database file, and alternative on-disk representations
+** of the same content (ex: UTF8 vs UTF16) do not affect the hash.  So,
+** for example, the database file page size, encoding, and auto_vacuum setting
+** can all be changed without changing the hash.
+*/
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "sqlite3.h"
+
+/* Context for the SHA1 hash */
+typedef struct SHA1Context SHA1Context;
+struct SHA1Context {
+  unsigned int state[5];
+  unsigned int count[2];
+  unsigned char buffer[64];
+};
+
+/*
+** All global variables are gathered into the "g" singleton.
+*/
+struct GlobalVars {
+  const char *zArgv0;       /* Name of program */
+  unsigned fDebug;          /* Debug flags */
+  sqlite3 *db;              /* The database connection */
+  SHA1Context cx;           /* SHA1 hash context */
+} g;
+
+/*
+** Debugging flags
+*/
+#define DEBUG_FULLTRACE   0x00000001   /* Trace hash to stderr */
+
+/******************************************************************************
+** The Hash Engine
+**
+** Modify these routines (and appropriate state fields in global variable 'g')
+** in order to compute a different (better?) hash of the database.
+*/
+/*
+ * blk0() and blk() perform the initial expand.
+ * I got the idea of expanding during the round function from SSLeay
+ *
+ * blk0le() for little-endian and blk0be() for big-endian.
+ */
+#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
+/*
+ * GCC by itself only generates left rotates.  Use right rotates if
+ * possible to be kinder to dinky implementations with iterative rotate
+ * instructions.
+ */
+#define SHA_ROT(op, x, k) \
+        ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
+#define rol(x,k) SHA_ROT("roll", x, k)
+#define ror(x,k) SHA_ROT("rorl", x, k)
+
+#else
+/* Generic C equivalent */
+#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
+#define rol(x,k) SHA_ROT(x,k,32-(k))
+#define ror(x,k) SHA_ROT(x,32-(k),k)
+#endif
+
+
+#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
+    |(rol(block[i],8)&0x00FF00FF))
+#define blk0be(i) block[i]
+#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
+    ^block[(i+2)&15]^block[i&15],1))
+
+/*
+ * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
+ *
+ * Rl0() for little-endian and Rb0() for big-endian.  Endianness is
+ * determined at run-time.
+ */
+#define Rl0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define Rb0(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R1(v,w,x,y,z,i) \
+    z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
+#define R2(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
+#define R3(v,w,x,y,z,i) \
+    z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
+#define R4(v,w,x,y,z,i) \
+    z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
+
+/*
+ * Hash a single 512-bit block. This is the core of the algorithm.
+ */
+#define a qq[0]
+#define b qq[1]
+#define c qq[2]
+#define d qq[3]
+#define e qq[4]
+
+void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
+  unsigned int qq[5]; /* a, b, c, d, e; */
+  static int one = 1;
+  unsigned int block[16];
+  memcpy(block, buffer, 64);
+  memcpy(qq,state,5*sizeof(unsigned int));
+
+  /* Copy g.cx.state[] to working vars */
+  /*
+  a = state[0];
+  b = state[1];
+  c = state[2];
+  d = state[3];
+  e = state[4];
+  */
+
+  /* 4 rounds of 20 operations each. Loop unrolled. */
+  if( 1 == *(unsigned char*)&one ){
+    Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
+    Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
+    Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
+    Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
+  }else{
+    Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
+    Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
+    Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
+    Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
+  }
+  R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
+  R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
+  R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
+  R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
+  R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
+  R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
+  R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
+  R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
+  R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
+  R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
+  R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
+  R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
+  R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
+  R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
+  R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
+  R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
+
+  /* Add the working vars back into context.state[] */
+  state[0] += a;
+  state[1] += b;
+  state[2] += c;
+  state[3] += d;
+  state[4] += e;
+}
+
+
+/* Initialize the SHA1 hash */
+static void hash_init(void){
+  /* SHA1 initialization constants */
+  g.cx.state[0] = 0x67452301;
+  g.cx.state[1] = 0xEFCDAB89;
+  g.cx.state[2] = 0x98BADCFE;
+  g.cx.state[3] = 0x10325476;
+  g.cx.state[4] = 0xC3D2E1F0;
+  g.cx.count[0] = g.cx.count[1] = 0;
+}
+
+/* Add new content to the SHA1 hash */
+static void hash_step(const unsigned char *data,  unsigned int len){
+  unsigned int i, j;
+
+  j = g.cx.count[0];
+  if( (g.cx.count[0] += len << 3) < j ){
+    g.cx.count[1] += (len>>29)+1;
+  }
+  j = (j >> 3) & 63;
+  if( (j + len) > 63 ){
+    (void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
+    SHA1Transform(g.cx.state, g.cx.buffer);
+    for(; i + 63 < len; i += 64){
+      SHA1Transform(g.cx.state, &data[i]);
+    }
+    j = 0;
+  }else{
+    i = 0;
+  }
+  (void)memcpy(&g.cx.buffer[j], &data[i], len - i);
+}
+
+
+/* Add padding and compute and output the message digest. */
+static void hash_finish(const char *zName){
+  unsigned int i;
+  unsigned char finalcount[8];
+  unsigned char digest[20];
+  static const char zEncode[] = "0123456789abcdef";
+  char zOut[41];
+
+  for (i = 0; i < 8; i++){
+    finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
+       >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
+  }
+  hash_step((const unsigned char *)"\200", 1);
+  while ((g.cx.count[0] & 504) != 448){
+    hash_step((const unsigned char *)"\0", 1);
+  }
+  hash_step(finalcount, 8);  /* Should cause a SHA1Transform() */
+  for (i = 0; i < 20; i++){
+    digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
+  }
+  for(i=0; i<20; i++){
+    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  }
+  zOut[i*2]= 0;
+  printf("%s %s\n", zOut, zName);
+}
+/* End of the hashing logic
+*******************************************************************************/
+  
+/*
+** Print an error resulting from faulting command-line arguments and
+** abort the program.
+*/
+static void cmdlineError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
+  exit(1);
+}
+
+/*
+** Print an error message for an error that occurs at runtime, then
+** abort the program.
+*/
+static void runtimeError(const char *zFormat, ...){
+  va_list ap;
+  fprintf(stderr, "%s: ", g.zArgv0);
+  va_start(ap, zFormat);
+  vfprintf(stderr, zFormat, ap);
+  va_end(ap);
+  fprintf(stderr, "\n");
+  exit(1);
+}
+
+/*
+** Prepare a new SQL statement.  Print an error and abort if anything
+** goes wrong.
+*/
+static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
+  char *zSql;
+  int rc;
+  sqlite3_stmt *pStmt;
+
+  zSql = sqlite3_vmprintf(zFormat, ap);
+  if( zSql==0 ) runtimeError("out of memory");
+  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
+  if( rc ){
+    runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
+                 zSql);
+  }
+  sqlite3_free(zSql);
+  return pStmt;
+}
+static sqlite3_stmt *db_prepare(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  return pStmt;
+}
+
+/*
+** Compute the hash for all rows of the query formed from the printf-style
+** zFormat and its argument.
+*/
+static void hash_one_query(const char *zFormat, ...){
+  va_list ap;
+  sqlite3_stmt *pStmt;        /* The query defined by zFormat and "..." */
+  int nCol;                   /* Number of columns in the result set */
+  int i;                      /* Loop counter */
+
+  /* Prepare the query defined by zFormat and "..." */
+  va_start(ap, zFormat);
+  pStmt = db_vprepare(zFormat, ap);
+  va_end(ap);
+  nCol = sqlite3_column_count(pStmt);
+
+  /* Compute a hash over the result of the query */
+  while( SQLITE_ROW==sqlite3_step(pStmt) ){
+    for(i=0; i=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"1",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_FLOAT: {
+          sqlite3_uint64 u;
+          int j;
+          unsigned char x[8];
+          double r = sqlite3_column_double(pStmt,i);
+          memcpy(&u, &r, 8);
+          for(j=7; j>=0; j--){
+            x[j] = u & 0xff;
+            u >>= 8;
+          }
+          hash_step((const unsigned char*)"2",1);
+          hash_step(x,8);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_TEXT: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_text(pStmt, i);
+          hash_step((const unsigned char*)"3", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
+          }
+          break;
+        }
+        case SQLITE_BLOB: {
+          int n = sqlite3_column_bytes(pStmt, i);
+          const unsigned char *z = sqlite3_column_blob(pStmt, i);
+          hash_step((const unsigned char*)"4", 1);
+          hash_step(z, n);
+          if( g.fDebug & DEBUG_FULLTRACE ){
+            fprintf(stderr, "BLOB (%d bytes)\n", n);
+          }
+          break;
+        }
+      }
+    }
+  }
+  sqlite3_finalize(pStmt);
+}
+
+
+/*
+** Print sketchy documentation for this utility program
+*/
+static void showHelp(void){
+  printf("Usage: %s [options] FILE ...\n", g.zArgv0);
+  printf(
+"Compute a SHA1 hash on the content of database FILE.  System tables such as\n"
+"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
+"Options:\n"
+"   --debug N           Set debugging flags to N (experts only)\n"
+"   --like PATTERN      Only hash tables whose name is LIKE the pattern\n"
+"   --schema-only       Only hash the schema - omit table content\n"
+"   --without-schema    Only hash table content - omit the schema\n"
+  );
+}
+
+int main(int argc, char **argv){
+  const char *zDb = 0;         /* Name of the database currently being hashed */
+  int i;                       /* Loop counter */
+  int rc;                      /* Subroutine return code */
+  char *zErrMsg;               /* Error message when opening database */
+  sqlite3_stmt *pStmt;         /* An SQLite query */
+  const char *zLike = 0;       /* LIKE pattern of tables to hash */
+  int omitSchema = 0;          /* True to compute hash on content only */
+  int omitContent = 0;         /* True to compute hash on schema only */
+  int nFile = 0;               /* Number of input filenames seen */
+
+  g.zArgv0 = argv[0];
+  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+  for(i=1; in + n >= p->nAlloc ){
+    char *zNew;
+    sqlite3_uint64 nNew;
+    if( p->oomErr ) return;
+    nNew = p->nAlloc*2 + 100 + n;
+    zNew = sqlite3_realloc(p->z, nNew);
+    if( zNew==0 ){
+      sqlite3_free(p->z);
+      memset(p, 0, sizeof(*p));
+      p->oomErr = 1;
+      return;
+    }
+    p->z = zNew;
+    p->nAlloc = nNew;
+  }
+  memcpy(p->z + p->n, z, n);
+  p->n += n;
+  p->z[p->n] = 0;
+}
+
+/* Return the current string content */
+static char *StrStr(Str *p){
+ return p->z;
+}
+
+/* Free the string */
+static void StrFree(Str *p){
+  sqlite3_free(p->z);
+  StrInit(p);
+}
+
 /***************************************************************************
 ** eval() implementation copied from ../ext/misc/eval.c
 */
@@ -312,6 +364,326 @@ static void sqlEvalFunc(
 /* End of the eval() implementation
 ******************************************************************************/
 
+/******************************************************************************
+** The generate_series(START,END,STEP) eponymous table-valued function.
+**
+** This code is copy/pasted from ext/misc/series.c in the SQLite source tree.
+*/
+/* series_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct series_cursor series_cursor;
+struct series_cursor {
+  sqlite3_vtab_cursor base;  /* Base class - must be first */
+  int isDesc;                /* True to count down rather than up */
+  sqlite3_int64 iRowid;      /* The rowid */
+  sqlite3_int64 iValue;      /* Current value ("value") */
+  sqlite3_int64 mnValue;     /* Mimimum value ("start") */
+  sqlite3_int64 mxValue;     /* Maximum value ("stop") */
+  sqlite3_int64 iStep;       /* Increment ("step") */
+};
+
+/*
+** The seriesConnect() method is invoked to create a new
+** series_vtab that describes the generate_series virtual table.
+**
+** Think of this routine as the constructor for series_vtab objects.
+**
+** All this routine needs to do is:
+**
+**    (1) Allocate the series_vtab object and initialize all fields.
+**
+**    (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+**        result set of queries against generate_series will look like.
+*/
+static int seriesConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  sqlite3_vtab *pNew;
+  int rc;
+
+/* Column numbers */
+#define SERIES_COLUMN_VALUE 0
+#define SERIES_COLUMN_START 1
+#define SERIES_COLUMN_STOP  2
+#define SERIES_COLUMN_STEP  3
+
+  rc = sqlite3_declare_vtab(db,
+     "CREATE TABLE x(value,start hidden,stop hidden,step hidden)");
+  if( rc==SQLITE_OK ){
+    pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) );
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(*pNew));
+  }
+  return rc;
+}
+
+/*
+** This method is the destructor for series_cursor objects.
+*/
+static int seriesDisconnect(sqlite3_vtab *pVtab){
+  sqlite3_free(pVtab);
+  return SQLITE_OK;
+}
+
+/*
+** Constructor for a new series_cursor object.
+*/
+static int seriesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+  series_cursor *pCur;
+  pCur = sqlite3_malloc( sizeof(*pCur) );
+  if( pCur==0 ) return SQLITE_NOMEM;
+  memset(pCur, 0, sizeof(*pCur));
+  *ppCursor = &pCur->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destructor for a series_cursor.
+*/
+static int seriesClose(sqlite3_vtab_cursor *cur){
+  sqlite3_free(cur);
+  return SQLITE_OK;
+}
+
+
+/*
+** Advance a series_cursor to its next row of output.
+*/
+static int seriesNext(sqlite3_vtab_cursor *cur){
+  series_cursor *pCur = (series_cursor*)cur;
+  if( pCur->isDesc ){
+    pCur->iValue -= pCur->iStep;
+  }else{
+    pCur->iValue += pCur->iStep;
+  }
+  pCur->iRowid++;
+  return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int seriesColumn(
+  sqlite3_vtab_cursor *cur,   /* The cursor */
+  sqlite3_context *ctx,       /* First argument to sqlite3_result_...() */
+  int i                       /* Which column to return */
+){
+  series_cursor *pCur = (series_cursor*)cur;
+  sqlite3_int64 x = 0;
+  switch( i ){
+    case SERIES_COLUMN_START:  x = pCur->mnValue; break;
+    case SERIES_COLUMN_STOP:   x = pCur->mxValue; break;
+    case SERIES_COLUMN_STEP:   x = pCur->iStep;   break;
+    default:                   x = pCur->iValue;  break;
+  }
+  sqlite3_result_int64(ctx, x);
+  return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row.  In this implementation, the
+** rowid is the same as the output value.
+*/
+static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  series_cursor *pCur = (series_cursor*)cur;
+  *pRowid = pCur->iRowid;
+  return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int seriesEof(sqlite3_vtab_cursor *cur){
+  series_cursor *pCur = (series_cursor*)cur;
+  if( pCur->isDesc ){
+    return pCur->iValue < pCur->mnValue;
+  }else{
+    return pCur->iValue > pCur->mxValue;
+  }
+}
+
+/* True to cause run-time checking of the start=, stop=, and/or step= 
+** parameters.  The only reason to do this is for testing the
+** constraint checking logic for virtual tables in the SQLite core.
+*/
+#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY
+# define SQLITE_SERIES_CONSTRAINT_VERIFY 0
+#endif
+
+/*
+** This method is called to "rewind" the series_cursor object back
+** to the first row of output.  This method is always called at least
+** once prior to any call to seriesColumn() or seriesRowid() or 
+** seriesEof().
+**
+** The query plan selected by seriesBestIndex is passed in the idxNum
+** parameter.  (idxStr is not used in this implementation.)  idxNum
+** is a bitmask showing which constraints are available:
+**
+**    1:    start=VALUE
+**    2:    stop=VALUE
+**    4:    step=VALUE
+**
+** Also, if bit 8 is set, that means that the series should be output
+** in descending order rather than in ascending order.
+**
+** This routine should initialize the cursor and position it so that it
+** is pointing at the first row, or pointing off the end of the table
+** (so that seriesEof() will return true) if the table is empty.
+*/
+static int seriesFilter(
+  sqlite3_vtab_cursor *pVtabCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  series_cursor *pCur = (series_cursor *)pVtabCursor;
+  int i = 0;
+  if( idxNum & 1 ){
+    pCur->mnValue = sqlite3_value_int64(argv[i++]);
+  }else{
+    pCur->mnValue = 0;
+  }
+  if( idxNum & 2 ){
+    pCur->mxValue = sqlite3_value_int64(argv[i++]);
+  }else{
+    pCur->mxValue = 0xffffffff;
+  }
+  if( idxNum & 4 ){
+    pCur->iStep = sqlite3_value_int64(argv[i++]);
+    if( pCur->iStep<1 ) pCur->iStep = 1;
+  }else{
+    pCur->iStep = 1;
+  }
+  if( idxNum & 8 ){
+    pCur->isDesc = 1;
+    pCur->iValue = pCur->mxValue;
+    if( pCur->iStep>0 ){
+      pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep;
+    }
+  }else{
+    pCur->isDesc = 0;
+    pCur->iValue = pCur->mnValue;
+  }
+  pCur->iRowid = 1;
+  return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table.  This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan.  idxStr is unused.
+**
+** The query plan is represented by bits in idxNum:
+**
+**  (1)  start = $value  -- constraint exists
+**  (2)  stop = $value   -- constraint exists
+**  (4)  step = $value   -- constraint exists
+**  (8)  output in descending order
+*/
+static int seriesBestIndex(
+  sqlite3_vtab *tab,
+  sqlite3_index_info *pIdxInfo
+){
+  int i;                 /* Loop over constraints */
+  int idxNum = 0;        /* The query plan bitmask */
+  int startIdx = -1;     /* Index of the start= constraint, or -1 if none */
+  int stopIdx = -1;      /* Index of the stop= constraint, or -1 if none */
+  int stepIdx = -1;      /* Index of the step= constraint, or -1 if none */
+  int nArg = 0;          /* Number of arguments that seriesFilter() expects */
+
+  const struct sqlite3_index_constraint *pConstraint;
+  pConstraint = pIdxInfo->aConstraint;
+  for(i=0; inConstraint; i++, pConstraint++){
+    if( pConstraint->usable==0 ) continue;
+    if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+    switch( pConstraint->iColumn ){
+      case SERIES_COLUMN_START:
+        startIdx = i;
+        idxNum |= 1;
+        break;
+      case SERIES_COLUMN_STOP:
+        stopIdx = i;
+        idxNum |= 2;
+        break;
+      case SERIES_COLUMN_STEP:
+        stepIdx = i;
+        idxNum |= 4;
+        break;
+    }
+  }
+  if( startIdx>=0 ){
+    pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( stopIdx>=0 ){
+    pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( stepIdx>=0 ){
+    pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
+    pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+  }
+  if( (idxNum & 3)==3 ){
+    /* Both start= and stop= boundaries are available.  This is the 
+    ** the preferred case */
+    pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0));
+    pIdxInfo->estimatedRows = 1000;
+    if( pIdxInfo->nOrderBy==1 ){
+      if( pIdxInfo->aOrderBy[0].desc ) idxNum |= 8;
+      pIdxInfo->orderByConsumed = 1;
+    }
+  }else{
+    /* If either boundary is missing, we have to generate a huge span
+    ** of numbers.  Make this case very expensive so that the query
+    ** planner will work hard to avoid it. */
+    pIdxInfo->estimatedCost = (double)2147483647;
+    pIdxInfo->estimatedRows = 2147483647;
+  }
+  pIdxInfo->idxNum = idxNum;
+  return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the 
+** generate_series virtual table.
+*/
+static sqlite3_module seriesModule = {
+  0,                         /* iVersion */
+  0,                         /* xCreate */
+  seriesConnect,             /* xConnect */
+  seriesBestIndex,           /* xBestIndex */
+  seriesDisconnect,          /* xDisconnect */
+  0,                         /* xDestroy */
+  seriesOpen,                /* xOpen - open a cursor */
+  seriesClose,               /* xClose - close a cursor */
+  seriesFilter,              /* xFilter - configure scan constraints */
+  seriesNext,                /* xNext - advance a cursor */
+  seriesEof,                 /* xEof - check for end of scan */
+  seriesColumn,              /* xColumn - read data */
+  seriesRowid,               /* xRowid - read data */
+  0,                         /* xUpdate */
+  0,                         /* xBegin */
+  0,                         /* xSync */
+  0,                         /* xCommit */
+  0,                         /* xRollback */
+  0,                         /* xFindMethod */
+  0,                         /* xRename */
+};
+/* END the generate_series(START,END,STEP) implementation
+*********************************************************************************/
+
 /*
 ** Print sketchy documentation for this utility program
 */
@@ -706,6 +1078,8 @@ int main(int argc, char **argv){
         oomCnt = 0;
       }
       do{
+        Str sql;
+        StrInit(&sql);
         if( zDbName ){
           rc = sqlite3_open_v2(zDbName, &db, SQLITE_OPEN_READWRITE, 0);
           if( rc!=SQLITE_OK ){
@@ -729,11 +1103,31 @@ int main(int argc, char **argv){
     #endif
         sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
         sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, sqlEvalFunc, 0, 0);
+        sqlite3_create_module(db, "generate_series", &seriesModule, 0);
         sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 1000000);
         if( zEncoding ) sqlexec(db, "PRAGMA encoding=%s", zEncoding);
         if( pageSize ) sqlexec(db, "PRAGMA pagesize=%d", pageSize);
         if( doAutovac ) sqlexec(db, "PRAGMA auto_vacuum=FULL");
         iStart = timeOfDay();
+
+        /* If using an input database file and that database contains a table
+        ** named "autoexec" with a column "sql", then replace the input SQL
+        ** with the concatenated text of the autoexec table.  In this way,
+        ** if the database file is the input being fuzzed, the SQL text is
+        ** fuzzed at the same time. */
+        if( sqlite3_table_column_metadata(db,0,"autoexec","sql",0,0,0,0,0)==0 ){
+          sqlite3_stmt *pStmt;
+          rc = sqlite3_prepare_v2(db, "SELECT sql FROM autoexec", -1, &pStmt, 0);
+          if( rc==SQLITE_OK ){
+            while( sqlite3_step(pStmt)==SQLITE_ROW ){
+              StrAppend(&sql, (const char*)sqlite3_column_text(pStmt, 0));
+              StrAppend(&sql, "\n");
+            }
+          }
+          sqlite3_finalize(pStmt);
+          zSql = StrStr(&sql);
+        }
+
         g.bOomEnable = 1;
         if( verboseFlag ){
           zErrMsg = 0;
@@ -747,6 +1141,7 @@ int main(int argc, char **argv){
         }
         g.bOomEnable = 0;
         iEnd = timeOfDay();
+        StrFree(&sql);
         rc = sqlite3_close(db);
         if( rc ){
           abendError("sqlite3_close() failed with rc=%d", rc);
diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh
new file mode 100644
index 0000000000..5f2c8345be
--- /dev/null
+++ b/tool/kvtest-speed.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# A script for running speed tests using kvtest.
+#
+# The test database must be set up first.  Recommended
+# command-line:
+#
+#    ./kvtest init kvtest.db --count 100K --size 12K --variance 5K
+
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD"
+KVARGS="--count 100K --stats"
+gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest
+
+# First run using SQL
+rm cachegrind.out.[1-9][0-9]*
+valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt
+mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME
+cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt
+
+# Second run using the sqlite3_blob object
+valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt
+mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME
+cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt
+
+# Diff the sqlite3_blob API analysis for non-trunk runs.
+if test "$NAME" != "trunk"; then
+  fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt &
+fi
diff --git a/tool/lemon.c b/tool/lemon.c
index cefdf80174..aa0f4e3e9c 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -263,7 +263,8 @@ struct symbol {
   int useCnt;              /* Number of times used */
   char *destructor;        /* Code which executes whenever this symbol is
                            ** popped from the stack during error processing */
-  int destLineno;          /* Line number for start of destructor */
+  int destLineno;          /* Line number for start of destructor.  Set to
+                           ** -1 for duplicate destructors. */
   char *datatype;          /* The data type of information held by this
                            ** object. Only used if type==NONTERMINAL */
   int dtnum;               /* The data type number.  In the parser, the value
@@ -288,9 +289,13 @@ struct rule {
   const char *code;        /* The code executed when this rule is reduced */
   const char *codePrefix;  /* Setup code before code[] above */
   const char *codeSuffix;  /* Breakdown code after code[] above */
+  int noCode;              /* True if this rule has no associated C code */
+  int codeEmitted;         /* True if the code has been emitted already */
   struct symbol *precsym;  /* Precedence symbol for this rule */
   int index;               /* An index number for this rule */
+  int iRule;               /* Rule number as used in the generated tables */
   Boolean canReduce;       /* True if this rule is ever reduced */
+  Boolean doesReduce;      /* Reduce actions occur after optimization */
   struct rule *nextlhs;    /* Next rule with the same LHS */
   struct rule *next;       /* Next rule in the global list */
 };
@@ -338,6 +343,7 @@ struct action {
     struct state *stp;     /* The new state, if a shift */
     struct rule *rp;       /* The rule, if a reduce */
   } x;
+  struct symbol *spOpt;    /* SHIFTREDUCE optimization to this symbol */
   struct action *next;     /* Next action for this state */
   struct action *collide;  /* Next action with the same hash */
 };
@@ -348,7 +354,7 @@ struct state {
   struct config *bp;       /* The basis configurations for this state */
   struct config *cfp;      /* All configurations in this set */
   int statenum;            /* Sequential number for this state */
-  struct action *ap;       /* Array of actions for this state */
+  struct action *ap;       /* List of actions for this state */
   int nTknAct, nNtAct;     /* Number of actions on terminals and nonterminals */
   int iTknOfst, iNtOfst;   /* yy_action[] offset for terminals and nonterms */
   int iDfltReduce;         /* Default action is to REDUCE by this rule */
@@ -372,6 +378,7 @@ struct plink {
 struct lemon {
   struct state **sorted;   /* Table of states sorted by state number */
   struct rule *rule;       /* List of all rules */
+  struct rule *startRule;  /* First rule */
   int nstate;              /* Number of states */
   int nxstate;             /* nstate with tail degenerate states removed */
   int nrule;               /* Number of rules */
@@ -528,6 +535,7 @@ void Action_add(
   *app = newaction;
   newaction->type = type;
   newaction->sp = sp;
+  newaction->spOpt = 0;
   if( type==SHIFT ){
     newaction->x.stp = (struct state *)arg;
   }else{
@@ -858,12 +866,12 @@ void FindStates(struct lemon *lemp)
       ErrorMsg(lemp->filename,0,
 "The specified start symbol \"%s\" is not \
 in a nonterminal of the grammar.  \"%s\" will be used as the start \
-symbol instead.",lemp->start,lemp->rule->lhs->name);
+symbol instead.",lemp->start,lemp->startRule->lhs->name);
       lemp->errorcnt++;
-      sp = lemp->rule->lhs;
+      sp = lemp->startRule->lhs;
     }
   }else{
-    sp = lemp->rule->lhs;
+    sp = lemp->startRule->lhs;
   }
 
   /* Make sure the start symbol doesn't occur on the right-hand side of
@@ -1117,9 +1125,9 @@ void FindActions(struct lemon *lemp)
   /* Add the accepting token */
   if( lemp->start ){
     sp = Symbol_find(lemp->start);
-    if( sp==0 ) sp = lemp->rule->lhs;
+    if( sp==0 ) sp = lemp->startRule->lhs;
   }else{
-    sp = lemp->rule->lhs;
+    sp = lemp->startRule->lhs;
   }
   /* Add to the first state (which is always the starting state of the
   ** finite state machine) an action to ACCEPT if the lookahead is the
@@ -1497,6 +1505,54 @@ static void handle_T_option(char *z){
   lemon_strcpy(user_templatename, z);
 }
 
+/* Merge together to lists of rules ordered by rule.iRule */
+static struct rule *Rule_merge(struct rule *pA, struct rule *pB){
+  struct rule *pFirst = 0;
+  struct rule **ppPrev = &pFirst;
+  while( pA && pB ){
+    if( pA->iRuleiRule ){
+      *ppPrev = pA;
+      ppPrev = &pA->next;
+      pA = pA->next;
+    }else{
+      *ppPrev = pB;
+      ppPrev = &pB->next;
+      pB = pB->next;
+    }
+  }
+  if( pA ){
+    *ppPrev = pA;
+  }else{
+    *ppPrev = pB;
+  }
+  return pFirst;
+}
+
+/*
+** Sort a list of rules in order of increasing iRule value
+*/
+static struct rule *Rule_sort(struct rule *rp){
+  int i;
+  struct rule *pNext;
+  struct rule *x[32];
+  memset(x, 0, sizeof(x));
+  while( rp ){
+    pNext = rp->next;
+    rp->next = 0;
+    for(i=0; iname[0]); i++);
   lem.nterminal = i;
 
+  /* Assign sequential rule numbers.  Start with 0.  Put rules that have no
+  ** reduce action C-code associated with them last, so that the switch()
+  ** statement that selects reduction actions will have a smaller jump table.
+  */
+  for(i=0, rp=lem.rule; rp; rp=rp->next){
+    rp->iRule = rp->code ? i++ : -1;
+  }
+  for(rp=lem.rule; rp; rp=rp->next){
+    if( rp->iRule<0 ) rp->iRule = i++;
+  }
+  lem.startRule = lem.rule;
+  lem.rule = Rule_sort(lem.rule);
+
   /* Generate a reprint of the grammar, if requested on the command line */
   if( rpflag ){
     Reprint(&lem);
@@ -2150,6 +2220,7 @@ to follow the previous rule.");
         }else{
           psp->prevrule->line = psp->tokenlineno;
           psp->prevrule->code = &x[1];
+          psp->prevrule->noCode = 0;
         }
       }else if( x[0]=='[' ){
         psp->state = PRECEDENCE_MARK_1;
@@ -2256,6 +2327,7 @@ to follow the previous rule.");
           rp->lhsalias = psp->lhsalias;
           rp->nrhs = psp->nrhs;
           rp->code = 0;
+          rp->noCode = 1;
           rp->precsym = 0;
           rp->index = psp->gp->nrule++;
           rp->nextlhs = rp->lhs->rule;
@@ -3054,13 +3126,13 @@ int PrintAction(
     }
     case REDUCE: {
       struct rule *rp = ap->x.rp;
-      fprintf(fp,"%*s reduce       %-7d",indent,ap->sp->name,rp->index);
+      fprintf(fp,"%*s reduce       %-7d",indent,ap->sp->name,rp->iRule);
       RulePrint(fp, rp, -1);
       break;
     }
     case SHIFTREDUCE: {
       struct rule *rp = ap->x.rp;
-      fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index);
+      fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule);
       RulePrint(fp, rp, -1);
       break;
     }
@@ -3073,7 +3145,7 @@ int PrintAction(
     case SRCONFLICT:
     case RRCONFLICT:
       fprintf(fp,"%*s reduce       %-7d ** Parsing conflict **",
-        indent,ap->sp->name,ap->x.rp->index);
+        indent,ap->sp->name,ap->x.rp->iRule);
       break;
     case SSCONFLICT:
       fprintf(fp,"%*s shift        %-7d ** Parsing conflict **", 
@@ -3090,7 +3162,7 @@ int PrintAction(
     case RD_RESOLVED:
       if( showPrecedenceConflict ){
         fprintf(fp,"%*s reduce %-7d -- dropped by precedence",
-                indent,ap->sp->name,ap->x.rp->index);
+                indent,ap->sp->name,ap->x.rp->iRule);
       }else{
         result = 0;
       }
@@ -3099,6 +3171,9 @@ int PrintAction(
       result = 0;
       break;
   }
+  if( result && ap->spOpt ){
+    fprintf(fp,"  /* because %s==%s */", ap->sp->name, ap->spOpt->name);
+  }
   return result;
 }
 
@@ -3121,7 +3196,7 @@ void ReportOutput(struct lemon *lemp)
     while( cfp ){
       char buf[20];
       if( cfp->dot==cfp->rp->nrhs ){
-        lemon_sprintf(buf,"(%d)",cfp->rp->index);
+        lemon_sprintf(buf,"(%d)",cfp->rp->iRule);
         fprintf(fp,"    %5s ",buf);
       }else{
         fprintf(fp,"          ");
@@ -3222,8 +3297,8 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap)
   int act;
   switch( ap->type ){
     case SHIFT:  act = ap->x.stp->statenum;                        break;
-    case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate;        break;
-    case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break;
+    case SHIFTREDUCE: act = ap->x.rp->iRule + lemp->nstate;        break;
+    case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break;
     case ERROR:  act = lemp->nstate + lemp->nrule*2;               break;
     case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1;           break;
     default:     act = -1; break;
@@ -3466,9 +3541,8 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
 }
 
 /*
-** zCode is a string that is the action associated with a rule.  Expand
-** the symbols in this string so that the refer to elements of the parser
-** stack.
+** Write and transform the rp->code string so that symbols are expanded.
+** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate.
 **
 ** Return 1 if the expanded code requires that "yylhsminor" local variable
 ** to be defined.
@@ -3492,17 +3566,17 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
     static char newlinestr[2] = { '\n', '\0' };
     rp->code = newlinestr;
     rp->line = rp->ruleline;
+    rp->noCode = 1;
+  }else{
+    rp->noCode = 0;
   }
 
 
-  if( rp->lhsalias==0 ){
-    /* There is no LHS value symbol. */
-    lhsdirect = 1;
-  }else if( rp->nrhs==0 ){
+  if( rp->nrhs==0 ){
     /* If there are no RHS symbols, then writing directly to the LHS is ok */
     lhsdirect = 1;
   }else if( rp->rhsalias[0]==0 ){
-    /* The left-most RHS symbol has not value.  LHS direct is ok.  But
+    /* The left-most RHS symbol has no value.  LHS direct is ok.  But
     ** we have to call the distructor on the RHS symbol first. */
     lhsdirect = 1;
     if( has_destructor(rp->rhs[0],lemp) ){
@@ -3510,7 +3584,11 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
       append_str("  yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
                  rp->rhs[0]->index,1-rp->nrhs);
       rp->codePrefix = Strsafe(append_str(0,0,0,0));
+      rp->noCode = 0;
     }
+  }else if( rp->lhsalias==0 ){
+    /* There is no LHS value symbol. */
+    lhsdirect = 1;
   }else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){
     /* The LHS symbol and the left-most RHS symbol are the same, so 
     ** direct writing is allowed */
@@ -3654,7 +3732,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
 
   /* Suffix code generation complete */
   cp = append_str(0,0,0,0);
-  if( cp ) rp->codeSuffix = Strsafe(cp);
+  if( cp && cp[0] ){
+    rp->codeSuffix = Strsafe(cp);
+    rp->noCode = 0;
+  }
 
   return rc;
 }
@@ -4073,6 +4154,18 @@ void ReportTable(
   }
   free(ax);
 
+  /* Mark rules that are actually used for reduce actions after all
+  ** optimizations have been applied
+  */
+  for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE;
+  for(i=0; inxstate; i++){
+    for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
+      if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){
+        ap->x.rp->doesReduce = i;
+      }
+    }
+  }
+
   /* Finish rendering the constants now that the action table has
   ** been computed */
   fprintf(out,"#define YYNSTATE             %d\n",lemp->nxstate);  lineno++;
@@ -4138,20 +4231,21 @@ void ReportTable(
   fprintf(out, "};\n"); lineno++;
 
   /* Output the yy_shift_ofst[] table */
-  fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++;
   n = lemp->nxstate;
   while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
-  fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++;
-  fprintf(out, "#define YY_SHIFT_MIN   (%d)\n", mnTknOfst); lineno++;
-  fprintf(out, "#define YY_SHIFT_MAX   (%d)\n", mxTknOfst); lineno++;
+  fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++;
+  fprintf(out, "#define YY_SHIFT_COUNT    (%d)\n", n-1); lineno++;
+  fprintf(out, "#define YY_SHIFT_MIN      (%d)\n", mnTknOfst); lineno++;
+  fprintf(out, "#define YY_SHIFT_MAX      (%d)\n", mxTknOfst); lineno++;
   fprintf(out, "static const %s yy_shift_ofst[] = {\n", 
-          minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++;
+       minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz));
+       lineno++;
   lemp->tablesize += n*sz;
   for(i=j=0; isorted[i];
     ofst = stp->iTknOfst;
-    if( ofst==NO_OFFSET ) ofst = mnTknOfst - 1;
+    if( ofst==NO_OFFSET ) ofst = lemp->nactiontab;
     if( j==0 ) fprintf(out," /* %5d */ ", i);
     fprintf(out, " %4d,", ofst);
     if( j==9 || i==n-1 ){
@@ -4241,7 +4335,7 @@ void ReportTable(
   ** when tracing REDUCE actions.
   */
   for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
-    assert( rp->index==i );
+    assert( rp->iRule==i );
     fprintf(out," /* %3d */ \"", i);
     writeRuleText(out, rp);
     fprintf(out,"\",\n"); lineno++;
@@ -4291,6 +4385,7 @@ void ReportTable(
   for(i=0; insymbol; i++){
     struct symbol *sp = lemp->symbols[i];
     if( sp==0 || sp->type==TERMINAL || sp->destructor==0 ) continue;
+    if( sp->destLineno<0 ) continue;  /* Already emitted */
     fprintf(out,"    case %d: /* %s */\n", sp->index, sp->name); lineno++;
 
     /* Combine duplicate destructors into a single case */
@@ -4301,7 +4396,7 @@ void ReportTable(
           && strcmp(sp->destructor,sp2->destructor)==0 ){
          fprintf(out,"    case %d: /* %s */\n",
                  sp2->index, sp2->name); lineno++;
-         sp2->destructor = 0;
+         sp2->destLineno = -1;  /* Avoid emitting this destructor again */
       }
     }
 
@@ -4335,32 +4430,41 @@ void ReportTable(
   /* First output rules other than the default: rule */
   for(rp=lemp->rule; rp; rp=rp->next){
     struct rule *rp2;               /* Other rules with the same action */
-    if( rp->code==0 ) continue;
-    if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */
-    fprintf(out,"      case %d: /* ", rp->index);
+    if( rp->codeEmitted ) continue;
+    if( rp->noCode ){
+      /* No C code actions, so this will be part of the "default:" rule */
+      continue;
+    }
+    fprintf(out,"      case %d: /* ", rp->iRule);
     writeRuleText(out, rp);
     fprintf(out, " */\n"); lineno++;
     for(rp2=rp->next; rp2; rp2=rp2->next){
-      if( rp2->code==rp->code ){
-        fprintf(out,"      case %d: /* ", rp2->index);
+      if( rp2->code==rp->code && rp2->codePrefix==rp->codePrefix
+             && rp2->codeSuffix==rp->codeSuffix ){
+        fprintf(out,"      case %d: /* ", rp2->iRule);
         writeRuleText(out, rp2);
-        fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->index); lineno++;
-        rp2->code = 0;
+        fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++;
+        rp2->codeEmitted = 1;
       }
     }
     emit_code(out,rp,lemp,&lineno);
     fprintf(out,"        break;\n"); lineno++;
-    rp->code = 0;
+    rp->codeEmitted = 1;
   }
   /* Finally, output the default: rule.  We choose as the default: all
   ** empty actions. */
   fprintf(out,"      default:\n"); lineno++;
   for(rp=lemp->rule; rp; rp=rp->next){
-    if( rp->code==0 ) continue;
-    assert( rp->code[0]=='\n' && rp->code[1]==0 );
-    fprintf(out,"      /* (%d) ", rp->index);
+    if( rp->codeEmitted ) continue;
+    assert( rp->noCode );
+    fprintf(out,"      /* (%d) ", rp->iRule);
     writeRuleText(out, rp);
-    fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->index); lineno++;
+    if( rp->doesReduce ){
+      fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
+    }else{
+      fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n",
+              rp->iRule); lineno++;
+    }
   }
   fprintf(out,"        break;\n"); lineno++;
   tplt_xfer(lemp->name,in,out,&lineno);
@@ -4431,7 +4535,7 @@ void ReportHeader(struct lemon *lemp)
 void CompressTables(struct lemon *lemp)
 {
   struct state *stp;
-  struct action *ap, *ap2;
+  struct action *ap, *ap2, *nextap;
   struct rule *rp, *rp2, *rbest;
   int nbest, n;
   int i;
@@ -4508,6 +4612,36 @@ void CompressTables(struct lemon *lemp)
       }
     }
   }
+
+  /* If a SHIFTREDUCE action specifies a rule that has a single RHS term
+  ** (meaning that the SHIFTREDUCE will land back in the state where it
+  ** started) and if there is no C-code associated with the reduce action,
+  ** then we can go ahead and convert the action to be the same as the
+  ** action for the RHS of the rule.
+  */
+  for(i=0; instate; i++){
+    stp = lemp->sorted[i];
+    for(ap=stp->ap; ap; ap=nextap){
+      nextap = ap->next;
+      if( ap->type!=SHIFTREDUCE ) continue;
+      rp = ap->x.rp;
+      if( rp->noCode==0 ) continue;
+      if( rp->nrhs!=1 ) continue;
+#if 1
+      /* Only apply this optimization to non-terminals.  It would be OK to
+      ** apply it to terminal symbols too, but that makes the parser tables
+      ** larger. */
+      if( ap->sp->indexnterminal ) continue;
+#endif
+      /* If we reach this point, it means the optimization can be applied */
+      nextap = ap;
+      for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){}
+      assert( ap2!=0 );
+      ap->spOpt = ap2->sp;
+      ap->type = ap2->type;
+      ap->x = ap2->x;
+    }
+  }
 }
 
 
diff --git a/tool/lempar.c b/tool/lempar.c
index db7f2dbf1b..21ed9d79ab 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -116,7 +116,7 @@
 **
 **   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
 **     and YY_MAX_REDUCE
-
+**
 **   N == YY_ERROR_ACTION               A syntax error has occurred.
 **
 **   N == YY_ACCEPT_ACTION              The parser accepts its input.
@@ -125,16 +125,20 @@
 **                                      slots in the yy_action[] table.
 **
 ** The action table is constructed as a single large table named yy_action[].
-** Given state S and lookahead X, the action is computed as
+** Given state S and lookahead X, the action is computed as either:
 **
-**      yy_action[ yy_shift_ofst[S] + X ]
+**    (A)   N = yy_action[ yy_shift_ofst[S] + X ]
+**    (B)   N = yy_default[S]
 **
-** If the index value yy_shift_ofst[S]+X is out of range or if the value
-** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S]
-** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table
-** and that yy_default[S] should be used instead.  
+** The (A) formula is preferred.  The B formula is used instead if:
+**    (1)  The yy_shift_ofst[S]+X value is out of range, or
+**    (2)  yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or
+**    (3)  yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT.
+** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that
+** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X.
+** Hence only tests (1) and (2) need to be evaluated.)
 **
-** The formula above is for computing the action when the lookahead is
+** The formulas above are for computing the action when the lookahead is
 ** a terminal symbol.  If the lookahead is a non-terminal (as occurs after
 ** a reduce action) then the yy_reduce_ofst[] array is used in place of
 ** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
@@ -203,9 +207,9 @@ typedef struct yyStackEntry yyStackEntry;
 /* The state of the parser is completely contained in an instance of
 ** the following structure */
 struct yyParser {
-  int yyidx;                    /* Index of top element in stack */
+  yyStackEntry *yytos;          /* Pointer to top element of the stack */
 #ifdef YYTRACKMAXSTACKDEPTH
-  int yyidxMax;                 /* Maximum value of yyidx */
+  int yyhwm;                    /* High-water mark of the stack */
 #endif
 #ifndef YYNOERRORRECOVERY
   int yyerrcnt;                 /* Shifts left before out of the error */
@@ -214,6 +218,7 @@ struct yyParser {
 #if YYSTACKDEPTH<=0
   int yystksz;                  /* Current side of the stack */
   yyStackEntry *yystack;        /* The parser's stack */
+  yyStackEntry yystk0;          /* First stack entry */
 #else
   yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
 #endif
@@ -271,24 +276,34 @@ static const char *const yyRuleName[] = {
 
 #if YYSTACKDEPTH<=0
 /*
-** Try to increase the size of the parser stack.
+** Try to increase the size of the parser stack.  Return the number
+** of errors.  Return 0 on success.
 */
-static void yyGrowStack(yyParser *p){
+static int yyGrowStack(yyParser *p){
   int newSize;
+  int idx;
   yyStackEntry *pNew;
 
   newSize = p->yystksz*2 + 100;
-  pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
+  if( p->yystack==&p->yystk0 ){
+    pNew = malloc(newSize*sizeof(pNew[0]));
+    if( pNew ) pNew[0] = p->yystk0;
+  }else{
+    pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
+  }
   if( pNew ){
     p->yystack = pNew;
-    p->yystksz = newSize;
+    p->yytos = &p->yystack[idx];
 #ifndef NDEBUG
     if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sStack grows to %d entries!\n",
-              yyTracePrompt, p->yystksz);
+      fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
+              yyTracePrompt, p->yystksz, newSize);
     }
 #endif
+    p->yystksz = newSize;
   }
+  return pNew==0; 
 }
 #endif
 
@@ -301,6 +316,31 @@ static void yyGrowStack(yyParser *p){
 # define YYMALLOCARGTYPE size_t
 #endif
 
+/* Initialize a new parser that has already been allocated.
+*/
+void ParseInit(void *yypParser){
+  yyParser *pParser = (yyParser*)yypParser;
+#ifdef YYTRACKMAXSTACKDEPTH
+  pParser->yyhwm = 0;
+#endif
+#if YYSTACKDEPTH<=0
+  pParser->yytos = NULL;
+  pParser->yystack = NULL;
+  pParser->yystksz = 0;
+  if( yyGrowStack(pParser) ){
+    pParser->yystack = &pParser->yystk0;
+    pParser->yystksz = 1;
+  }
+#endif
+#ifndef YYNOERRORRECOVERY
+  pParser->yyerrcnt = -1;
+#endif
+  pParser->yytos = pParser->yystack;
+  pParser->yystack[0].stateno = 0;
+  pParser->yystack[0].major = 0;
+}
+
+#ifndef Parse_ENGINEALWAYSONSTACK
 /* 
 ** This function allocates a new parser.
 ** The only argument is a pointer to a function which works like
@@ -316,19 +356,11 @@ static void yyGrowStack(yyParser *p){
 void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
   yyParser *pParser;
   pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
-  if( pParser ){
-    pParser->yyidx = -1;
-#ifdef YYTRACKMAXSTACKDEPTH
-    pParser->yyidxMax = 0;
-#endif
-#if YYSTACKDEPTH<=0
-    pParser->yystack = NULL;
-    pParser->yystksz = 0;
-    yyGrowStack(pParser);
-#endif
-  }
+  if( pParser ) ParseInit(pParser);
   return pParser;
 }
+#endif /* Parse_ENGINEALWAYSONSTACK */
+
 
 /* The following function deletes the "minor type" or semantic value
 ** associated with a symbol.  The symbol can be either a terminal
@@ -369,8 +401,9 @@ static void yy_destructor(
 */
 static void yy_pop_parser_stack(yyParser *pParser){
   yyStackEntry *yytos;
-  assert( pParser->yyidx>=0 );
-  yytos = &pParser->yystack[pParser->yyidx--];
+  assert( pParser->yytos!=0 );
+  assert( pParser->yytos > pParser->yystack );
+  yytos = pParser->yytos--;
 #ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sPopping %s\n",
@@ -381,6 +414,18 @@ static void yy_pop_parser_stack(yyParser *pParser){
   yy_destructor(pParser, yytos->major, &yytos->minor);
 }
 
+/*
+** Clear all secondary memory allocations from the parser
+*/
+void ParseFinalize(void *p){
+  yyParser *pParser = (yyParser*)p;
+  while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
+#if YYSTACKDEPTH<=0
+  if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
+#endif
+}
+
+#ifndef Parse_ENGINEALWAYSONSTACK
 /* 
 ** Deallocate and destroy a parser.  Destructors are called for
 ** all stack elements before shutting the parser down.
@@ -393,16 +438,13 @@ void ParseFree(
   void *p,                    /* The parser to be deleted */
   void (*freeProc)(void*)     /* Function used to reclaim memory */
 ){
-  yyParser *pParser = (yyParser*)p;
 #ifndef YYPARSEFREENEVERNULL
-  if( pParser==0 ) return;
+  if( p==0 ) return;
 #endif
-  while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
-#if YYSTACKDEPTH<=0
-  free(pParser->yystack);
-#endif
-  (*freeProc)((void*)pParser);
+  ParseFinalize(p);
+  (*freeProc)(p);
 }
+#endif /* Parse_ENGINEALWAYSONSTACK */
 
 /*
 ** Return the peak depth of the stack for a parser.
@@ -410,7 +452,7 @@ void ParseFree(
 #ifdef YYTRACKMAXSTACKDEPTH
 int ParseStackPeak(void *p){
   yyParser *pParser = (yyParser*)p;
-  return pParser->yyidxMax;
+  return pParser->yyhwm;
 }
 #endif
 
@@ -418,61 +460,58 @@ int ParseStackPeak(void *p){
 ** Find the appropriate action for a parser given the terminal
 ** look-ahead token iLookAhead.
 */
-static int yy_find_shift_action(
+static unsigned int yy_find_shift_action(
   yyParser *pParser,        /* The parser */
   YYCODETYPE iLookAhead     /* The look-ahead token */
 ){
   int i;
-  int stateno = pParser->yystack[pParser->yyidx].stateno;
+  int stateno = pParser->yytos->stateno;
  
   if( stateno>=YY_MIN_REDUCE ) return stateno;
   assert( stateno <= YY_SHIFT_COUNT );
   do{
     i = yy_shift_ofst[stateno];
-    if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
     assert( iLookAhead!=YYNOCODE );
     i += iLookAhead;
     if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
-      if( iLookAhead>0 ){
 #ifdef YYFALLBACK
-        YYCODETYPE iFallback;            /* Fallback token */
-        if( iLookAhead %s\n",
-               yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
-          }
-#endif
-          assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
-          iLookAhead = iFallback;
-          continue;
+        if( yyTraceFILE ){
+          fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
+             yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
         }
+#endif
+        assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
+        iLookAhead = iFallback;
+        continue;
+      }
 #endif
 #ifdef YYWILDCARD
-        {
-          int j = i - iLookAhead + YYWILDCARD;
-          if( 
+      {
+        int j = i - iLookAhead + YYWILDCARD;
+        if( 
 #if YY_SHIFT_MIN+YYWILDCARD<0
-            j>=0 &&
+          j>=0 &&
 #endif
 #if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
-            j0
+        ){
 #ifndef NDEBUG
-            if( yyTraceFILE ){
-              fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
-                 yyTracePrompt, yyTokenName[iLookAhead],
-                 yyTokenName[YYWILDCARD]);
-            }
-#endif /* NDEBUG */
-            return yy_action[j];
+          if( yyTraceFILE ){
+            fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
+               yyTracePrompt, yyTokenName[iLookAhead],
+               yyTokenName[YYWILDCARD]);
           }
+#endif /* NDEBUG */
+          return yy_action[j];
         }
-#endif /* YYWILDCARD */
       }
+#endif /* YYWILDCARD */
       return yy_default[stateno];
     }else{
       return yy_action[i];
@@ -516,13 +555,12 @@ static int yy_find_reduce_action(
 */
 static void yyStackOverflow(yyParser *yypParser){
    ParseARG_FETCH;
-   yypParser->yyidx--;
 #ifndef NDEBUG
    if( yyTraceFILE ){
      fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
    }
 #endif
-   while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+   while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
    /* Here code is inserted which will execute if the parser
    ** stack every overflows */
 /******** Begin %stack_overflow code ******************************************/
@@ -539,11 +577,11 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){
   if( yyTraceFILE ){
     if( yyNewStateyystack[yypParser->yyidx].major],
+         yyTracePrompt,yyTokenName[yypParser->yytos->major],
          yyNewState);
     }else{
       fprintf(yyTraceFILE,"%sShift '%s'\n",
-         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]);
+         yyTracePrompt,yyTokenName[yypParser->yytos->major]);
     }
   }
 }
@@ -561,27 +599,32 @@ static void yy_shift(
   ParseTOKENTYPE yyMinor        /* The minor token to shift in */
 ){
   yyStackEntry *yytos;
-  yypParser->yyidx++;
+  yypParser->yytos++;
 #ifdef YYTRACKMAXSTACKDEPTH
-  if( yypParser->yyidx>yypParser->yyidxMax ){
-    yypParser->yyidxMax = yypParser->yyidx;
+  if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+    yypParser->yyhwm++;
+    assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
   }
 #endif
 #if YYSTACKDEPTH>0 
-  if( yypParser->yyidx>=YYSTACKDEPTH ){
+  if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){
+    yypParser->yytos--;
     yyStackOverflow(yypParser);
     return;
   }
 #else
-  if( yypParser->yyidx>=yypParser->yystksz ){
-    yyGrowStack(yypParser);
-    if( yypParser->yyidx>=yypParser->yystksz ){
+  if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
+    if( yyGrowStack(yypParser) ){
+      yypParser->yytos--;
       yyStackOverflow(yypParser);
       return;
     }
   }
 #endif
-  yytos = &yypParser->yystack[yypParser->yyidx];
+  if( yyNewState > YY_MAX_SHIFT ){
+    yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+  }
+  yytos = yypParser->yytos;
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor.yy0 = yyMinor;
@@ -606,17 +649,16 @@ static void yy_accept(yyParser*);  /* Forward Declaration */
 */
 static void yy_reduce(
   yyParser *yypParser,         /* The parser */
-  int yyruleno                 /* Number of the rule by which to reduce */
+  unsigned int yyruleno        /* Number of the rule by which to reduce */
 ){
   int yygoto;                     /* The next state */
   int yyact;                      /* The next action */
   yyStackEntry *yymsp;            /* The top of the parser's stack */
   int yysize;                     /* Amount to pop the stack */
   ParseARG_FETCH;
-  yymsp = &yypParser->yystack[yypParser->yyidx];
+  yymsp = yypParser->yytos;
 #ifndef NDEBUG
-  if( yyTraceFILE && yyruleno>=0 
-        && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
+  if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
     yysize = yyRuleInfo[yyruleno].nrhs;
     fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt,
       yyRuleName[yyruleno], yymsp[-yysize].stateno);
@@ -628,22 +670,23 @@ static void yy_reduce(
   ** enough on the stack to push the LHS value */
   if( yyRuleInfo[yyruleno].nrhs==0 ){
 #ifdef YYTRACKMAXSTACKDEPTH
-    if( yypParser->yyidx>yypParser->yyidxMax ){
-      yypParser->yyidxMax = yypParser->yyidx;
+    if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
+      yypParser->yyhwm++;
+      assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
     }
 #endif
 #if YYSTACKDEPTH>0 
-    if( yypParser->yyidx>=YYSTACKDEPTH-1 ){
+    if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){
       yyStackOverflow(yypParser);
       return;
     }
 #else
-    if( yypParser->yyidx>=yypParser->yystksz-1 ){
-      yyGrowStack(yypParser);
-      if( yypParser->yyidx>=yypParser->yystksz-1 ){
+    if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
+      if( yyGrowStack(yypParser) ){
         yyStackOverflow(yypParser);
         return;
       }
+      yymsp = yypParser->yytos;
     }
 #endif
   }
@@ -661,20 +704,22 @@ static void yy_reduce(
 %%
 /********** End reduce actions ************************************************/
   };
-  assert( yyruleno>=0 && yyrulenoYY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-    yypParser->yyidx -= yysize - 1;
+    if( yyact>YY_MAX_SHIFT ){
+      yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+    }
     yymsp -= yysize-1;
+    yypParser->yytos = yymsp;
     yymsp->stateno = (YYACTIONTYPE)yyact;
     yymsp->major = (YYCODETYPE)yygoto;
     yyTraceShift(yypParser, yyact);
   }else{
     assert( yyact == YY_ACCEPT_ACTION );
-    yypParser->yyidx -= yysize;
+    yypParser->yytos -= yysize;
     yy_accept(yypParser);
   }
 }
@@ -692,7 +737,7 @@ static void yy_parse_failed(
     fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+  while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
   /* Here code is inserted which will be executed whenever the
   ** parser fails */
 /************ Begin %parse_failure code ***************************************/
@@ -730,7 +775,10 @@ static void yy_accept(
     fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
   }
 #endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
+#ifndef YYNOERRORRECOVERY
+  yypParser->yyerrcnt = -1;
+#endif
+  assert( yypParser->yytos==yypParser->yystack );
   /* Here code is inserted which will be executed whenever the
   ** parser accepts */
 /*********** Begin %parse_accept code *****************************************/
@@ -765,7 +813,7 @@ void Parse(
   ParseARG_PDECL               /* Optional %extra_argument parameter */
 ){
   YYMINORTYPE yyminorunion;
-  int yyact;            /* The parser action. */
+  unsigned int yyact;   /* The parser action. */
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   int yyendofinput;     /* True if we are at the end of input */
 #endif
@@ -774,28 +822,8 @@ void Parse(
 #endif
   yyParser *yypParser;  /* The parser */
 
-  /* (re)initialize the parser, if necessary */
   yypParser = (yyParser*)yyp;
-  if( yypParser->yyidx<0 ){
-#if YYSTACKDEPTH<=0
-    if( yypParser->yystksz <=0 ){
-      yyStackOverflow(yypParser);
-      return;
-    }
-#endif
-    yypParser->yyidx = 0;
-#ifndef YYNOERRORRECOVERY
-    yypParser->yyerrcnt = -1;
-#endif
-    yypParser->yystack[0].stateno = 0;
-    yypParser->yystack[0].major = 0;
-#ifndef NDEBUG
-    if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
-              yyTracePrompt);
-    }
-#endif
-  }
+  assert( yypParser->yytos!=0 );
 #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
   yyendofinput = (yymajor==0);
 #endif
@@ -810,7 +838,6 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
       yy_shift(yypParser,yyact,yymajor,yyminor);
 #ifndef YYNOERRORRECOVERY
       yypParser->yyerrcnt--;
@@ -852,7 +879,7 @@ void Parse(
       if( yypParser->yyerrcnt<0 ){
         yy_syntax_error(yypParser,yymajor,yyminor);
       }
-      yymx = yypParser->yystack[yypParser->yyidx].major;
+      yymx = yypParser->yytos->major;
       if( yymx==YYERRORSYMBOL || yyerrorhit ){
 #ifndef NDEBUG
         if( yyTraceFILE ){
@@ -863,18 +890,20 @@ void Parse(
         yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion);
         yymajor = YYNOCODE;
       }else{
-        while(
-          yypParser->yyidx >= 0 &&
-          yymx != YYERRORSYMBOL &&
-          (yyact = yy_find_reduce_action(
-                        yypParser->yystack[yypParser->yyidx].stateno,
+        while( yypParser->yytos >= yypParser->yystack
+            && yymx != YYERRORSYMBOL
+            && (yyact = yy_find_reduce_action(
+                        yypParser->yytos->stateno,
                         YYERRORSYMBOL)) >= YY_MIN_REDUCE
         ){
           yy_pop_parser_stack(yypParser);
         }
-        if( yypParser->yyidx < 0 || yymajor==0 ){
+        if( yypParser->yytos < yypParser->yystack || yymajor==0 ){
           yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
           yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+          yypParser->yyerrcnt = -1;
+#endif
           yymajor = YYNOCODE;
         }else if( yymx!=YYERRORSYMBOL ){
           yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor);
@@ -911,18 +940,23 @@ void Parse(
       yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
       if( yyendofinput ){
         yy_parse_failed(yypParser);
+#ifndef YYNOERRORRECOVERY
+        yypParser->yyerrcnt = -1;
+#endif
       }
       yymajor = YYNOCODE;
 #endif
     }
-  }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
+  }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack );
 #ifndef NDEBUG
   if( yyTraceFILE ){
-    int i;
+    yyStackEntry *i;
+    char cDiv = '[';
     fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
-    for(i=1; i<=yypParser->yyidx; i++)
-      fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', 
-              yyTokenName[yypParser->yystack[i].major]);
+    for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
+      fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
+      cDiv = ' ';
+    }
     fprintf(yyTraceFILE,"]\n");
   }
 #endif
diff --git a/tool/libvers.c b/tool/libvers.c
new file mode 100644
index 0000000000..6911dbd07e
--- /dev/null
+++ b/tool/libvers.c
@@ -0,0 +1,15 @@
+/*
+** Compile this program against an SQLite library of unknown version
+** and then run this program, and it will print out the SQLite version
+** information.
+*/
+#include 
+
+extern const char *sqlite3_libversion(void);
+extern const char *sqlite3_sourceid(void);
+
+int main(int argc, char **argv){
+  printf("SQLite version %s\n", sqlite3_libversion());
+  printf("SQLite source  %s\n", sqlite3_sourceid());
+  return 0;
+}
diff --git a/tool/logest.c b/tool/logest.c
index 347fa68a4f..e936e02cbe 100644
--- a/tool/logest.c
+++ b/tool/logest.c
@@ -147,7 +147,7 @@ int main(int argc, char **argv){
     }else if( strcmp(z,"inv")==0 ){
       if( n>0 ) a[n-1] = -a[n-1];
     }else if( z[0]=='^' ){
-      a[n++] = atoi(z+1);
+      a[n++] = (LogEst)atoi(z+1);
     }else if( isInteger(z) ){
       a[n++] = logEstFromInteger(atoi(z));
     }else if( isFloat(z) && z[0]!='-' ){
diff --git a/tool/max-limits.c b/tool/max-limits.c
new file mode 100644
index 0000000000..d019974426
--- /dev/null
+++ b/tool/max-limits.c
@@ -0,0 +1,41 @@
+/*
+** Link this program against an SQLite library of unknown provenance in order
+** to display the compile-time maximum values for various settings.
+*/
+#include "sqlite3.h"
+#include 
+
+static const struct {
+  int eCode;
+  char *zName;
+} aLimit[] = {
+  { SQLITE_LIMIT_LENGTH,                "SQLITE_MAX_LENGTH"               },
+  { SQLITE_LIMIT_SQL_LENGTH,            "SQLITE_MAX_SQL_LENGTH"           },
+  { SQLITE_LIMIT_COLUMN,                "SQLITE_MAX_COLUMN"               },
+  { SQLITE_LIMIT_EXPR_DEPTH,            "SQLITE_MAX_EXPR_DEPTH"           },
+  { SQLITE_LIMIT_COMPOUND_SELECT,       "SQLITE_MAX_COMPOUND_SELECT"      },
+  { SQLITE_LIMIT_VDBE_OP,               "SQLITE_MAX_VDBE_OP"              },
+  { SQLITE_LIMIT_FUNCTION_ARG,          "SQLITE_MAX_FUNCTION_ARG"         },
+  { SQLITE_LIMIT_ATTACHED,              "SQLITE_MAX_ATTACHED"             },
+  { SQLITE_LIMIT_LIKE_PATTERN_LENGTH,   "SQLITE_MAX_LIKE_PATTERN_LENGTH"  },
+  { SQLITE_LIMIT_VARIABLE_NUMBER,       "SQLITE_MAX_VARIABLE_NUMBER"      },
+  { SQLITE_LIMIT_TRIGGER_DEPTH,         "SQLITE_MAX_TRIGGER_DEPTH"        },
+  { SQLITE_LIMIT_WORKER_THREADS,        "SQLITE_MAX_WORKER_THREADS"       },
+};
+
+static int maxLimit(sqlite3 *db, int eCode){
+  int iOrig = sqlite3_limit(db, eCode, 0x7fffffff);
+  return sqlite3_limit(db, eCode, iOrig);
+}
+
+int main(int argc, char **argv){
+  sqlite3 *db;
+  int j, rc;
+  rc = sqlite3_open(":memory:", &db);
+  if( rc==SQLITE_OK ){
+    for(j=0; j sqlite3.def
 	dumpbin /all $(LIBOBJ) \\
-		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\
+		| .\Replace.exe "^\s+/EXPORT:_?(sqlite3(?:session|changeset)?_[^@,]*)(?:@\d+|,DATA)?$$" $$1 true \\
 		| sort >> sqlite3.def
 }]]
 
diff --git a/tool/mkopcodeh.tcl b/tool/mkopcodeh.tcl
index 053c7f8984..3c2a59789c 100644
--- a/tool/mkopcodeh.tcl
+++ b/tool/mkopcodeh.tcl
@@ -20,8 +20,7 @@
 # during code generation, we need to generate corresponding opcodes like
 # OP_Add and OP_Divide.  By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide,
 # code to translate from one to the other is avoided.  This makes the
-# code generator run (infinitesimally) faster and more importantly it makes
-# the library footprint smaller.
+# code generator smaller and faster.
 #
 # This script also scans for lines of the form:
 #
@@ -159,7 +158,29 @@ for {set i 0} {$i<$nOp} {incr i} {
   }
 }
 
-# Generate the numeric values for remaining opcodes
+# Assign the next group of values to JUMP opcodes
+#
+for {set i 0} {$i<$nOp} {incr i} {
+  set name $order($i)
+  if {$op($name)>=0} continue
+  if {!$jump($name)} continue
+  incr cnt
+  while {[info exists used($cnt)]} {incr cnt}
+  set op($name) $cnt
+  set used($cnt) 1
+  set def($cnt) $name
+}
+
+# Find the numeric value for the largest JUMP opcode
+#
+set mxJump -1
+for {set i 0} {$i<$nOp} {incr i} {
+  set name $order($i)
+  if {$jump($name) && $op($name)>$mxJump} {set mxJump $op($name)}
+}
+
+
+# Generate the numeric values for all remaining opcodes
 #
 for {set i 0} {$i<$nOp} {incr i} {
   set name $order($i)
@@ -232,3 +253,11 @@ for {set i 0} {$i<=$max} {incr i} {
   }
 }
 puts "\175"
+puts ""
+puts "/* The sqlite3P2Values() routine is able to run faster if it knows"
+puts "** the value of the largest JUMP opcode.  The smaller the maximum"
+puts "** JUMP opcode the better, so the mkopcodeh.tcl script that"
+puts "** generated this include file strives to group all JUMP opcodes"
+puts "** together near the beginning of the list."
+puts "*/"
+puts "#define SQLITE_MX_JUMP_OPCODE  $mxJump  /* Maximum JUMP opcode */"
diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl
index 145a365c54..59b245cc76 100644
--- a/tool/mkpragmatab.tcl
+++ b/tool/mkpragmatab.tcl
@@ -10,6 +10,16 @@
 # new pragma in ../src/pragma.c.
 #
 
+# Flag meanings:
+set flagMeaning(NeedSchema) {Force schema load before running}
+set flagMeaning(ReadOnly)   {Read-only HEADER_VALUE}
+set flagMeaning(Result0)    {Acts as query when no argument}
+set flagMeaning(Result1)    {Acts as query when has one argument}
+set flagMeaning(SchemaReq)  {Schema required - "main" is default}
+set flagMeaning(SchemaOpt)  {Schema restricts name search if present}
+set flagMeaning(NoColumns)  {OP_ResultRow called with zero columns}
+set flagMeaning(NoColumns1) {zero columns if RHS argument is present}
+
 set pragma_def {
   NAME: full_column_names
   TYPE: FLAG
@@ -47,6 +57,7 @@ set pragma_def {
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
 
   NAME: cache_spill
+  FLAG: Result0 SchemaReq NoColumns1
   IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)
 
   NAME: reverse_unordered_selects
@@ -139,108 +150,128 @@ set pragma_def {
   ARG:  SQLITE_CellSizeCk
 
   NAME: default_cache_size
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq NoColumns1
+  COLS: cache_size
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED)
 
   NAME: page_size
+  FLAG: Result0 SchemaReq NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: secure_delete
+  FLAG: Result0
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: page_count
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: max_page_count
   TYPE: PAGE_COUNT
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: locking_mode
+  FLAG: Result0 SchemaReq
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: journal_mode
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: journal_size_limit
+  FLAG: Result0 SchemaReq
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: cache_size
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: mmap_size
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: auto_vacuum
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq NoColumns1
   IF:   !defined(SQLITE_OMIT_AUTOVACUUM)
 
   NAME: incremental_vacuum
-  FLAG: NeedSchema
+  FLAG: NeedSchema NoColumns
   IF:   !defined(SQLITE_OMIT_AUTOVACUUM)
 
   NAME: temp_store
+  FLAG: Result0 NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: temp_store_directory
+  FLAG: NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: data_store_directory
+  FLAG: NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN
 
   NAME: lock_proxy_file
+  FLAG: NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE
 
   NAME: synchronous
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0 SchemaReq NoColumns1
   IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)
 
   NAME: table_info
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result1 SchemaOpt
+  COLS: cid name type notnull dflt_value pk
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: stats
-  FLAG: NeedSchema
-  IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
+  FLAG: NeedSchema Result0 SchemaReq
+  COLS: tbl idx wdth hght flgs
+  IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG)
 
   NAME: index_info
   TYPE: INDEX_INFO
   ARG:  0
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result1 SchemaOpt
+  COLS: seqno cid name
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: index_xinfo
   TYPE: INDEX_INFO
   ARG:  1
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result1 SchemaOpt
+  COLS: seqno cid name desc coll key
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: index_list
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result1 SchemaOpt
+  COLS: seq name unique origin partial
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: database_list
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result0
+  COLS: seq name file
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: collation_list
+  FLAG: Result0
+  COLS: seq name
   IF:   !defined(SQLITE_OMIT_SCHEMA_PRAGMAS)
 
   NAME: foreign_key_list
-  FLAG: NeedSchema
+  FLAG: NeedSchema Result1 SchemaOpt
+  COLS: id seq table from to on_update on_delete match
   IF:   !defined(SQLITE_OMIT_FOREIGN_KEY)
 
   NAME: foreign_key_check
   FLAG: NeedSchema
+  COLS: table rowid parent fkid
   IF:   !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER)
 
   NAME: parser_trace
   IF:   defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE)
 
   NAME: case_sensitive_like
+  FLAG: NoColumns
 
   NAME: integrity_check
   FLAG: NeedSchema
@@ -252,50 +283,61 @@ set pragma_def {
   IF:   !defined(SQLITE_OMIT_INTEGRITY_CHECK)
 
   NAME: encoding
+  FLAG: Result0 NoColumns1
   IF:   !defined(SQLITE_OMIT_UTF16)
 
   NAME: schema_version
   TYPE: HEADER_VALUE
   ARG:  BTREE_SCHEMA_VERSION
+  FLAG: NoColumns1 Result0
   IF:   !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
 
   NAME: user_version
   TYPE: HEADER_VALUE
   ARG:  BTREE_USER_VERSION
+  FLAG: NoColumns1 Result0
   IF:   !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
 
   NAME: data_version
   TYPE: HEADER_VALUE
   ARG:  BTREE_DATA_VERSION
-  FLAG: ReadOnly
+  FLAG: ReadOnly Result0
   IF:   !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
 
   NAME: freelist_count
   TYPE: HEADER_VALUE
   ARG:  BTREE_FREE_PAGE_COUNT
-  FLAG: ReadOnly
+  FLAG: ReadOnly Result0
   IF:   !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
 
   NAME: application_id
   TYPE: HEADER_VALUE
   ARG:  BTREE_APPLICATION_ID
+  FLAG: NoColumns1 Result0
   IF:   !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS)
 
   NAME: compile_options
+  FLAG: Result0
   IF:   !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS)
 
   NAME: wal_checkpoint
   FLAG: NeedSchema
+  COLS: busy log checkpointed
   IF:   !defined(SQLITE_OMIT_WAL)
 
   NAME: wal_autocheckpoint
   IF:   !defined(SQLITE_OMIT_WAL)
 
   NAME: shrink_memory
+  FLAG: NoColumns
 
   NAME: busy_timeout
+  FLAG: Result0
+  COLS: timeout
 
   NAME: lock_status
+  FLAG: Result0
+  COLS: database status
   IF:   defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
 
   NAME: key
@@ -315,8 +357,13 @@ set pragma_def {
   IF:   defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD)
 
   NAME: soft_heap_limit
+  FLAG: Result0
 
   NAME: threads
+  FLAG: Result0
+
+  NAME: optimize
+  FLAG: Result1
 }
 
 # Open the output file
@@ -336,15 +383,29 @@ set name {}
 set type {}
 set if {}
 set flags {}
+set cols {}
+set cols_list {}
 set arg 0
 proc record_one {} {
-  global name type if arg allbyname typebyif flags
+  global name type if arg allbyname typebyif flags cols allcols
+  global cols_list colUsedBy
   if {$name==""} return
-  set allbyname($name) [list $type $arg $if $flags]
+  if {$cols!=""} {
+    if {![info exists allcols($cols)]} {
+      lappend cols_list $cols
+      set allcols($cols) [llength $cols_list]
+    }
+    set cx $allcols($cols)
+    lappend colUsedBy($cols) $name
+  } else {
+    set cx 0
+  }
+  set allbyname($name) [list $type $arg $if $flags $cx]
   set name {}
   set type {}
   set if {}
   set flags {}
+  set cols {}
   set arg 0
 }
 foreach line [split $pragma_def \n] {
@@ -358,8 +419,13 @@ foreach line [split $pragma_def \n] {
     set type [string toupper $val]
   } elseif {$id=="TYPE"} {
     set type $val
+    if {$type=="FLAG"} {
+      lappend flags Result0 NoColumns1
+    }
   } elseif {$id=="ARG"} {
     set arg $val
+  } elseif {$id=="COLS"} {
+    set cols $val
   } elseif {$id=="IF"} {
     lappend if $val
   } elseif {$id=="FLAG"} {
@@ -378,6 +444,7 @@ set allnames [lsort [array names allbyname]]
 # omit in default builds (defined(SQLITE_DEBUG) and defined(SQLITE_HAS_CODEC))
 # at the end.
 #
+puts $fd "\n/* The various pragma types */"
 set pnum 0
 foreach name $allnames {
   set type [lindex $allbyname($name) 0]
@@ -407,25 +474,58 @@ foreach name $allnames {
 
 # Generate #defines for flags
 #
+puts $fd "\n/* Property flags associated with various pragma. */"
 set fv 1
 foreach f [lsort [array names allflags]] {
-  puts $fd [format {#define PragFlag_%-20s 0x%02x} $f $fv]
+  puts $fd [format {#define PragFlg_%-10s 0x%02x /* %s */} \
+             $f $fv $flagMeaning($f)]
   set fv [expr {$fv*2}]
 }
 
+# Generate the array of column names used by pragmas that act like
+# queries.
+#
+puts $fd "\n/* Names of columns for pragmas that return multi-column result"
+puts $fd "** or that return single-column results where the name of the"
+puts $fd "** result column is different from the name of the pragma\n*/"
+puts $fd "static const char *const pragCName\[\] = {"
+set offset 0
+foreach cols $cols_list {
+  set cols_offset($allcols($cols)) $offset
+  set ub " /* Used by: $colUsedBy($cols) */"
+  foreach c $cols {
+    puts $fd [format "  /* %3d */ %-14s%s" $offset \"$c\", $ub]
+    set ub ""
+    incr offset
+  }
+}
+puts $fd "\175;"
+
 # Generate the lookup table
 #
-puts $fd "static const struct sPragmaNames \173"
-puts $fd "  const char *const zName;  /* Name of pragma */"
-puts $fd "  u8 ePragTyp;              /* PragTyp_XXX value */"
-puts $fd "  u8 mPragFlag;             /* Zero or more PragFlag_XXX values */"
-puts $fd "  u32 iArg;                 /* Extra argument */"
-puts $fd "\175 aPragmaNames\[\] = \173"
+puts $fd "\n/* Definitions of all built-in pragmas */"
+puts $fd "typedef struct PragmaName \173"
+puts $fd "  const char *const zName; /* Name of pragma */"
+puts $fd "  u8 ePragTyp;             /* PragTyp_XXX value */"
+puts $fd "  u8 mPragFlg;             /* Zero or more PragFlg_XXX values */"
+puts $fd {  u8 iPragCName;           /* Start of column names in pragCName[] */}
+puts $fd "  u8 nPragCName;          \
+/* Num of col names. 0 means use pragma name */"
+puts $fd "  u32 iArg;                /* Extra argument */"
+puts $fd "\175 PragmaName;"
+puts $fd "static const PragmaName aPragmaName\[\] = \173"
 
 set current_if {}
 set spacer [format {    %26s } {}]
 foreach name $allnames {
-  foreach {type arg if flag} $allbyname($name) break
+  foreach {type arg if flag cx} $allbyname($name) break
+  if {$cx==0} {
+    set cy 0
+    set nx 0
+  } else {
+    set cy $cols_offset($cx)
+    set nx [llength [lindex $cols_list [expr {$cx-1}]]]
+  }
   if {$if!=$current_if} {
     if {$current_if!=""} {
       foreach this_if $current_if {
@@ -443,12 +543,13 @@ foreach name $allnames {
   if {$flag==""} {
     set flagx "0"
   } else {
-    set flagx PragFlag_[join $flag {|PragFlag_}]
+    set flagx PragFlg_[join $flag {|PragFlg_}]
   }
-  puts $fd "  \173 /* zName:     */ \"$name\","
-  puts $fd "    /* ePragTyp:  */ PragTyp_$type,"
-  puts $fd "    /* ePragFlag: */ $flagx,"
-  puts $fd "    /* iArg:      */ $arg \175,"
+  puts $fd " \173/* zName:     */ \"$name\","
+  puts $fd "  /* ePragTyp:  */ PragTyp_$type,"
+  puts $fd "  /* ePragFlg:  */ $flagx,"
+  puts $fd "  /* ColNames:  */ $cy, $nx,"
+  puts $fd "  /* iArg:      */ $arg \175,"
 }
 if {$current_if!=""} {
   foreach this_if $current_if {
diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl
index 601b8cce8c..a16b0591eb 100644
--- a/tool/mksqlite3c-noext.tcl
+++ b/tool/mksqlite3c-noext.tcl
@@ -1,7 +1,7 @@
 #!/usr/bin/tclsh
 #
 # To build a single huge source file holding all of SQLite (or at
-# least the core components - the test harness, shell, and TCL 
+# least the core components - the test harness, shell, and TCL
 # interface are omitted.) first do
 #
 #      make target_source
@@ -11,7 +11,7 @@
 # there and will not work if they are not found.)  There are a few
 # generated C code files that are also added to the tsrc directory.
 # For example, the "parse.c" and "parse.h" files to implement the
-# the parser are derived from "parse.y" using lemon.  And the 
+# the parser are derived from "parse.y" using lemon.  And the
 # "keywordhash.h" files is generated by a program named "mkkeywordhash".
 #
 # After the "tsrc" directory has been created and populated, run
@@ -26,15 +26,20 @@
 # from in this file.  The version number is needed to generate the header
 # comment of the amalgamation.
 #
-if {[lsearch $argv --nostatic]>=0} {
-  set addstatic 0
-} else {
-  set addstatic 1
-}
-if {[lsearch $argv --linemacros]>=0} {
-  set linemacros 1
-} else {
-  set linemacros 0
+set addstatic 1
+set linemacros 0
+set useapicall 0
+for {set i 0} {$i<[llength $argv]} {incr i} {
+  set x [lindex $argv $i]
+  if {[regexp {^-+nostatic$} $x]} {
+    set addstatic 0
+  } elseif {[regexp {^-+linemacros} $x]} {
+    set linemacros 1
+  } elseif {[regexp {^-+useapicall} $x]} {
+    set useapicall 1
+  } else {
+    error "unknown command-line option: $x"
+  }
 }
 set in [open tsrc/sqlite3.h]
 set cnt 0
@@ -57,7 +62,7 @@ set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version $VERSION.  By combining all the individual C code files into this 
+** version $VERSION.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -66,7 +71,7 @@ puts $out [subst \
 **
 ** This file is all you need to compile SQLite.  To use SQLite in other
 ** programs, you need this file and the "sqlite3.h" header file that defines
-** the programming interface to the SQLite library.  (If you do not have 
+** the programming interface to the SQLite library.  (If you do not have
 ** the "sqlite3.h" header file at hand, you will find a copy embedded within
 ** the text of this file.  Search for "Begin file sqlite3.h" to find the start
 ** of the embedded sqlite3.h header file.) Additional code files may be needed
@@ -83,7 +88,7 @@ if {$addstatic} {
 #endif}
 }
 
-# These are the header files used by SQLite.  The first time any of these 
+# These are the header files used by SQLite.  The first time any of these
 # files are seen in a #include statement in the C code, include the complete
 # text of the file in-line.  The file only needs to be included once.
 #
@@ -104,8 +109,8 @@ foreach hdr {
    parse.h
    pcache.h
    pragma.h
-   sqlite3ext.h
    sqlite3.h
+   sqlite3ext.h
    sqliteicu.h
    sqliteInt.h
    sqliteLimit.h
@@ -155,7 +160,8 @@ proc section_comment {text} {
 # process them appropriately.
 #
 proc copy_file {filename} {
-  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
+  global seen_hdr available_hdr varonly_hdr cdecllist out
+  global addstatic linemacros useapicall
   set ln 0
   set tail [file tail $filename]
   section_comment "Begin file $tail"
@@ -203,7 +209,8 @@ proc copy_file {filename} {
       puts $out "#if 0"
     } elseif {!$linemacros && [regexp {^#line} $line]} {
       # Skip #line directives.
-    } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} {
+    } elseif {$addstatic
+               && ![regexp {^(static|typedef|SQLITE_PRIVATE)} $line]} {
       # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before
       # functions if this header file does not need it.
       if {![info exists varonly_hdr($tail)]
@@ -211,18 +218,20 @@ proc copy_file {filename} {
         regsub {^SQLITE_API } $line {} line
         # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
         # so that linkage can be modified at compile-time.
-        if {[regexp {^sqlite3_} $funcname]} {
+        if {[regexp {^sqlite3[a-z]*_} $funcname]} {
           set line SQLITE_API
           append line " " [string trim $rettype]
           if {[string index $rettype end] ne "*"} {
             append line " "
           }
-          if {[lsearch -exact $cdecllist $funcname] >= 0} {
-            append line SQLITE_CDECL
-          } else {
-            append line SQLITE_STDCALL
+          if {$useapicall} {
+            if {[lsearch -exact $cdecllist $funcname] >= 0} {
+              append line SQLITE_CDECL " "
+            } else {
+              append line SQLITE_APICALL " "
+            }
           }
-          append line " " $funcname $rest
+          append line $funcname $rest
           puts $out $line
         } else {
           puts $out "SQLITE_PRIVATE $line"
@@ -285,6 +294,7 @@ foreach file {
    mutex_w32.c
    malloc.c
    printf.c
+   treeview.c
    random.c
    threads.c
    utf.c
@@ -313,7 +323,6 @@ foreach file {
    vdbe.c
    vdbeblob.c
    vdbesort.c
-   journal.c
    memjournal.c
 
    walker.c
@@ -339,6 +348,8 @@ foreach file {
    update.c
    vacuum.c
    vtab.c
+   wherecode.c
+   whereexpr.c
    where.c
 
    parse.c
diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl
index 7bc05d2f75..55179c4b80 100644
--- a/tool/mksqlite3c.tcl
+++ b/tool/mksqlite3c.tcl
@@ -1,7 +1,7 @@
 #!/usr/bin/tclsh
 #
 # To build a single huge source file holding all of SQLite (or at
-# least the core components - the test harness, shell, and TCL 
+# least the core components - the test harness, shell, and TCL
 # interface are omitted.) first do
 #
 #      make target_source
@@ -11,7 +11,7 @@
 # there and will not work if they are not found.)  There are a few
 # generated C code files that are also added to the tsrc directory.
 # For example, the "parse.c" and "parse.h" files to implement the
-# the parser are derived from "parse.y" using lemon.  And the 
+# the parser are derived from "parse.y" using lemon.  And the
 # "keywordhash.h" files is generated by a program named "mkkeywordhash".
 #
 # After the "tsrc" directory has been created and populated, run
@@ -28,12 +28,15 @@
 #
 set addstatic 1
 set linemacros 0
+set useapicall 0
 for {set i 0} {$i<[llength $argv]} {incr i} {
   set x [lindex $argv $i]
   if {[regexp {^-+nostatic$} $x]} {
     set addstatic 0
   } elseif {[regexp {^-+linemacros} $x]} {
     set linemacros 1
+  } elseif {[regexp {^-+useapicall} $x]} {
+    set useapicall 1
   } else {
     error "unknown command-line option: $x"
   }
@@ -59,7 +62,7 @@ set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
 puts $out [subst \
 {/******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version $VERSION.  By combining all the individual C code files into this 
+** version $VERSION.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -68,7 +71,7 @@ puts $out [subst \
 **
 ** This file is all you need to compile SQLite.  To use SQLite in other
 ** programs, you need this file and the "sqlite3.h" header file that defines
-** the programming interface to the SQLite library.  (If you do not have 
+** the programming interface to the SQLite library.  (If you do not have
 ** the "sqlite3.h" header file at hand, you will find a copy embedded within
 ** the text of this file.  Search for "Begin file sqlite3.h" to find the start
 ** of the embedded sqlite3.h header file.) Additional code files may be needed
@@ -85,7 +88,7 @@ if {$addstatic} {
 #endif}
 }
 
-# These are the header files used by SQLite.  The first time any of these 
+# These are the header files used by SQLite.  The first time any of these
 # files are seen in a #include statement in the C code, include the complete
 # text of the file in-line.  The file only needs to be included once.
 #
@@ -111,6 +114,7 @@ foreach hdr {
    pcache.h
    pragma.h
    rtree.h
+   sqlite3session.h
    sqlite3.h
    sqlite3ext.h
    sqlite3rbu.h
@@ -126,6 +130,7 @@ foreach hdr {
   set available_hdr($hdr) 1
 }
 set available_hdr(sqliteInt.h) 0
+set available_hdr(sqlite3session.h) 0
 
 # These headers should be copied into the amalgamation without modifying any
 # of their function declarations or definitions.
@@ -163,7 +168,8 @@ proc section_comment {text} {
 # process them appropriately.
 #
 proc copy_file {filename} {
-  global seen_hdr available_hdr varonly_hdr cdecllist out addstatic linemacros
+  global seen_hdr available_hdr varonly_hdr cdecllist out
+  global addstatic linemacros useapicall
   set ln 0
   set tail [file tail $filename]
   section_comment "Begin file $tail"
@@ -220,18 +226,20 @@ proc copy_file {filename} {
         regsub {^SQLITE_API } $line {} line
         # Add the SQLITE_PRIVATE or SQLITE_API keyword before functions.
         # so that linkage can be modified at compile-time.
-        if {[regexp {^sqlite3(_|rbu_)} $funcname]} {
+        if {[regexp {^sqlite3[a-z]*_} $funcname]} {
           set line SQLITE_API
           append line " " [string trim $rettype]
           if {[string index $rettype end] ne "*"} {
             append line " "
           }
-          if {[lsearch -exact $cdecllist $funcname] >= 0} {
-            append line SQLITE_CDECL
-          } else {
-            append line SQLITE_STDCALL
+          if {$useapicall} {
+            if {[lsearch -exact $cdecllist $funcname] >= 0} {
+              append line SQLITE_CDECL " "
+            } else {
+              append line SQLITE_APICALL " "
+            }
           }
-          append line " " $funcname $rest
+          append line $funcname $rest
           puts $out $line
         } else {
           puts $out "SQLITE_PRIVATE $line"
@@ -378,6 +386,7 @@ foreach file {
    fts3_icu.c
    sqlite3rbu.c
    dbstat.c
+   sqlite3session.c
    json1.c
    fts5.c
 } {
diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl
index 3f59aef467..9d307d1b1e 100644
--- a/tool/mksqlite3h.tcl
+++ b/tool/mksqlite3h.tcl
@@ -10,17 +10,19 @@
 #
 # Run this script by specifying the root directory of the source tree
 # on the command-line.
-# 
+#
 # This script performs processing on src/sqlite.h.in. It:
 #
 #   1) Adds SQLITE_EXTERN in front of the declaration of global variables,
 #   2) Adds SQLITE_API in front of the declaration of API functions,
-#   3) Replaces the string --VERS-- with the current library version, 
+#   3) Replaces the string --VERS-- with the current library version,
 #      formatted as a string (e.g. "3.6.17"), and
 #   4) Replaces the string --VERSION-NUMBER-- with current library version,
 #      formatted as an integer (e.g. "3006017").
-#   5) Replaces the string --SOURCE-ID-- with the date and time and sha1 
+#   5) Replaces the string --SOURCE-ID-- with the date and time and sha1
 #      hash of the fossil-scm manifest for the source tree.
+#   6) Adds the SQLITE_CALLBACK calling convention macro in front of all
+#      callback declarations.
 #
 # This script outputs to stdout.
 #
@@ -34,6 +36,14 @@
 #
 set TOP [lindex $argv 0]
 
+# Enable use of SQLITE_APICALL macros at the right points?
+#
+set useapicall 0
+
+if {[lsearch -regexp [lrange $argv 1 end] {^-+useapicall}] != -1} {
+  set useapicall 1
+}
+
 # Get the SQLite version number (ex: 3.6.18) from the $TOP/VERSION file.
 #
 set in [open $TOP/VERSION]
@@ -63,7 +73,13 @@ close $in
 # Set up patterns for recognizing API declarations.
 #
 set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)}
-set declpattern {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$}
+set declpattern1 {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3_[_a-zA-Z0-9]+)(\(.*)$}
+
+set declpattern2 \
+    {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3session_[_a-zA-Z0-9]+)(\(.*)$}
+
+set declpattern3 \
+    {^ *([a-zA-Z][a-zA-Z_0-9 ]+ \**)(sqlite3changeset_[_a-zA-Z0-9]+)(\(.*)$}
 
 # Force the output to use unix line endings, even on Windows.
 fconfigure stdout -translation lf
@@ -71,6 +87,7 @@ fconfigure stdout -translation lf
 set filelist [subst {
   $TOP/src/sqlite.h.in
   $TOP/ext/rtree/sqlite3rtree.h
+  $TOP/ext/session/sqlite3session.h
   $TOP/ext/fts5/fts5.h
 }]
 
@@ -91,15 +108,18 @@ set cdecllist {
 #
 foreach file $filelist {
   set in [open $file]
+  if {![regexp {sqlite\.h\.in} $file]} {
+    puts "/******** Begin file [file tail $file] *********/"
+  }
   while {![eof $in]} {
-  
+
     set line [gets $in]
 
     # File sqlite3rtree.h contains a line "#include ". Omit this
     # line when copying sqlite3rtree.h into sqlite3.h.
     #
     if {[string match {*#include*[<"]sqlite3.h[>"]*} $line]} continue
-  
+
     regsub -- --VERS--           $line $zVersion line
     regsub -- --VERSION-NUMBER-- $line $nVersion line
     regsub -- --SOURCE-ID--      $line "$zDate $zUuid" line
@@ -107,21 +127,33 @@ foreach file $filelist {
     if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} {
       set line "SQLITE_API $line"
     } else {
-      if {[regexp $declpattern $line all rettype funcname rest]} {
+      if {[regexp $declpattern1 $line all rettype funcname rest] || \
+          [regexp $declpattern2 $line all rettype funcname rest] || \
+          [regexp $declpattern3 $line all rettype funcname rest]} {
         set line SQLITE_API
         append line " " [string trim $rettype]
         if {[string index $rettype end] ne "*"} {
           append line " "
         }
-        if {[lsearch -exact $cdecllist $funcname] >= 0} {
-          append line SQLITE_CDECL
-        } else {
-          append line SQLITE_STDCALL
+        if {$useapicall} {
+          if {[lsearch -exact $cdecllist $funcname] >= 0} {
+            append line SQLITE_CDECL " "
+          } else {
+            append line SQLITE_APICALL " "
+          }
         }
-        append line " " $funcname $rest
+        append line $funcname $rest
       }
     }
+    if {$useapicall} {
+      set line [string map [list (*sqlite3_syscall_ptr) \
+          "(SQLITE_SYSAPI *sqlite3_syscall_ptr)"] $line]
+      regsub {\(\*} $line {(SQLITE_CALLBACK *} line
+    }
     puts $line
   }
   close $in
+  if {![regexp {sqlite\.h\.in} $file]} {
+    puts "/******** End of [file tail $file] *********/"
+  }
 }
diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl
index b7350f9c84..c874d538fb 100644
--- a/tool/mkvsix.tcl
+++ b/tool/mkvsix.tcl
@@ -77,7 +77,7 @@
 # command prompt:
 #
 #                         CD /D C:\dev\sqlite\core
-#                         tclsh85 tool\mkvsix.tcl C:\Temp
+#                         tclsh tool\mkvsix.tcl C:\Temp
 #
 # In the example above, "C:\dev\sqlite\core" represents the root of the source
 # tree for SQLite and "C:\Temp" represents the top-level directory containing
diff --git a/tool/opcodesum.tcl b/tool/opcodesum.tcl
new file mode 100644
index 0000000000..47dff32b90
--- /dev/null
+++ b/tool/opcodesum.tcl
@@ -0,0 +1,34 @@
+#!/usr/bin/tclsh
+#
+# Run this script, redirecting input from cachegrind output, to compute the
+# number of CPU cycles used by each VDBE opcode.
+#
+# The cachegrind output should be configured so that it reports a single
+# column of Ir at the left margin. Ex:
+#
+#    cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl
+#
+set currentop x
+set ncycle(x) 0
+while {![eof stdin]} {
+  set line [string map {\173 x \175 x \042 x} [gets stdin]]
+  if {[regexp {  \.  case OP_.*:} $line]} {
+    regexp {OP_(.+):} $line all currentop
+    set ncycle($currentop) 0
+  } elseif {[lindex $line 1]=="default:"
+            && [regexp {really OP_Noop and OP_Explain} $line]} {
+    break
+  } elseif {[lindex $line 0]!="."} {
+    regsub -all {[^0-9]} [lindex $line 0] {} n
+    if {$n!=""} {incr ncycle($currentop) $n}
+  }
+}
+unset ncycle(x)
+set results {}
+foreach op [lsort [array names ncycle]] {
+  if {$ncycle($op)==0} continue
+  lappend results [list $ncycle($op) $op]
+}
+foreach entry [lsort -index 0 -int -decr $results] {
+  puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]]
+}
diff --git a/tool/replace.tcl b/tool/replace.tcl
index b01a83accb..5a1ac5983c 100644
--- a/tool/replace.tcl
+++ b/tool/replace.tcl
@@ -4,16 +4,19 @@
 # only lines successfully modified with a regular
 # expression.
 #
+fconfigure stdout -translation binary -encoding binary
+fconfigure stderr -translation binary -encoding binary
 set mode [string tolower [lindex $argv 0]]
 set from [lindex $argv 1]
 set to [lindex $argv 2]
-if {$mode ni [list exact include]} {exit 1}
+if {$mode ni [list exact regsub include]} {exit 1}
 if {[string length $from]==0} {exit 2}
 while {![eof stdin]} {
   set line [gets stdin]
   if {[eof stdin]} break
   switch -exact $mode {
     exact {set line [string map [list $from $to] $line]}
+    regsub {regsub -all -- $from $line $to line}
     include {if {[regsub -all -- $from $line $to line]==0} continue}
   }
   puts stdout $line
diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh
index ee2ceac660..0e970ea0f6 100644
--- a/tool/run-speed-test.sh
+++ b/tool/run-speed-test.sh
@@ -12,7 +12,7 @@
 #
 #     summary-$BASE.txt           # Copy of standard output
 #     cout-$BASE.txt              # cachegrind output
-#     explain-$BASE.txt           # EXPLAIN listings
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
 #
 if test "$1" = ""
 then
@@ -21,9 +21,10 @@ then
 fi
 NAME=$1
 shift
-CC_OPTS="-DSQLITE_ENABLE_RTREE"
-SPEEDTEST_OPTS="--shrink-memory --reprepare"
+CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
 SIZE=5
+doExplain=0
 while test "$1" != ""; do
   case $1 in
     --reprepare)
@@ -35,12 +36,29 @@ while test "$1" != ""; do
     --utf16be)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
         ;;
+    --stats)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
     --without-rowid)
         SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
         ;;
+    --nomemstat)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --wal)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        ;;
     --size)
         shift; SIZE=$1
         ;;
+    --explain)
+        doExplain=1
+        ;;
+    --heap)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
+        ;;
     *)
         CC_OPTS="$CC_OPTS $1"
         ;;
@@ -54,9 +72,11 @@ echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
 rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
 gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
 size sqlite3.o | tee -a summary-$NAME.txt
-gcc -g -Os -Wall -I. $CC_OPTS \
-   -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-   ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+if test $doExplain -eq 1; then
+  gcc -g -Os -Wall -I. $CC_OPTS \
+     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+fi
 SRC=./speedtest1.c
 gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
 ls -l speedtest1 | tee -a summary-$NAME.txt
@@ -65,4 +85,6 @@ valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
 size sqlite3.o | tee -a summary-$NAME.txt
 wc sqlite3.c
 cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
-./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+if test $doExplain -eq 1; then
+  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+fi
diff --git a/tool/showdb.c b/tool/showdb.c
index 06cd36cd2c..ba7a362258 100644
--- a/tool/showdb.c
+++ b/tool/showdb.c
@@ -151,7 +151,7 @@ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){
       exit(1);
     }
   }else{
-    lseek(g.dbfd, ofst, SEEK_SET);
+    lseek(g.dbfd, (long)ofst, SEEK_SET);
     got = read(g.dbfd, aData, nByte);
     if( got>0 && got "
+    } else {
+      puts -nonewline "% "
+    }
+    flush stdout
+    append line [gets stdin]
+    if {[info complete $line]} {
+      if {[catch {uplevel #0 $line} result]} {
+        puts stderr "Error: $result"
+      } elseif {$result!=""} {
+        puts $result
+      }
+      set line {}
+    } else {
+      append line \n
+    }
+  }
+}
+
+
 # Get the name of the database to analyze
 #
 proc usage {} {
@@ -34,22 +61,37 @@ information for the database and its constituent tables and indexes.
 
 Options:
 
-   --stats        Output SQL text that creates a new database containing
-                  statistics about the database that was analyzed
+   --pageinfo   Show how each page of the database-file is used
 
-   --pageinfo     Show how each page of the database-file is used
+   --stats      Output SQL text that creates a new database containing
+                statistics about the database that was analyzed
+
+   --tclsh      Run the built-in TCL interpreter interactively (for debugging)
+
+   --version    Show the version number of SQLite
 }
   exit 1
 }
 set file_to_analyze {}
 set flags(-pageinfo) 0
 set flags(-stats) 0
+set flags(-debug) 0
 append argv {}
 foreach arg $argv {
   if {[regexp {^-+pageinfo$} $arg]} {
     set flags(-pageinfo) 1
   } elseif {[regexp {^-+stats$} $arg]} {
     set flags(-stats) 1
+  } elseif {[regexp {^-+debug$} $arg]} {
+    set flags(-debug) 1
+  } elseif {[regexp {^-+tclsh$} $arg]} {
+    tclsh
+    exit 0
+  } elseif {[regexp {^-+version$} $arg]} {
+    sqlite3 mem :memory:
+    puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
+    mem close
+    exit 0
   } elseif {[regexp {^-} $arg]} {
     puts stderr "Unknown option: $arg"
     usage
@@ -100,6 +142,10 @@ if {[catch {sqlite3 db $file_to_analyze -uri 1} msg]} {
   puts stderr "error trying to open $file_to_analyze: $msg"
   exit 1
 }
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  db trace ::dbtrace
+}
 
 db eval {SELECT count(*) FROM sqlite_master}
 set pageSize [expr {wide([db one {PRAGMA page_size}])}]
@@ -142,16 +188,22 @@ if {$flags(-stats)} {
   exit 0
 }
 
+
 # In-memory database for collecting statistics. This script loops through
 # the tables and indices in the database being analyzed, adding a row for each
 # to an in-memory database (for which the schema is shown below). It then
 # queries the in-memory db to produce the space-analysis report.
 #
 sqlite3 mem :memory:
+if {$flags(-debug)} {
+  proc dbtrace {txt} {puts $txt; flush stdout;}
+  mem trace ::dbtrace
+}
 set tabledef {CREATE TABLE space_used(
    name clob,        -- Name of a table or index in the database file
    tblname clob,     -- Name of associated table
    is_index boolean, -- TRUE if it is an index, false for a table
+   is_without_rowid boolean, -- TRUE if WITHOUT ROWID table  
    nentry int,       -- Number of entries in the BTree
    leaf_entries int, -- Number of leaf entries
    depth int,        -- Depth of the b-tree
@@ -184,7 +236,7 @@ set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
 foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
 
   set is_index [expr {$name!=$tblname}]
-  set idx_btree [expr {$is_index || [is_without_rowid $name]}]
+  set is_without_rowid [is_without_rowid $name]
   db eval {
     SELECT 
       sum(ncell) AS nentry,
@@ -235,6 +287,7 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
       $name,
       $tblname,
       $is_index,
+      $is_without_rowid,
       $nentry,
       $leaf_entries,
       $depth,
@@ -330,12 +383,15 @@ proc subreport {title where showFrag} {
   # following query returns exactly one row (because it is an aggregate).
   #
   # The results of the query are stored directly by SQLite into local 
-  # variables (i.e. $nentry, $nleaf etc.).
+  # variables (i.e. $nentry, $payload etc.).
   #
   mem eval "
     SELECT
-      int(sum(nentry)) AS nentry,
-      int(sum(leaf_entries)) AS nleaf,
+      int(sum(
+        CASE WHEN (is_without_rowid OR is_index) THEN nentry 
+             ELSE leaf_entries 
+        END
+      )) AS nentry,
       int(sum(payload)) AS payload,
       int(sum(ovfl_payload)) AS ovfl_payload,
       max(mx_payload) AS mx_payload,
@@ -375,8 +431,8 @@ proc subreport {title where showFrag} {
   set storage [expr {$total_pages*$pageSize}]
   set payload_percent [percent $payload $storage {of storage consumed}]
   set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
-  set avg_payload [divide $payload $nleaf]
-  set avg_unused [divide $total_unused $nleaf]
+  set avg_payload [divide $payload $nentry]
+  set avg_unused [divide $total_unused $nentry]
   if {$int_pages>0} {
     # TODO: Is this formula correct?
     set nTab [mem eval "
@@ -390,12 +446,12 @@ proc subreport {title where showFrag} {
     "]
     set avg_fanout [format %.2f $avg_fanout]
   }
-  set ovfl_cnt_percent [percent $ovfl_cnt $nleaf {of all entries}]
+  set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}]
 
   # Print out the sub-report statistics.
   #
   statline {Percentage of total database} $total_pages_percent
-  statline {Number of entries} $nleaf
+  statline {Number of entries} $nentry
   statline {Bytes of storage consumed} $storage
   if {$compressed_size!=$storage} {
     set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]
diff --git a/tool/speed-check.sh b/tool/speed-check.sh
new file mode 100644
index 0000000000..1e1389f2fd
--- /dev/null
+++ b/tool/speed-check.sh
@@ -0,0 +1,147 @@
+#!/bin/bash
+#
+# This is a template for a script used for day-to-day size and 
+# performance monitoring of SQLite.  Typical usage:
+#
+#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
+#     sh run-speed-test.sh x1     # Measure some experimental change
+#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
+#
+# There are multiple output files, all with a base name given by
+# the first argument:
+#
+#     summary-$BASE.txt           # Copy of standard output
+#     cout-$BASE.txt              # cachegrind output
+#     explain-$BASE.txt           # EXPLAIN listings (only with --explain)
+#
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
+CC_OPTS="-DSQLITE_ENABLE_MEMSYS5"
+CC=gcc
+SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64"
+SIZE=5
+LEAN_OPTS="-DSQLITE_THREADSAFE=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOB"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE"
+LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA"
+doExplain=0
+doCachegrind=1
+while test "$1" != ""; do
+  case $1 in
+    --reprepare)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --autovacuum)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --utf16be)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --stats)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --without-rowid)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --nomemstat)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --temp)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
+        ;;
+    --wal)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
+        ;;
+    --size)
+        shift; SIZE=$1
+        ;;
+    --cachesize)
+        shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1"
+        ;;
+    --explain)
+        doExplain=1
+        ;;
+    --vdbeprofile)
+        rm -f vdbe_profile.out
+        CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
+        doCachegrind=0
+        ;;
+    --lean)
+        CC_OPTS="$CC_OPTS $LEAN_OPTS"
+        ;;
+    --clang)
+        CC=clang
+        ;;
+    --heap)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
+        ;;
+    --lookaside)
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2"
+        shift;
+        ;;
+    --repeat)
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE"
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --repeat $1"
+        ;;
+    --mmap)
+        shift;
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1"
+        ;;
+    --rtree)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree"
+        CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE"
+        ;;
+    *)
+        CC_OPTS="$CC_OPTS $1"
+        ;;
+  esac
+  shift
+done
+SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
+echo "NAME           = $NAME" | tee summary-$NAME.txt
+echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
+echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
+rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
+size sqlite3.o | tee -a summary-$NAME.txt
+if test $doExplain -eq 1; then
+  $CC -g -Os -Wall -I. $CC_OPTS \
+     -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+    ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+fi
+SRC=./speedtest1.c
+$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
+ls -l speedtest1 | tee -a summary-$NAME.txt
+if test $doCachegrind -eq 1; then
+  valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
+      $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+else
+  ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+fi
+size sqlite3.o | tee -a summary-$NAME.txt
+wc sqlite3.c
+if test $doCachegrind -eq 1; then
+  cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
+fi
+if test $doExplain -eq 1; then
+  ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
+fi
+if test "$NAME" != "trunk"; then
+  fossil test-diff --tk cout-trunk.txt cout-$NAME.txt
+fi
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index ae01cd3c4d..67f3197bbf 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -33,6 +33,7 @@ struct GlobalVars {
   const char *zArgv0;       /* Name of program */
   int bSchemaOnly;          /* Only show schema differences */
   int bSchemaPK;            /* Use the schema-defined PK, not the true PK */
+  int bHandleVtab;          /* Handle fts3, fts4, fts5 and rtree vtabs */
   unsigned fDebug;          /* Debug flags */
   sqlite3 *db;              /* The database connection */
 } g;
@@ -402,7 +403,8 @@ static void printQuoted(FILE *out, sqlite3_value *X){
         }
         fprintf(out, "'");
       }else{
-        fprintf(out, "NULL");
+        /* Could be an OOM, could be a zero-byte blob */
+        fprintf(out, "X''");
       }
       break;
     }
@@ -683,7 +685,7 @@ static void diff_one_table(const char *zTab, FILE *out){
 
   /* Run the query and output differences */
   if( !g.bSchemaOnly ){
-    pStmt = db_prepare(sql.z);
+    pStmt = db_prepare("%s", sql.z);
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       int iType = sqlite3_column_int(pStmt, nPk);
       if( iType==1 || iType==2 ){
@@ -1178,8 +1180,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n)");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Deleted rows: */
   strPrintf(pSql, "\nUNION ALL\nSELECT ");
@@ -1193,8 +1196,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n) ");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Updated rows. If all table columns are part of the primary key, there 
   ** can be no updates. In this case this part of the compound SELECT can
@@ -1225,7 +1229,7 @@ static void getRbudiffQuery(
     );
 
     strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
-    strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+    strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
     strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
   }
 
@@ -1244,6 +1248,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
   Str sql = {0, 0, 0};            /* Query to find differences */
   Str insert = {0, 0, 0};         /* First part of output INSERT statement */
   sqlite3_stmt *pStmt = 0;
+  int nRow = 0;                   /* Total rows in data_xxx table */
 
   /* --rbu mode must use real primary keys. */
   g.bSchemaPK = 1;
@@ -1289,6 +1294,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
 
     /* Output the first part of the INSERT statement */
     fprintf(out, "%s", insert.z);
+    nRow++;
 
     if( sqlite3_column_type(pStmt, nCol)==SQLITE_INTEGER ){
       for(i=0; i<=nCol; i++){
@@ -1299,7 +1305,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
       char *zOtaControl;
       int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
 
-      zOtaControl = (char*)sqlite3_malloc(nOtaControl);
+      zOtaControl = (char*)sqlite3_malloc(nOtaControl+1);
       memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
 
       for(i=0; i0 ){
+    Str cnt = {0, 0, 0};
+    strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow);
+    fprintf(out, "%s\n", cnt.z);
+    strFree(&cnt);
+  }
 
   strFree(&ct);
   strFree(&sql);
@@ -1452,7 +1464,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
   }
 
   /* Run the query and output difference summary */
-  pStmt = db_prepare(sql.z);
+  pStmt = db_prepare("%s", sql.z);
   nUpdate = 0;
   nInsert = 0;
   nDelete = 0;
@@ -1727,6 +1739,144 @@ end_changeset_one_table:
   sqlite3_free(zId);
 }
 
+/*
+** Extract the next SQL keyword or quoted string from buffer zIn and copy it
+** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
+** Return a pointer to the character within zIn immediately following 
+** the token or quoted string just extracted.
+*/
+const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
+  const char *p = zIn;
+  char *pOut = zBuf;
+  char *pEnd = &pOut[nBuf-1];
+  char q = 0;                     /* quote character, if any */
+
+  if( p==0 ) return 0;
+  while( *p==' ' ) p++;
+  switch( *p ){
+    case '"': q = '"'; break;
+    case '\'': q = '\''; break;
+    case '`': q = '`'; break;
+    case '[': q = ']'; break;
+  }
+
+  if( q ){
+    p++;
+    while( *p && pOut