Updates to the 'vtshim' extension.

FossilOrigin-Name: 163c6cdf93451cb4ae7655266654d4aec2b94c4ceb0a6fc68859ecc73bf393a9

Makefile.in

@@ -503,6 +503,7 @@ TESTSRC2 = \
   $(TOP)/src/pcache1.c \
   $(TOP)/src/select.c \
   $(TOP)/src/tokenize.c \
+  $(TOP)/src/treeview.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vdbeapi.c \
@@ -609,7 +610,7 @@ TESTOPTS = --verbose=file --output=test-out.txt
 
 # Extra compiler options for various shell tools
 #
-SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4
+SHELL_OPT = -DSQLITE_ENABLE_FTS4
 #SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
@@ -619,8 +620,8 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZERSHELL_OPT = 
+FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
@@ -630,7 +631,7 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
 FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
+FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c
 DBFUZZ_OPT =
 
 # This is the default Makefile target.  The objects listed here
@@ -874,7 +875,7 @@ hash.lo:	$(TOP)/src/hash.c $(HDR)
 insert.lo:	$(TOP)/src/insert.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/insert.c
 
-json.lo:	$(TOP)/src/json.c
+json.lo:	$(TOP)/src/json.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/json.c
 
 legacy.lo:	$(TOP)/src/legacy.c $(HDR)
@@ -1280,6 +1281,12 @@ valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessio
 tcltest:	./testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/veryquick.test $(TESTOPTS)
 
+# Runs all the same tests cases as the "tcltest" target but uses
+# the testrunner.tcl script to run them in multiple cores
+# concurrently.
+testrunner:	testfixture$(TEXE)
+	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl
+
 # Minimal testing that runs in less than 3 minutes
 #
 quicktest:	./testfixture$(TEXE)
@@ -1512,3 +1519,135 @@ sqlite3.def: $(REAL_LIBOBJ)
 sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def
 	$(TCC) -shared -o $@ sqlite3.def \
 		-Wl,"--strip-all" $(REAL_LIBOBJ)
+
+
+#
+# fiddle/wasm section
+#
+fiddle_dir = ext/fiddle
+fiddle_dir_abs = $(TOP)/$(fiddle_dir)
+# ^^^ some emcc opts require absolute paths
+fiddle_html = $(fiddle_dir)/fiddle.html
+fiddle_module_js = $(fiddle_dir)/fiddle-module.js
+sqlite3_wasm_js = $(fiddle_dir)/sqlite3.js
+sqlite3_wasm = $(fiddle_dir)/sqlite3.wasm
+#emcc_opt = -O0
+#emcc_opt = -O1
+#emcc_opt = -O2
+#emcc_opt = -O3
+emcc_opt = -Oz
+emcc_flags = $(emcc_opt) -sALLOW_TABLE_GROWTH -sSTRICT_JS \
+        -sENVIRONMENT=web -sMODULARIZE \
+        -sEXPORTED_RUNTIME_METHODS=@$(fiddle_dir_abs)/EXPORTED_RUNTIME_METHODS \
+        -sDYNAMIC_EXECUTION=0 \
+        -I. $(SHELL_OPT)
+$(fiddle_module_js): Makefile sqlite3.c shell.c \
+    $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \
+    $(fiddle_dir)/EXPORTED_FUNCTIONS.fiddle
+	emcc -o $@ $(emcc_flags) \
+        -sEXPORT_NAME=initFiddleModule \
+        -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.fiddle \
+        -DSQLITE_SHELL_FIDDLE \
+        sqlite3.c shell.c
+	gzip < $@ > $@.gz
+	gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz
+$(sqlite3_wasm_js): Makefile sqlite3.c \
+    $(fiddle_dir)/sqlite3-api.js \
+    $(fiddle_dir)/EXPORTED_RUNTIME_METHODS \
+    $(fiddle_dir)/EXPORTED_FUNCTIONS.sqlite3-api
+	emcc -o $@ $(emcc_flags) \
+        -sEXPORT_NAME=initSqlite3Module \
+        -sEXPORTED_FUNCTIONS=@$(fiddle_dir_abs)/EXPORTED_FUNCTIONS.sqlite3-api \
+        --post-js=$(fiddle_dir)/sqlite3-api.js \
+        --no-entry \
+        sqlite3.c
+	gzip < $@ > $@.gz
+	gzip < $(sqlite3_wasm) > $(sqlite3_wasm).gz
+	gzip < $(fiddle_dir)/sqlite3-api.js > $(fiddle_dir)/sqlite3-api.js.gz
+$(fiddle_dir)/fiddle.js.gz: $(fiddle_dir)/fiddle.js
+	gzip < $< > $@
+$(fiddle_dir)/sqlite3-api.js.gz: $(fiddle_dir)/sqlite3-api.js
+	gzip < $< > $@
+
+fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \
+                   $(fiddle_dir)/fiddle-module.wasm \
+                   $(fiddle_dir)/fiddle-module.wasm.gz \
+                   $(fiddle_dir)/fiddle.js.gz
+sqlite3_wasm_generated = \
+                   $(sqlite3_wasm) $(sqlite3_wasm).gz \
+                   $(sqlite3_wasm_js) $(sqlite3_wasm_js).gz \
+                   $(fiddle_dir)/sqlite3.js.gz \
+                   $(fiddle_dir)/sqlite3-api.js.gz
+
+clean-wasm:
+	rm -f $(fiddle_generated) $(sqlite3_wasm_generated)
+clean: clean-wasm
+fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz
+sqlite3-wasm: $(sqlite3_wasm_js)
+wasm: fiddle sqlite3-wasm
+########################################################################
+# Explanation of the emcc build flags follows. Full docs for these can
+# be found at:
+#
+#  https://github.com/emscripten-core/emscripten/blob/main/src/settings.js
+#
+# -sENVIRONMENT=web: elides bootstrap code related to non-web JS
+#  environments like node.js. Removing this makes the output a tiny
+#  tick larger but hypothetically makes it more portable to
+#  non-browser JS environments.
+#
+# -sMODULARIZE: changes how the generated code is structured to avoid
+#  declaring a global Module object and instead installing a function
+#  which loads and initialized the module. The function is named...
+#
+# -sEXPORT_NAME=jsFunctionName (see -sMODULARIZE)
+#
+# -sEXPORTED_RUNTIME_METHODS=@/absolute/path/to/file: a file
+#  containing a list of emscripten-supplied APIs, one per line, which
+#  must be exported into the generated JS. Must be an absolute path!
+#
+# -sEXPORTED_FUNCTIONS=@/absolute/path/to/file: a file containing a
+#  list of C functions, one per line, which must be exported via wasm
+#  so they're visible to JS. C symbols names in that file must all
+#  start with an underscore for reasons known only to the emcc
+#  developers. e.g., _sqlite3_open_v2 and _sqlite3_finalize. Must be
+#  an absolute path!
+#
+# -sSTRICT_JS ensures that the emitted JS code includes the 'use
+#  strict' option. Note that -sSTRICT is more broadly-scoped and
+#  results in build errors.
+#
+# -sALLOW_TABLE_GROWTH is required for (at a minimum) the UDF-binding
+#  feature.
+#
+# -sDYNAMIC_EXECUTION=0 disables eval() and the Function constructor.
+#  If the build runs without these, it's preferable to use this flag
+#  because certain execution environments disallow those constructs.
+#  This flag is not strictly necessary, however.
+#
+# -sWASM_BIGINT is UNTESTED but "should" allow the int64-using C APIs
+#  to work with JS/wasm, insofar as the JS environment supports the
+#  BigInt type. That support requires an extremely recent browser:
+#  Safari didn't get that support until late 2020.
+#
+# --no-entry: for compiling library code with no main(). If this is
+#  not supplied and the code has a main(), it is called as part of the
+#  module init process. Note that main() is #if'd out of shell.c
+#  (renamed) when building in wasm mode.
+#
+# --pre-js/--post-js=FILE relative or absolute paths to JS files to
+#  prepend/append to the emcc-generated bootstrapping JS. It's
+#  easier/faster to develop with separate JS files (reduces rebuilding
+#  requirements) but certain configurations, namely -sMODULARIZE, may
+#  require using at least a --pre-js file. They can be used
+#  individually and need not be paired.
+#
+# -O0..-O3 and -Oz: optimization levels affect not only C-style
+#  optimization but whether or not the resulting generated JS code
+#  gets minified. -O0 compiles _much_ more quickly than -O3 or -Oz,
+#  and doesn't minimize any JS code, so is recommended for
+#  development. -O3 or -Oz are recommended for deployment, but
+#  primarily because -Oz will shrink the wasm file notably. JS-side
+#  minification makes little difference in terms of overall
+#  distributable size.
+########################################################################

Makefile.msc

@@ -363,7 +363,6 @@ SQLITE_TCL_DEP =
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1
@@ -1696,9 +1695,9 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 # <<mark>>
 # Extra compiler options for various test tools.
 #
-MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5
-FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
+MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5
+FUZZERSHELL_COMPILE_OPTS =
+FUZZCHECK_OPTS = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
@@ -1706,7 +1705,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
 
-FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c
+FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c $(TOP)\test\fuzzinvariants.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
 KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
@@ -2393,7 +2392,6 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
 
@@ -2477,6 +2475,12 @@ tcltest: testfixture.exe
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS)
 
+# Runs all the same tests cases as the "tcltest" target but uses
+# the testrunner.tcl script to run them in multiple cores
+# concurrently.
+testrunner:	testfixture.exe
+	.\testfixture.exe $(TOP)\test\testrunner.tcl
+
 smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)

README.md

@@ -1,7 +1,7 @@
 <h1 align="center">SQLite Source Repository</h1>
 
-This repository contains the complete source code for the 
-[SQLite database engine](https://sqlite.org/).  Some test scripts 
+This repository contains the complete source code for the
+[SQLite database engine](https://sqlite.org/).  Some test scripts
 are also included.  However, many other test scripts
 and most of the documentation are managed separately.
 
@@ -15,7 +15,7 @@ The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
 If you are reading this on GitHub or some other Git repository or service,
 then you are looking at a mirror.  The names of check-ins and
 other artifacts in a Git mirror are different from the official
-names for those objects.  The offical names for check-ins are
+names for those objects.  The official names for check-ins are
 found in a footer on the check-in comment for authorized mirrors.
 The official check-in name can also be seen in the `manifest.uuid` file
 in the root of the tree.  Always use the official name, not  the
@@ -30,7 +30,7 @@ verify its integrity, there are hints on how to do that in the
 If you do not want to use Fossil, you can download tarballs or ZIP
 archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
 
-  *  Lastest trunk check-in as
+  *  Latest trunk check-in as
      [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz),
      [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or
      [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar).
@@ -47,11 +47,11 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
      then click on the "Tarball" or "ZIP Archive" links on the information
      page.
 
-If you do want to use Fossil to check out the source tree, 
+If you do want to use Fossil to check out the source tree,
 first install Fossil version 2.0 or later.
 (Source tarballs and precompiled binaries available
 [here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
-a stand-alone program.  To install, simply download or build the single 
+a stand-alone program.  To install, simply download or build the single
 executable file and put that file someplace on your $PATH.)
 Then run commands like this:
 
@@ -61,7 +61,7 @@ Then run commands like this:
         fossil open ~/Fossils/sqlite.fossil
 
 After setting up a repository using the steps above, you can always
-update to the lastest version using:
+update to the latest version using:
 
         fossil update trunk   ;# latest trunk check-in
         fossil update release ;# latest official release
@@ -136,7 +136,7 @@ the "tclsqlite.c" file which implements the
 extension and only later escaped to the wild as an independent library.)
 
 Test scripts and programs are found in the **test/** subdirectory.
-Addtional test code is found in other source repositories.
+Additional test code is found in other source repositories.
 See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
 additional information.
 
@@ -170,7 +170,7 @@ at just the right spots. Note that comment text in the sqlite3.h file is
 used to generate much of the SQLite API documentation.  The Tcl scripts
 used to generate that documentation are in a separate source repository.
 
-The SQL language parser is **parse.c** which is generate from a grammar in
+The SQL language parser is **parse.c** which is generated from a grammar in
 the src/parse.y file.  The conversion of "parse.y" into "parse.c" is done
 by the [lemon](./doc/lemon.html) LALR(1) parser generator.  The source code
 for lemon is at tool/lemon.c.  Lemon uses the tool/lempar.c file as a
@@ -180,7 +180,7 @@ generates parse.c.
 
 The **opcodes.h** header file contains macros that define the numbers
 corresponding to opcodes in the "VDBE" virtual machine.  The opcodes.h
-file is generated by the scanning the src/vdbe.c source file.  The
+file is generated by scanning the src/vdbe.c source file.  The
 Tcl script at ./mkopcodeh.tcl does this scan and generates opcodes.h.
 A second Tcl script, ./mkopcodec.tcl, then scans opcodes.h to generate
 the **opcodes.c** source file, which contains a reverse mapping from
@@ -237,7 +237,7 @@ prepared statements, the description of
 [how transactions work](http://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](http://www.sqlite.org/optoverview.html).
 
-Years of effort have gone into optimizating SQLite, both
+Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
 complex code.  So there is a lot of complexity in the current SQLite
 implementation.  It will not be the easiest library in the world to hack.
@@ -294,11 +294,11 @@ Key files:
      building the "testfixture.exe" program.  The testfixture.exe program is
      an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
      test/ folder to validate the core SQLite code.  The testfixture program
-     (and some other test programs too) is build and run when you type
+     (and some other test programs too) is built and run when you type
      "make test".
 
   *  **ext/misc/json1.c** - This file implements the various JSON functions
-     that are build into SQLite.
+     that are built into SQLite.
 
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
@@ -307,7 +307,7 @@ describes its purpose and role within the larger system.
 ## Verifying Code Authenticity
 
 The `manifest` file at the root directory of the source tree
-contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for 
+contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for
 older files) for every source file in the repository.
 The name of the version of the entire source tree is just the
 SHA3-256 hash of the `manifest` file itself, possibly with the

VERSION

@@ -1 +1 @@
-3.38.0
+3.40.0

autoconf/Makefile.msc

@@ -285,7 +285,6 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_JSON1=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1

autoconf/README.txt

@@ -105,7 +105,7 @@ may be specified in this manner as some require the amalgamation to be built
 with them enabled (see http://www.sqlite.org/compile.html). For example, the
 following will work:
 
-  "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_ENABLE_JSON1=1"
+  "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_JSON=1"
 
 However, the following will not compile unless the amalgamation was built
 with it enabled:

autoconf/configure.ac

@@ -173,21 +173,6 @@ else
 fi
 #-----------------------------------------------------------------------
 
-#-----------------------------------------------------------------------
-#   --enable-json1
-#
-AC_ARG_ENABLE(json1, [AS_HELP_STRING(
-  [--enable-json1], [include json1 support [default=yes]])], 
-  [],[enable_json1=yes])
-AC_MSG_CHECKING([JSON functions])
-if test x"$enable_json1" = "xyes"; then
-  BUILD_CFLAGS="$BUILD_CFLAGS -DSQLITE_ENABLE_JSON1"
-  AC_MSG_RESULT([enabled])
-else
-  AC_MSG_RESULT([disabled])
-fi
-#-----------------------------------------------------------------------
-
 #-----------------------------------------------------------------------
 #   --enable-rtree
 #

autoconf/tea/Makefile.in

@@ -11,8 +11,6 @@
 #
 # See the file "license.terms" for information on usage and redistribution
 # of this file, and for a DISCLAIMER OF ALL WARRANTIES.
-#
-# RCS: @(#) $Id: Makefile.in,v 1.59 2005/07/26 19:17:02 mdejong Exp $
 
 #========================================================================
 # Add additional lines to handle any additional AC_SUBST cases that
@@ -60,6 +58,8 @@ PKG_HEADERS	= @PKG_HEADERS@
 #========================================================================
 
 PKG_LIB_FILE	= @PKG_LIB_FILE@
+PKG_LIB_FILE8	= @PKG_LIB_FILE8@
+PKG_LIB_FILE9	= @PKG_LIB_FILE9@
 PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@
 
 lib_BINARIES	= $(PKG_LIB_FILE)
@@ -73,10 +73,11 @@ exec_prefix	= @exec_prefix@
 
 bindir		= @bindir@
 libdir		= @libdir@
+includedir	= @includedir@
 datarootdir	= @datarootdir@
+runstatedir	= @runstatedir@
 datadir		= @datadir@
 mandir		= @mandir@
-includedir	= @includedir@
 
 DESTDIR		=
 
@@ -85,24 +86,25 @@ pkgdatadir	= $(datadir)/$(PKG_DIR)
 pkglibdir	= $(libdir)/$(PKG_DIR)
 pkgincludedir	= $(includedir)/$(PKG_DIR)
 
-top_builddir	= .
+top_builddir	= @abs_top_builddir@
 
-INSTALL		= @INSTALL@
-INSTALL_PROGRAM	= @INSTALL_PROGRAM@
+INSTALL_OPTIONS	=
+INSTALL		= @INSTALL@ $(INSTALL_OPTIONS)
+INSTALL_DATA_DIR = @INSTALL_DATA_DIR@
 INSTALL_DATA	= @INSTALL_DATA@
+INSTALL_PROGRAM	= @INSTALL_PROGRAM@
 INSTALL_SCRIPT	= @INSTALL_SCRIPT@
+INSTALL_LIBRARY	= @INSTALL_LIBRARY@
 
 PACKAGE_NAME	= @PACKAGE_NAME@
 PACKAGE_VERSION	= @PACKAGE_VERSION@
 CC		= @CC@
+CCLD		= @CCLD@
 CFLAGS_DEFAULT	= @CFLAGS_DEFAULT@
 CFLAGS_WARNING	= @CFLAGS_WARNING@
-CLEANFILES	= @CLEANFILES@
 EXEEXT		= @EXEEXT@
 LDFLAGS_DEFAULT	= @LDFLAGS_DEFAULT@
 MAKE_LIB	= @MAKE_LIB@
-MAKE_SHARED_LIB	= @MAKE_SHARED_LIB@
-MAKE_STATIC_LIB	= @MAKE_STATIC_LIB@
 MAKE_STUB_LIB	= @MAKE_STUB_LIB@
 OBJEXT		= @OBJEXT@
 RANLIB		= @RANLIB@
@@ -117,8 +119,6 @@ TCL_SRC_DIR	= @TCL_SRC_DIR@
 #TK_BIN_DIR	= @TK_BIN_DIR@
 #TK_SRC_DIR	= @TK_SRC_DIR@
 
-# This is no longer necessary even for packages that use private Tcl headers
-#TCL_TOP_DIR_NATIVE	= @TCL_TOP_DIR_NATIVE@
 # Not used, but retained for reference of what libs Tcl required
 #TCL_LIBS	= @TCL_LIBS@
 
@@ -132,41 +132,52 @@ TCL_SRC_DIR	= @TCL_SRC_DIR@
 EXTRA_PATH	= $(top_builddir):$(TCL_BIN_DIR)
 #EXTRA_PATH	= $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR)
 TCLLIBPATH	= $(top_builddir)
-TCLSH_ENV	= TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` \
-		  @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \
+TCLSH_ENV	= TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library`
+PKG_ENV		= @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \
 		  PATH="$(EXTRA_PATH):$(PATH)" \
 		  TCLLIBPATH="$(TCLLIBPATH)"
-#		  TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library`
 
 TCLSH_PROG	= @TCLSH_PROG@
-TCLSH	= $(TCLSH_ENV) $(TCLSH_PROG)
+TCLSH		= $(TCLSH_ENV) $(PKG_ENV) $(TCLSH_PROG)
 
+#WISH_ENV	= TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library`
 #WISH_PROG	= @WISH_PROG@
-#WISH	= $(TCLSH_ENV) $(WISH_PROG)
-
+#WISH		= $(TCLSH_ENV) $(WISH_ENV) $(PKG_ENV) $(WISH_PROG)
 
 SHARED_BUILD	= @SHARED_BUILD@
 
-INCLUDES	= @PKG_INCLUDES@ @TCL_INCLUDES@ -I$(srcdir)/..
+INCLUDES	= @PKG_INCLUDES@ @TCL_INCLUDES@ -I. -I$(srcdir)/..
 #INCLUDES	= @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@
 
 PKG_CFLAGS	= @PKG_CFLAGS@
 
 # TCL_DEFS is not strictly need here, but if you remove it, then you
-# must make sure that configure.in checks for the necessary components
+# must make sure that configure.ac checks for the necessary components
 # that your library may use.  TCL_DEFS can actually be a problem if
 # you do not compile with a similar machine setup as the Tcl core was
 # compiled with.
 #DEFS		= $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS)
 DEFS		= @DEFS@ $(PKG_CFLAGS)
 
+# Move pkgIndex.tcl to 'BINARIES' var if it is generated in the Makefile
 CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl
+CLEANFILES	= @CLEANFILES@
 
 CPPFLAGS	= @CPPFLAGS@
 LIBS		= @PKG_LIBS@ @LIBS@
 AR		= @AR@
 CFLAGS		= @CFLAGS@
-COMPILE		= $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LDFLAGS		= @LDFLAGS@
+LDFLAGS_DEFAULT			= @LDFLAGS_DEFAULT@
+COMPILE		= $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) \
+			  $(CFLAGS_DEFAULT) $(CFLAGS_WARNING) $(SHLIB_CFLAGS) $(CFLAGS)
+
+GDB		= gdb
+VALGRIND	= valgrind
+VALGRINDARGS	= --tool=memcheck --num-callers=8 --leak-resolution=high \
+		  --leak-check=yes --show-reachable=yes -v
+
+.SUFFIXES: .c .$(OBJEXT)
 
 #========================================================================
 # Start of user-definable TARGETS section
@@ -174,7 +185,7 @@ COMPILE		= $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(C
 
 #========================================================================
 # TEA TARGETS.  Please note that the "libraries:" target refers to platform
-# independent files, and the "binaries:" target inclues executable programs and
+# independent files, and the "binaries:" target includes executable programs and
 # platform-dependent libraries.  Modify these targets so that they install
 # the various pieces of your package.  The make and install rules
 # for the BINARIES that you specified above have already been done.
@@ -193,7 +204,6 @@ binaries: $(BINARIES)
 
 libraries:
 
-
 #========================================================================
 # Your doc target should differentiate from doc builds (by the developer)
 # and doc installs (see install-doc), which just install the docs on the
@@ -216,11 +226,11 @@ install-binaries: binaries install-lib-binaries install-bin-binaries
 #========================================================================
 
 install-libraries: libraries
-	@mkdir -p $(DESTDIR)$(includedir)
+	@$(INSTALL_DATA_DIR) "$(DESTDIR)$(includedir)"
 	@echo "Installing header files in $(DESTDIR)$(includedir)"
 	@list='$(PKG_HEADERS)'; for i in $$list; do \
 	    echo "Installing $(srcdir)/$$i" ; \
-	    $(INSTALL_DATA) $(srcdir)/$$i $(DESTDIR)$(includedir) ; \
+	    $(INSTALL_DATA) $(srcdir)/$$i "$(DESTDIR)$(includedir)" ; \
 	done;
 
 #========================================================================
@@ -229,12 +239,11 @@ install-libraries: libraries
 #========================================================================
 
 install-doc: doc
-	@mkdir -p $(DESTDIR)$(mandir)/mann
+	@$(INSTALL_DATA_DIR) "$(DESTDIR)$(mandir)/mann"
 	@echo "Installing documentation in $(DESTDIR)$(mandir)"
 	@list='$(srcdir)/doc/*.n'; for i in $$list; do \
 	    echo "Installing $$i"; \
-	    rm -f $(DESTDIR)$(mandir)/mann/`basename $$i`; \
-	    $(INSTALL_DATA) $$i $(DESTDIR)$(mandir)/mann ; \
+	    $(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/mann" ; \
 	done
 
 test: binaries libraries
@@ -244,7 +253,21 @@ shell: binaries libraries
 	@$(TCLSH) $(SCRIPT)
 
 gdb:
-	$(TCLSH_ENV) gdb $(TCLSH_PROG) $(SCRIPT)
+	$(TCLSH_ENV) $(PKG_ENV) $(GDB) $(TCLSH_PROG) $(SCRIPT)
+
+gdb-test: binaries libraries
+	$(TCLSH_ENV) $(PKG_ENV) $(GDB) \
+	    --args $(TCLSH_PROG) `@CYGPATH@ $(srcdir)/tests/all.tcl` \
+	    $(TESTFLAGS) -singleproc 1 \
+	    -load "package ifneeded $(PACKAGE_NAME) $(PACKAGE_VERSION) \
+		[list load `@CYGPATH@ $(PKG_LIB_FILE)` [string totitle $(PACKAGE_NAME)]]"
+
+valgrind: binaries libraries
+	$(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) \
+	    `@CYGPATH@ $(srcdir)/tests/all.tcl` $(TESTFLAGS)
+
+valgrindshell: binaries libraries
+	$(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) $(SCRIPT)
 
 depend:
 
@@ -283,49 +306,70 @@ $(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS)
 # As necessary, add $(srcdir):$(srcdir)/compat:....
 #========================================================================
 
-VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win
+VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macosx
 
 .c.@OBJEXT@:
 	$(COMPILE) -c `@CYGPATH@ $<` -o $@
 
+tclsample.@OBJEXT@:	sampleUuid.h
+
+$(srcdir)/manifest.uuid:
+	printf "git-" >$(srcdir)/manifest.uuid
+	(cd $(srcdir); git rev-parse HEAD >>$(srcdir)/manifest.uuid || \
+	    (printf "svn-r" >$(srcdir)/manifest.uuid ; \
+	    svn info --show-item last-changed-revision >>$(srcdir)/manifest.uuid) || \
+	    printf "unknown" >$(srcdir)/manifest.uuid)
+
+sampleUuid.h:	$(srcdir)/manifest.uuid
+	echo "#define SAMPLE_VERSION_UUID \\" >$@
+	cat $(srcdir)/manifest.uuid >>$@
+	echo "" >>$@
+
 #========================================================================
 # Distribution creation
 # You may need to tweak this target to make it work correctly.
 #========================================================================
 
 #COMPRESS	= tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar
-COMPRESS	= gtar zcvf $(PKG_DIR).tar.gz $(PKG_DIR)
+COMPRESS	= tar zcvf $(PKG_DIR).tar.gz $(PKG_DIR)
 DIST_ROOT	= /tmp/dist
 DIST_DIR	= $(DIST_ROOT)/$(PKG_DIR)
 
+DIST_INSTALL_DATA	= CPPROG='cp -p' $(INSTALL) -m 644
+DIST_INSTALL_SCRIPT	= CPPROG='cp -p' $(INSTALL) -m 755
+
 dist-clean:
 	rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.*
 
-dist: dist-clean
-	mkdir -p $(DIST_DIR)
-	cp -p $(srcdir)/README* $(srcdir)/license* \
-		$(srcdir)/aclocal.m4 $(srcdir)/configure $(srcdir)/*.in \
-		$(DIST_DIR)/
-	chmod 664 $(DIST_DIR)/Makefile.in $(DIST_DIR)/aclocal.m4
-	chmod 775 $(DIST_DIR)/configure $(DIST_DIR)/configure.in
+dist: dist-clean $(srcdir)/manifest.uuid
+	$(INSTALL_DATA_DIR) $(DIST_DIR)
 
-	for i in $(srcdir)/*.[ch]; do \
-	    if [ -f $$i ]; then \
-		cp -p $$i $(DIST_DIR)/ ; \
-	    fi; \
-	done;
+	# TEA files
+	$(DIST_INSTALL_DATA) $(srcdir)/Makefile.in \
+	    $(srcdir)/aclocal.m4 $(srcdir)/configure.ac \
+	    $(DIST_DIR)/
+	$(DIST_INSTALL_SCRIPT) $(srcdir)/configure $(DIST_DIR)/
 
-	mkdir $(DIST_DIR)/tclconfig
-	cp $(srcdir)/tclconfig/install-sh $(srcdir)/tclconfig/tcl.m4 \
-		$(DIST_DIR)/tclconfig/
-	chmod 664 $(DIST_DIR)/tclconfig/tcl.m4
-	chmod +x $(DIST_DIR)/tclconfig/install-sh
+	$(INSTALL_DATA_DIR) $(DIST_DIR)/tclconfig
+	$(DIST_INSTALL_DATA) $(srcdir)/tclconfig/README.txt \
+	    $(srcdir)/manifest.uuid \
+	    $(srcdir)/tclconfig/tcl.m4 $(srcdir)/tclconfig/install-sh \
+	    $(DIST_DIR)/tclconfig/
 
-	list='demos doc generic library mac tests unix win'; \
+	# Extension files
+	$(DIST_INSTALL_DATA) \
+	    $(srcdir)/ChangeLog \
+	    $(srcdir)/README.sha \
+	    $(srcdir)/license.terms \
+	    $(srcdir)/README \
+	    $(srcdir)/pkgIndex.tcl.in \
+	    $(DIST_DIR)/
+
+	list='demos doc generic library macosx tests unix win'; \
 	for p in $$list; do \
 	    if test -d $(srcdir)/$$p ; then \
-		mkdir $(DIST_DIR)/$$p; \
-		cp -p $(srcdir)/$$p/*.* $(DIST_DIR)/$$p/; \
+		$(INSTALL_DATA_DIR) $(DIST_DIR)/$$p; \
+		$(DIST_INSTALL_DATA) $(srcdir)/$$p/* $(DIST_DIR)/$$p/; \
 	    fi; \
 	done
 
@@ -337,10 +381,10 @@ dist: dist-clean
 
 #========================================================================
 # Don't modify the file to clean here.  Instead, set the "CLEANFILES"
-# variable in configure.in
+# variable in configure.ac
 #========================================================================
 
-clean:  
+clean:
 	-test -z "$(BINARIES)" || rm -f $(BINARIES)
 	-rm -f *.$(OBJEXT) core *.core
 	-test -z "$(CLEANFILES)" || rm -f $(CLEANFILES)
@@ -348,7 +392,7 @@ clean:
 distclean: clean
 	-rm -f *.tab.c
 	-rm -f $(CONFIG_CLEAN_FILES)
-	-rm -f config.h config.cache config.log config.status
+	-rm -f config.cache config.log config.status
 
 #========================================================================
 # Install binary object libraries.  On Windows this includes both .dll and
@@ -362,25 +406,17 @@ distclean: clean
 #========================================================================
 
 install-lib-binaries: binaries
-	@mkdir -p $(DESTDIR)$(pkglibdir)
+	@$(INSTALL_DATA_DIR) "$(DESTDIR)$(pkglibdir)"
 	@list='$(lib_BINARIES)'; for p in $$list; do \
 	  if test -f $$p; then \
-	    echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p"; \
-	    $(INSTALL_PROGRAM) $$p $(DESTDIR)$(pkglibdir)/$$p; \
-	    stub=`echo $$p|sed -e "s/.*\(stub\).*/\1/"`; \
-	    if test "x$$stub" = "xstub"; then \
-		echo " $(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p"; \
-		$(RANLIB_STUB) $(DESTDIR)$(pkglibdir)/$$p; \
-	    else \
-		echo " $(RANLIB) $(DESTDIR)$(pkglibdir)/$$p"; \
-		$(RANLIB) $(DESTDIR)$(pkglibdir)/$$p; \
-	    fi; \
+	    echo " $(INSTALL_LIBRARY) $$p $(DESTDIR)$(pkglibdir)/$$p"; \
+	    $(INSTALL_LIBRARY) $$p "$(DESTDIR)$(pkglibdir)/$$p"; \
 	    ext=`echo $$p|sed -e "s/.*\.//"`; \
 	    if test "x$$ext" = "xdll"; then \
 		lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \
 		if test -f $$lib; then \
 		    echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \
-	            $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib; \
+	            $(INSTALL_DATA) $$lib "$(DESTDIR)$(pkglibdir)/$$lib"; \
 		fi; \
 	    fi; \
 	  fi; \
@@ -389,12 +425,12 @@ install-lib-binaries: binaries
 	  if test -f $(srcdir)/$$p; then \
 	    destp=`basename $$p`; \
 	    echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \
-	    $(INSTALL_DATA) $(srcdir)/$$p $(DESTDIR)$(pkglibdir)/$$destp; \
+	    $(INSTALL_DATA) $(srcdir)/$$p "$(DESTDIR)$(pkglibdir)/$$destp"; \
 	  fi; \
 	done
 	@if test "x$(SHARED_BUILD)" = "x1"; then \
 	    echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \
-	    $(INSTALL_DATA) pkgIndex.tcl $(DESTDIR)$(pkglibdir); \
+	    $(INSTALL_DATA) pkgIndex.tcl "$(DESTDIR)$(pkglibdir)"; \
 	fi
 
 #========================================================================
@@ -407,33 +443,32 @@ install-lib-binaries: binaries
 #========================================================================
 
 install-bin-binaries: binaries
-	@mkdir -p $(DESTDIR)$(bindir)
+	@$(INSTALL_DATA_DIR) "$(DESTDIR)$(bindir)"
 	@list='$(bin_BINARIES)'; for p in $$list; do \
 	  if test -f $$p; then \
 	    echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \
-	    $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p; \
+	    $(INSTALL_PROGRAM) $$p "$(DESTDIR)$(bindir)/$$p"; \
 	  fi; \
 	done
 
-.SUFFIXES: .c .$(OBJEXT)
-
 Makefile: $(srcdir)/Makefile.in  $(top_builddir)/config.status
 	cd $(top_builddir) \
 	  && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
 
 uninstall-binaries:
 	list='$(lib_BINARIES)'; for p in $$list; do \
-	  rm -f $(DESTDIR)$(pkglibdir)/$$p; \
+	  rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \
 	done
 	list='$(PKG_TCL_SOURCES)'; for p in $$list; do \
 	  p=`basename $$p`; \
-	  rm -f $(DESTDIR)$(pkglibdir)/$$p; \
+	  rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \
 	done
 	list='$(bin_BINARIES)'; for p in $$list; do \
-	  rm -f $(DESTDIR)$(bindir)/$$p; \
+	  rm -f "$(DESTDIR)$(bindir)/$$p"; \
 	done
 
 .PHONY: all binaries clean depend distclean doc install libraries test
+.PHONY: gdb gdb-test valgrind valgrindshell
 
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.

autoconf/tea/configure.ac

@@ -2,11 +2,9 @@
 dnl	This file is an input file used by the GNU "autoconf" program to
 dnl	generate the file "configure", which is run during Tcl installation
 dnl	to configure the system for the local environment.
-#
-# RCS: @(#) $Id: configure.in,v 1.43 2005/07/26 19:17:05 mdejong Exp $
 
 #-----------------------------------------------------------------------
-# Sample configure.in for Tcl Extensions.  The only places you should
+# Sample configure.ac for Tcl Extensions.  The only places you should
 # need to modify this file are marked by the string __CHANGE__
 #-----------------------------------------------------------------------
 
@@ -17,9 +15,11 @@ dnl	to configure the system for the local environment.
 # This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION
 # set as provided.  These will also be added as -D defs in your Makefile
 # so you can encode the package version directly into the source files.
+# This will also define a special symbol for Windows (BUILD_<PACKAGE_NAME>
+# so that we create the export library with the dll.
 #-----------------------------------------------------------------------
 
-AC_INIT([sqlite], [3.32.0])
+AC_INIT([sqlite],[3.40.0])
 
 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -27,7 +27,7 @@ AC_INIT([sqlite], [3.32.0])
 # as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE.
 #--------------------------------------------------------------------
 
-TEA_INIT([3.9])
+TEA_INIT()
 
 AC_CONFIG_AUX_DIR(tclconfig)
 
@@ -55,8 +55,8 @@ TEA_PREFIX
 #-----------------------------------------------------------------------
 # Standard compiler checks.
 # This sets up CC by using the CC env var, or looks for gcc otherwise.
-# This also calls AC_PROG_CC, AC_PROG_INSTALL and a few others to create
-# the basic setup necessary to compile executables.
+# This also calls AC_PROG_CC and a few others to create the basic setup
+# necessary to compile executables.
 #-----------------------------------------------------------------------
 
 TEA_SETUP_COMPILER
@@ -73,11 +73,19 @@ TEA_SETUP_COMPILER
 
 TEA_ADD_SOURCES([tclsqlite3.c])
 TEA_ADD_HEADERS([])
-TEA_ADD_INCLUDES([-I\"`\${CYGPATH} \${srcdir}/generic`\"])
+TEA_ADD_INCLUDES([])
 TEA_ADD_LIBS([])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS4=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS5=1])
 TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1])
 TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_GEOPOLY=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_MATH_FUNCTIONS=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DESERIALIZE=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBPAGE_VTAB=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_BYTECODE_VTAB=1])
+TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBSTAT_VTAB=1])
 TEA_ADD_STUB_SOURCES([])
 TEA_ADD_TCL_SOURCES([])
 
@@ -99,6 +107,31 @@ if test x$with_system_sqlite != xno; then
       LIBS="$LIBS -lsqlite3"])])
 fi
 
+#--------------------------------------------------------------------
+# __CHANGE__
+#
+# You can add more files to clean if your extension creates any extra
+# files by extending CLEANFILES.
+# Add pkgIndex.tcl if it is generated in the Makefile instead of ./configure
+# and change Makefile.in to move it from CONFIG_CLEAN_FILES to BINARIES var.
+#
+# A few miscellaneous platform-specific items:
+# TEA_ADD_* any platform specific compiler/build info here.
+#--------------------------------------------------------------------
+
+#CLEANFILES="$CLEANFILES pkgIndex.tcl"
+if test "${TEA_PLATFORM}" = "windows" ; then
+    # Ensure no empty if clauses
+    :
+    #TEA_ADD_SOURCES([win/winFile.c])
+    #TEA_ADD_INCLUDES([-I\"$(${CYGPATH} ${srcdir}/win)\"])
+else
+    # Ensure no empty else clauses
+    :
+    #TEA_ADD_SOURCES([unix/unixFile.c])
+    #TEA_ADD_LIBS([-lsuperfly])
+fi
+
 #--------------------------------------------------------------------
 # __CHANGE__
 # Choose which headers you need.  Extension authors should try very
@@ -151,28 +184,6 @@ TEA_CONFIG_CFLAGS
 
 TEA_ENABLE_SYMBOLS
 
-#--------------------------------------------------------------------
-# Everyone should be linking against the Tcl stub library.  If you
-# can't for some reason, remove this definition.  If you aren't using
-# stubs, you also need to modify the SHLIB_LD_LIBS setting below to
-# link against the non-stubbed Tcl library.  Add Tk too if necessary.
-#--------------------------------------------------------------------
-
-AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs])
-#AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
-
-
-#--------------------------------------------------------------------
-# Redefine fdatasync as fsync on systems that lack fdatasync
-#--------------------------------------------------------------------
-#
-#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync))
-# Check for library functions that SQLite can optionally use.
-AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r])
-
-AC_FUNC_STRERROR_R
-
-
 #--------------------------------------------------------------------
 # This macro generates a line to use when building a library.  It
 # depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS,
@@ -193,9 +204,24 @@ TEA_PROG_TCLSH
 #TEA_PROG_WISH
 
 #--------------------------------------------------------------------
-# Finally, substitute all of the various values into the Makefile.
-# You may alternatively have a special pkgIndex.tcl.in or other files
-# which require substituting th AC variables in.  Include these here.
+# Setup a *Config.sh.in configuration file.
 #--------------------------------------------------------------------
 
-AC_OUTPUT([Makefile pkgIndex.tcl])
+#TEA_EXPORT_CONFIG([sample])
+#AC_SUBST(SAMPLE_VAR)
+
+#--------------------------------------------------------------------
+# Specify files to substitute AC variables in. You may alternatively
+# have a special pkgIndex.tcl.in or other files which require
+# substituting the AC variables in. Include these here.
+#--------------------------------------------------------------------
+
+AC_CONFIG_FILES([Makefile pkgIndex.tcl])
+#AC_CONFIG_FILES([sampleConfig.sh])
+
+#--------------------------------------------------------------------
+# Finally, substitute all of the various values into the files
+# specified with AC_CONFIG_FILES.
+#--------------------------------------------------------------------
+
+AC_OUTPUT

autoconf/tea/pkgIndex.tcl.in

@@ -1,7 +1,10 @@
+# -*- tcl -*-
+# Tcl package index file, version 1.1
 #
-# Tcl package index file
-#
-# Note sqlite*3* init specifically
-#
-package ifneeded sqlite3 @PACKAGE_VERSION@ \
-    [list load [file join $dir @PKG_LIB_FILE@] Sqlite3]
+if {[package vsatisfies [package provide Tcl] 9.0-]} {
+    package ifneeded sqlite3 @PACKAGE_VERSION@ \
+	    [list load [file join $dir @PKG_LIB_FILE9@] sqlite3]
+} else {
+    package ifneeded sqlite3 @PACKAGE_VERSION@ \
+	    [list load [file join $dir @PKG_LIB_FILE8@] sqlite3]
+}

autoconf/tea/win/makefile.vc

@@ -251,7 +251,18 @@ INCLUDES	= $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \
                   -I"$(GENERICDIR)" -I"$(ROOT)\.."
 BASE_CLFAGS	= $(cflags) $(cdebug) $(crt) $(INCLUDES) \
                   -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \
-                  -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1
+                  -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 \
+                  -DSQLITE_ENABLE_FTS4=1 \
+                  -DSQLITE_ENABLE_FTS5=1 \
+                  -DSQLITE_3_SUFFIX_ONLY=1 \
+                  -DSQLITE_ENABLE_RTREE=1 \
+                  -DSQLITE_ENABLE_GEOPOLY=1 \
+                  -DSQLITE_ENABLE_MATH_FUNCTIONS=1 \
+                  -DSQLITE_ENABLE_DESERIALIZE=1 \
+                  -DSQLITE_ENABLE_DBPAGE_VTAB=1 \
+                  -DSQLITE_ENABLE_BYTECODE_VTAB=1 \
+                  -DSQLITE_ENABLE_DBSTAT_VTAB=1 
+
 CON_CFLAGS	= $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1
 TCL_CFLAGS	= -DBUILD_sqlite -DUSE_TCL_STUBS \
                   -DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.38.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.40.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.38.0'
-PACKAGE_STRING='sqlite 3.38.0'
+PACKAGE_VERSION='3.40.0'
+PACKAGE_STRING='sqlite 3.40.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1468,7 +1468,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.38.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.40.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1533,7 +1533,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.38.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.40.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1661,7 +1661,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.38.0
+sqlite configure 3.40.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2080,7 +2080,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.38.0, which was
+It was created by sqlite $as_me 3.40.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -12390,7 +12390,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.38.0, which was
+This file was extended by sqlite $as_me 3.40.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12456,7 +12456,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.38.0
+sqlite config.status 3.40.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

ext/expert/sqlite3expert.c

@@ -662,6 +662,7 @@ static int idxRegisterVtab(sqlite3expert *p){
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
     0,                            /* xShadowName */
+    0                             /* xShadowName2 */
   };
 
   return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);

ext/fiddle/EXPORTED_FUNCTIONS.fiddle

@@ -0,0 +1,7 @@
+_fiddle_exec
+_fiddle_interrupt
+_fiddle_experiment
+_fiddle_the_db
+_fiddle_db_arg
+_fiddle_db_filename
+_fiddle_reset_db

ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api

@@ -0,0 +1,51 @@
+_sqlite3_bind_blob
+_sqlite3_bind_double
+_sqlite3_bind_int
+_sqlite3_bind_int64
+_sqlite3_bind_null
+_sqlite3_bind_parameter_count
+_sqlite3_bind_parameter_index
+_sqlite3_bind_text
+_sqlite3_changes
+_sqlite3_clear_bindings
+_sqlite3_close_v2
+_sqlite3_column_blob
+_sqlite3_column_bytes
+_sqlite3_column_count
+_sqlite3_column_count
+_sqlite3_column_double
+_sqlite3_column_int
+_sqlite3_column_int64
+_sqlite3_column_name
+_sqlite3_column_text
+_sqlite3_column_type
+_sqlite3_compileoption_get
+_sqlite3_compileoption_used
+_sqlite3_create_function_v2
+_sqlite3_data_count
+_sqlite3_db_filename
+_sqlite3_errmsg
+_sqlite3_exec
+_sqlite3_finalize
+_sqlite3_interrupt
+_sqlite3_libversion
+_sqlite3_open
+_sqlite3_open_v2
+_sqlite3_prepare_v2
+_sqlite3_prepare_v2
+_sqlite3_reset
+_sqlite3_result_blob
+_sqlite3_result_double
+_sqlite3_result_error
+_sqlite3_result_int
+_sqlite3_result_null
+_sqlite3_result_text
+_sqlite3_sourceid
+_sqlite3_sql
+_sqlite3_step
+_sqlite3_value_blob
+_sqlite3_value_bytes
+_sqlite3_value_double
+_sqlite3_value_text
+_sqlite3_value_type
+_free

ext/fiddle/EXPORTED_RUNTIME_METHODS

@@ -0,0 +1,14 @@
+ALLOC_NORMAL
+FS
+UTF8ToString
+addFunction
+allocate
+allocateUTF8OnStack
+ccall
+cwrap
+getValue
+removeFunction
+setValue
+stackAlloc
+stackRestore
+stackSave

ext/fiddle/Makefile

@@ -0,0 +1,34 @@
+# This GNU makefile exists primarily to simplify/speed up development
+# from emacs. It is not part of the canonical build process.
+default:
+	$(MAKE) -C ../.. wasm -e emcc_opt=-O0
+
+clean:
+	$(MAKE) -C ../../ clean-wasm
+
+fiddle_files = emscripten.css fiddle.html \
+             fiddle.js fiddle-module.js \
+             fiddle-module.wasm fiddle-worker.js \
+             $(wildcard *.wasm.gz) $(wildcard *.js.gz)
+
+# fiddle_remote is the remote destination for the fiddle app. It
+# must be a [user@]HOST:/path for rsync.
+# Note that the target "should probably" contain a symlink of
+# index.html -> fiddle.html.
+fiddle_remote ?=
+ifeq (,$(fiddle_remote))
+ifneq (,$(wildcard /home/stephan))
+  fiddle_remote = wh2:www/wh/sqlite3/.
+else ifneq (,$(wildcard /home/drh))
+  #fiddle_remote = if appropriate, add that user@host:/path here
+endif
+endif
+
+$(fiddle_files): default
+
+push-fiddle: $(fiddle_files)
+	@if [ x = "x$(fiddle_remote)" ]; then \
+		echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \
+		exit 1; \
+	fi
+	rsync -va $(fiddle_files) $(fiddle_remote)

ext/fiddle/SqliteTestUtil.js

@@ -0,0 +1,144 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This file contains bootstrapping code used by various test scripts
+  which live in this file's directory.
+*/
+(function(){
+    /* querySelectorAll() proxy */
+    const EAll = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelectorAll(arguments[arguments.length-1]);
+    };
+    /* querySelector() proxy */
+    const E = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelector(arguments[arguments.length-1]);
+    };
+
+    /**
+       Helpers for writing sqlite3-specific tests.
+    */
+    self/*window or worker*/.SqliteTestUtil = {
+        /** Running total of the number of tests run via
+            this API. */
+        counter: 0,
+        /**
+           If expr is a function, it is called and its result
+           is returned, coerced to a bool, else expr, coerced to
+           a bool, is returned.
+        */
+        toBool: function(expr){
+            return (expr instanceof Function) ? !!expr() : !!expr;
+        },
+        /** abort() if expr is false. If expr is a function, it
+            is called and its result is evaluated.
+        */
+        assert: function f(expr, msg){
+            if(!f._){
+                f._ = ('undefined'===typeof abort
+                       ? (msg)=>{throw new Error(msg)}
+                       : abort);
+            }
+            ++this.counter;
+            if(!this.toBool(expr)){
+                f._(msg || "Assertion failed.");
+            }
+            return this;
+        },
+        /** Identical to assert() but throws instead of calling
+            abort(). */
+        affirm: function(expr, msg){
+            ++this.counter;
+            if(!this.toBool(expr)) throw new Error(msg || "Affirmation failed.");
+            return this;
+        },
+        /** Calls f() and squelches any exception it throws. If it
+            does not throw, this function throws. */
+        mustThrow: function(f, msg){
+            ++this.counter;
+            let err;
+            try{ f(); } catch(e){err=e;}
+            if(!err) throw new Error(msg || "Expected exception.");
+            return this;
+        },
+        /** Throws if expr is truthy or expr is a function and expr()
+            returns truthy. */
+        throwIf: function(expr, msg){
+            ++this.counter;
+            if(this.toBool(expr)) throw new Error(msg || "throwIf() failed");
+            return this;
+        },
+        /** Throws if expr is falsy or expr is a function and expr()
+            returns falsy. */
+        throwUnless: function(expr, msg){
+            ++this.counter;
+            if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed");
+            return this;
+        }
+    };
+
+    
+    /**
+       This is a module object for use with the emscripten-installed
+       initSqlite3Module() factory function.
+    */
+    self.sqlite3TestModule = {
+        postRun: [
+            /* function(theModule){...} */
+        ],
+        //onRuntimeInitialized: function(){},
+        /* Proxy for C-side stdout output. */
+        print: function(){
+            console.log.apply(console, Array.prototype.slice.call(arguments));
+        },
+        /* Proxy for C-side stderr output. */
+        printErr: function(){
+            console.error.apply(console, Array.prototype.slice.call(arguments));
+        },
+        /**
+           Called by the module init bits to report loading
+           progress. It gets passed an empty argument when loading is
+           done (after onRuntimeInitialized() and any this.postRun
+           callbacks have been run).
+        */
+        setStatus: function f(text){
+            if(!f.last){
+                f.last = { text: '', step: 0 };
+                f.ui = {
+                    status: E('#module-status'),
+                    progress: E('#module-progress'),
+                    spinner: E('#module-spinner')
+                };
+            }
+            if(text === f.last.text) return;
+            f.last.text = text;
+            if(f.ui.progress){
+                f.ui.progress.value = f.last.step;
+                f.ui.progress.max = f.last.step + 1;
+            }
+            ++f.last.step;
+            if(text) {
+                f.ui.status.classList.remove('hidden');
+                f.ui.status.innerText = text;
+            }else{
+                if(f.ui.progress){
+                    f.ui.progress.remove();
+                    f.ui.spinner.remove();
+                    delete f.ui.progress;
+                    delete f.ui.spinner;
+                }
+                f.ui.status.classList.add('hidden');
+            }
+        }
+    };
+})(self/*window or worker*/);

ext/fiddle/emscripten.css

@@ -0,0 +1,24 @@
+/* emcscript-related styling, used during the module load/intialization processes... */
+.emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; }
+div.emscripten { text-align: center; }
+div.emscripten_border { border: 1px solid black; }
+#module-spinner { overflow: visible; }
+#module-spinner > * {
+    margin-top: 1em;
+}
+.spinner {
+    height: 50px;
+    width: 50px;
+    margin: 0px auto;
+    animation: rotation 0.8s linear infinite;
+    border-left: 10px solid rgb(0,150,240);
+    border-right: 10px solid rgb(0,150,240);
+    border-bottom: 10px solid rgb(0,150,240);
+    border-top: 10px solid rgb(100,0,200);
+    border-radius: 100%;
+    background-color: rgb(200,100,250);
+}
+@keyframes rotation {
+    from {transform: rotate(0deg);}
+    to {transform: rotate(360deg);}
+}

ext/fiddle/fiddle-worker.js

@@ -0,0 +1,301 @@
+/*
+  2022-05-20
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This is the JS Worker file for the sqlite3 fiddle app. It loads the
+  sqlite3 wasm module and offers access to the db via the Worker
+  message-passing interface.
+
+  Forewarning: this API is still very much Under Construction and
+  subject to any number of changes as experience reveals what those
+  need to be.
+
+  Because we can have only a single message handler, as opposed to an
+  arbitrary number of discrete event listeners like with DOM elements,
+  we have to define a lower-level message API. Messages abstractly
+  look like:
+
+  { type: string, data: type-specific value }
+
+  Where 'type' is used for dispatching and 'data' is a
+  'type'-dependent value.
+
+  The 'type' values expected by each side of the main/worker
+  connection vary. The types are described below but subject to
+  change at any time as this experiment evolves.
+
+  Workers-to-Main types
+
+  - stdout, stderr: indicate stdout/stderr output from the wasm
+    layer. The data property is the string of the output, noting
+    that the emscripten binding emits these one line at a time. Thus,
+    if a C-side puts() emits multiple lines in a single call, the JS
+    side will see that as multiple calls. Example:
+
+    {type:'stdout', data: 'Hi, world.'}
+
+  - module: Status text. This is intended to alert the main thread
+    about module loading status so that, e.g., the main thread can
+    update a progress widget and DTRT when the module is finished
+    loading and available for work. Status messages come in the form
+    
+    {type:'module', data:{
+        type:'status',
+        data: {text:string|null, step:1-based-integer}
+    }
+
+    with an incrementing step value for each subsequent message. When
+    the module loading is complete, a message with a text value of
+    null is posted.
+
+  - working: data='start'|'end'. Indicates that work is about to be
+    sent to the module or has just completed. This can be used, e.g.,
+    to disable UI elements which should not be activated while work
+    is pending. Example:
+
+    {type:'working', data:'start'}
+
+  Main-to-Worker types:
+
+  - shellExec: data=text to execute as if it had been entered in the
+    sqlite3 CLI shell app (as opposed to sqlite3_exec()). This event
+    causes the worker to emit a 'working' event (data='start') before
+    it starts and a 'working' event (data='end') when it finished. If
+    called while work is currently being executed it emits stderr
+    message instead of doing actual work, as the underlying db cannot
+    handle concurrent tasks. Example:
+
+    {type:'shellExec', data: 'select * from sqlite_master'}
+
+  - More TBD as the higher-level db layer develops.
+*/
+
+/*
+  Apparent browser(s) bug: console messages emitted may be duplicated
+  in the console, even though they're provably only run once. See:
+
+  https://stackoverflow.com/questions/49659464
+
+  Noting that it happens in Firefox as well as Chrome. Harmless but
+  annoying.
+*/
+"use strict";
+(function(){
+    /**
+       Posts a message in the form {type,data} unless passed more than 2
+       args, in which case it posts {type, data:[arg1...argN]}.
+    */
+    const wMsg = function(type,data){
+        postMessage({
+            type,
+            data: arguments.length<3
+                ? data
+                : Array.prototype.slice.call(arguments,1)
+        });
+    };
+
+    const stdout = function(){wMsg('stdout', Array.prototype.slice.call(arguments));};
+    const stderr = function(){wMsg('stderr', Array.prototype.slice.call(arguments));};
+
+    self.onerror = function(/*message, source, lineno, colno, error*/) {
+        const err = arguments[4];
+        if(err && 'ExitStatus'==err.name){
+            /* This is relevant for the sqlite3 shell binding but not the
+               lower-level binding. */
+            fiddleModule.isDead = true;
+            stderr("FATAL ERROR:", err.message);
+            stderr("Restarting the app requires reloading the page.");
+            wMsg('error', err);
+        }
+        console.error(err);
+        fiddleModule.setStatus('Exception thrown, see JavaScript console: '+err);
+    };
+
+    const Sqlite3Shell = {
+        /** Returns the name of the currently-opened db. */
+        dbFilename: function f(){
+            if(!f._) f._ = fiddleModule.cwrap('fiddle_db_filename', "string", ['string']);
+            return f._();
+        },
+        /**
+           Runs the given text through the shell as if it had been typed
+           in by a user. Fires a working/start event before it starts and
+           working/end event when it finishes.
+        */
+        exec: function f(sql){
+            if(!f._) f._ = fiddleModule.cwrap('fiddle_exec', null, ['string']);
+            if(fiddleModule.isDead){
+                stderr("shell module has exit()ed. Cannot run SQL.");
+                return;
+            }
+            wMsg('working','start');
+            try {
+                if(f._running){
+                    stderr('Cannot run multiple commands concurrently.');
+                }else{
+                    f._running = true;
+                    f._(sql);
+                }
+            } finally {
+                delete f._running;
+                wMsg('working','end');
+            }
+        },
+        resetDb: function f(){
+            if(!f._) f._ = fiddleModule.cwrap('fiddle_reset_db', null);
+            stdout("Resetting database.");
+            f._();
+            stdout("Reset",this.dbFilename());
+        },
+        /* Interrupt can't work: this Worker is tied up working, so won't get the
+           interrupt event which would be needed to perform the interrupt. */
+        interrupt: function f(){
+            if(!f._) f._ = fiddleModule.cwrap('fiddle_interrupt', null);
+            stdout("Requesting interrupt.");
+            f._();
+        }
+    };
+
+    self.onmessage = function f(ev){
+        ev = ev.data;
+        if(!f.cache){
+            f.cache = {
+                prevFilename: null
+            };
+        }
+        //console.debug("worker: onmessage.data",ev);
+        switch(ev.type){
+            case 'shellExec': Sqlite3Shell.exec(ev.data); return;
+            case 'db-reset': Sqlite3Shell.resetDb(); return;
+            case 'interrupt': Sqlite3Shell.interrupt(); return;
+                /** Triggers the export of the current db. Fires an
+                    event in the form:
+
+                    {type:'db-export',
+                    data:{
+                    filename: name of db,
+                    buffer: contents of the db file (Uint8Array),
+                    error: on error, a message string and no buffer property.
+                    }
+                    }
+                */
+            case 'db-export': {
+                const fn = Sqlite3Shell.dbFilename();
+                stdout("Exporting",fn+".");
+                const fn2 = fn ? fn.split(/[/\\]/).pop() : null;
+                try{
+                    if(!fn2) throw new Error("DB appears to be closed.");
+                    wMsg('db-export',{
+                        filename: fn2,
+                        buffer: fiddleModule.FS.readFile(fn, {encoding:"binary"})
+                    });
+                }catch(e){
+                    /* Post a failure message so that UI elements disabled
+                       during the export can be re-enabled. */
+                    wMsg('db-export',{
+                        filename: fn,
+                        error: e.message
+                    });
+                }
+                return;
+            }
+            case 'open': {
+                /* Expects: {
+                   buffer: ArrayBuffer | Uint8Array,
+                   filename: for logging/informational purposes only
+                   } */
+                const opt = ev.data;
+                let buffer = opt.buffer;
+                if(buffer instanceof Uint8Array){
+                }else if(buffer instanceof ArrayBuffer){
+                    buffer = new Uint8Array(buffer);
+                }else{
+                    stderr("'open' expects {buffer:Uint8Array} containing an uploaded db.");
+                    return;
+                }
+                const fn = (
+                    opt.filename
+                        ? opt.filename.split(/[/\\]/).pop().replace('"','_')
+                        : ("db-"+((Math.random() * 10000000) | 0)+
+                           "-"+((Math.random() * 10000000) | 0)+".sqlite3")
+                );
+                /* We cannot delete the existing db file until the new one
+                   is installed, which means that we risk overflowing our
+                   quota (if any) by having both the previous and current
+                   db briefly installed in the virtual filesystem. */
+                fiddleModule.FS.createDataFile("/", fn, buffer, true, true);
+                const oldName = Sqlite3Shell.dbFilename();
+                Sqlite3Shell.exec('.open "/'+fn+'"');
+                if(oldName && oldName !== fn){
+                    try{fiddleModule.FS.unlink(oldName);}
+                    catch(e){/*ignored*/}
+                }
+                stdout("Replaced DB with",fn+".");
+                return;
+            }
+        };
+        console.warn("Unknown fiddle-worker message type:",ev);
+    };
+   
+    /**
+       emscripten module for use with build mode -sMODULARIZE.
+    */
+    const fiddleModule = {
+        print: stdout,
+        printErr: stderr,
+        /**
+           Intercepts status updates from the emscripting module init
+           and fires worker events with a type of 'status' and a
+           payload of:
+
+           {
+           text: string | null, // null at end of load process
+           step: integer // starts at 1, increments 1 per call
+           }
+
+           We have no way of knowing in advance how many steps will
+           be processed/posted, so creating a "percentage done" view is
+           not really practical. One can be approximated by giving it a
+           current value of message.step and max value of message.step+1,
+           though.
+
+           When work is finished, a message with a text value of null is
+           submitted.
+
+           After a message with text==null is posted, the module may later
+           post messages about fatal problems, e.g. an exit() being
+           triggered, so it is recommended that UI elements for posting
+           status messages not be outright removed from the DOM when
+           text==null, and that they instead be hidden until/unless
+           text!=null.
+        */
+        setStatus: function f(text){
+            if(!f.last) f.last = { step: 0, text: '' };
+            else if(text === f.last.text) return;
+            f.last.text = text;
+            wMsg('module',{
+                type:'status',
+                data:{step: ++f.last.step, text: text||null}
+            });
+        }
+    };
+
+    importScripts('fiddle-module.js');
+    /**
+       initFiddleModule() is installed via fiddle-module.js due to
+       building with:
+
+       emcc ... -sMODULARIZE=1 -sEXPORT_NAME=initFiddleModule
+    */
+    initFiddleModule(fiddleModule).then(function(thisModule){
+        wMsg('fiddle-ready');
+    });
+})();

ext/fiddle/fiddle.html

@@ -0,0 +1,283 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <title>SQLite3 Fiddle</title>
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <!-- to add a togglable terminal-style view, uncomment the following
+         two lines and ensure that these files are on the web server. -->
+    <!--script src="jqterm/jqterm-bundle.min.js"></script>
+    <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"/-->
+    <link rel="stylesheet" href="emscripten.css"/>
+    <style>
+      /* The following styles are for app-level use. */
+      :root {
+          --sqlite-blue: #044a64;
+          --textarea-color1: #044a64;
+          --textarea-color2: white;
+      }
+      textarea {
+          font-family: monospace;
+          flex: 1 1 auto;
+          background-color: var(--textarea-color1);
+          color: var(--textarea-color2);
+      }
+      textarea#input {
+          color: var(--textarea-color1);
+          background-color: var(--textarea-color2);
+      }
+      header {
+          display: flex;
+          justify-content: space-between;
+          align-items: center;
+          background-color: var(--sqlite-blue);
+          color: white;
+          font-size: 120%;
+          font-weight: bold;
+          border-radius: 0.25em;
+          padding: 0.2em 0.5em;
+      }
+      header > .powered-by {
+          font-size: 80%;
+      }
+      header a, header a:visited, header a:hover {
+          color: inherit;
+      }
+      #main-wrapper {
+          display: flex;
+          flex-direction: column-reverse;
+          flex: 1 1 auto;
+          margin: 0.5em 0;
+          overflow: hidden;
+      }
+      #main-wrapper.side-by-side {
+          flex-direction: row;
+      }
+      #main-wrapper.side-by-side > fieldset {
+          margin-left: 0.25em;
+          margin-right: 0.25em;
+      }
+      #main-wrapper:not(.side-by-side) > fieldset {
+          margin-bottom: 0.25em;
+      }
+      #main-wrapper.swapio {
+          flex-direction: column;
+      }
+      #main-wrapper.side-by-side.swapio {
+          flex-direction: row-reverse;
+      }
+      .zone-wrapper{
+          display: flex;
+          margin: 0;
+          flex: 1 1 0%;
+          border-radius: 0.5em;
+          min-width: inherit/*important: resolves inability to scroll fieldset child element!*/;
+          padding: 0.35em 0 0 0;
+      }
+      .zone-wrapper textarea {
+          border-radius: 0.5em;
+          flex: 1 1 auto;
+          /*min/max width resolve an inexplicable margin on the RHS.  The -1em
+            is for the padding, else we overlap the parent boundaries.*/
+          /*min-width: calc(100% - 1em);
+          max-width: calc(100% - 1em);
+          padding: 0 0.5em;*/
+      }
+
+      .zone-wrapper.input { flex: 10 1 auto; }
+      .zone-wrapper.output { flex: 20 1 auto; }
+      .zone-wrapper > div {
+          display:flex;
+          flex: 1 1 0%;
+      }
+      .zone-wrapper.output {}
+      .button-bar {
+          display: flex;
+          flex-wrap: wrap;
+          align-items: center;
+          align-content: space-between;
+          justify-content: flex-start;
+      }
+      .button-bar > * {
+          margin: 0.05em 0.5em 0.05em 0;
+          flex: 0 1 auto;
+          align-self: auto;
+      }
+      label[for] {
+          cursor: pointer;
+      }
+      .error {
+          color: red;
+          background-color: yellow;
+      }
+      .hidden, .initially-hidden {
+          position: absolute !important;
+          opacity: 0 !important;
+          pointer-events: none !important;
+          display: none !important;
+      }
+      fieldset {
+          border-radius: 0.5em;
+          border: 1px inset;
+          padding: 0.25em;
+      }
+      fieldset.options {
+          font-size: 80%;
+          margin-top: 0.5em;
+      }
+      fieldset:not(.options) > legend {
+          font-size: 80%;
+      }
+      fieldset.options > div {
+          display: flex;
+          flex-wrap: wrap;
+      }
+      fieldset button {
+          font-size: inherit;
+      }
+      fieldset.collapsible > legend > .fieldset-toggle::after {
+          content: " [hide]";
+          position: relative;
+      }
+      fieldset.collapsible.collapsed > legend > .fieldset-toggle::after {
+          content: " [show]";
+          position: relative;
+      }
+      span.labeled-input {
+          padding: 0.25em;
+          margin: 0.05em 0.25em;
+          border-radius: 0.25em;
+          white-space: nowrap;
+          background: #0002;
+          display: flex;
+          align-items: center;
+      }
+      span.labeled-input > *:nth-child(2) {
+          margin-left: 0.3em;
+      }
+      .center { text-align: center; }
+      body.terminal-mode {
+          max-height: calc(100% - 2em);
+          display: flex;
+          flex-direction: column;
+          align-items: stretch;
+      }
+      #view-terminal {}
+      .app-view {
+          flex: 20 1 auto;
+      }
+      #view-split {
+          display: flex;
+          flex-direction: column-reverse;
+      }
+    </style>
+  </head>
+  <body>
+    <header id='titlebar'>
+      <span>SQLite3 Fiddle</span>
+      <span class='powered-by'>Powered by
+        <a href='https://sqlite.org'>SQLite3</a></span>
+    </header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+
+    <div id='view-terminal' class='app-view hidden initially-hidden'>
+      This is a placeholder for a terminal-like view which is not in
+      the default build.
+    </div>
+
+    <div id='view-split' class='app-view initially-hidden'>
+      <fieldset class='options collapsible'>
+        <legend><button class='fieldset-toggle'>Options</button></legend>
+        <div class=''>
+          <span class='labeled-input'>
+            <input type='checkbox' id='opt-cb-sbs'
+                   data-csstgt='#main-wrapper'
+                   data-cssclass='side-by-side'
+                   data-config='sideBySide'>
+            <label for='opt-cb-sbs'>Side-by-side</label>
+          </span>
+          <span class='labeled-input'>
+            <input type='checkbox' id='opt-cb-swapio'
+                   data-csstgt='#main-wrapper'
+                   data-cssclass='swapio'
+                   data-config='swapInOut'>
+            <label for='opt-cb-swapio'>Swap in/out</label>
+          </span>
+          <span class='labeled-input'>
+            <input type='checkbox' id='opt-cb-autoscroll'
+                   data-config='autoScrollOutput'>
+            <label for='opt-cb-autoscroll'>Auto-scroll output</label>
+          </span>
+          <span class='labeled-input'>
+            <input type='checkbox' id='opt-cb-autoclear'
+                   data-config='autoClearOutput'>
+            <label for='opt-cb-autoclear'>Auto-clear output</label>
+          </span>
+          <span class='labeled-input'>
+            <input type='file' id='load-db' class='hidden'/>
+            <button id='btn-load-db'>Load DB...</button>
+          </span>
+          <span class='labeled-input'>
+            <button id='btn-export'>Download DB</button>
+          </span>
+          <span class='labeled-input'>
+            <button id='btn-reset'>Reset DB</button>
+          </span>
+        </div>
+      </fieldset><!-- .options -->
+      <div id='main-wrapper' class=''>
+        <fieldset class='zone-wrapper input'>
+          <legend><div class='button-bar'>
+            <button id='btn-shell-exec'>Run</button>
+            <button id='btn-clear'>Clear Input</button>
+            <!--button data-cmd='.help'>Help</button-->
+            <select id='select-examples'></select>
+          </div></legend>
+          <div><textarea id="input"
+                         placeholder="Shell input. Ctrl-enter/shift-enter runs it.">
+-- ==================================================
+-- Use ctrl-enter or shift-enter to execute sqlite3
+-- shell commands and SQL.
+-- If a subset of the text is currently selected,
+-- only that part is executed.
+-- ==================================================
+.nullvalue NULL
+.headers on
+</textarea></div>
+        </fieldset>
+        <fieldset class='zone-wrapper output'>
+          <legend><div class='button-bar'>
+            <button id='btn-clear-output'>Clear Output</button>
+            <button id='btn-interrupt' class='hidden' disabled>Interrupt</button>
+            <!-- interruption cannot work in the current configuration
+                 because we cannot send an interrupt message when work
+                 is currently underway. At that point the Worker is
+                 tied up and will not receive the message. -->
+          </div></legend>
+          <div><textarea id="output" readonly
+                         placeholder="Shell output."></textarea></div>
+        </fieldset>
+      </div>
+    </div> <!-- #view-split -->
+    <!-- Maintenance notes:
+
+        ... TODO... currently being refactored...
+
+    -->
+    <script src="fiddle.js"></script>
+  </body>
+</html>

ext/fiddle/fiddle.js

@@ -0,0 +1,809 @@
+/*
+  2022-05-20
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This is the main entry point for the sqlite3 fiddle app. It sets up the
+  various UI bits, loads a Worker for the db connection, and manages the
+  communication between the UI and worker.
+*/
+(function(){
+    'use strict';
+    /* Recall that the 'self' symbol, except where locally
+       overwritten, refers to the global window or worker object. */
+
+    const storage = (function(NS/*namespace object in which to store this module*/){
+        /* Pedantic licensing note: this code originated in the Fossil SCM
+           source tree, where it has a different license, but the person who
+           ported it into sqlite is the same one who wrote it for fossil. */
+        'use strict';
+        NS = NS||{};
+
+        /**
+           This module provides a basic wrapper around localStorage
+           or sessionStorage or a dummy proxy object if neither
+           of those are available.
+        */
+        const tryStorage = function f(obj){
+            if(!f.key) f.key = 'storage.access.check';
+            try{
+                obj.setItem(f.key, 'f');
+                const x = obj.getItem(f.key);
+                obj.removeItem(f.key);
+                if(x!=='f') throw new Error(f.key+" failed")
+                return obj;
+            }catch(e){
+                return undefined;
+            }
+        };
+
+        /** Internal storage impl for this module. */
+        const $storage =
+              tryStorage(window.localStorage)
+              || tryStorage(window.sessionStorage)
+              || tryStorage({
+                  // A basic dummy xyzStorage stand-in
+                  $$$:{},
+                  setItem: function(k,v){this.$$$[k]=v},
+                  getItem: function(k){
+                      return this.$$$.hasOwnProperty(k) ? this.$$$[k] : undefined;
+                  },
+                  removeItem: function(k){delete this.$$$[k]},
+                  clear: function(){this.$$$={}}
+              });
+
+        /**
+           For the dummy storage we need to differentiate between
+           $storage and its real property storage for hasOwnProperty()
+           to work properly...
+        */
+        const $storageHolder = $storage.hasOwnProperty('$$$') ? $storage.$$$ : $storage;
+
+        /**
+           A prefix which gets internally applied to all storage module
+           property keys so that localStorage and sessionStorage across the
+           same browser profile instance do not "leak" across multiple apps
+           being hosted by the same origin server. Such cross-polination is
+           still there but, with this key prefix applied, it won't be
+           immediately visible via the storage API.
+
+           With this in place we can justify using localStorage instead of
+           sessionStorage.
+
+           One implication of using localStorage and sessionStorage is that
+           their scope (the same "origin" and client application/profile)
+           allows multiple apps on the same origin to use the same
+           storage. Thus /appA/foo could then see changes made via
+           /appB/foo. The data do not cross user- or browser boundaries,
+           though, so it "might" arguably be called a
+           feature. storageKeyPrefix was added so that we can sandbox that
+           state for each separate app which shares an origin.
+
+           See: https://fossil-scm.org/forum/forumpost/4afc4d34de
+
+           Sidebar: it might seem odd to provide a key prefix and stick all
+           properties in the topmost level of the storage object. We do that
+           because adding a layer of object to sandbox each app would mean
+           (de)serializing that whole tree on every storage property change.
+           e.g. instead of storageObject.projectName.foo we have
+           storageObject[storageKeyPrefix+'foo']. That's soley for
+           efficiency's sake (in terms of battery life and
+           environment-internal storage-level effort).
+        */
+        const storageKeyPrefix = (
+            $storageHolder===$storage/*localStorage or sessionStorage*/
+                ? (
+                    (NS.config ?
+                     (NS.config.projectCode || NS.config.projectName
+                      || NS.config.shortProjectName)
+                     : false)
+                        || window.location.pathname
+                )+'::' : (
+                    '' /* transient storage */
+                )
+        );
+
+        /**
+           A proxy for localStorage or sessionStorage or a
+           page-instance-local proxy, if neither one is availble.
+
+           Which exact storage implementation is uses is unspecified, and
+           apps must not rely on it.
+        */
+        NS.storage = {
+            storageKeyPrefix: storageKeyPrefix,
+            /** Sets the storage key k to value v, implicitly converting
+                it to a string. */
+            set: (k,v)=>$storage.setItem(storageKeyPrefix+k,v),
+            /** Sets storage key k to JSON.stringify(v). */
+            setJSON: (k,v)=>$storage.setItem(storageKeyPrefix+k,JSON.stringify(v)),
+            /** Returns the value for the given storage key, or
+                dflt if the key is not found in the storage. */
+            get: (k,dflt)=>$storageHolder.hasOwnProperty(
+                storageKeyPrefix+k
+            ) ? $storage.getItem(storageKeyPrefix+k) : dflt,
+            /** Returns true if the given key has a value of "true".  If the
+                key is not found, it returns true if the boolean value of dflt
+                is "true". (Remember that JS persistent storage values are all
+                strings.) */
+            getBool: function(k,dflt){
+                return 'true'===this.get(k,''+(!!dflt));
+            },
+            /** Returns the JSON.parse()'d value of the given
+                storage key's value, or dflt is the key is not
+                found or JSON.parse() fails. */
+            getJSON: function f(k,dflt){
+                try {
+                    const x = this.get(k,f);
+                    return x===f ? dflt : JSON.parse(x);
+                }
+                catch(e){return dflt}
+            },
+            /** Returns true if the storage contains the given key,
+                else false. */
+            contains: (k)=>$storageHolder.hasOwnProperty(storageKeyPrefix+k),
+            /** Removes the given key from the storage. Returns this. */
+            remove: function(k){
+                $storage.removeItem(storageKeyPrefix+k);
+                return this;
+            },
+            /** Clears ALL keys from the storage. Returns this. */
+            clear: function(){
+                this.keys().forEach((k)=>$storage.removeItem(/*w/o prefix*/k));
+                return this;
+            },
+            /** Returns an array of all keys currently in the storage. */
+            keys: ()=>Object.keys($storageHolder).filter((v)=>(v||'').startsWith(storageKeyPrefix)),
+            /** Returns true if this storage is transient (only available
+                until the page is reloaded), indicating that fileStorage
+                and sessionStorage are unavailable. */
+            isTransient: ()=>$storageHolder!==$storage,
+            /** Returns a symbolic name for the current storage mechanism. */
+            storageImplName: function(){
+                if($storage===window.localStorage) return 'localStorage';
+                else if($storage===window.sessionStorage) return 'sessionStorage';
+                else return 'transient';
+            },
+
+            /**
+               Returns a brief help text string for the currently-selected
+               storage type.
+            */
+            storageHelpDescription: function(){
+                return {
+                    localStorage: "Browser-local persistent storage with an "+
+                        "unspecified long-term lifetime (survives closing the browser, "+
+                        "but maybe not a browser upgrade).",
+                    sessionStorage: "Storage local to this browser tab, "+
+                        "lost if this tab is closed.",
+                    "transient": "Transient storage local to this invocation of this page."
+                }[this.storageImplName()];
+            }
+        };
+        return NS.storage;
+    })({})/*storage API setup*/;
+
+
+    /** Name of the stored copy of SqliteFiddle.config. */
+    const configStorageKey = 'sqlite3-fiddle-config';
+
+    /**
+       The SqliteFiddle object is intended to be the primary
+       app-level object for the main-thread side of the sqlite
+       fiddle application. It uses a worker thread to load the
+       sqlite WASM module and communicate with it.
+    */
+    const SF/*local convenience alias*/
+    = window.SqliteFiddle/*canonical name*/ = {
+        /* Config options. */
+        config: {
+            /* If true, SqliteFiddle.echo() will auto-scroll the
+               output widget to the bottom when it receives output,
+               else it won't. */
+            autoScrollOutput: true,
+            /* If true, the output area will be cleared before each
+               command is run, else it will not. */
+            autoClearOutput: false,
+            /* If true, SqliteFiddle.echo() will echo its output to
+               the console, in addition to its normal output widget.
+               That slows it down but is useful for testing. */
+            echoToConsole: false,
+            /* If true, display input/output areas side-by-side. */
+            sideBySide: true,
+            /* If true, swap positions of the input/output areas. */
+            swapInOut: false
+        },
+        /**
+           Emits the given text, followed by a line break, to the
+           output widget.  If given more than one argument, they are
+           join()'d together with a space between each. As a special
+           case, if passed a single array, that array is used in place
+           of the arguments array (this is to facilitate receiving
+           lists of arguments via worker events).
+        */
+        echo: function f(text) {
+            /* Maintenance reminder: we currently require/expect a textarea
+               output element. It might be nice to extend this to behave
+               differently if the output element is a non-textarea element,
+               in which case it would need to append the given text as a TEXT
+               node and add a line break. */
+            if(!f._){
+                f._ = document.getElementById('output');
+                f._.value = ''; // clear browser cache
+            }
+            if(arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
+            else if(1===arguments.length && Array.isArray(text)) text = text.join(' ');
+            // These replacements are necessary if you render to raw HTML
+            //text = text.replace(/&/g, "&");
+            //text = text.replace(/</g, "&lt;");
+            //text = text.replace(/>/g, "&gt;");
+            //text = text.replace('\n', '<br>', 'g');
+            if(null===text){/*special case: clear output*/
+                f._.value = '';
+                return;
+            }else if(this.echo._clearPending){
+                delete this.echo._clearPending;
+                f._.value = '';
+            }
+            if(this.config.echoToConsole) console.log(text);
+            if(this.jqTerm) this.jqTerm.echo(text);
+            f._.value += text + "\n";
+            if(this.config.autoScrollOutput){
+                f._.scrollTop = f._.scrollHeight;
+            }
+        },
+        _msgMap: {},
+        /** Adds a worker message handler for messages of the given
+            type. */
+        addMsgHandler: function f(type,callback){
+            if(Array.isArray(type)){
+                type.forEach((t)=>this.addMsgHandler(t, callback));
+                return this;
+            }
+            (this._msgMap.hasOwnProperty(type)
+             ? this._msgMap[type]
+             : (this._msgMap[type] = [])).push(callback);
+            return this;
+        },
+        /** Given a worker message, runs all handlers for msg.type. */
+        runMsgHandlers: function(msg){
+            const list = (this._msgMap.hasOwnProperty(msg.type)
+                          ? this._msgMap[msg.type] : false);
+            if(!list){
+                console.warn("No handlers found for message type:",msg);
+                return false;
+            }
+            //console.debug("runMsgHandlers",msg);
+            list.forEach((f)=>f(msg));
+            return true;
+        },
+        /** Removes all message handlers for the given message type. */
+        clearMsgHandlers: function(type){
+            delete this._msgMap[type];
+            return this;
+        },
+        /* Posts a message in the form {type, data} to the db worker. Returns this. */
+        wMsg: function(type,data){
+            this.worker.postMessage({type, data});
+            return this;
+        },
+        /**
+           Prompts for confirmation and, if accepted, deletes
+           all content and tables in the (transient) database.
+        */
+        resetDb: function(){
+            if(window.confirm("Really destroy all content and tables "
+                              +"in the (transient) db?")){
+                this.wMsg('db-reset');
+            }
+            return this;
+        },
+        /** Stores this object's config in the browser's storage. */
+        storeConfig: function(){
+            storage.setJSON(configStorageKey,this.config);
+        }
+    };
+
+    if(1){ /* Restore SF.config */
+        const storedConfig = storage.getJSON(configStorageKey);
+        if(storedConfig){
+            /* Copy all properties to SF.config which are currently in
+               storedConfig. We don't bother copying any other
+               properties: those have been removed from the app in the
+               meantime. */
+            Object.keys(SF.config).forEach(function(k){
+                if(storedConfig.hasOwnProperty(k)){
+                    SF.config[k] = storedConfig[k];
+                }
+            });
+        }
+    }
+
+    SF.worker = new Worker('fiddle-worker.js');
+    SF.worker.onmessage = (ev)=>SF.runMsgHandlers(ev.data);
+    SF.addMsgHandler(['stdout', 'stderr'], (ev)=>SF.echo(ev.data));
+
+    /* querySelectorAll() proxy */
+    const EAll = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelectorAll(arguments[arguments.length-1]);
+    };
+    /* querySelector() proxy */
+    const E = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelector(arguments[arguments.length-1]);
+    };
+
+    /** Handles status updates from the Module object. */
+    SF.addMsgHandler('module', function f(ev){
+        ev = ev.data;
+        if('status'!==ev.type){
+            console.warn("Unexpected module-type message:",ev);
+            return;
+        }
+        if(!f.ui){
+            f.ui = {
+                status: E('#module-status'),
+                progress: E('#module-progress'),
+                spinner: E('#module-spinner')
+            };
+        }
+        const msg = ev.data;
+        if(f.ui.progres){
+            progress.value = msg.step;
+            progress.max = msg.step + 1/*we don't know how many steps to expect*/;
+        }
+        if(1==msg.step){
+            f.ui.progress.classList.remove('hidden');
+            f.ui.spinner.classList.remove('hidden');
+        }
+        if(msg.text){
+            f.ui.status.classList.remove('hidden');
+            f.ui.status.innerText = msg.text;
+        }else{
+            if(f.ui.progress){
+                f.ui.progress.remove();
+                f.ui.spinner.remove();
+                delete f.ui.progress;
+                delete f.ui.spinner;
+            }
+            f.ui.status.classList.add('hidden');
+            /* The module can post messages about fatal problems,
+               e.g. an exit() being triggered or assertion failure,
+               after the last "load" message has arrived, so
+               leave f.ui.status and message listener intact. */
+        }
+    });
+
+    /**
+       The 'fiddle-ready' event is fired (with no payload) when the
+       wasm module has finished loading. Interestingly, that happens
+       _before_ the final module:status event */
+    SF.addMsgHandler('fiddle-ready', function(){
+        SF.clearMsgHandlers('fiddle-ready');
+        self.onSFLoaded();
+    });
+
+    /**
+       Performs all app initialization which must wait until after the
+       worker module is loaded. This function removes itself when it's
+       called.
+    */
+    self.onSFLoaded = function(){
+        delete this.onSFLoaded;
+        // Unhide all elements which start out hidden
+        EAll('.initially-hidden').forEach((e)=>e.classList.remove('initially-hidden'));
+        E('#btn-reset').addEventListener('click',()=>SF.resetDb());
+        const taInput = E('#input');
+        const btnClearIn = E('#btn-clear');
+        btnClearIn.addEventListener('click',function(){
+            taInput.value = '';
+        },false);
+        // Ctrl-enter and shift-enter both run the current SQL.
+        taInput.addEventListener('keydown',function(ev){
+            if((ev.ctrlKey || ev.shiftKey) && 13 === ev.keyCode){
+                ev.preventDefault();
+                ev.stopPropagation();
+                btnShellExec.click();
+            }
+        }, false);
+        const taOutput = E('#output');
+        const btnClearOut = E('#btn-clear-output');
+        btnClearOut.addEventListener('click',function(){
+            taOutput.value = '';
+            if(SF.jqTerm) SF.jqTerm.clear();
+        },false);
+        const btnShellExec = E('#btn-shell-exec');
+        btnShellExec.addEventListener('click',function(ev){
+            let sql;
+            ev.preventDefault();
+            if(taInput.selectionStart<taInput.selectionEnd){
+                sql = taInput.value.substring(taInput.selectionStart,taInput.selectionEnd).trim();
+            }else{
+                sql = taInput.value.trim();
+            }
+            if(sql) SF.dbExec(sql);
+        },false);
+
+        const btnInterrupt = E("#btn-interrupt");
+        //btnInterrupt.classList.add('hidden');
+        /** To be called immediately before work is sent to the
+            worker. Updates some UI elements. The 'working'/'end'
+            event will apply the inverse, undoing the bits this
+            function does. This impl is not in the 'working'/'start'
+            event handler because that event is given to us
+            asynchronously _after_ we need to have performed this
+            work.
+        */
+        const preStartWork = function f(){
+            if(!f._){
+                const title = E('title');
+                f._ = {
+                    btnLabel: btnShellExec.innerText,
+                    pageTitle: title,
+                    pageTitleOrig: title.innerText
+                };
+            }
+            f._.pageTitle.innerText = "[working...] "+f._.pageTitleOrig;
+            btnShellExec.setAttribute('disabled','disabled');
+            btnInterrupt.removeAttribute('disabled','disabled');
+        };
+
+        /* Sends the given text to the db module to evaluate as if it
+           had been entered in the sqlite3 CLI shell. If it's null or
+           empty, this is a no-op except that the very first call will
+           initialize the db and output an informational header. */
+        SF.dbExec = function f(sql){
+            if(this.config.autoClearOutput){
+                this.echo._clearPending = true;
+            }
+            preStartWork();
+            this.wMsg('shellExec',sql);
+        };
+
+        SF.addMsgHandler('working',function f(ev){
+            switch(ev.data){
+                case 'start': /* See notes in preStartWork(). */; return;
+                case 'end':
+                    preStartWork._.pageTitle.innerText = preStartWork._.pageTitleOrig;
+                    btnShellExec.innerText = preStartWork._.btnLabel;
+                    btnShellExec.removeAttribute('disabled');
+                    btnInterrupt.setAttribute('disabled','disabled');
+                    return;
+            }
+            console.warn("Unhandled 'working' event:",ev.data);
+        });
+
+        /* For each checkbox with data-csstgt, set up a handler which
+           toggles the given CSS class on the element matching
+           E(data-csstgt). */
+        EAll('input[type=checkbox][data-csstgt]')
+            .forEach(function(e){
+                const tgt = E(e.dataset.csstgt);
+                const cssClass = e.dataset.cssclass || 'error';
+                e.checked = tgt.classList.contains(cssClass);
+                e.addEventListener('change', function(){
+                    tgt.classList[
+                        this.checked ? 'add' : 'remove'
+                    ](cssClass)
+                }, false);
+            });
+        /* For each checkbox with data-config=X, set up a binding to
+           SF.config[X]. These must be set up AFTER data-csstgt
+           checkboxes so that those two states can be synced properly. */
+        EAll('input[type=checkbox][data-config]')
+            .forEach(function(e){
+                const confVal = !!SF.config[e.dataset.config];
+                if(e.checked !== confVal){
+                    /* Ensure that data-csstgt mappings (if any) get
+                       synced properly. */
+                    e.checked = confVal;
+                    e.dispatchEvent(new Event('change'));
+                }
+                e.addEventListener('change', function(){
+                    SF.config[this.dataset.config] = this.checked;
+                    SF.storeConfig();
+                }, false);
+            });
+        /* For each button with data-cmd=X, map a click handler which
+           calls SF.dbExec(X). */
+        const cmdClick = function(){SF.dbExec(this.dataset.cmd);};
+        EAll('button[data-cmd]').forEach(
+            e => e.addEventListener('click', cmdClick, false)
+        );
+
+        btnInterrupt.addEventListener('click',function(){
+            SF.wMsg('interrupt');
+        });
+
+        /** Initiate a download of the db. */
+        const btnExport = E('#btn-export');
+        const eLoadDb = E('#load-db');
+        const btnLoadDb = E('#btn-load-db');
+        btnLoadDb.addEventListener('click', ()=>eLoadDb.click());
+        /**
+           Enables (if passed true) or disables all UI elements which
+           "might," if timed "just right," interfere with an
+           in-progress db import/export/exec operation.
+        */
+        const enableMutatingElements = function f(enable){
+            if(!f._elems){
+                f._elems = [
+                    /* UI elements to disable while import/export are
+                       running. Normally the export is fast enough
+                       that this won't matter, but we really don't
+                       want to be reading (from outside of sqlite) the
+                       db when the user taps btnShellExec. */
+                    btnShellExec, btnExport, eLoadDb
+                ];
+            }
+            f._elems.forEach( enable
+                              ? (e)=>e.removeAttribute('disabled')
+                              : (e)=>e.setAttribute('disabled','disabled') );
+        };
+        btnExport.addEventListener('click',function(){
+            enableMutatingElements(false);
+            SF.wMsg('db-export');
+        });
+        SF.addMsgHandler('db-export', function(ev){
+            enableMutatingElements(true);
+            ev = ev.data;
+            if(ev.error){
+                SF.echo("Export failed:",ev.error);
+                return;
+            }
+            const blob = new Blob([ev.buffer], {type:"application/x-sqlite3"});
+            const a = document.createElement('a');
+            document.body.appendChild(a);
+            a.href = window.URL.createObjectURL(blob);
+            a.download = ev.filename;
+            a.addEventListener('click',function(){
+                setTimeout(function(){
+                    SF.echo("Exported (possibly auto-downloaded):",ev.filename);
+                    window.URL.revokeObjectURL(a.href);
+                    a.remove();
+                },500);
+            });
+            a.click();
+        });
+        /**
+           Handle load/import of an external db file.
+        */
+        eLoadDb.addEventListener('change',function(){
+            const f = this.files[0];
+            const r = new FileReader();
+            const status = {loaded: 0, total: 0};
+            enableMutatingElements(false);
+            r.addEventListener('loadstart', function(){
+                SF.echo("Loading",f.name,"...");
+            });
+            r.addEventListener('progress', function(ev){
+                SF.echo("Loading progress:",ev.loaded,"of",ev.total,"bytes.");
+            });
+            const that = this;
+            r.addEventListener('load', function(){
+                enableMutatingElements(true);
+                SF.echo("Loaded",f.name+". Opening db...");
+                SF.wMsg('open',{
+                    filename: f.name,
+                    buffer: this.result
+                });
+            });
+            r.addEventListener('error',function(){
+                enableMutatingElements(true);
+                SF.echo("Loading",f.name,"failed for unknown reasons.");
+            });
+            r.addEventListener('abort',function(){
+                enableMutatingElements(true);
+                SF.echo("Cancelled loading of",f.name+".");
+            });
+            r.readAsArrayBuffer(f);
+        });
+
+        EAll('fieldset.collapsible').forEach(function(fs){
+            const btnToggle = E(fs,'legend > .fieldset-toggle'),
+                  content = EAll(fs,':scope > div');
+            btnToggle.addEventListener('click', function(){
+                fs.classList.toggle('collapsed');
+                content.forEach((d)=>d.classList.toggle('hidden'));
+            }, false);
+        });
+
+        /**
+           Given a DOM element, this routine measures its "effective
+           height", which is the bounding top/bottom range of this element
+           and all of its children, recursively. For some DOM structure
+           cases, a parent may have a reported height of 0 even though
+           children have non-0 sizes.
+
+           Returns 0 if !e or if the element really has no height.
+        */
+        const effectiveHeight = function f(e){
+            if(!e) return 0;
+            if(!f.measure){
+                f.measure = function callee(e, depth){
+                    if(!e) return;
+                    const m = e.getBoundingClientRect();
+                    if(0===depth){
+                        callee.top = m.top;
+                        callee.bottom = m.bottom;
+                    }else{
+                        callee.top = m.top ? Math.min(callee.top, m.top) : callee.top;
+                        callee.bottom = Math.max(callee.bottom, m.bottom);
+                    }
+                    Array.prototype.forEach.call(e.children,(e)=>callee(e,depth+1));
+                    if(0===depth){
+                        //console.debug("measure() height:",e.className, callee.top, callee.bottom, (callee.bottom - callee.top));
+                        f.extra += callee.bottom - callee.top;
+                    }
+                    return f.extra;
+                };
+            }
+            f.extra = 0;
+            f.measure(e,0);
+            return f.extra;
+        };
+
+        /**
+           Returns a function, that, as long as it continues to be invoked,
+           will not be triggered. The function will be called after it stops
+           being called for N milliseconds. If `immediate` is passed, call
+           the callback immediately and hinder future invocations until at
+           least the given time has passed.
+
+           If passed only 1 argument, or passed a falsy 2nd argument,
+           the default wait time set in this function's $defaultDelay
+           property is used.
+
+           Source: underscore.js, by way of https://davidwalsh.name/javascript-debounce-function
+        */
+        const debounce = function f(func, wait, immediate) {
+            var timeout;
+            if(!wait) wait = f.$defaultDelay;
+            return function() {
+                const context = this, args = Array.prototype.slice.call(arguments);
+                const later = function() {
+                    timeout = undefined;
+                    if(!immediate) func.apply(context, args);
+                };
+                const callNow = immediate && !timeout;
+                clearTimeout(timeout);
+                timeout = setTimeout(later, wait);
+                if(callNow) func.apply(context, args);
+            };
+        };
+        debounce.$defaultDelay = 500 /*arbitrary*/;
+
+        const ForceResizeKludge = (function(){
+            /* Workaround for Safari mayhem regarding use of vh CSS
+               units....  We cannot use vh units to set the main view
+               size because Safari chokes on that, so we calculate
+               that height here. Larger than ~95% is too big for
+               Firefox on Android, causing the input area to move
+               off-screen. */
+            const appViews = EAll('.app-view');
+            const elemsToCount = [
+                /* Elements which we need to always count in the
+                   visible body size. */
+                E('body > header'),
+                E('body > footer')
+            ];
+            const resized = function f(){
+                if(f.$disabled) return;
+                const wh = window.innerHeight;
+                var ht;
+                var extra = 0;
+                elemsToCount.forEach((e)=>e ? extra += effectiveHeight(e) : false);
+                ht = wh - extra;
+                appViews.forEach(function(e){
+                    e.style.height =
+                        e.style.maxHeight = [
+                            "calc(", (ht>=100 ? ht : 100), "px",
+                            " - 2em"/*fudge value*/,")"
+                            /* ^^^^ hypothetically not needed, but both
+                               Chrome/FF on Linux will force scrollbars on the
+                               body if this value is too small. */
+                        ].join('');
+                });
+            };
+            resized.$disabled = true/*gets deleted when setup is finished*/;
+            window.addEventListener('resize', debounce(resized, 250), false);
+            return resized;
+        })();
+
+        /** Set up a selection list of examples */
+        (function(){
+            const xElem = E('#select-examples');
+            const examples = [
+                {name: "Help", sql:
+`-- ================================================
+-- Use ctrl-enter or shift-enter to execute sqlite3
+-- shell commands and SQL.
+-- If a subset of the text is currently selected,
+-- only that part is executed.
+-- ================================================
+.help`},
+                {name: "Timer on", sql: ".timer on"},
+                {name: "Setup table T", sql:`.nullvalue NULL
+CREATE TABLE t(a,b);
+INSERT INTO t(a,b) VALUES('abc',123),('def',456),(NULL,789),('ghi',012);
+SELECT * FROM t;`},
+                {name: "Table list", sql: ".tables"},
+                {name: "Box Mode", sql: ".mode box"},
+                {name: "JSON Mode", sql: ".mode json"},
+                {name: "Mandlebrot", sql: `WITH RECURSIVE
+  xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2),
+  yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0),
+  m(iter, cx, cy, x, y) AS (
+    SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis
+    UNION ALL
+    SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m 
+     WHERE (x*x + y*y) < 4.0 AND iter<28
+  ),
+  m2(iter, cx, cy) AS (
+    SELECT max(iter), cx, cy FROM m GROUP BY cx, cy
+  ),
+  a(t) AS (
+    SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') 
+    FROM m2 GROUP BY cy
+  )
+SELECT group_concat(rtrim(t),x'0a') as Mandelbrot FROM a;`}
+            ];
+            const newOpt = function(lbl,val){
+                const o = document.createElement('option');
+                o.value = val;
+                if(!val) o.setAttribute('disabled',true);
+                o.appendChild(document.createTextNode(lbl));
+                xElem.appendChild(o);
+            };
+            newOpt("Examples (replaces input!)");
+            examples.forEach((o)=>newOpt(o.name, o.sql));
+            //xElem.setAttribute('disabled',true);
+            xElem.selectedIndex = 0;
+            xElem.addEventListener('change', function(){
+                taInput.value = '-- ' +
+                    this.selectedOptions[0].innerText +
+                    '\n' + this.value;
+                SF.dbExec(this.value);
+            });
+        })()/* example queries */;
+
+        SF.echo(null/*clear any output generated by the init process*/);
+        if(window.jQuery && window.jQuery.terminal){
+            /* Set up the terminal-style view... */
+            const eTerm = window.jQuery('#view-terminal').empty();
+            SF.jqTerm = eTerm.terminal(SF.dbExec.bind(SF),{
+                prompt: 'sqlite> ',
+                greetings: false /* note that the docs incorrectly call this 'greeting' */
+            });
+            /* Set up a button to toggle the views... */
+            const head = E('header#titlebar');
+            const btnToggleView = document.createElement('button');
+            btnToggleView.appendChild(document.createTextNode("Toggle View"));
+            head.appendChild(btnToggleView);
+            btnToggleView.addEventListener('click',function f(){
+                EAll('.app-view').forEach(e=>e.classList.toggle('hidden'));
+                if(document.body.classList.toggle('terminal-mode')){
+                    ForceResizeKludge();
+                }
+            }, false);
+            btnToggleView.click()/*default to terminal view*/;
+        }
+        SF.dbExec(null/*init the db and output the header*/);
+        SF.echo('This experimental app is provided in the hope that it',
+                'may prove interesting or useful but is not an officially',
+                'supported deliverable of the sqlite project. It is subject to',
+                'any number of changes or outright removal at any time.\n');
+        delete ForceResizeKludge.$disabled;
+        ForceResizeKludge();
+
+        btnShellExec.click();
+    }/*onSFLoaded()*/;
+})();

ext/fiddle/index.md

@@ -0,0 +1,94 @@
+This directory houses a "fiddle"-style application which embeds a
+[Web Assembly (WASM)](https://en.wikipedia.org/wiki/WebAssembly)
+build of the sqlite3 shell app into an HTML page, effectively running
+the shell in a client-side browser.
+
+It requires [emscripten][] and that the build environment be set up for
+emscripten. A mini-HOWTO for setting that up follows...
+
+First, install the Emscripten SDK, as documented
+[here](https://emscripten.org/docs/getting_started/downloads.html) and summarized
+below for Linux environments:
+
+```
+# Clone the emscripten repository:
+$ git clone https://github.com/emscripten-core/emsdk.git
+$ cd emsdk
+
+# Download and install the latest SDK tools:
+$ ./emsdk install latest
+
+# Make the "latest" SDK "active" for the current user:
+$ ./emsdk activate latest
+```
+
+Those parts only need to be run once. The following needs to be run for each
+shell instance which needs the `emcc` compiler:
+
+```
+# Activate PATH and other environment variables in the current terminal:
+$ source ./emsdk_env.sh
+
+$ which emcc
+/path/to/emsdk/upstream/emscripten/emcc
+```
+
+That `env` script needs to be sourced for building this application from the
+top of the sqlite3 build tree:
+
+```
+$ make fiddle
+```
+
+Or:
+
+```
+$ cd ext/fiddle
+$ make
+```
+
+That will generate the fiddle application under
+[ext/fiddle](/dir/ext/fiddle), as `fiddle.html`. That application
+cannot, due to XMLHttpRequest security limitations, run if the HTML
+file is opened directly in the browser (i.e. if it is opened using a
+`file://` URL), so it needs to be served via an HTTP server.  For
+example, using [althttpd][]:
+
+```
+$ cd ext/fiddle
+$ althttpd -debug 1 -jail 0 -port 9090 -root .
+```
+
+Then browse to `http://localhost:9090/fiddle.html`.
+
+Note that when serving this app via [althttpd][], it must be a version
+from 2022-05-17 or newer so that it recognizes the `.wasm` file
+extension and responds with the mimetype `application/wasm`, as the
+WASM loader is pedantic about that detail.
+
+# Known Quirks and Limitations
+
+Some "impedence mismatch" between C and WASM/JavaScript is to be
+expected.
+
+## No I/O
+
+sqlite3 shell commands which require file I/O or pipes are disabled in
+the WASM build.
+
+## `exit()` Triggered from C
+
+When C code calls `exit()`, as happens (for example) when running an
+"unsafe" command when safe mode is active, WASM's connection to the
+sqlite3 shell environment has no sensible choice but to shut down
+because `exit()` leaves it in a state we can no longer recover
+from. The JavaScript-side application attempts to recognize this and
+warn the user that restarting the application is necessary. Currently
+the only way to restart it is to reload the page. Restructuring the
+shell code such that it could be "rebooted" without restarting the
+JS app would require some invasive changes which are not currently
+on any TODO list but have not been entirely ruled out long-term.
+
+
+[emscripten]: https://emscripten.org
+[althttpd]: https://sqlite.org/althttpd

ext/fiddle/sqlite3-worker.js

@@ -0,0 +1,44 @@
+/*
+  2022-05-23
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This is a JS Worker file for the main sqlite3 api. It loads
+  sqlite3.js, initializes the module, and postMessage()'s a message
+  after the module is initialized:
+
+  {type: 'sqlite3-api', data: 'ready'}
+
+  This seemingly superfluous level of indirection is necessary when
+  loading sqlite3.js via a Worker. Loading sqlite3.js from the main
+  window thread elides the Worker-specific API. Instantiating a worker
+  with new Worker("sqlite.js") will not (cannot) call
+  initSqlite3Module() to initialize the module due to a
+  timing/order-of-operations conflict (and that symbol is not exported
+  in a way that a Worker loading it that way can see it).  Thus JS
+  code wanting to load the sqlite3 Worker-specific API needs to pass
+  _this_ file (or equivalent) to the Worker constructor and then
+  listen for an event in the form shown above in order to know when
+  the module has completed initialization. sqlite3.js will fire a
+  similar event, with data:'loaded' as the final step in its loading
+  process. Whether or not we _really_ need both 'loaded' and 'ready'
+  events is unclear, but they are currently separate events primarily
+  for the sake of clarity in the timing of when it's okay to use the
+  loaded module. At the time the 'loaded' event is fired, it's
+  possible (but unknown and unknowable) that the emscripten-generated
+  module-setup infrastructure still has work to do. Thus it is
+  hypothesized that client code is better off waiting for the 'ready'
+  even before using the API.
+*/
+"use strict";
+importScripts('sqlite3.js');
+initSqlite3Module().then(function(){
+    setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'ready'}), 0);
+});

ext/fiddle/testing.css

@@ -0,0 +1,31 @@
+textarea {
+    font-family: monospace;
+}
+header {
+    font-size: 130%;
+    font-weight: bold;
+}
+.hidden, .initially-hidden {
+    position: absolute !important;
+    opacity: 0 !important;
+    pointer-events: none !important;
+    display: none !important;
+}
+fieldset.options {
+    font-size: 75%;
+}
+fieldset > legend {
+    padding: 0 0.5em;
+}
+span.labeled-input {
+    padding: 0.25em;
+    margin: 0.25em 0.5em;
+    border-radius: 0.25em;
+    white-space: nowrap;
+    background: #0002;
+}
+.center { text-align: center; }
+.error {
+    color: red;
+    background-color: yellow;
+}

ext/fiddle/testing1.html

@@ -0,0 +1,33 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="emscripten.css"/>
+    <link rel="stylesheet" href="testing.css"/>
+    <title>sqlite3-api.js tests</title>
+    <style></style>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api.js tests</span></header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+    <div>Everything on this page happens in the dev console.</div>
+    <script src="sqlite3.js"></script>
+    <script src="SqliteTestUtil.js"></script>
+    <script src="testing1.js"></script>
+  </body>
+</html>

ext/fiddle/testing1.js

@@ -0,0 +1,185 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic test script for sqlite3-api.js. This file must be run in
+  main JS thread and sqlite3.js must have been loaded before it.
+*/
+(function(){
+    const T = self.SqliteTestUtil;
+    const log = console.log.bind(console);
+
+    const assert = function(condition, text) {
+        if (!condition) {
+            throw new Error('Assertion failed' + (text ? ': ' + text : ''));
+        }
+    };
+
+    const test1 = function(db,sqlite3){
+        const api = sqlite3.api;
+        log("Basic sanity tests...");
+        T.assert(db._pDb);
+        let st = db.prepare("select 3 as a");
+        //log("statement =",st);
+        T.assert(st._pStmt)
+            .assert(!st._mayGet)
+            .assert('a' === st.getColumnName(0))
+            .assert(st === db._statements[st._pStmt])
+            .assert(1===st.columnCount)
+            .assert(0===st.parameterCount)
+            .mustThrow(()=>st.bind(1,null))
+            .assert(true===st.step())
+            .assert(3 === st.get(0))
+            .mustThrow(()=>st.get(1))
+            .mustThrow(()=>st.get(0,~api.SQLITE_INTEGER))
+            .assert(3 === st.get(0,api.SQLITE_INTEGER))
+            .assert(3 === st.getInt(0))
+            .assert('3' === st.get(0,api.SQLITE_TEXT))
+            .assert('3' === st.getString(0))
+            .assert(3.0 === st.get(0,api.SQLITE_FLOAT))
+            .assert(3.0 === st.getFloat(0))
+            .assert(st.get(0,api.SQLITE_BLOB) instanceof Uint8Array)
+            .assert(st.getBlob(0) instanceof Uint8Array)
+            .assert(3 === st.get([])[0])
+            .assert(3 === st.get({}).a)
+            .assert(3 === st.getJSON(0))
+            .assert(st._mayGet)
+            .assert(false===st.step())
+            .assert(!st._mayGet)
+        ;
+        let pId = st._pStmt;
+        st.finalize();
+        T.assert(!st._pStmt)
+            .assert(!db._statements[pId]);
+
+        let list = [];
+        db.exec({
+            sql:`CREATE TABLE t(a,b);
+INSERT INTO t(a,b) VALUES(1,2),(3,4),(?,?);`,
+            multi: true,
+            saveSql: list,
+            bind: [5,6]
+        });
+        T.assert(2 === list.length);
+        //log("Exec'd SQL:", list);
+        let counter = 0, colNames = [];
+        list.length = 0;
+        db.exec("SELECT a a, b b FROM t",{
+            rowMode: 'object',
+            resultRows: list,
+            columnNames: colNames,
+            callback: function(row,stmt){
+                ++counter;
+                T.assert(row.a%2 && row.a<6);
+            }
+        });
+        T.assert(2 === colNames.length)
+            .assert('a' === colNames[0])
+            .assert(3 === counter)
+            .assert(3 === list.length);
+        list.length = 0;
+        db.exec("SELECT a a, b b FROM t",{
+            rowMode: 'array',
+            callback: function(row,stmt){
+                ++counter;
+                T.assert(Array.isArray(row))
+                    .assert(0===row[1]%2 && row[1]<7);
+            }
+        });
+        T.assert(6 === counter);
+    };
+
+    const testUDF = function(db){
+        log("Testing UDF...");
+        db.createFunction("foo",function(a,b){return a+b});
+        T.assert(7===db.selectValue("select foo(3,4)")).
+            assert(5===db.selectValue("select foo(3,?)",2)).
+            assert(5===db.selectValue("select foo(?,?)",[1,4])).
+            assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5}));
+        db.createFunction("bar", {
+            arity: -1,
+            callback: function(){
+                var rc = 0;
+                for(let i = 0; i < arguments.length; ++i) rc += arguments[i];
+                return rc;
+            }
+        });
+
+        log("Testing DB::selectValue() w/ UDF...");
+        T.assert(0===db.selectValue("select bar()")).
+            assert(1===db.selectValue("select bar(1)")).
+            assert(3===db.selectValue("select bar(1,2)")).
+            assert(-1===db.selectValue("select bar(1,2,-4)"));
+
+        const eqApprox = function(v1,v2,factor=0.05){
+            return v1>=(v2-factor) && v1<=(v2+factor);
+        };
+        
+        T.assert('hi' === db.selectValue("select ?",'hi')).
+            assert(null===db.selectValue("select null")).
+            assert(null === db.selectValue("select ?",null)).
+            assert(null === db.selectValue("select ?",[null])).
+            assert(null === db.selectValue("select $a",{$a:null})).
+            assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1")))
+        ;
+    };
+
+    const testAttach = function(db){
+        log("Testing ATTACH...");
+        db.exec({
+            sql:[
+                "attach 'foo.db' as foo",
+                "create table foo.bar(a)",
+                "insert into foo.bar(a) values(1),(2),(3)"
+            ].join(';'),
+            multi: true
+        });
+        T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a'));
+        db.exec("detach foo");
+        T.mustThrow(()=>db.exec("select * from foo.bar"));
+    };
+
+    const runTests = function(Module){
+        T.assert(Module._free instanceof Function).
+            assert(Module.allocate instanceof Function).
+            assert(Module.addFunction instanceof Function).
+            assert(Module.removeFunction instanceof Function);
+        const sqlite3 = Module.sqlite3;
+        const api = sqlite3.api;
+        const oo = sqlite3.SQLite3;
+        console.log("Loaded module:",api.sqlite3_libversion(),
+                    api.sqlite3_sourceid());
+        log("Build options:",oo.compileOptionUsed());
+        const db = new oo.DB();
+        try {
+            log("DB:",db.filename);
+            [
+                test1, testUDF, testAttach
+            ].forEach((f)=>{
+                const t = T.counter;
+                f(db, sqlite3);
+                log("Test count:",T.counter - t);
+            });
+        }finally{
+            db.close();
+        }
+        log("Total Test count:",T.counter);
+    };
+
+    initSqlite3Module(self.sqlite3TestModule).then(function(theModule){
+        /** Use a timeout so that we are (hopefully) out from
+            under the module init stack when our setup gets
+            run. Just on principle, not because we _need_ to
+            be. */
+        //console.debug("theModule =",theModule);
+        setTimeout(()=>runTests(theModule), 0);
+    });
+})();

ext/fiddle/testing2.html

@@ -0,0 +1,32 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="emscripten.css"/>
+    <link rel="stylesheet" href="testing.css"/>
+    <title>sqlite3-worker.js tests</title>
+    <style></style>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-worker.js tests</span></header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+    <div>Everything on this page happens in the dev console.</div>
+    <script src="testing-common.js"></script>
+    <script src="testing2.js"></script>
+  </body>
+</html>

ext/fiddle/testing2.js

@@ -0,0 +1,235 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic test script for sqlite3-worker.js.
+*/
+(function(){
+    const T = self.SqliteTestUtil;
+    const SW = new Worker("sqlite3-worker.js");
+    /** Posts a worker message as {type:type, data:data}. */
+    const wMsg = function(type,data){
+        SW.postMessage({type, data});
+        return SW;
+    };
+    const log = console.log.bind(console);
+    const warn = console.warn.bind(console);
+    const error = console.error.bind(console);
+
+    SW.onerror = function(event){
+        error("onerror",event);
+    };
+
+    /**
+       A queue for callbacks which are to be run in response to async
+       DB commands. See the notes in runTests() for why we need
+       this. The event-handling plumbing of this file requires that
+       any DB command which includes a `messageId` property also have
+       a queued callback entry, as the existence of that property in
+       response payloads is how it knows whether or not to shift an
+       entry off of the queue.
+    */
+    const MsgHandlerQueue = {
+        queue: [],
+        id: 0,
+        push: function(type,callback){
+            this.queue.push(callback);
+            return type + '-' + (++this.id);
+        },
+        shift: function(){
+            return this.queue.shift();
+        }
+    };
+
+    const testCount = ()=>log("Total test count:",T.counter);
+
+    const runOneTest = function(eventType, eventData, callback){
+        T.assert(eventData && 'object'===typeof eventData);
+        /* ^^^ that is for the testing and messageId-related code, not
+           a hard requirement of all of the Worker-exposed APIs. */
+        eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){
+            log("runOneTest",eventType,"result",ev.data);
+            if(callback instanceof Function){
+                callback(ev);
+                testCount();
+            }
+        });
+        wMsg(eventType, eventData);
+    };
+
+    /** Methods which map directly to onmessage() event.type keys.
+        They get passed the inbound event.data. */
+    const dbMsgHandler = {
+        open: function(ev){
+            log("open result",ev.data);
+        },
+        exec: function(ev){
+            log("exec result",ev.data);
+        },
+        export: function(ev){
+            log("exec result",ev.data);
+        },
+        error: function(ev){
+            error("ERROR from the worker:",ev.data);
+        },
+        resultRowTest1: function f(ev){
+            if(undefined === f.counter) f.counter = 0;
+            if(ev.data) ++f.counter;
+            //log("exec() result row:",ev.data);
+            T.assert(null===ev.data || 'number' === typeof ev.data.b);
+        }
+    };
+
+    const runTests = function(){
+        const mustNotReach = ()=>{
+            throw new Error("This is not supposed to be reached.");
+        };
+        /**
+           "The problem" now is that the test results are async. We
+           know, however, that the messages posted to the worker will
+           be processed in the order they are passed to it, so we can
+           create a queue of callbacks to handle them. The problem
+           with that approach is that it's not error-handling
+           friendly, in that an error can cause us to bypass a result
+           handler queue entry. We have to perform some extra
+           acrobatics to account for that.
+        */
+        runOneTest('open', {filename:'testing2.sqlite3'}, function(ev){
+            //log("open result",ev);
+            T.assert('testing2.sqlite3'===ev.data.filename)
+                .assert(ev.data.messageId);
+        });
+        runOneTest('exec',{
+            sql: ["create table t(a,b)",
+                  "insert into t(a,b) values(1,2),(3,4),(5,6)"
+                 ].join(';'),
+            multi: true,
+            resultRows: [], columnNames: []
+        }, function(ev){
+            ev = ev.data;
+            T.assert(0===ev.resultRows.length)
+                .assert(0===ev.columnNames.length);
+        });
+        runOneTest('exec',{
+            sql: 'select a a, b b from t order by a',
+            resultRows: [], columnNames: [],
+        }, function(ev){
+            ev = ev.data;
+            T.assert(3===ev.resultRows.length)
+                .assert(1===ev.resultRows[0][0])
+                .assert(6===ev.resultRows[2][1])
+                .assert(2===ev.columnNames.length)
+                .assert('b'===ev.columnNames[1]);
+        });
+        runOneTest('exec',{
+            sql: 'select a a, b b from t order by a',
+            resultRows: [], columnNames: [],
+            rowMode: 'object'
+        }, function(ev){
+            ev = ev.data;
+            T.assert(3===ev.resultRows.length)
+                .assert(1===ev.resultRows[0].a)
+                .assert(6===ev.resultRows[2].b)
+        });
+        runOneTest('exec',{sql:'intentional_error'}, mustNotReach);
+        // Ensure that the message-handler queue survives ^^^ that error...
+        runOneTest('exec',{
+            sql:'select 1',
+            resultRows: [],
+            //rowMode: 'array', // array is the default in the Worker interface
+        }, function(ev){
+            ev = ev.data;
+            T.assert(1 === ev.resultRows.length)
+                .assert(1 === ev.resultRows[0][0]);
+        });
+        runOneTest('exec',{
+            sql: 'select a a, b b from t order by a',
+            callback: 'resultRowTest1',
+            rowMode: 'object'
+        }, function(ev){
+            T.assert(3===dbMsgHandler.resultRowTest1.counter);
+            dbMsgHandler.resultRowTest1.counter = 0;
+        });
+        runOneTest('exec',{sql: 'delete from t where a>3'});
+        runOneTest('exec',{
+            sql: 'select count(a) from t',
+            resultRows: []
+        },function(ev){
+            ev = ev.data;
+            T.assert(1===ev.resultRows.length)
+                .assert(2===ev.resultRows[0][0]);
+        });
+        runOneTest('export',{}, function(ev){
+            ev = ev.data;
+            T.assert('string' === typeof ev.filename)
+                .assert(ev.buffer instanceof Uint8Array)
+                .assert(ev.buffer.length > 1024)
+                .assert('application/x-sqlite3' === ev.mimetype);
+        });
+
+        /***** close() tests must come last. *****/
+        runOneTest('close',{unlink:true},function(ev){
+            ev = ev.data;
+            T.assert('string' === typeof ev.filename);
+        });
+        runOneTest('close',{unlink:true},function(ev){
+            ev = ev.data;
+            T.assert(undefined === ev.filename);
+        });
+    };
+
+    SW.onmessage = function(ev){
+        if(!ev.data || 'object'!==typeof ev.data){
+            warn("Unknown sqlite3-worker message type:",ev);
+            return;
+        }
+        ev = ev.data/*expecting a nested object*/;
+        //log("main window onmessage:",ev);
+        if(ev.data && ev.data.messageId){
+            /* We're expecting a queued-up callback handler. */
+            const f = MsgHandlerQueue.shift();
+            if('error'===ev.type){
+                dbMsgHandler.error(ev);
+                return;
+            }
+            T.assert(f instanceof Function);
+            f(ev);
+            return;
+        }
+        switch(ev.type){
+            case 'sqlite3-api':
+                switch(ev.data){
+                    case 'loaded':
+                        log("Message:",ev); return;
+                    case 'ready':
+                        log("Message:",ev);
+                        self.sqlite3TestModule.setStatus(null);
+                        setTimeout(runTests, 0);
+                        return;
+                    default:
+                        warn("Unknown sqlite3-api message type:",ev);
+                        return;
+                }
+            default:
+                if(dbMsgHandler.hasOwnProperty(ev.type)){
+                    try{dbMsgHandler[ev.type](ev);}
+                    catch(err){
+                        error("Exception while handling db result message",
+                              ev,":",err);
+                    }
+                    return;
+                }
+                warn("Unknown sqlite3-api message type:",ev);
+        }
+    };
+
+    log("Init complete, but async init bits may still be running.");
+})();

ext/fts3/fts3.c

@@ -3980,8 +3980,13 @@ static int fts3ShadowName(const char *zName){
   return 0;
 }
 
+static int fts3ShadowName2(sqlite3_vtab *pVtab, const char *zName){
+  UNUSED_PARAMETER(pVtab);
+  return fts3ShadowName(zName);
+}
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 3,
+  /* iVersion      */ 4,
   /* xCreate       */ fts3CreateMethod,
   /* xConnect      */ fts3ConnectMethod,
   /* xBestIndex    */ fts3BestIndexMethod,
@@ -4000,11 +4005,12 @@ static const sqlite3_module fts3Module = {
   /* xCommit       */ fts3CommitMethod,
   /* xRollback     */ fts3RollbackMethod,
   /* xFindFunction */ fts3FindFunctionMethod,
-  /* xRename */       fts3RenameMethod,
+  /* xRename       */ fts3RenameMethod,
   /* xSavepoint    */ fts3SavepointMethod,
   /* xRelease      */ fts3ReleaseMethod,
   /* xRollbackTo   */ fts3RollbackToMethod,
   /* xShadowName   */ fts3ShadowName,
+  /* xShadowName2  */ fts3ShadowName2
 };
 
 /*
@@ -4308,8 +4314,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
   char *aPoslist = 0;             /* Position list for deferred tokens */
   int nPoslist = 0;               /* Number of bytes in aPoslist */
   int iPrev = -1;                 /* Token number of previous deferred token */
-
-  assert( pPhrase->doclist.bFreeList==0 );
+  char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0);
 
   for(iToken=0; iToken<pPhrase->nToken; iToken++){
     Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
@@ -4323,6 +4328,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
 
       if( pList==0 ){
         sqlite3_free(aPoslist);
+        sqlite3_free(aFree);
         pPhrase->doclist.pList = 0;
         pPhrase->doclist.nList = 0;
         return SQLITE_OK;
@@ -4343,6 +4349,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
         nPoslist = (int)(aOut - aPoslist);
         if( nPoslist==0 ){
           sqlite3_free(aPoslist);
+          sqlite3_free(aFree);
           pPhrase->doclist.pList = 0;
           pPhrase->doclist.nList = 0;
           return SQLITE_OK;
@@ -4382,6 +4389,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
       }
       
       pPhrase->doclist.pList = aOut;
+      assert( p1 && p2 );
       if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
         pPhrase->doclist.bFreeList = 1;
         pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
@@ -4394,6 +4402,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
     }
   }
 
+  if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree);
   return SQLITE_OK;
 }
 #endif /* SQLITE_DISABLE_FTS4_DEFERRED */
@@ -5568,11 +5577,10 @@ static int fts3EvalTestExpr(
 
       default: {
 #ifndef SQLITE_DISABLE_FTS4_DEFERRED
-        if( pCsr->pDeferred 
-         && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
-        ){
+        if( pCsr->pDeferred && (pExpr->bDeferred || (
+            pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList
+        ))){
           Fts3Phrase *pPhrase = pExpr->pPhrase;
-          assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
           if( pExpr->bDeferred ){
             fts3EvalInvalidatePoslist(pPhrase);
           }

ext/fts3/fts3_aux.c

@@ -545,7 +545,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xShadowName2  */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_term.c

@@ -362,7 +362,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xShadowName2  */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_tokenize_vtab.c

@@ -445,7 +445,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xShadowName2  */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_write.c

@@ -3899,6 +3899,8 @@ static int fts3IncrmergePush(
           pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix);
         }
         pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix);
+        assert( nPrefix+nSuffix<=nTerm );
+        assert( nPrefix>=0 );
         memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix);
         pBlk->n += nSuffix;
 
@@ -4021,6 +4023,7 @@ static int fts3IncrmergeAppend(
   pLeaf = &pWriter->aNodeWriter[0];
   nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm);
   nSuffix = nTerm - nPrefix;
+  if(nSuffix<=0 ) return FTS_CORRUPT_VTAB;
 
   nSpace  = sqlite3Fts3VarintLen(nPrefix);
   nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;

ext/fts5/fts5_expr.c

@@ -1627,6 +1627,9 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
   }else{
     if( pRet->nPhrase>0 ){
       Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
+      assert( pParse!=0 );
+      assert( pParse->apPhrase!=0 );
+      assert( pParse->nPhrase>=2 );
       assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
       if( pPhrase->nTerm==0 ){
         fts5ExprPhraseFree(pPhrase);

ext/fts5/fts5_index.c

@@ -296,7 +296,7 @@ struct Fts5Index {
   sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
   sqlite3_stmt *pDeleter;         /* "DELETE FROM %_data ... id>=? AND id<=?" */
   sqlite3_stmt *pIdxWriter;       /* "INSERT ... %_idx VALUES(?,?,?,?)" */
-  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=? */
+  sqlite3_stmt *pIdxDeleter;      /* "DELETE FROM %_idx WHERE segid=?" */
   sqlite3_stmt *pIdxSelect;
   int nRead;                      /* Total number of blocks read */
 

ext/fts5/fts5_main.c

@@ -804,7 +804,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
   rc = sqlite3_step(pSorter->pStmt);
   if( rc==SQLITE_DONE ){
     rc = SQLITE_OK;
-    CsrFlagSet(pCsr, FTS5CSR_EOF);
+    CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT);
   }else if( rc==SQLITE_ROW ){
     const u8 *a;
     const u8 *aBlob;
@@ -2811,9 +2811,14 @@ static int fts5ShadowName(const char *zName){
   return 0;
 }
 
+static int fts5ShadowName2(sqlite3_vtab *pVtab, const char *zName){
+  UNUSED_PARAMETER(pVtab);
+  return fts5ShadowName(zName);
+}
+
 static int fts5Init(sqlite3 *db){
   static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 3,
+    /* iVersion      */ 4,
     /* xCreate       */ fts5CreateMethod,
     /* xConnect      */ fts5ConnectMethod,
     /* xBestIndex    */ fts5BestIndexMethod,
@@ -2836,7 +2841,8 @@ static int fts5Init(sqlite3 *db){
     /* xSavepoint    */ fts5SavepointMethod,
     /* xRelease      */ fts5ReleaseMethod,
     /* xRollbackTo   */ fts5RollbackToMethod,
-    /* xShadowName   */ fts5ShadowName
+    /* xShadowName   */ fts5ShadowName,
+    /* xShadowName2  */ fts5ShadowName2
   };
 
   int rc;

ext/fts5/fts5_test_tok.c

@@ -472,7 +472,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xShadowName2  */
   };
   int rc;                         /* Return code */
 

ext/fts5/fts5_vocab.c

@@ -783,7 +783,8 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
     /* xSavepoint    */ 0,
     /* xRelease      */ 0,
     /* xRollbackTo   */ 0,
-    /* xShadowName   */ 0
+    /* xShadowName   */ 0,
+    /* xShadowName2  */ 0
   };
   void *p = (void*)pGlobal;
 

ext/fts5/test/fts5ah.test

@@ -11,6 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this script is testing the FTS5 module.
 #
+# TESTRUNNER: slow
 
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah

ext/fts5/test/fts5corrupt3.test

@@ -15366,6 +15366,160 @@ do_catchsql_test 79.2 {
   INSERT INTO t1(t1) SELECT 'merge' FROM t2;
 } {1 {query aborted}}
 
+#-------------------------------------------------------------------------
+reset_db
+do_test 80.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-f928a9c1ec68dd.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c   ..........n....L
+|    112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50   .....$...r.....P
+|    128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00   .....n..........
+|   2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d   ..............C.
+|   2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43   .....utablet4t4C
+|   2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41   REATE VIRTUAL TA
+|   2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73   BLE t4 USING fts
+|   2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f   5vocab('t2', 'ro
+|   2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65   w')C......utable
+|   3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55   t3t3CREATE VIRTU
+|   3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e   AL TABLE t3 USIN
+|   3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27   G fts5vocab('t1'
+|   3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d   , 'row')V.......
+|   3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32   tablet2_configt2
+|   3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54   _config.CREATE T
+|   3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27   ABLE 't2_config'
+|   3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20   (k PRIMARY KEY, 
+|   3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   v) WITHOUT ROWID
+|   3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32   ^...!!...tablet2
+|   3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65   _contentt2_conte
+|   3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20   nt.CREATE TABLE 
+|   3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20   't2_content'(id 
+|   3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20   INTEGER PRIMARY 
+|   3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29   KEY, c0, c1, c2)
+|   3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32   i.......-tablet2
+|   3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54   _idxt2_idx.CREAT
+|   3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27   E TABLE 't2_idx'
+|   3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67   (segid, term, pg
+|   3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28   no, PRIMARY KEY(
+|   3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49   segid, term)) WI
+|   3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b   THOUT ROWIDU....
+|   3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61   ....tablet2_data
+|   3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54   t2_data.CREATE T
+|   3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69   ABLE 't2_data'(i
+|   3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52   d INTEGER PRIMAR
+|   3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f   Y KEY, block BLO
+|   3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65   B)X........table
+|   3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55   t2t2CREATE VIRTU
+|   3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e   AL TABLE t2 USIN
+|   3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22   G fts5('a',[b],.
+|   3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63   c.,detail=none,c
+|   3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17   olumnsize=0)V...
+|   3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66   ....tablet1_conf
+|   3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41   igt1_config.CREA
+|   3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e   TE TABLE 't1_con
+|   3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62   OWID[...!!...tab
+|   3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64   let1_docsizet1_d
+|   3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41   ocsize.CREATE TA
+|   3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27   BLE 't1_docsize'
+|   3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42   ARY KEY, sz BLOB
+|   3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74   )^...!!...tablet
+|   3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
+|   3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
+|   3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64    't1_content'(id
+|   3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59    INTEGER PRIMARY
+|   3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32    KEY, c0, c1, c2
+|   3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74   )i.......-tablet
+|   3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41   1_idxt1_idx.CREA
+|   3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78   TE TABLE 't1_idx
+|   3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70   '(segid, term, p
+|   3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59   gno, PRIMARY KEY
+|   3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57   (segid, term)) W
+|   3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17   ITHOUT ROWIDU...
+|   3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74   .....tablet1_dat
+|   3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20   at1_data.CREATE 
+|   3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28   TABLE 't1_data'(
+|   3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41   id INTEGER PRIMA
+|   3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c   RY KEY, block BL
+|   4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c   OB)[.......#tabl
+|   4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
+|   4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
+|   4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e   NG fts5(a,b unin
+|   4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65   dexed,c,tokenize
+|   4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29   =.porter ascii.)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 01 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 00 00 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 12   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 00 00 00 00   ........vers....
+| end crash-f928a9c1ec68dd.db
+}]} {}
+
+do_catchsql_test 80.1 {
+SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank);
+} {1 {database disk image is malformed}}
+
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test
 

ext/fts5/test/fts5fault4.test

@@ -16,12 +16,16 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault4
 
-# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
 }
 
+set ::TMPDBERROR [list 1 \
+  {unable to open a temporary database file for storing temporary tables}
+]
+
 #-------------------------------------------------------------------------
 # An OOM while dropping an fts5 table.
 #
@@ -391,7 +395,7 @@ do_faultsim_test 14.1 -faults oom-t* -prep {
 } -body {
   db eval { ALTER TABLE "tbl one" RENAME TO "tbl two" }
 } -test {
-  faultsim_test_result {0 {}}
+  faultsim_test_result {0 {}} $::TMPDBERROR
 }
 
 finish_test

ext/icu/README.txt

@@ -153,18 +153,12 @@ SQLite. Documentation follows.
     the ICU extension LIKE operator is 50000, defined in source 
     file "icu.c".
 
-  3.3 Collation Sequence Security Issue
+  3.3 Collation Sequence Security
 
     Internally, SQLite assumes that indices stored in database files
     are sorted according to the collation sequence indicated by the
     SQL schema. Changing the definition of a collation sequence after
     an index has been built is therefore equivalent to database
-    corruption. The SQLite library is not very well tested under
-    these conditions, and may contain potential buffer overruns
-    or other programming errors that could be exploited by a malicious
-    programmer.
-
-    If the ICU extension is used in an environment where potentially
-    malicious users may execute arbitrary SQL (i.e. gears), they
-    should be prevented from invoking the icu_load_collation() function,
-    possibly using the authorisation callback.
+    corruption. The SQLite library is well tested for robustness in
+    the fact of database corruption.  Database corruption may well
+    lead to incorrect answers, but should not cause memory errors.

ext/lsm1/lsm_vtab.c

@@ -26,7 +26,7 @@
 **
 ** The virtual table contains read-only hidden columns:
 **
-**     lsm1_key	      A BLOB which is the raw LSM key.  If the "keytype"
+**     lsm1_key       A BLOB which is the raw LSM key.  If the "keytype"
 **                    is BLOB or TEXT then this column is exactly the
 **                    same as the key.  For the UINT keytype, this column
 **                    will be a variable-length integer encoding of the key.

ext/misc/amatch.c

@@ -1475,7 +1475,8 @@ static sqlite3_module amatchModule = {
   0,                      /* xSavepoint */
   0,                      /* xRelease */
   0,                      /* xRollbackTo */
-  0                       /* xShadowName */
+  0,                      /* xShadowName */
+  0                       /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/btreeinfo.c

@@ -411,7 +411,8 @@ int sqlite3BinfoRegister(sqlite3 *db){
     0,                           /* xSavepoint */
     0,                           /* xRelease */
     0,                           /* xRollbackTo */
-    0                            /* xShadowName */
+    0,                           /* xShadowName */
+    0                            /* xShadowName2 */
   };
   return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
 }

ext/misc/closure.c

@@ -939,7 +939,8 @@ static sqlite3_module closureModule = {
   0,                      /* xSavepoint */
   0,                      /* xRelease */
   0,                      /* xRollbackTo */
-  0                       /* xShadowName */
+  0,                      /* xShadowName */
+  0                       /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/completion.c

@@ -470,7 +470,8 @@ static sqlite3_module completionModule = {
   0,                         /* xSavepoint */
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
-  0                          /* xShadowName */
+  0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/csv.c

@@ -280,6 +280,7 @@ static char *csv_read_one_field(CsvReader *p){
     }
     p->cTerm = (char)c;
   }
+  assert( p->z==0 || p->n<p->nAlloc );
   if( p->z ) p->z[p->n] = 0;
   p->bNotFirst = 1;
   return p->z;
@@ -750,7 +751,7 @@ static int csvtabNext(sqlite3_vtab_cursor *cur){
       i++;
     }
   }while( pCur->rdr.cTerm==',' );
-  if( z==0 || (pCur->rdr.cTerm==EOF && i<pTab->nCol) ){
+  if( z==0 && i==0 ){
     pCur->iRowid = -1;
   }else{
     pCur->iRowid++;
@@ -811,6 +812,12 @@ static int csvtabFilter(
   CsvCursor *pCur = (CsvCursor*)pVtabCursor;
   CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab;
   pCur->iRowid = 0;
+
+  /* Ensure the field buffer is always allocated. Otherwise, if the
+  ** first field is zero bytes in size, this may be mistaken for an OOM
+  ** error in csvtabNext(). */
+  if( csv_append(&pCur->rdr, 0) ) return SQLITE_NOMEM;
+
   if( pCur->rdr.in==0 ){
     assert( pCur->rdr.zIn==pTab->zData );
     assert( pTab->iStart>=0 );
@@ -941,7 +948,7 @@ int sqlite3_csv_init(
   char **pzErrMsg, 
   const sqlite3_api_routines *pApi
 ){
-#ifndef SQLITE_OMIT_VIRTUALTABLE	
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   int rc;
   SQLITE_EXTENSION_INIT2(pApi);
   rc = sqlite3_create_module(db, "csv", &CsvModule, 0);

ext/misc/dbdata.c

@@ -828,7 +828,8 @@ static int sqlite3DbdataRegister(sqlite3 *db){
     0,                            /* xSavepoint */
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
-    0                             /* xShadowName */
+    0,                            /* xShadowName */
+    0                             /* xShadowName2 */
   };
 
   int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0);

ext/misc/explain.c

@@ -293,6 +293,7 @@ static sqlite3_module explainModule = {
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
   0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/fileio.c

@@ -983,6 +983,7 @@ static int fsdirRegister(sqlite3 *db){
     0,                         /* xRelease */
     0,                         /* xRollbackTo */
     0,                         /* xShadowName */
+    0                          /* xShadowName2 */
   };
 
   int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);

ext/misc/fossildelta.c

@@ -1035,30 +1035,31 @@ static int deltaparsevtabBestIndex(
 ** virtual table.
 */
 static sqlite3_module deltaparsevtabModule = {
-  /* iVersion    */ 0,
-  /* xCreate     */ 0,
-  /* xConnect    */ deltaparsevtabConnect,
-  /* xBestIndex  */ deltaparsevtabBestIndex,
-  /* xDisconnect */ deltaparsevtabDisconnect,
-  /* xDestroy    */ 0,
-  /* xOpen       */ deltaparsevtabOpen,
-  /* xClose      */ deltaparsevtabClose,
-  /* xFilter     */ deltaparsevtabFilter,
-  /* xNext       */ deltaparsevtabNext,
-  /* xEof        */ deltaparsevtabEof,
-  /* xColumn     */ deltaparsevtabColumn,
-  /* xRowid      */ deltaparsevtabRowid,
-  /* xUpdate     */ 0,
-  /* xBegin      */ 0,
-  /* xSync       */ 0,
-  /* xCommit     */ 0,
-  /* xRollback   */ 0,
-  /* xFindMethod */ 0,
-  /* xRename     */ 0,
-  /* xSavepoint  */ 0,
-  /* xRelease    */ 0,
-  /* xRollbackTo */ 0,
-  /* xShadowName */ 0
+  /* iVersion     */ 0,
+  /* xCreate      */ 0,
+  /* xConnect     */ deltaparsevtabConnect,
+  /* xBestIndex   */ deltaparsevtabBestIndex,
+  /* xDisconnect  */ deltaparsevtabDisconnect,
+  /* xDestroy     */ 0,
+  /* xOpen        */ deltaparsevtabOpen,
+  /* xClose       */ deltaparsevtabClose,
+  /* xFilter      */ deltaparsevtabFilter,
+  /* xNext        */ deltaparsevtabNext,
+  /* xEof         */ deltaparsevtabEof,
+  /* xColumn      */ deltaparsevtabColumn,
+  /* xRowid       */ deltaparsevtabRowid,
+  /* xUpdate      */ 0,
+  /* xBegin       */ 0,
+  /* xSync        */ 0,
+  /* xCommit      */ 0,
+  /* xRollback    */ 0,
+  /* xFindMethod  */ 0,
+  /* xRename      */ 0,
+  /* xSavepoint   */ 0,
+  /* xRelease     */ 0,
+  /* xRollbackTo  */ 0,
+  /* xShadowName  */ 0,
+  /* xShadowName2 */ 0
 };
 
 

ext/misc/ieee754.c

@@ -284,7 +284,7 @@ int sqlite3_ieee_init(
   SQLITE_EXTENSION_INIT2(pApi);
   (void)pzErrMsg;  /* Unused parameter */
   for(i=0; i<sizeof(aFunc)/sizeof(aFunc[0]) && rc==SQLITE_OK; i++){
-    rc = sqlite3_create_function(db, aFunc[i].zFName, aFunc[i].nArg,	
+    rc = sqlite3_create_function(db, aFunc[i].zFName, aFunc[i].nArg,
                                SQLITE_UTF8|SQLITE_INNOCUOUS,
                                (void*)&aFunc[i].iAux,
                                aFunc[i].xFunc, 0, 0);

ext/misc/memstat.c

@@ -396,6 +396,7 @@ static sqlite3_module memstatModule = {
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
   0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/prefixes.c

@@ -225,30 +225,31 @@ static int prefixesBestIndex(
 ** virtual table.
 */
 static sqlite3_module prefixesModule = {
-  /* iVersion    */ 0,
-  /* xCreate     */ 0,
-  /* xConnect    */ prefixesConnect,
-  /* xBestIndex  */ prefixesBestIndex,
-  /* xDisconnect */ prefixesDisconnect,
-  /* xDestroy    */ 0,
-  /* xOpen       */ prefixesOpen,
-  /* xClose      */ prefixesClose,
-  /* xFilter     */ prefixesFilter,
-  /* xNext       */ prefixesNext,
-  /* xEof        */ prefixesEof,
-  /* xColumn     */ prefixesColumn,
-  /* xRowid      */ prefixesRowid,
-  /* xUpdate     */ 0,
-  /* xBegin      */ 0,
-  /* xSync       */ 0,
-  /* xCommit     */ 0,
-  /* xRollback   */ 0,
-  /* xFindMethod */ 0,
-  /* xRename     */ 0,
-  /* xSavepoint  */ 0,
-  /* xRelease    */ 0,
-  /* xRollbackTo */ 0,
-  /* xShadowName */ 0
+  /* iVersion     */ 0,
+  /* xCreate      */ 0,
+  /* xConnect     */ prefixesConnect,
+  /* xBestIndex   */ prefixesBestIndex,
+  /* xDisconnect  */ prefixesDisconnect,
+  /* xDestroy     */ 0,
+  /* xOpen        */ prefixesOpen,
+  /* xClose       */ prefixesClose,
+  /* xFilter      */ prefixesFilter,
+  /* xNext        */ prefixesNext,
+  /* xEof         */ prefixesEof,
+  /* xColumn      */ prefixesColumn,
+  /* xRowid       */ prefixesRowid,
+  /* xUpdate      */ 0,
+  /* xBegin       */ 0,
+  /* xSync        */ 0,
+  /* xCommit      */ 0,
+  /* xRollback    */ 0,
+  /* xFindMethod  */ 0,
+  /* xRename      */ 0,
+  /* xSavepoint   */ 0,
+  /* xRelease     */ 0,
+  /* xRollbackTo  */ 0,
+  /* xShadowName  */ 0,
+  /* xShadowName2 */ 0
 };
 
 /*

ext/misc/qpvtab.c

@@ -40,7 +40,7 @@
 **
 ** Option 1 is the default behavior.  2 is use if there is a usable
 ** constraint on "flags" with an integer right-hand side that where the
-** value of the right-hand side has its 0x01 bit set.
+** value of the right-hand side has its 0x001 bit set.
 **
 ** All constraints on columns "a" through "e" are marked as "omit".
 **
@@ -48,6 +48,12 @@
 ** is an integer and that integer has its 0x02 bit set, then the
 ** orderByConsumed flag is set.
 **
+** FLAGS SUMMARY:
+**
+**   0x001               Columns 'a' through 'e' have INT values
+**   0x002               orderByConsumed is set
+**   0x004               OFFSET and LIMIT have omit set
+**
 ** COMPILE:
 **
 **   gcc -Wall -g -shared -fPIC -I. qpvtab.c -o qqvtab.so
@@ -103,7 +109,8 @@ static const char *azColname[] = {
   "ux",
   "rhs",
   "a", "b", "c", "d", "e",
-  "flags"
+  "flags",
+  ""
 };
 
 /*
@@ -143,6 +150,7 @@ static int qpvtabConnect(
 #define QPVTAB_D       9
 #define QPVTAB_E      10
 #define QPVTAB_FLAGS  11
+#define QPVTAB_NONE   12
   if( rc==SQLITE_OK ){
     pNew = sqlite3_malloc( sizeof(*pNew) );
     *ppVtab = (sqlite3_vtab*)pNew;
@@ -226,7 +234,7 @@ static int qpvtabColumn(
       sqlite3_result_text64(ctx, z, zEnd-z, SQLITE_TRANSIENT, SQLITE_UTF8);
     }
   }else if( i>=QPVTAB_A && i<=QPVTAB_E ){
-    if( pCur->flags & 1 ){
+    if( pCur->flags & 0x001 ){
       sqlite3_result_int(ctx, i-QPVTAB_A+1);
     }else{
       char x = 'a'+i-QPVTAB_A;
@@ -338,19 +346,25 @@ static int qpvtabBestIndex(
   for(i=0; i<pIdxInfo->nConstraint; i++){
     sqlite3_value *pVal;
     int iCol = pIdxInfo->aConstraint[i].iColumn;
+    int op = pIdxInfo->aConstraint[i].op;
     if( iCol==QPVTAB_FLAGS &&  pIdxInfo->aConstraint[i].usable ){
       pVal = 0;
       rc = sqlite3_vtab_rhs_value(pIdxInfo, i, &pVal);
       assert( rc==SQLITE_OK || pVal==0 );
       if( pVal ){
         pIdxInfo->idxNum = sqlite3_value_int(pVal);
-        if( pIdxInfo->idxNum & 2 ) pIdxInfo->orderByConsumed = 1;
+        if( pIdxInfo->idxNum & 0x002 ) pIdxInfo->orderByConsumed = 1;
       }
     }
+    if( op==SQLITE_INDEX_CONSTRAINT_LIMIT
+     || op==SQLITE_INDEX_CONSTRAINT_OFFSET
+    ){
+      iCol = QPVTAB_NONE;
+    }
     sqlite3_str_appendf(pStr,"aConstraint,%d,%s,%d,%d,",
        i,
        azColname[iCol],
-       pIdxInfo->aConstraint[i].op,
+       op,
        pIdxInfo->aConstraint[i].usable);
     pVal = 0;
     rc = sqlite3_vtab_rhs_value(pIdxInfo, i, &pVal);
@@ -359,9 +373,20 @@ static int qpvtabBestIndex(
       qpvtabStrAppendValue(pStr, pVal);
     }
     sqlite3_str_append(pStr, "\n", 1);
+  }
+  for(i=0; i<pIdxInfo->nConstraint; i++){
+    int iCol = pIdxInfo->aConstraint[i].iColumn;
+    int op = pIdxInfo->aConstraint[i].op;
+    if( op==SQLITE_INDEX_CONSTRAINT_LIMIT
+     || op==SQLITE_INDEX_CONSTRAINT_OFFSET
+    ){
+      iCol = QPVTAB_NONE;
+    }
     if( iCol>=QPVTAB_A && pIdxInfo->aConstraint[i].usable ){
-      pIdxInfo->aConstraintUsage[i].argvIndex = ++k;   
-      pIdxInfo->aConstraintUsage[i].omit = 1;
+      pIdxInfo->aConstraintUsage[i].argvIndex = ++k;
+      if( iCol<=QPVTAB_FLAGS || (pIdxInfo->idxNum & 0x004)!=0 ){
+        pIdxInfo->aConstraintUsage[i].omit = 1;
+      }
     }
   }
   sqlite3_str_appendf(pStr, "nOrderBy,%d,,,,\n", pIdxInfo->nOrderBy);
@@ -391,30 +416,31 @@ static int qpvtabBestIndex(
 ** virtual table.
 */
 static sqlite3_module qpvtabModule = {
-  /* iVersion    */ 0,
-  /* xCreate     */ 0,
-  /* xConnect    */ qpvtabConnect,
-  /* xBestIndex  */ qpvtabBestIndex,
-  /* xDisconnect */ qpvtabDisconnect,
-  /* xDestroy    */ 0,
-  /* xOpen       */ qpvtabOpen,
-  /* xClose      */ qpvtabClose,
-  /* xFilter     */ qpvtabFilter,
-  /* xNext       */ qpvtabNext,
-  /* xEof        */ qpvtabEof,
-  /* xColumn     */ qpvtabColumn,
-  /* xRowid      */ qpvtabRowid,
-  /* xUpdate     */ 0,
-  /* xBegin      */ 0,
-  /* xSync       */ 0,
-  /* xCommit     */ 0,
-  /* xRollback   */ 0,
-  /* xFindMethod */ 0,
-  /* xRename     */ 0,
-  /* xSavepoint  */ 0,
-  /* xRelease    */ 0,
-  /* xRollbackTo */ 0,
-  /* xShadowName */ 0
+  /* iVersion     */ 0,
+  /* xCreate      */ 0,
+  /* xConnect     */ qpvtabConnect,
+  /* xBestIndex   */ qpvtabBestIndex,
+  /* xDisconnect  */ qpvtabDisconnect,
+  /* xDestroy     */ 0,
+  /* xOpen        */ qpvtabOpen,
+  /* xClose       */ qpvtabClose,
+  /* xFilter      */ qpvtabFilter,
+  /* xNext        */ qpvtabNext,
+  /* xEof         */ qpvtabEof,
+  /* xColumn      */ qpvtabColumn,
+  /* xRowid       */ qpvtabRowid,
+  /* xUpdate      */ 0,
+  /* xBegin       */ 0,
+  /* xSync        */ 0,
+  /* xCommit      */ 0,
+  /* xRollback    */ 0,
+  /* xFindMethod  */ 0,
+  /* xRename      */ 0,
+  /* xSavepoint   */ 0,
+  /* xRelease     */ 0,
+  /* xRollbackTo  */ 0,
+  /* xShadowName  */ 0,
+  /* xShadowName2 */ 0
 };
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 

ext/misc/regexp.c

@@ -72,6 +72,7 @@ SQLITE_EXTENSION_INIT1
 
 /* The end-of-input character */
 #define RE_EOF            0    /* End of input */
+#define RE_START  0xfffffff    /* Start of input - larger than an UTF-8 */
 
 /* The NFA is implemented as sequence of opcodes taken from the following
 ** set.  Each opcode has a single integer argument.
@@ -93,6 +94,33 @@ SQLITE_EXTENSION_INIT1
 #define RE_OP_SPACE      15    /* space:  [ \t\n\r\v\f] */
 #define RE_OP_NOTSPACE   16    /* Not a digit */
 #define RE_OP_BOUNDARY   17    /* Boundary between word and non-word */
+#define RE_OP_ATSTART    18    /* Currently at the start of the string */
+
+#if defined(SQLITE_DEBUG)
+/* Opcode names used for symbolic debugging */
+static const char *ReOpName[] = {
+  "EOF",
+  "MATCH",
+  "ANY",
+  "ANYSTAR",
+  "FORK",
+  "GOTO",
+  "ACCEPT",
+  "CC_INC",
+  "CC_EXC",
+  "CC_VALUE",
+  "CC_RANGE",
+  "WORD",
+  "NOTWORD",
+  "DIGIT",
+  "NOTDIGIT",
+  "SPACE",
+  "NOTSPACE",
+  "BOUNDARY",
+  "ATSTART",
+};
+#endif /* SQLITE_DEBUG */
+
 
 /* Each opcode is a "state" in the NFA */
 typedef unsigned short ReStateNumber;
@@ -127,7 +155,7 @@ struct ReCompiled {
   int *aArg;                  /* Arguments to each operator */
   unsigned (*xNextChar)(ReInput*);  /* Next character function */
   unsigned char zInit[12];    /* Initial text to match */
-  int nInit;                  /* Number of characters in zInit */
+  int nInit;                  /* Number of bytes in zInit */
   unsigned nState;            /* Number of entries in aOp[] and aArg[] */
   unsigned nAlloc;            /* Slots allocated for aOp[] and aArg[] */
 };
@@ -200,7 +228,7 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
   ReStateNumber *pToFree;
   unsigned int i = 0;
   unsigned int iSwap = 0;
-  int c = RE_EOF+1;
+  int c = RE_START;
   int cPrev = 0;
   int rc = 0;
   ReInput in;
@@ -219,6 +247,7 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
       in.i++;
     }
     if( in.i+pRe->nInit>in.mx ) return 0;
+    c = RE_START-1;
   }
 
   if( pRe->nState<=(sizeof(aSpace)/(sizeof(aSpace[0])*2)) ){
@@ -247,6 +276,10 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
           if( pRe->aArg[x]==c ) re_add_state(pNext, x+1);
           break;
         }
+        case RE_OP_ATSTART: {
+          if( cPrev==RE_START ) re_add_state(pThis, x+1);
+          break;
+        }
         case RE_OP_ANY: {
           if( c!=0 ) re_add_state(pNext, x+1);
           break;
@@ -328,7 +361,9 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){
     }
   }
   for(i=0; i<pNext->nState; i++){
-    if( pRe->aOp[pNext->aState[i]]==RE_OP_ACCEPT ){ rc = 1; break; }
+    int x = pNext->aState[i];
+    while( pRe->aOp[x]==RE_OP_GOTO ) x += pRe->aArg[x];
+    if( pRe->aOp[x]==RE_OP_ACCEPT ){ rc = 1; break; }
   }
 re_match_end:
   sqlite3_free(pToFree);
@@ -483,7 +518,6 @@ static const char *re_subcompile_string(ReCompiled *p){
     iStart = p->nState;
     switch( c ){
       case '|':
-      case '$':
       case ')': {
         p->sIn.i--;
         return 0;
@@ -520,6 +554,14 @@ static const char *re_subcompile_string(ReCompiled *p){
         re_insert(p, iPrev, RE_OP_FORK, p->nState - iPrev+1);
         break;
       }
+      case '$': {
+        re_append(p, RE_OP_MATCH, RE_EOF);
+        break;
+      }
+      case '^': {
+        re_append(p, RE_OP_ATSTART, 0);
+        break;
+      }
       case '{': {
         int m = 0, n = 0;
         int sz, j;
@@ -538,6 +580,7 @@ static const char *re_subcompile_string(ReCompiled *p){
         if( m==0 ){
           if( n==0 ) return "both m and n are zero in '{m,n}'";
           re_insert(p, iPrev, RE_OP_FORK, sz+1);
+          iPrev++;
           n--;
         }else{
           for(j=1; j<m; j++) re_copy(p, iPrev, sz);
@@ -656,11 +699,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
     re_free(pRe);
     return zErr;
   }
-  if( rePeek(pRe)=='$' && pRe->sIn.i+1>=pRe->sIn.mx ){
-    re_append(pRe, RE_OP_MATCH, RE_EOF);
-    re_append(pRe, RE_OP_ACCEPT, 0);
-    *ppRe = pRe;
-  }else if( pRe->sIn.i>=pRe->sIn.mx ){
+  if( pRe->sIn.i>=pRe->sIn.mx ){
     re_append(pRe, RE_OP_ACCEPT, 0);
     *ppRe = pRe;
   }else{
@@ -685,7 +724,7 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
         pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
         pRe->zInit[j++] = 0x80 | (x&0x3f);
       }else if( x<=0xffff ){
-        pRe->zInit[j++] = (unsigned char)(0xd0 | (x>>12));
+        pRe->zInit[j++] = (unsigned char)(0xe0 | (x>>12));
         pRe->zInit[j++] = 0x80 | ((x>>6)&0x3f);
         pRe->zInit[j++] = 0x80 | (x&0x3f);
       }else{
@@ -744,6 +783,67 @@ static void re_sql_func(
   }
 }
 
+#if defined(SQLITE_DEBUG)
+/*
+** This function is used for testing and debugging only.  It is only available
+** if the SQLITE_DEBUG compile-time option is used.
+**
+** Compile a regular expression and then convert the compiled expression into
+** text and return that text.
+*/
+static void re_bytecode_func(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zPattern;
+  const char *zErr;
+  ReCompiled *pRe;
+  sqlite3_str *pStr;
+  int i;
+  int n;
+  char *z;
+
+  zPattern = (const char*)sqlite3_value_text(argv[0]);
+  if( zPattern==0 ) return;
+  zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0);
+  if( zErr ){
+    re_free(pRe);
+    sqlite3_result_error(context, zErr, -1);
+    return;
+  }
+  if( pRe==0 ){
+    sqlite3_result_error_nomem(context);
+    return;
+  }
+  pStr = sqlite3_str_new(0);
+  if( pStr==0 ) goto re_bytecode_func_err;
+  if( pRe->nInit>0 ){
+    sqlite3_str_appendf(pStr, "INIT     ");
+    for(i=0; i<pRe->nInit; i++){
+      sqlite3_str_appendf(pStr, "%02x", pRe->zInit[i]);
+    }
+    sqlite3_str_appendf(pStr, "\n");
+  }
+  for(i=0; (unsigned)i<pRe->nState; i++){
+    sqlite3_str_appendf(pStr, "%-8s %4d\n",
+         ReOpName[(unsigned char)pRe->aOp[i]], pRe->aArg[i]);
+  }
+  n = sqlite3_str_length(pStr);
+  z = sqlite3_str_finish(pStr);
+  if( n==0 ){
+    sqlite3_free(z);
+  }else{
+    sqlite3_result_text(context, z, n-1, sqlite3_free);
+  }
+
+re_bytecode_func_err:
+  re_free(pRe);
+}
+
+#endif /* SQLITE_DEBUG */
+
+
 /*
 ** Invoke this routine to register the regexp() function with the
 ** SQLite database connection.
@@ -768,6 +868,13 @@ int sqlite3_regexp_init(
     rc = sqlite3_create_function(db, "regexpi", 2,
                             SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
                             (void*)db, re_sql_func, 0, 0);
+#if defined(SQLITE_DEBUG)
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(db, "regexp_bytecode", 1,
+                            SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
+                            0, re_bytecode_func, 0, 0);
+    }
+#endif /* SQLITE_DEBUG */
   }
   return rc;
 }

ext/misc/series.c

@@ -432,7 +432,8 @@ static sqlite3_module seriesModule = {
   0,                         /* xSavepoint */
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
-  0                          /* xShadowName */
+  0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/shathree.c

@@ -19,7 +19,7 @@
 ** The sha3(X) function computes the SHA3 hash of the input X, or NULL if
 ** X is NULL.
 **
-** The sha3_query(Y) function evalutes all queries in the SQL statements of Y
+** The sha3_query(Y) function evaluates all queries in the SQL statements of Y
 ** and returns a hash of their results.
 **
 ** The SIZE argument is optional.  If omitted, the SHA3-256 hash algorithm

ext/misc/stmt.c

@@ -30,6 +30,16 @@ SQLITE_EXTENSION_INIT1
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
+
+#define STMT_NUM_INTEGER_COLUMN 10
+typedef struct StmtRow StmtRow;
+struct StmtRow {
+  sqlite3_int64 iRowid;                /* Rowid value */
+  char *zSql;                          /* column "sql" */
+  int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */
+  StmtRow *pNext;                      /* Next row to return */
+};
+
 /* stmt_vtab is a subclass of sqlite3_vtab which will
 ** serve as the underlying representation of a stmt virtual table
 */
@@ -47,8 +57,7 @@ typedef struct stmt_cursor stmt_cursor;
 struct stmt_cursor {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
   sqlite3 *db;               /* Database connection for this cursor */
-  sqlite3_stmt *pStmt;       /* Statement cursor is currently pointing at */
-  sqlite3_int64 iRowid;      /* The rowid */
+  StmtRow *pRow;             /* Current row */
 };
 
 /*
@@ -92,7 +101,7 @@ static int stmtConnect(
      "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep,"
                     "reprep,run,mem)");
   if( rc==SQLITE_OK ){
-    pNew = sqlite3_malloc( sizeof(*pNew) );
+    pNew = sqlite3_malloc64( sizeof(*pNew) );
     *ppVtab = (sqlite3_vtab*)pNew;
     if( pNew==0 ) return SQLITE_NOMEM;
     memset(pNew, 0, sizeof(*pNew));
@@ -114,7 +123,7 @@ static int stmtDisconnect(sqlite3_vtab *pVtab){
 */
 static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   stmt_cursor *pCur;
-  pCur = sqlite3_malloc( sizeof(*pCur) );
+  pCur = sqlite3_malloc64( sizeof(*pCur) );
   if( pCur==0 ) return SQLITE_NOMEM;
   memset(pCur, 0, sizeof(*pCur));
   pCur->db = ((stmt_vtab*)p)->db;
@@ -122,10 +131,21 @@ static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   return SQLITE_OK;
 }
 
+static void stmtCsrReset(stmt_cursor *pCur){
+  StmtRow *pRow = 0;
+  StmtRow *pNext = 0;
+  for(pRow=pCur->pRow; pRow; pRow=pNext){
+    pNext = pRow->pNext;
+    sqlite3_free(pRow);
+  }
+  pCur->pRow = 0;
+}
+
 /*
 ** Destructor for a stmt_cursor.
 */
 static int stmtClose(sqlite3_vtab_cursor *cur){
+  stmtCsrReset((stmt_cursor*)cur);
   sqlite3_free(cur);
   return SQLITE_OK;
 }
@@ -136,8 +156,9 @@ static int stmtClose(sqlite3_vtab_cursor *cur){
 */
 static int stmtNext(sqlite3_vtab_cursor *cur){
   stmt_cursor *pCur = (stmt_cursor*)cur;
-  pCur->iRowid++;
-  pCur->pStmt = sqlite3_next_stmt(pCur->db, pCur->pStmt);
+  StmtRow *pNext = pCur->pRow->pNext;
+  sqlite3_free(pCur->pRow);
+  pCur->pRow = pNext;
   return SQLITE_OK;
 }
 
@@ -151,39 +172,11 @@ static int stmtColumn(
   int i                       /* Which column to return */
 ){
   stmt_cursor *pCur = (stmt_cursor*)cur;
-  switch( i ){
-    case STMT_COLUMN_SQL: {
-      sqlite3_result_text(ctx, sqlite3_sql(pCur->pStmt), -1, SQLITE_TRANSIENT);
-      break;
-    }
-    case STMT_COLUMN_NCOL: {
-      sqlite3_result_int(ctx, sqlite3_column_count(pCur->pStmt));
-      break;
-    }
-    case STMT_COLUMN_RO: {
-      sqlite3_result_int(ctx, sqlite3_stmt_readonly(pCur->pStmt));
-      break;
-    }
-    case STMT_COLUMN_BUSY: {
-      sqlite3_result_int(ctx, sqlite3_stmt_busy(pCur->pStmt));
-      break;
-    }
-    default: {
-      assert( i==STMT_COLUMN_MEM );
-      i = SQLITE_STMTSTATUS_MEMUSED + 
-            STMT_COLUMN_NSCAN - SQLITE_STMTSTATUS_FULLSCAN_STEP;
-      /* Fall thru */
-    }
-    case STMT_COLUMN_NSCAN:
-    case STMT_COLUMN_NSORT:
-    case STMT_COLUMN_NAIDX:
-    case STMT_COLUMN_NSTEP:
-    case STMT_COLUMN_REPREP:
-    case STMT_COLUMN_RUN: {
-      sqlite3_result_int(ctx, sqlite3_stmt_status(pCur->pStmt,
-                      i-STMT_COLUMN_NSCAN+SQLITE_STMTSTATUS_FULLSCAN_STEP, 0));
-      break;
-    }
+  StmtRow *pRow = pCur->pRow;
+  if( i==STMT_COLUMN_SQL ){
+    sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_int(ctx, pRow->aCol[i]);
   }
   return SQLITE_OK;
 }
@@ -194,7 +187,7 @@ static int stmtColumn(
 */
 static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
   stmt_cursor *pCur = (stmt_cursor*)cur;
-  *pRowid = pCur->iRowid;
+  *pRowid = pCur->pRow->iRowid;
   return SQLITE_OK;
 }
 
@@ -204,7 +197,7 @@ static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
 */
 static int stmtEof(sqlite3_vtab_cursor *cur){
   stmt_cursor *pCur = (stmt_cursor*)cur;
-  return pCur->pStmt==0;
+  return pCur->pRow==0;
 }
 
 /*
@@ -219,9 +212,53 @@ static int stmtFilter(
   int argc, sqlite3_value **argv
 ){
   stmt_cursor *pCur = (stmt_cursor *)pVtabCursor;
-  pCur->pStmt = 0;
-  pCur->iRowid = 0;
-  return stmtNext(pVtabCursor);
+  sqlite3_stmt *p = 0;
+  sqlite3_int64 iRowid = 1;
+  StmtRow **ppRow = 0;
+
+  stmtCsrReset(pCur);
+  ppRow = &pCur->pRow;
+  for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){
+    const char *zSql = sqlite3_sql(p);
+    sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0;
+    StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql);
+
+    if( pNew==0 ) return SQLITE_NOMEM;
+    memset(pNew, 0, sizeof(StmtRow));
+    if( zSql ){
+      pNew->zSql = (char*)&pNew[1];
+      memcpy(pNew->zSql, zSql, nSql);
+    }
+    pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p);
+    pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p);
+    pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p);
+    pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0
+    );
+    pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_SORT, 0
+    );
+    pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_AUTOINDEX, 0
+    );
+    pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_VM_STEP, 0
+    );
+    pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_REPREPARE, 0
+    );
+    pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_RUN, 0
+    );
+    pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status(
+        p, SQLITE_STMTSTATUS_MEMUSED, 0
+    );
+    pNew->iRowid = iRowid++;
+    *ppRow = pNew;
+    ppRow = &pNew->pNext;
+  }
+
+  return SQLITE_OK;
 }
 
 /*
@@ -268,6 +305,7 @@ static sqlite3_module stmtModule = {
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
   0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #endif /* SQLITE_OMIT_VIRTUALTABLE */

ext/misc/templatevtab.c

@@ -226,30 +226,31 @@ static int templatevtabBestIndex(
 ** virtual table.
 */
 static sqlite3_module templatevtabModule = {
-  /* iVersion    */ 0,
-  /* xCreate     */ 0,
-  /* xConnect    */ templatevtabConnect,
-  /* xBestIndex  */ templatevtabBestIndex,
-  /* xDisconnect */ templatevtabDisconnect,
-  /* xDestroy    */ 0,
-  /* xOpen       */ templatevtabOpen,
-  /* xClose      */ templatevtabClose,
-  /* xFilter     */ templatevtabFilter,
-  /* xNext       */ templatevtabNext,
-  /* xEof        */ templatevtabEof,
-  /* xColumn     */ templatevtabColumn,
-  /* xRowid      */ templatevtabRowid,
-  /* xUpdate     */ 0,
-  /* xBegin      */ 0,
-  /* xSync       */ 0,
-  /* xCommit     */ 0,
-  /* xRollback   */ 0,
-  /* xFindMethod */ 0,
-  /* xRename     */ 0,
-  /* xSavepoint  */ 0,
-  /* xRelease    */ 0,
-  /* xRollbackTo */ 0,
-  /* xShadowName */ 0
+  /* iVersion     */ 0,
+  /* xCreate      */ 0,
+  /* xConnect     */ templatevtabConnect,
+  /* xBestIndex   */ templatevtabBestIndex,
+  /* xDisconnect  */ templatevtabDisconnect,
+  /* xDestroy     */ 0,
+  /* xOpen        */ templatevtabOpen,
+  /* xClose       */ templatevtabClose,
+  /* xFilter      */ templatevtabFilter,
+  /* xNext        */ templatevtabNext,
+  /* xEof         */ templatevtabEof,
+  /* xColumn      */ templatevtabColumn,
+  /* xRowid       */ templatevtabRowid,
+  /* xUpdate      */ 0,
+  /* xBegin       */ 0,
+  /* xSync        */ 0,
+  /* xCommit      */ 0,
+  /* xRollback    */ 0,
+  /* xFindMethod  */ 0,
+  /* xRename      */ 0,
+  /* xSavepoint   */ 0,
+  /* xRelease     */ 0,
+  /* xRollbackTo  */ 0,
+  /* xShadowName  */ 0,
+  /* xShadowName2 */ 0
 };
 
 

ext/misc/unionvtab.c

@@ -1351,7 +1351,8 @@ static int createUnionVtab(sqlite3 *db){
     0,                            /* xSavepoint */
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
-    0                             /* xShadowName */
+    0,                            /* xShadowName */
+    0                             /* xShadowName2 */
   };
   int rc;
 

ext/misc/vtablog.c

@@ -493,6 +493,7 @@ static sqlite3_module vtablogModule = {
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
   0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 #ifdef _WIN32

ext/misc/vtshim.c

@@ -425,6 +425,16 @@ static int vtshimRollbackTo(sqlite3_vtab *pBase, int n){
   return rc;
 }
 
+static int vtshimShadowName2(sqlite3_vtab *pBase, const char *zName){
+  vtshim_vtab *pVtab = (vtshim_vtab*)pBase;
+  vtshim_aux *pAux = pVtab->pAux;
+  int rc;
+  if( pAux->bDisposed ) return 0;
+  rc = pAux->pMod->xShadowName2(pVtab->pChild, zName);
+  VTSHIM_COPY_ERRMSG();
+  return rc;
+}
+
 /* The destructor function for a disposible module */
 static void vtshimAuxDestructor(void *pXAux){
   vtshim_aux *pAux = (vtshim_aux*)pXAux;
@@ -481,7 +491,7 @@ void *sqlite3_create_disposable_module(
   pAux->zName = sqlite3_mprintf("%s", zName);
   pAux->bDisposed = 0;
   pAux->pAllVtab = 0;
-  pAux->sSelf.iVersion = p->iVersion<=2 ? p->iVersion : 2;
+  pAux->sSelf.iVersion = p->iVersion<=4 ? p->iVersion : 4;
   pAux->sSelf.xCreate = p->xCreate ? vtshimCreate : 0;
   pAux->sSelf.xConnect = p->xConnect ? vtshimConnect : 0;
   pAux->sSelf.xBestIndex = p->xBestIndex ? vtshimBestIndex : 0;
@@ -510,6 +520,12 @@ void *sqlite3_create_disposable_module(
     pAux->sSelf.xRelease = 0;
     pAux->sSelf.xRollbackTo = 0;
   }
+  pAux->sSelf.xShadowName = 0; /* v3 */
+  if( p->iVersion>=4 ){
+    pAux->sSelf.xShadowName2 = p->xShadowName2 ? vtshimShadowName2 : 0;
+  }else{
+    pAux->sSelf.xShadowName2 = 0;
+  }
   rc = sqlite3_create_module_v2(db, zName, &pAux->sSelf,
                                 pAux, vtshimAuxDestructor);
   return rc==SQLITE_OK ? (void*)pAux : 0;

ext/misc/zipfile.c

@@ -1262,9 +1262,14 @@ static int zipfileFilter(
     zipfileCursorErr(pCsr, "zipfile() function requires an argument");
     return SQLITE_ERROR;
   }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+    static const u8 aEmptyBlob = 0;
     const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]);
     int nBlob = sqlite3_value_bytes(argv[0]);
     assert( pTab->pFirstEntry==0 );
+    if( aBlob==0 ){
+      aBlob = &aEmptyBlob;
+      nBlob = 0;
+    }
     rc = zipfileLoadDirectory(pTab, aBlob, nBlob);
     pCsr->pFreeEntry = pTab->pFirstEntry;
     pTab->pFirstEntry = pTab->pLastEntry = 0;
@@ -2170,6 +2175,11 @@ static int zipfileRegister(sqlite3 *db){
     zipfileRollback,           /* xRollback */
     zipfileFindFunction,       /* xFindMethod */
     0,                         /* xRename */
+    0,                         /* xSavepoint */
+    0,                         /* xRelease */
+    0,                         /* xRollback */
+    0,                         /* xShadowName */
+    0                          /* xShadowName2 */
   };
 
   int rc = sqlite3_create_module(db, "zipfile"  , &zipfileModule, 0);

ext/rbu/rbubusy.test

@@ -0,0 +1,87 @@
+# 2014 August 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbubusy
+
+db close
+sqlite3_shutdown
+test_sqlite3_log xLog
+reset_db
+
+set db_sql {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  INSERT INTO t1 VALUES(1000, 2000, 3000);
+}
+
+set rbu_sql {
+  CREATE TABLE data_t1(a, b, c, rbu_control);
+  INSERT INTO data_t1 VALUES(1, 2, 3, 0);
+  INSERT INTO data_t1 VALUES(4, 5, 6, 0);
+  INSERT INTO data_t1 VALUES(7, 8, 9, 0);
+}
+
+do_test 1.1 {
+  forcedelete rbu.db
+  sqlite3 rbu rbu.db
+  rbu eval $rbu_sql
+  rbu close
+
+  db eval $db_sql
+} {}
+
+do_execsql_test 1.2 {
+  BEGIN;
+    SELECT * FROM t1
+} {1000 2000 3000}
+
+do_test 1.3 {
+  sqlite3rbu rbu test.db rbu.db
+  rbu step
+} {SQLITE_OK}
+
+do_test 1.4 {
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  set rc
+} {SQLITE_BUSY}
+
+do_test 1.5 {
+  rbu step
+} {SQLITE_BUSY}
+
+do_test 1.6 {
+  db eval COMMIT
+  rbu step
+} {SQLITE_BUSY}
+catch { rbu close }
+
+do_test 1.7 {
+  sqlite3rbu rbu test.db rbu.db
+  while 1 {
+    set rc [rbu step]
+    if {$rc!="SQLITE_OK"} break
+  }
+  set rc
+} {SQLITE_DONE}
+
+rbu close
+
+db close
+sqlite3_shutdown
+test_sqlite3_log 
+sqlite3_initialize
+finish_test
+

ext/repair/sqlite3_checker.c.in

@@ -4,7 +4,6 @@
 */
 #define TCLSH_INIT_PROC sqlite3_checker_init_proc
 #define SQLITE_ENABLE_DBPAGE_VTAB 1
-#define SQLITE_ENABLE_JSON1 1
 #undef SQLITE_THREADSAFE
 #define SQLITE_THREADSAFE 0
 #undef SQLITE_ENABLE_COLUMN_METADATA

ext/rtree/geopoly.c

@@ -1736,7 +1736,7 @@ static int geopolyFindFunction(
 
 
 static sqlite3_module geopolyModule = {
-  3,                          /* iVersion */
+  4,                          /* iVersion */
   geopolyCreate,              /* xCreate - create a table */
   geopolyConnect,             /* xConnect - connect to an existing table */
   geopolyBestIndex,           /* xBestIndex - Determine search strategy */
@@ -1759,7 +1759,8 @@ static sqlite3_module geopolyModule = {
   rtreeSavepoint,             /* xSavepoint */
   0,                          /* xRelease */
   0,                          /* xRollbackTo */
-  rtreeShadowName             /* xShadowName */
+  rtreeShadowName,            /* xShadowName */
+  rtreeShadowName2            /* xShadowName2 */
 };
 
 static int sqlite3_geopoly_init(sqlite3 *db){

ext/rtree/rtree.c

@@ -3477,8 +3477,13 @@ static int rtreeShadowName(const char *zName){
   return 0;
 }
 
+static int rtreeShadowName2(sqlite3_vtab *pVtab, const char *zName){
+  UNUSED_PARAMETER(pVtab);
+  return rtreeShadowName(zName);
+}
+
 static sqlite3_module rtreeModule = {
-  3,                          /* iVersion */
+  4,                          /* iVersion */
   rtreeCreate,                /* xCreate - create a table */
   rtreeConnect,               /* xConnect - connect to an existing table */
   rtreeBestIndex,             /* xBestIndex - Determine search strategy */
@@ -3501,7 +3506,8 @@ static sqlite3_module rtreeModule = {
   rtreeSavepoint,             /* xSavepoint */
   0,                          /* xRelease */
   0,                          /* xRollbackTo */
-  rtreeShadowName             /* xShadowName */
+  rtreeShadowName,            /* xShadowName */
+  rtreeShadowName2            /* xShadowName2 */
 };
 
 static int rtreeSqlInit(

ext/rtree/rtree1.test

@@ -733,4 +733,27 @@ do_execsql_test 19.1 {
   SELECT v0.x FROM v0, rt0 WHERE v0.x = rt0.b;
 } {0.0}
 
+# 2022-06-20 https://sqlite.org/forum/forumpost/57bdf2217d
+#
+reset_db
+do_execsql_test 20.0 {
+  CREATE VIRTUAL TABLE rt0 USING rtree(id, x0, x1);
+  CREATE TABLE t0(a INT);
+  CREATE TABLE t1(b INT);
+  INSERT INTO rt0 VALUES(0, 0, 0);
+}
+do_catchsql_test 20.1 {
+  SELECT * FROM t1 JOIN t0 ON x0>a RIGHT JOIN rt0 ON true WHERE +x0 = 0;
+} {1 {ON clause references tables to its right}}
+do_catchsql_test 20.2 {
+  SELECT * FROM t1 JOIN t0 ON x0>a RIGHT JOIN rt0 ON true WHERE x0 = 0;
+} {1 {ON clause references tables to its right}}
+db null -
+do_execsql_test 20.3 {
+  SELECT * FROM t1 JOIN t0 ON true RIGHT JOIN rt0 ON x0>a WHERE +x0 = 0;
+} {- - 0 0.0 0.0}
+do_execsql_test 20.4 {
+  SELECT * FROM t1 JOIN t0 ON true RIGHT JOIN rt0 ON x0>a WHERE x0 = 0;
+} {- - 0 0.0 0.0}
+
 finish_test

ext/rtree/rtree3.test

@@ -23,6 +23,10 @@ ifcapable !rtree {
   return
 }
 
+set ::TMPDBERROR [list 1 \
+  {unable to open a temporary database file for storing temporary tables}
+]
+
 # Test summary:
 #
 #   rtree3-1: Test OOM in simple CREATE TABLE, INSERT, DELETE and SELECT 
@@ -196,9 +200,9 @@ do_test rtree3-7.prep {
 do_faultsim_test rtree3-7 -faults oom-* -prep {
   faultsim_restore_and_reopen
 } -body {
-  execsql { ALTER TABLE rt RENAME TO rt2 }
+  execsql { ALTER TABLE rt RENAME TO rt2 } 
 } -test {
-  faultsim_test_result {0 {}}
+  faultsim_test_result {0 {}} $::TMPDBERROR
 }
 
 do_faultsim_test rtree3-8 -faults oom-* -prep {

ext/rtree/rtreeC.test

@@ -342,7 +342,7 @@ do_eqp_execsql_test 7.2 {
   QUERY PLAN
   |--SCAN xdir
   |--SCAN rt VIRTUAL TABLE INDEX 2:B0D1
-  `--SCAN ydir
+  `--SCAN ydir LEFT-JOIN
 } {
   5 1 2 7 12 14 {}
   5 2 2 7  8 12 10

ext/rtree/test_rtreedoc.c

@@ -324,7 +324,7 @@ static int SQLITE_TCLAPI register_box_query(
   }
   if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
 
-  pCtx = (BoxQueryCtx*)ckalloc(sizeof(BoxQueryCtx*));
+  pCtx = (BoxQueryCtx*)ckalloc(sizeof(BoxQueryCtx));
   pCtx->interp = interp;
   pCtx->pScript = Tcl_DuplicateObj(objv[2]);
   Tcl_IncrRefCount(pCtx->pScript);

main.mk

@@ -423,6 +423,7 @@ TESTSRC2 = \
   $(TOP)/src/select.c \
   $(TOP)/src/threads.c \
   $(TOP)/src/tokenize.c \
+  $(TOP)/src/treeview.c \
   $(TOP)/src/utf.c \
   $(TOP)/src/util.c \
   $(TOP)/src/vdbeapi.c \
@@ -529,7 +530,7 @@ TESTOPTS = --verbose=file --output=test-out.txt
 
 # Extra compiler options for various shell tools
 #
-SHELL_OPT += -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
+SHELL_OPT += -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
 SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
@@ -538,8 +539,7 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1
-FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5
+FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
@@ -547,6 +547,9 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
 FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
+FUZZSRC += $(TOP)/test/fuzzcheck.c
+FUZZSRC += $(TOP)/test/ossfuzz.c
+FUZZSRC += $(TOP)/test/fuzzinvariants.c
 DBFUZZ_OPT =
 KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
 ST_OPT = -DSQLITE_THREADSAFE=0
@@ -605,10 +608,10 @@ dbfuzz2$(EXE):	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
 	$(TCCX) -I. -g -O0 -DSTANDALONE -o dbfuzz2$(EXE) \
 	  $(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c  $(TLIBS) $(THREADLIB)
 
-fuzzcheck$(EXE):	$(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(TOP)/test/ossfuzz.c
+fuzzcheck$(EXE):	$(FUZZSRC) sqlite3.c sqlite3.h
 	$(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
 		-DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) -DSQLITE_OSS_FUZZ \
-		$(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) $(THREADLIB)
+		$(FUZZSRC) sqlite3.c $(TLIBS) $(THREADLIB)
 
 ossshell$(EXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(TCCX) -o ossshell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \
@@ -955,6 +958,12 @@ valgrindfuzz:	fuzzcheck$(EXE) $(FUZZDATA) sessionfuzz$(EXE) $(TOP)/test/sessionf
 tcltest:	./testfixture$(EXE)
 	./testfixture$(EXE) $(TOP)/test/veryquick.test $(TESTOPTS)
 
+# Runs all the same tests cases as the "tcltest" target but uses
+# the testrunner.tcl script to run them in multiple cores
+# concurrently.
+testrunner:	testfixture$(EXE)
+	./testfixture$(EXE) $(TOP)/test/testrunner.tcl
+
 # A very quick test using only testfixture and omitting all the slower
 # tests.  Designed to run in under 3 minutes on a workstation.
 #
@@ -965,6 +974,7 @@ quicktest:	./testfixture$(EXE)
 # and fuzz tests, and sqlite3_analyzer and sqldiff tests.
 test:	fuzztest sourcetest $(TESTPROGS) tcltest
 
+
 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
 #

manifest.uuid

@@ -1 +1 @@
-4289edf3c5e32a05b51f232020099b33f6f5e79b0ceca2b96baf1186168d9af6
+163c6cdf93451cb4ae7655266654d4aec2b94c4ceb0a6fc68859ecc73bf393a9

src/alter.c

@@ -324,7 +324,9 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){
   int r1;                   /* Temporary registers */
 
   db = pParse->db;
-  if( pParse->nErr || db->mallocFailed ) return;
+  assert( db->pParse==pParse );
+  if( pParse->nErr ) return;
+  assert( db->mallocFailed==0 );
   pNew = pParse->pNewTable;
   assert( pNew );
 
@@ -629,7 +631,7 @@ void sqlite3AlterRenameColumn(
     if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break;
   }
   if( iCol==pTab->nCol ){
-    sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld);
+    sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld);
     goto exit_rename_column;
   }
 
@@ -735,7 +737,9 @@ struct RenameCtx {
 ** following a valid object, it may not be used in comparison operations.
 */
 static void renameTokenCheckAll(Parse *pParse, const void *pPtr){
-  if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){
+  assert( pParse==pParse->db->pParse );
+  assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 );
+  if( pParse->nErr==0 ){
     const RenameToken *p;
     u8 i = 0;
     for(p=pParse->pRename; p; p=p->pNext){
@@ -854,11 +858,10 @@ static void unmapColumnIdlistNames(
   Parse *pParse,
   const IdList *pIdList
 ){
-  if( pIdList ){
-    int ii;
-    for(ii=0; ii<pIdList->nId; ii++){
-      sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName);
-    }
+  int ii;
+  assert( pIdList!=0 );
+  for(ii=0; ii<pIdList->nId; ii++){
+    sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName);
   }
 }
 
@@ -877,7 +880,7 @@ static int renameUnmapSelectCb(Walker *pWalker, Select *p){
   if( ALWAYS(p->pEList) ){
     ExprList *pList = p->pEList;
     for(i=0; i<pList->nExpr; i++){
-      if( pList->a[i].zEName && pList->a[i].eEName==ENAME_NAME ){
+      if( pList->a[i].zEName && pList->a[i].fg.eEName==ENAME_NAME ){
         sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName);
       }
     }
@@ -886,8 +889,11 @@ static int renameUnmapSelectCb(Walker *pWalker, Select *p){
     SrcList *pSrc = p->pSrc;
     for(i=0; i<pSrc->nSrc; i++){
       sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName);
-      sqlite3WalkExpr(pWalker, pSrc->a[i].pOn);
-      unmapColumnIdlistNames(pParse, pSrc->a[i].pUsing);
+      if( pSrc->a[i].fg.isUsing==0 ){
+        sqlite3WalkExpr(pWalker, pSrc->a[i].u3.pOn);
+      }else{
+        unmapColumnIdlistNames(pParse, pSrc->a[i].u3.pUsing);
+      }
     }
   }
 
@@ -923,7 +929,7 @@ void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){
     sWalker.xExprCallback = renameUnmapExprCb;
     sqlite3WalkExprList(&sWalker, pEList);
     for(i=0; i<pEList->nExpr; i++){
-      if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) ){
+      if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME) ){
         sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName);
       }
     }
@@ -1057,12 +1063,12 @@ static void renameColumnParseError(
   const char *zN = (const char*)sqlite3_value_text(pObject);
   char *zErr;
 
-  zErr = sqlite3_mprintf("error in %s %s%s%s: %s", 
+  zErr = sqlite3MPrintf(pParse->db, "error in %s %s%s%s: %s", 
       zT, zN, (zWhen[0] ? " " : ""), zWhen,
       pParse->zErrMsg
   );
   sqlite3_result_error(pCtx, zErr, -1);
-  sqlite3_free(zErr);
+  sqlite3DbFree(pParse->db, zErr);
 }
 
 /*
@@ -1081,7 +1087,7 @@ static void renameColumnElistNames(
     int i;
     for(i=0; i<pEList->nExpr; i++){
       const char *zName = pEList->a[i].zEName;
-      if( ALWAYS(pEList->a[i].eEName==ENAME_NAME)
+      if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME)
        && ALWAYS(zName!=0)
        && 0==sqlite3_stricmp(zName, zOld)
       ){
@@ -1127,19 +1133,21 @@ static int renameParseSql(
 ){
   int rc;
 
+  sqlite3ParseObjectInit(p, db);
+  if( zSql==0 ){
+    return SQLITE_NOMEM;
+  }
+  if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){
+    return SQLITE_CORRUPT_BKPT;
+  }
   db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb);
-
-  /* Parse the SQL statement passed as the first argument. If no error
-  ** occurs and the parse does not result in a new table, index or
-  ** trigger object, the database must be corrupt. */
-  memset(p, 0, sizeof(Parse));
   p->eParseMode = PARSE_MODE_RENAME;
   p->db = db;
   p->nQueryLoop = 1;
-  rc = zSql ? sqlite3RunParser(p, zSql) : SQLITE_NOMEM;
+  rc = sqlite3RunParser(p, zSql);
   if( db->mallocFailed ) rc = SQLITE_NOMEM;
   if( rc==SQLITE_OK 
-   && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 
+   && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0)
   ){
     rc = SQLITE_CORRUPT_BKPT;
   }
@@ -1309,27 +1317,33 @@ static int renameResolveTrigger(Parse *pParse){
     if( rc==SQLITE_OK && pStep->zTarget ){
       SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep);
       if( pSrc ){
-        int i;
-        for(i=0; i<pSrc->nSrc && rc==SQLITE_OK; i++){
-          SrcItem *p = &pSrc->a[i];
-          p->iCursor = pParse->nTab++;
-          if( p->pSelect ){
-            sqlite3SelectPrep(pParse, p->pSelect, 0);
-            sqlite3ExpandSubquery(pParse, p);
-            assert( i>0 );
-            assert( pStep->pFrom->a[i-1].pSelect );
-            sqlite3SelectPrep(pParse, pStep->pFrom->a[i-1].pSelect, 0);
-          }else{
-            p->pTab = sqlite3LocateTableItem(pParse, 0, p);
-            if( p->pTab==0 ){
-              rc = SQLITE_ERROR;
-            }else{
-              p->pTab->nTabRef++;
-              rc = sqlite3ViewGetColumnNames(pParse, p->pTab);
+        Select *pSel = sqlite3SelectNew(
+            pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0
+        );
+        if( pSel==0 ){
+          pStep->pExprList = 0;
+          pSrc = 0;
+          rc = SQLITE_NOMEM;
+        }else{
+          sqlite3SelectPrep(pParse, pSel, 0);
+          rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK;
+          assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList );
+          assert( pSrc==pSel->pSrc );
+          if( pStep->pExprList ) pSel->pEList = 0;
+          pSel->pSrc = 0;
+          sqlite3SelectDelete(db, pSel);
+        }
+        if( pStep->pFrom ){
+          int i;
+          for(i=0; i<pStep->pFrom->nSrc && rc==SQLITE_OK; i++){
+            SrcItem *p = &pStep->pFrom->a[i];
+            if( p->pSelect ){
+              sqlite3SelectPrep(pParse, p->pSelect, 0);
             }
           }
         }
-        if( rc==SQLITE_OK && db->mallocFailed ){
+
+        if(  db->mallocFailed ){
           rc = SQLITE_NOMEM;
         }
         sNC.pSrcList = pSrc;
@@ -1417,13 +1431,13 @@ static void renameParseCleanup(Parse *pParse){
   sqlite3DeleteTrigger(db, pParse->pNewTrigger);
   sqlite3DbFree(db, pParse->zErrMsg);
   renameTokenFree(db, pParse->pRename);
-  sqlite3ParserReset(pParse);
+  sqlite3ParseObjectReset(pParse);
 }
 
 /*
 ** SQL function:
 **
-**     sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld)
+**     sqlite_rename_column(SQL,TYPE,OBJ,DB,TABLE,COL,NEWNAME,QUOTE,TEMP)
 **
 **   0. zSql:     SQL statement to rewrite
 **   1. type:     Type of object ("table", "view" etc.)
@@ -1441,7 +1455,8 @@ static void renameParseCleanup(Parse *pParse){
 **
 ** This function is used internally by the ALTER TABLE RENAME COLUMN command.
 ** It is only accessible to SQL created using sqlite3NestedParse().  It is
-** not reachable from ordinary SQL passed into sqlite3_prepare().
+** not reachable from ordinary SQL passed into sqlite3_prepare() unless the
+** SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test setting is enabled.
 */
 static void renameColumnFunc(
   sqlite3_context *context,
@@ -1590,7 +1605,9 @@ static void renameColumnFunc(
 
 renameColumnFunc_done:
   if( rc!=SQLITE_OK ){
-    if( sParse.zErrMsg ){
+    if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){
+      sqlite3_result_value(context, argv[0]);
+    }else if( sParse.zErrMsg ){
       renameColumnParseError(context, "", argv[1], argv[2], &sParse);
     }else{
       sqlite3_result_error_code(context, rc);
@@ -1778,6 +1795,15 @@ static void renameTableFunc(
               if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){
                 renameTokenFind(&sParse, &sCtx, pStep->zTarget);
               }
+              if( pStep->pFrom ){
+                int i;
+                for(i=0; i<pStep->pFrom->nSrc; i++){
+                  SrcItem *pItem = &pStep->pFrom->a[i];
+                  if( 0==sqlite3_stricmp(pItem->zName, zOld) ){
+                    renameTokenFind(&sParse, &sCtx, pItem->zName);
+                  }
+                }
+              }
             }
           }
         }
@@ -1789,7 +1815,9 @@ static void renameTableFunc(
       rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote);
     }
     if( rc!=SQLITE_OK ){
-      if( sParse.zErrMsg ){
+      if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){
+        sqlite3_result_value(context, argv[3]);
+      }else if( sParse.zErrMsg ){
         renameColumnParseError(context, "", argv[1], argv[2], &sParse);
       }else{
         sqlite3_result_error_code(context, rc);
@@ -1814,10 +1842,10 @@ static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){
   return WRC_Continue;
 }
 
-/*
-** The implementation of an SQL scalar function that rewrites DDL statements
-** so that any string literals that use double-quotes are modified so that
-** they use single quotes.
+/* SQL function: sqlite_rename_quotefix(DB,SQL)
+**
+** Rewrite the DDL statement "SQL" so that any string literals that use
+** double-quotes use single quotes instead.
 **
 ** Two arguments must be passed:
 **
@@ -1836,6 +1864,10 @@ static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){
 ** returns the string:
 ** 
 **   CREATE VIEW v1 AS SELECT "a", 'string' FROM t1
+**
+** If there is a error in the input SQL, then raise an error, except
+** if PRAGMA writable_schema=ON, then just return the input string
+** unmodified following an error.
 */
 static void renameQuotefixFunc(
   sqlite3_context *context,
@@ -1910,7 +1942,11 @@ static void renameQuotefixFunc(
       renameTokenFree(db, sCtx.pList);
     }
     if( rc!=SQLITE_OK ){
-      sqlite3_result_error_code(context, rc);
+      if( sqlite3WritableSchema(db) && rc==SQLITE_ERROR ){
+        sqlite3_result_value(context, argv[1]);
+      }else{
+        sqlite3_result_error_code(context, rc);
+      }
     }
     renameParseCleanup(&sParse);
   }
@@ -1922,7 +1958,8 @@ static void renameQuotefixFunc(
   sqlite3BtreeLeaveAll(db);
 }
 
-/*
+/* Function:  sqlite_rename_test(DB,SQL,TYPE,NAME,ISTEMP,WHEN,DQS)
+**
 ** An SQL user function that checks that there are no parse or symbol
 ** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement.
 ** After an ALTER TABLE .. RENAME operation is performed and the schema
@@ -1937,11 +1974,13 @@ static void renameQuotefixFunc(
 **   5: "when" part of error message.
 **   6: True to disable the DQS quirk when parsing SQL.
 **
-** Unless it finds an error, this function normally returns NULL. However, it
-** returns integer value 1 if:
+** The return value is computed as follows:
 **
-**   * the SQL argument creates a trigger, and
-**   * the table that the trigger is attached to is in database zDb.
+**   A. If an error is seen and not in PRAGMA writable_schema=ON mode,
+**      then raise the error.
+**   B. Else if a trigger is created and the the table that the trigger is
+**      attached to is in database zDb, then return 1.
+**   C. Otherwise return NULL.
 */
 static void renameTableTest(
   sqlite3_context *context,
@@ -1986,12 +2025,16 @@ static void renameTableTest(
         if( rc==SQLITE_OK ){
           int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema);
           int i2 = sqlite3FindDbName(db, zDb);
-          if( i1==i2 ) sqlite3_result_int(context, 1);
+          if( i1==i2 ){
+            /* Handle output case B */
+            sqlite3_result_int(context, 1);
+          }
         }
       }
     }
 
-    if( rc!=SQLITE_OK && zWhen ){
+    if( rc!=SQLITE_OK && zWhen && !sqlite3WritableSchema(db) ){
+      /* Output case A */
       renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse);
     }
     renameParseCleanup(&sParse);
@@ -2107,7 +2150,7 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){
   }
   iCol = sqlite3ColumnIndex(pTab, zCol);
   if( iCol<0 ){
-    sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zCol);
+    sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pName);
     goto exit_drop_column;
   }
 

src/analyze.c

@@ -847,9 +847,14 @@ static void statGet(
     **   * "WHERE a=? AND b=?" matches 2 rows.
     **
     ** If D is the count of distinct values and K is the total number of 
-    ** rows, then each estimate is computed as:
+    ** rows, then each estimate is usually computed as:
     **
     **        I = (K+D-1)/D
+    **
+    ** In other words, I is K/D rounded up to the next whole integer.
+    ** However, if I is between 1.0 and 1.1 (in other words if I is
+    ** close to 1.0 but just a little larger) then do not round up but
+    ** instead keep the I value at 1.0.
     */
     sqlite3_str sStat;   /* Text of the constructed "stat" line */
     int i;               /* Loop counter */
@@ -860,6 +865,7 @@ static void statGet(
     for(i=0; i<p->nKeyCol; i++){
       u64 nDistinct = p->current.anDLt[i] + 1;
       u64 iVal = (p->nRow + nDistinct - 1) / nDistinct;
+      if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1;
       sqlite3_str_appendf(&sStat, " %llu", iVal);
       assert( p->current.anEq[i] );
     }
@@ -1023,7 +1029,7 @@ static void analyzeOneTable(
     memcpy(pStat1->zName, "sqlite_stat1", 13);
     pStat1->nCol = 3;
     pStat1->iPKey = -1;
-    sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNBLOB);
+    sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNAMIC);
   }
 #endif
 

src/attach.c

@@ -354,7 +354,7 @@ static void codeAttach(
   }
 
 #ifndef SQLITE_OMIT_AUTHORIZATION
-  if( pAuthArg ){
+  if( ALWAYS(pAuthArg) ){
     char *zAuthArg;
     if( pAuthArg->op==TK_STRING ){
       assert( !ExprHasProperty(pAuthArg, EP_IntValue) );
@@ -480,7 +480,11 @@ static int fixSelectCb(Walker *p, Select *pSelect){
       pItem->fg.fromDDL = 1;
     }
 #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER)
-    if( sqlite3WalkExpr(&pFix->w, pList->a[i].pOn) ) return WRC_Abort;
+    if( pList->a[i].fg.isUsing==0
+     && sqlite3WalkExpr(&pFix->w, pList->a[i].u3.pOn)
+    ){
+      return WRC_Abort;
+    }
 #endif
   }
   if( pSelect->pWith ){

src/backup.c

@@ -85,14 +85,13 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){
   if( i==1 ){
     Parse sParse;
     int rc = 0;
-    memset(&sParse, 0, sizeof(sParse));
-    sParse.db = pDb;
+    sqlite3ParseObjectInit(&sParse,pDb);
     if( sqlite3OpenTempDatabase(&sParse) ){
       sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg);
       rc = SQLITE_ERROR;
     }
     sqlite3DbFree(pErrorDb, sParse.zErrMsg);
-    sqlite3ParserReset(&sParse);
+    sqlite3ParseObjectReset(&sParse);
     if( rc ){
       return 0;
     }

src/btree.c

@@ -214,7 +214,7 @@ static int hasSharedCacheTableLock(
     int bSeen = 0;
     for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){
       Index *pIdx = (Index *)sqliteHashData(p);
-      if( pIdx->tnum==(int)iRoot ){
+      if( pIdx->tnum==iRoot ){
         if( bSeen ){
           /* Two or more indexes share the same root page.  There must
           ** be imposter tables.  So just return true.  The assert is not
@@ -807,7 +807,7 @@ void sqlite3BtreeClearCursor(BtCursor *pCur){
 /*
 ** In this version of BtreeMoveto, pKey is a packed index record
 ** such as is generated by the OP_MakeRecord opcode.  Unpack the
-** record and then call BtreeMovetoUnpacked() to do the work.
+** record and then call sqlite3BtreeIndexMoveto() to do the work.
 */
 static int btreeMoveto(
   BtCursor *pCur,     /* Cursor open on the btree to be searched */
@@ -1327,6 +1327,7 @@ static void btreeParseCell(
 ** the space used by the cell pointer.
 **
 ** cellSizePtrNoPayload()    =>   table internal nodes
+** cellSizePtrTableLeaf()    =>   table leaf nodes
 ** cellSizePtr()             =>   all index nodes & table leaf nodes
 */
 static u16 cellSizePtr(MemPage *pPage, u8 *pCell){
@@ -1352,13 +1353,6 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){
     }while( *(pIter)>=0x80 && pIter<pEnd );
   }
   pIter++;
-  if( pPage->intKey ){
-    /* pIter now points at the 64-bit integer key value, a variable length 
-    ** integer. The following block moves pIter to point at the first byte
-    ** past the end of the key value. */
-    pEnd = &pIter[9];
-    while( (*pIter++)&0x80 && pIter<pEnd );
-  }
   testcase( nSize==pPage->maxLocal );
   testcase( nSize==(u32)pPage->maxLocal+1 );
   if( nSize<=pPage->maxLocal ){
@@ -1398,6 +1392,58 @@ static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){
   assert( debuginfo.nSize==(u16)(pIter - pCell) || CORRUPT_DB );
   return (u16)(pIter - pCell);
 }
+static u16 cellSizePtrTableLeaf(MemPage *pPage, u8 *pCell){
+  u8 *pIter = pCell;   /* For looping over bytes of pCell */
+  u8 *pEnd;            /* End mark for a varint */
+  u32 nSize;           /* Size value to return */
+
+#ifdef SQLITE_DEBUG
+  /* The value returned by this function should always be the same as
+  ** the (CellInfo.nSize) value found by doing a full parse of the
+  ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
+  ** this function verifies that this invariant is not violated. */
+  CellInfo debuginfo;
+  pPage->xParseCell(pPage, pCell, &debuginfo);
+#endif
+
+  nSize = *pIter;
+  if( nSize>=0x80 ){
+    pEnd = &pIter[8];
+    nSize &= 0x7f;
+    do{
+      nSize = (nSize<<7) | (*++pIter & 0x7f);
+    }while( *(pIter)>=0x80 && pIter<pEnd );
+  }
+  pIter++;
+  /* pIter now points at the 64-bit integer key value, a variable length 
+  ** integer. The following block moves pIter to point at the first byte
+  ** past the end of the key value. */
+  if( (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80
+   && (*pIter++)&0x80 ){ pIter++; }
+  testcase( nSize==pPage->maxLocal );
+  testcase( nSize==(u32)pPage->maxLocal+1 );
+  if( nSize<=pPage->maxLocal ){
+    nSize += (u32)(pIter - pCell);
+    if( nSize<4 ) nSize = 4;
+  }else{
+    int minLocal = pPage->minLocal;
+    nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4);
+    testcase( nSize==pPage->maxLocal );
+    testcase( nSize==(u32)pPage->maxLocal+1 );
+    if( nSize>pPage->maxLocal ){
+      nSize = minLocal;
+    }
+    nSize += 4 + (u16)(pIter - pCell);
+  }
+  assert( nSize==debuginfo.nSize || CORRUPT_DB );
+  return (u16)nSize;
+}
 
 
 #ifdef SQLITE_DEBUG
@@ -1411,7 +1457,7 @@ static u16 cellSize(MemPage *pPage, int iCell){
 #ifndef SQLITE_OMIT_AUTOVACUUM
 /*
 ** The cell pCell is currently part of page pSrc but will ultimately be part
-** of pPage.  (pSrc and pPager are often the same.)  If pCell contains a
+** of pPage.  (pSrc and pPage are often the same.)  If pCell contains a
 ** pointer to an overflow page, insert an entry into the pointer-map for
 ** the overflow page that will be valid after pCell has been moved to pPage.
 */
@@ -1467,7 +1513,6 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
   assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE );
   assert( pPage->nOverflow==0 );
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
-  temp = 0;
   src = data = pPage->aData;
   hdr = pPage->hdrOffset;
   cellOffset = pPage->cellOffset;
@@ -1522,39 +1567,38 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
   cbrk = usableSize;
   iCellLast = usableSize - 4;
   iCellStart = get2byte(&data[hdr+5]);
-  for(i=0; i<nCell; i++){
-    u8 *pAddr;     /* The i-th cell pointer */
-    pAddr = &data[cellOffset + i*2];
-    pc = get2byte(pAddr);
-    testcase( pc==iCellFirst );
-    testcase( pc==iCellLast );
-    /* These conditions have already been verified in btreeInitPage()
-    ** if PRAGMA cell_size_check=ON.
-    */
-    if( pc<iCellStart || pc>iCellLast ){
-      return SQLITE_CORRUPT_PAGE(pPage);
+  if( nCell>0 ){
+    temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
+    memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart);
+    src = temp;
+    for(i=0; i<nCell; i++){
+      u8 *pAddr;     /* The i-th cell pointer */
+      pAddr = &data[cellOffset + i*2];
+      pc = get2byte(pAddr);
+      testcase( pc==iCellFirst );
+      testcase( pc==iCellLast );
+      /* These conditions have already been verified in btreeInitPage()
+      ** if PRAGMA cell_size_check=ON.
+      */
+      if( pc<iCellStart || pc>iCellLast ){
+        return SQLITE_CORRUPT_PAGE(pPage);
+      }
+      assert( pc>=iCellStart && pc<=iCellLast );
+      size = pPage->xCellSize(pPage, &src[pc]);
+      cbrk -= size;
+      if( cbrk<iCellStart || pc+size>usableSize ){
+        return SQLITE_CORRUPT_PAGE(pPage);
+      }
+      assert( cbrk+size<=usableSize && cbrk>=iCellStart );
+      testcase( cbrk+size==usableSize );
+      testcase( pc+size==usableSize );
+      put2byte(pAddr, cbrk);
+      memcpy(&data[cbrk], &src[pc], size);
     }
-    assert( pc>=iCellStart && pc<=iCellLast );
-    size = pPage->xCellSize(pPage, &src[pc]);
-    cbrk -= size;
-    if( cbrk<iCellStart || pc+size>usableSize ){
-      return SQLITE_CORRUPT_PAGE(pPage);
-    }
-    assert( cbrk+size<=usableSize && cbrk>=iCellStart );
-    testcase( cbrk+size==usableSize );
-    testcase( pc+size==usableSize );
-    put2byte(pAddr, cbrk);
-    if( temp==0 ){
-      if( cbrk==pc ) continue;
-      temp = sqlite3PagerTempSpace(pPage->pBt->pPager);
-      memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart);
-      src = temp;
-    }
-    memcpy(&data[cbrk], &src[pc], size);
   }
   data[hdr+7] = 0;
 
- defragment_out:
+defragment_out:
   assert( pPage->nFree>=0 );
   if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){
     return SQLITE_CORRUPT_PAGE(pPage);
@@ -1586,7 +1630,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
   const int hdr = pPg->hdrOffset;            /* Offset to page header */
   u8 * const aData = pPg->aData;             /* Page data */
   int iAddr = hdr + 1;                       /* Address of ptr to pc */
-  int pc = get2byte(&aData[iAddr]);          /* Address of a free slot */
+  u8 *pTmp = &aData[iAddr];                  /* Temporary ptr into aData[] */
+  int pc = get2byte(pTmp);                   /* Address of a free slot */
   int x;                                     /* Excess size of the slot */
   int maxPC = pPg->pBt->usableSize - nByte;  /* Max address for a usable slot */
   int size;                                  /* Size of the free slot */
@@ -1596,7 +1641,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
     /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each
     ** freeblock form a big-endian integer which is the size of the freeblock
     ** in bytes, including the 4-byte header. */
-    size = get2byte(&aData[pc+2]);
+    pTmp = &aData[pc+2];
+    size = get2byte(pTmp);
     if( (x = size - nByte)>=0 ){
       testcase( x==4 );
       testcase( x==3 );
@@ -1609,6 +1655,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
         ** fragmented bytes within the page. */
         memcpy(&aData[iAddr], &aData[pc], 2);
         aData[hdr+7] += (u8)x;
+        testcase( pc+x>maxPC );
+        return &aData[pc];
       }else if( x+pc > maxPC ){
         /* This slot extends off the end of the usable part of the page */
         *pRc = SQLITE_CORRUPT_PAGE(pPg);
@@ -1621,10 +1669,11 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
       return &aData[pc + x];
     }
     iAddr = pc;
-    pc = get2byte(&aData[pc]);
-    if( pc<=iAddr+size ){
+    pTmp = &aData[pc];
+    pc = get2byte(pTmp);
+    if( pc<=iAddr ){
       if( pc ){
-        /* The next slot in the chain is not past the end of the current slot */
+        /* The next slot in the chain comes before the current slot */
         *pRc = SQLITE_CORRUPT_PAGE(pPg);
       }
       return 0;
@@ -1655,6 +1704,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
   u8 * const data = pPage->aData;      /* Local cache of pPage->aData */
   int top;                             /* First byte of cell content area */
   int rc = SQLITE_OK;                  /* Integer return code */
+  u8 *pTmp;                            /* Temp ptr into data[] */
   int gap;        /* First byte of gap between cell pointers and cell content */
   
   assert( sqlite3PagerIswriteable(pPage->pDbPage) );
@@ -1673,7 +1723,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
   ** then the cell content offset of an empty page wants to be 65536.
   ** However, that integer is too large to be stored in a 2-byte unsigned
   ** integer, so a value of 0 is used in its place. */
-  top = get2byte(&data[hdr+5]);
+  pTmp = &data[hdr+5];
+  top = get2byte(pTmp);
   assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */
   if( gap>top ){
     if( top==0 && pPage->pBt->usableSize==65536 ){
@@ -1755,6 +1806,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   u16 x;                                /* Offset to cell content area */
   u32 iEnd = iStart + iSize;            /* First byte past the iStart buffer */
   unsigned char *data = pPage->aData;   /* Page content */
+  u8 *pTmp;                             /* Temporary ptr into data[] */
 
   assert( pPage->pBt!=0 );
   assert( sqlite3PagerIswriteable(pPage->pDbPage) );
@@ -1773,7 +1825,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     iFreeBlk = 0;  /* Shortcut for the case when the freelist is empty */
   }else{
     while( (iFreeBlk = get2byte(&data[iPtr]))<iStart ){
-      if( iFreeBlk<iPtr+4 ){
+      if( iFreeBlk<=iPtr ){
         if( iFreeBlk==0 ) break; /* TH3: corrupt082.100 */
         return SQLITE_CORRUPT_PAGE(pPage);
       }
@@ -1782,7 +1834,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */
       return SQLITE_CORRUPT_PAGE(pPage);
     }
-    assert( iFreeBlk>iPtr || iFreeBlk==0 );
+    assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB );
   
     /* At this point:
     **    iFreeBlk:   First freeblock after iStart, or zero if none
@@ -1817,7 +1869,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
     if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage);
     data[hdr+7] -= nFrag;
   }
-  x = get2byte(&data[hdr+5]);
+  pTmp = &data[hdr+5];
+  x = get2byte(pTmp);
   if( iStart<=x ){
     /* The new freeblock is at the beginning of the cell content area,
     ** so just extend the cell content area rather than create another
@@ -1861,7 +1914,6 @@ static int decodeFlags(MemPage *pPage, int flagByte){
   pPage->leaf = (u8)(flagByte>>3);  assert( PTF_LEAF == 1<<3 );
   flagByte &= ~PTF_LEAF;
   pPage->childPtrSize = 4-4*pPage->leaf;
-  pPage->xCellSize = cellSizePtr;
   pBt = pPage->pBt;
   if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){
     /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an
@@ -1873,6 +1925,7 @@ static int decodeFlags(MemPage *pPage, int flagByte){
     pPage->intKey = 1;
     if( pPage->leaf ){
       pPage->intKeyLeaf = 1;
+      pPage->xCellSize = cellSizePtrTableLeaf;
       pPage->xParseCell = btreeParseCellPtr;
     }else{
       pPage->intKeyLeaf = 0;
@@ -1890,12 +1943,17 @@ static int decodeFlags(MemPage *pPage, int flagByte){
     assert( (PTF_ZERODATA|PTF_LEAF)==10 );
     pPage->intKey = 0;
     pPage->intKeyLeaf = 0;
+    pPage->xCellSize = cellSizePtr;
     pPage->xParseCell = btreeParseCellPtrIndex;
     pPage->maxLocal = pBt->maxLocal;
     pPage->minLocal = pBt->minLocal;
   }else{
     /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is
     ** an error. */
+    pPage->intKey = 0;
+    pPage->intKeyLeaf = 0;
+    pPage->xCellSize = cellSizePtr;
+    pPage->xParseCell = btreeParseCellPtrIndex;
     return SQLITE_CORRUPT_PAGE(pPage);
   }
   pPage->max1bytePayload = pBt->max1bytePayload;
@@ -2053,7 +2111,7 @@ static int btreeInitPage(MemPage *pPage){
   pPage->nOverflow = 0;
   pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize;
   pPage->aCellIdx = data + pPage->childPtrSize + 8;
-  pPage->aDataEnd = pPage->aData + pBt->usableSize;
+  pPage->aDataEnd = pPage->aData + pBt->pageSize;
   pPage->aDataOfst = pPage->aData + pPage->childPtrSize;
   /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the
   ** number of cells on the page. */
@@ -2088,7 +2146,7 @@ static void zeroPage(MemPage *pPage, int flags){
   u8 hdr = pPage->hdrOffset;
   u16 first;
 
-  assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno );
+  assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB );
   assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage );
   assert( sqlite3PagerGetData(pPage->pDbPage) == data );
   assert( sqlite3PagerIswriteable(pPage->pDbPage) );
@@ -2104,7 +2162,7 @@ static void zeroPage(MemPage *pPage, int flags){
   pPage->nFree = (u16)(pBt->usableSize - first);
   decodeFlags(pPage, flags);
   pPage->cellOffset = first;
-  pPage->aDataEnd = &data[pBt->usableSize];
+  pPage->aDataEnd = &data[pBt->pageSize];
   pPage->aCellIdx = &data[first];
   pPage->aDataOfst = &data[pPage->childPtrSize];
   pPage->nOverflow = 0;
@@ -2230,7 +2288,7 @@ static int getAndInitPage(
       goto getAndInitPage_error2;
     }
   }
-  assert( (*ppPage)->pgno==pgno );
+  assert( (*ppPage)->pgno==pgno || CORRUPT_DB );
   assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) );
 
   /* If obtaining a child page for a cursor, we must verify that the page is
@@ -2249,7 +2307,7 @@ getAndInitPage_error1:
     pCur->pPage = pCur->apPage[pCur->iPage];
   }
   testcase( pgno==0 );
-  assert( pgno!=0 || rc==SQLITE_CORRUPT );
+  assert( pgno!=0 || rc!=SQLITE_OK );
   return rc;
 }
 
@@ -3871,12 +3929,17 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){
       }
       do {
         MemPage *pFreePg;
+        Pgno dbSize = btreePagecount(pBt);
         rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode);
         if( rc!=SQLITE_OK ){
           releasePage(pLastPg);
           return rc;
         }
         releasePage(pFreePg);
+        if( iFreePg>dbSize ){
+          releasePage(pLastPg);
+          return SQLITE_CORRUPT_BKPT;
+        }
       }while( bCommit && iFreePg>nFin );
       assert( iFreePg<iLastPg );
       
@@ -5186,8 +5249,6 @@ const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){
 ** vice-versa).
 */
 static int moveToChild(BtCursor *pCur, u32 newPgno){
-  BtShared *pBt = pCur->pBt;
-
   assert( cursorOwnsBtShared(pCur) );
   assert( pCur->eState==CURSOR_VALID );
   assert( pCur->iPage<BTCURSOR_MAX_DEPTH );
@@ -5201,7 +5262,8 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){
   pCur->apPage[pCur->iPage] = pCur->pPage;
   pCur->ix = 0;
   pCur->iPage++;
-  return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags);
+  return getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur,
+                        pCur->curPagerFlags);
 }
 
 #ifdef SQLITE_DEBUG
@@ -5307,7 +5369,7 @@ static int moveToRoot(BtCursor *pCur){
       }
       sqlite3BtreeClearCursor(pCur);
     }
-    rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage,
+    rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage,
                         0, pCur->curPagerFlags);
     if( rc!=SQLITE_OK ){
       pCur->eState = CURSOR_INVALID;
@@ -5317,7 +5379,7 @@ static int moveToRoot(BtCursor *pCur){
     pCur->curIntKey = pCur->pPage->intKey;
   }
   pRoot = pCur->pPage;
-  assert( pRoot->pgno==pCur->pgnoRoot );
+  assert( pRoot->pgno==pCur->pgnoRoot || CORRUPT_DB );
 
   /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor
   ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is
@@ -5638,6 +5700,69 @@ moveto_table_finish:
   return rc;
 }
 
+/*
+** Compare the "idx"-th cell on the page the cursor pCur is currently
+** pointing to to pIdxKey using xRecordCompare.  Return negative or
+** zero if the cell is less than or equal pIdxKey.  Return positive
+** if unknown.
+**
+**    Return value negative:     Cell at pCur[idx] less than pIdxKey
+**
+**    Return value is zero:      Cell at pCur[idx] equals pIdxKey
+**
+**    Return value positive:     Nothing is known about the relationship
+**                               of the cell at pCur[idx] and pIdxKey.
+**
+** This routine is part of an optimization.  It is always safe to return
+** a positive value as that will cause the optimization to be skipped.
+*/
+static int indexCellCompare(
+  BtCursor *pCur,
+  int idx,
+  UnpackedRecord *pIdxKey,
+  RecordCompare xRecordCompare
+){
+  MemPage *pPage = pCur->pPage;
+  int c;
+  int nCell;  /* Size of the pCell cell in bytes */
+  u8 *pCell = findCellPastPtr(pPage, idx);
+
+  nCell = pCell[0];
+  if( nCell<=pPage->max1bytePayload ){
+    /* This branch runs if the record-size field of the cell is a
+    ** single byte varint and the record fits entirely on the main
+    ** b-tree page.  */
+    testcase( pCell+nCell+1==pPage->aDataEnd );
+    c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey);
+  }else if( !(pCell[1] & 0x80) 
+    && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal
+  ){
+    /* The record-size field is a 2 byte varint and the record 
+    ** fits entirely on the main b-tree page.  */
+    testcase( pCell+nCell+2==pPage->aDataEnd );
+    c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey);
+  }else{
+    /* If the record extends into overflow pages, do not attempt
+    ** the optimization. */
+    c = 99;
+  }
+  return c;
+}
+
+/*
+** Return true (non-zero) if pCur is current pointing to the last
+** page of a table.
+*/
+static int cursorOnLastPage(BtCursor *pCur){
+  int i;
+  assert( pCur->eState==CURSOR_VALID );
+  for(i=0; i<pCur->iPage; i++){
+    MemPage *pPage = pCur->apPage[i];
+    if( pCur->aiIdx[i]<pPage->nCell ) return 0;
+  }
+  return 1;
+}
+
 /* Move the cursor so that it points to an entry in an index table
 ** near the key pIdxKey.   Return a success code.
 **
@@ -5688,6 +5813,43 @@ int sqlite3BtreeIndexMoveto(
        || pIdxKey->default_rc==-1
   );
 
+
+  /* Check to see if we can skip a lot of work.  Two cases:
+  **
+  **    (1) If the cursor is already pointing to the very last cell
+  **        in the table and the pIdxKey search key is greater than or
+  **        equal to that last cell, then no movement is required.
+  **
+  **    (2) If the cursor is on the last page of the table and the first
+  **        cell on that last page is less than or equal to the pIdxKey
+  **        search key, then we can start the search on the current page
+  **        without needing to go back to root.
+  */
+  if( pCur->eState==CURSOR_VALID
+   && pCur->pPage->leaf
+   && cursorOnLastPage(pCur)
+  ){
+    int c;
+    if( pCur->ix==pCur->pPage->nCell-1
+     && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0
+     && pIdxKey->errCode==SQLITE_OK
+    ){
+      *pRes = c;
+      return SQLITE_OK;  /* Cursor already pointing at the correct spot */
+    }
+    if( pCur->iPage>0 
+     && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0
+     && pIdxKey->errCode==SQLITE_OK
+    ){
+      pCur->curFlags &= ~BTCF_ValidOvfl;
+      if( !pCur->pPage->isInit ){
+        return SQLITE_CORRUPT_BKPT;
+      }
+      goto bypass_moveto_root;  /* Start search on the current page */
+    }
+    pIdxKey->errCode = SQLITE_OK;
+  }
+
   rc = moveToRoot(pCur);
   if( rc ){
     if( rc==SQLITE_EMPTY ){
@@ -5697,12 +5859,14 @@ int sqlite3BtreeIndexMoveto(
     }
     return rc;
   }
+
+bypass_moveto_root:
   assert( pCur->pPage );
   assert( pCur->pPage->isInit );
   assert( pCur->eState==CURSOR_VALID );
   assert( pCur->pPage->nCell > 0 );
-  assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey );
-  assert( pCur->curIntKey || pIdxKey );
+  assert( pCur->curIntKey==0 );
+  assert( pIdxKey!=0 );
   for(;;){
     int lwr, upr, idx, c;
     Pgno chldPg;
@@ -5716,7 +5880,7 @@ int sqlite3BtreeIndexMoveto(
     ** be the right kind (index or table) of b-tree page. Otherwise
     ** a moveToChild() or moveToRoot() call would have detected corruption.  */
     assert( pPage->nCell>0 );
-    assert( pPage->intKey==(pIdxKey==0) );
+    assert( pPage->intKey==0 );
     lwr = 0;
     upr = pPage->nCell-1;
     idx = upr>>1; /* idx = (lwr+upr)/2; */
@@ -5808,7 +5972,7 @@ int sqlite3BtreeIndexMoveto(
     assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) );
     assert( pPage->isInit );
     if( pPage->leaf ){
-      assert( pCur->ix<pCur->pPage->nCell );
+      assert( pCur->ix<pCur->pPage->nCell || CORRUPT_DB );
       pCur->ix = (u16)idx;
       *pRes = c;
       rc = SQLITE_OK;
@@ -6411,7 +6575,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   assert( CORRUPT_DB || iPage>1 );
   assert( !pMemPage || pMemPage->pgno==iPage );
 
-  if( NEVER(iPage<2) || iPage>pBt->nPage ){
+  if( iPage<2 || iPage>pBt->nPage ){
     return SQLITE_CORRUPT_BKPT;
   }
   if( pMemPage ){
@@ -8326,7 +8490,7 @@ static int balance_nonroot(
     iOvflSpace += sz;
     assert( sz<=pBt->maxLocal+23 );
     assert( iOvflSpace <= (int)pBt->pageSize );
-    for(k=0; b.ixNx[k]<=i && ALWAYS(k<NB*2); k++){}
+    for(k=0; b.ixNx[k]<=j && ALWAYS(k<NB*2); k++){}
     pSrcEnd = b.apEnd[k];
     if( SQLITE_WITHIN(pSrcEnd, pCell, pCell+sz) ){
       rc = SQLITE_CORRUPT_BKPT;
@@ -8577,7 +8741,6 @@ static int anotherValidCursor(BtCursor *pCur){
 */
 static int balance(BtCursor *pCur){
   int rc = SQLITE_OK;
-  const int nMin = pCur->pBt->usableSize * 2 / 3;
   u8 aBalanceQuickSpace[13];
   u8 *pFree = 0;
 
@@ -8589,7 +8752,11 @@ static int balance(BtCursor *pCur){
     MemPage *pPage = pCur->pPage;
 
     if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break;
-    if( pPage->nOverflow==0 && pPage->nFree<=nMin ){
+    if( pPage->nOverflow==0 && pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){
+      /* No rebalance required as long as:
+      **   (1) There are no overflow cells
+      **   (2) The amount of free space on the page is less than 2/3rds of
+      **       the total usable space on the page. */
       break;
     }else if( (iPage = pCur->iPage)==0 ){
       if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){
@@ -8809,7 +8976,7 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){
 ** pX.pData,nData,nZero fields must be zero.
 **
 ** If the seekResult parameter is non-zero, then a successful call to
-** MovetoUnpacked() to seek cursor pCur to (pKey,nKey) has already
+** sqlite3BtreeIndexMoveto() to seek cursor pCur to (pKey,nKey) has already
 ** been performed.  In other words, if seekResult!=0 then the cursor
 ** is currently pointing to a cell that will be adjacent to the cell
 ** to be inserted.  If seekResult<0 then pCur points to a cell that is
@@ -8827,7 +8994,7 @@ int sqlite3BtreeInsert(
   BtCursor *pCur,                /* Insert data into the table of this cursor */
   const BtreePayload *pX,        /* Content of the row to be inserted */
   int flags,                     /* True if this is likely an append */
-  int seekResult                 /* Result of prior MovetoUnpacked() call */
+  int seekResult                 /* Result of prior IndexMoveto() call */
 ){
   int rc;
   int loc = seekResult;          /* -1: before desired location  +1: after */
@@ -8842,24 +9009,6 @@ int sqlite3BtreeInsert(
   assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags );
   assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 );
 
-  if( pCur->eState==CURSOR_FAULT ){
-    assert( pCur->skipNext!=SQLITE_OK );
-    return pCur->skipNext;
-  }
-
-  assert( cursorOwnsBtShared(pCur) );
-  assert( (pCur->curFlags & BTCF_WriteFlag)!=0
-              && pBt->inTransaction==TRANS_WRITE
-              && (pBt->btsFlags & BTS_READ_ONLY)==0 );
-  assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
-
-  /* Assert that the caller has been consistent. If this cursor was opened
-  ** expecting an index b-tree, then the caller should be inserting blob
-  ** keys with no associated data. If the cursor was opened expecting an
-  ** intkey table, the caller should be inserting integer keys with a
-  ** blob of associated data.  */
-  assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) );
-
   /* Save the positions of any other cursors open on this table.
   **
   ** In some cases, the call to btreeMoveto() below is a no-op. For
@@ -8884,6 +9033,29 @@ int sqlite3BtreeInsert(
     }
   }
 
+  /* Ensure that the cursor is not in the CURSOR_FAULT state and that it
+  ** points to a valid cell.
+  */
+  if( pCur->eState>=CURSOR_REQUIRESEEK ){
+    testcase( pCur->eState==CURSOR_REQUIRESEEK );
+    testcase( pCur->eState==CURSOR_FAULT );
+    rc = moveToRoot(pCur);
+    if( rc && rc!=SQLITE_EMPTY ) return rc;
+  }
+
+  assert( cursorOwnsBtShared(pCur) );
+  assert( (pCur->curFlags & BTCF_WriteFlag)!=0
+              && pBt->inTransaction==TRANS_WRITE
+              && (pBt->btsFlags & BTS_READ_ONLY)==0 );
+  assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
+
+  /* Assert that the caller has been consistent. If this cursor was opened
+  ** expecting an index b-tree, then the caller should be inserting blob
+  ** keys with no associated data. If the cursor was opened expecting an
+  ** intkey table, the caller should be inserting integer keys with a
+  ** blob of associated data.  */
+  assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) );
+
   if( pCur->pKeyInfo==0 ){
     assert( pX->pKey==0 );
     /* If this is an insert into a table b-tree, invalidate any incrblob 
@@ -8972,14 +9144,14 @@ int sqlite3BtreeInsert(
     }
   }
   assert( pCur->eState==CURSOR_VALID 
-       || (pCur->eState==CURSOR_INVALID && loc)
-       || CORRUPT_DB );
+       || (pCur->eState==CURSOR_INVALID && loc) );
 
   pPage = pCur->pPage;
   assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) );
   assert( pPage->leaf || !pPage->intKey );
   if( pPage->nFree<0 ){
     if( NEVER(pCur->eState>CURSOR_INVALID) ){
+     /* ^^^^^--- due to the moveToRoot() call above */
       rc = SQLITE_CORRUPT_BKPT;
     }else{
       rc = btreeComputeFreeSpace(pPage);
@@ -8990,7 +9162,7 @@ int sqlite3BtreeInsert(
   TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n",
           pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno,
           loc==0 ? "overwrite" : "new entry"));
-  assert( pPage->isInit );
+  assert( pPage->isInit || CORRUPT_DB );
   newCell = pBt->pTmpSpace;
   assert( newCell!=0 );
   if( flags & BTREE_PREFORMAT ){
@@ -9141,7 +9313,11 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){
   u32 nRem;                     /* Bytes of data still to copy */
 
   getCellInfo(pSrc);
-  aOut += putVarint32(aOut, pSrc->info.nPayload);
+  if( pSrc->info.nPayload<0x80 ){
+    *(aOut++) = pSrc->info.nPayload;
+  }else{
+    aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload);
+  }
   if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey);
   nIn = pSrc->info.nLocal;
   aIn = pSrc->info.pPayload;
@@ -9260,12 +9436,16 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) );
   assert( !hasReadConflicts(p, pCur->pgnoRoot) );
   assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 );
-  if( pCur->eState==CURSOR_REQUIRESEEK ){
-    rc = btreeRestoreCursorPosition(pCur);
-    assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID );
-    if( rc || pCur->eState!=CURSOR_VALID ) return rc;
+  if( pCur->eState!=CURSOR_VALID ){
+    if( pCur->eState>=CURSOR_REQUIRESEEK ){
+      rc = btreeRestoreCursorPosition(pCur);
+      assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID );
+      if( rc || pCur->eState!=CURSOR_VALID ) return rc;
+    }else{
+      return SQLITE_CORRUPT_BKPT;
+    }
   }
-  assert( CORRUPT_DB || pCur->eState==CURSOR_VALID );
+  assert( pCur->eState==CURSOR_VALID );
 
   iCellDepth = pCur->iPage;
   iCellIdx = pCur->ix;
@@ -9297,7 +9477,8 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   bPreserve = (flags & BTREE_SAVEPOSITION)!=0;
   if( bPreserve ){
     if( !pPage->leaf 
-     || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3)
+     || (pPage->nFree+pPage->xCellSize(pPage,pCell)+2) >
+                                                   (int)(pBt->usableSize*2/3)
      || pPage->nCell==1  /* See dbfuzz001.test for a test case */
     ){
       /* A b-tree rebalance will be required after deleting this entry.
@@ -9393,7 +9574,15 @@ int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   ** been corrected, so be it. Otherwise, after balancing the leaf node,
   ** walk the cursor up the tree to the internal node and balance it as 
   ** well.  */
-  rc = balance(pCur);
+  assert( pCur->pPage->nOverflow==0 );
+  assert( pCur->pPage->nFree>=0 );
+  if( pCur->pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){
+    /* Optimization: If the free space is less than 2/3rds of the page,
+    ** then balance() will always be a no-op.  No need to invoke it. */
+    rc = SQLITE_OK;
+  }else{
+    rc = balance(pCur);
+  }
   if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){
     releasePageNotNull(pCur->pPage);
     pCur->iPage--;
@@ -9616,7 +9805,7 @@ static int clearDatabasePage(
   rc = getAndInitPage(pBt, pgno, &pPage, 0, 0);
   if( rc ) return rc;
   if( (pBt->openFlags & BTREE_SINGLE)==0 
-   && sqlite3PagerPageRefcount(pPage->pDbPage)!=1
+   && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1))
   ){
     rc = SQLITE_CORRUPT_BKPT;
     goto cleardatabasepage_out;

src/btreeInt.h

@@ -293,7 +293,9 @@ struct MemPage {
   u8 *apOvfl[4];       /* Pointers to the body of overflow cells */
   BtShared *pBt;       /* Pointer to BtShared that this page is part of */
   u8 *aData;           /* Pointer to disk image of the page data */
-  u8 *aDataEnd;        /* One byte past the end of usable data */
+  u8 *aDataEnd;        /* One byte past the end of the entire page - not just
+                       ** the usable space, the entire page.  Used to prevent
+                       ** corruption-induced buffer overflow. */
   u8 *aCellIdx;        /* The cell index area */
   u8 *aDataOfst;       /* Same as aData for leaves.  aData+4 for interior */
   DbPage *pDbPage;     /* Pager page handle */
@@ -598,7 +600,7 @@ struct BtCursor {
 /* 
 ** The database page the PENDING_BYTE occupies. This page is never used.
 */
-# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt)
+#define PENDING_BYTE_PAGE(pBt)  ((Pgno)((PENDING_BYTE/((pBt)->pageSize))+1))
 
 /*
 ** These macros define the location of the pointer-map entry for a 

src/build.c

@@ -143,11 +143,13 @@ void sqlite3FinishCoding(Parse *pParse){
 
   assert( pParse->pToplevel==0 );
   db = pParse->db;
+  assert( db->pParse==pParse );
   if( pParse->nested ) return;
-  if( db->mallocFailed || pParse->nErr ){
-    if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR;
+  if( pParse->nErr ){
+    if( db->mallocFailed ) pParse->rc = SQLITE_NOMEM;
     return;
   }
+  assert( db->mallocFailed==0 );
 
   /* Begin by generating some termination code at the end of the
   ** vdbe program
@@ -170,9 +172,7 @@ void sqlite3FinishCoding(Parse *pParse){
       int i;
       int reg;
 
-      if( NEVER(pReturning->nRetCol==0) ){
-        assert( CORRUPT_DB );
-      }else{
+      if( pReturning->nRetCol ){
         sqlite3VdbeAddOp0(v, OP_FkCheck);
         addrRewind =
            sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur);
@@ -212,7 +212,9 @@ void sqlite3FinishCoding(Parse *pParse){
       int iDb, i;
       assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init );
       sqlite3VdbeJumpHere(v, 0);
-      for(iDb=0; iDb<db->nDb; iDb++){
+      assert( db->nDb>0 );
+      iDb = 0;
+      do{
         Schema *pSchema;
         if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue;
         sqlite3VdbeUsesBtree(v, iDb);
@@ -227,7 +229,7 @@ void sqlite3FinishCoding(Parse *pParse){
         if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1);
         VdbeComment((v,
               "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite));
-      }
+      }while( ++iDb<db->nDb );
 #ifndef SQLITE_OMIT_VIRTUALTABLE
       for(i=0; i<pParse->nVtabLock; i++){
         char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]);
@@ -266,9 +268,7 @@ void sqlite3FinishCoding(Parse *pParse){
 
       if( pParse->bReturning ){
         Returning *pRet = pParse->u1.pReturning;
-        if( NEVER(pRet->nRetCol==0) ){
-          assert( CORRUPT_DB );
-        }else{
+        if( pRet->nRetCol ){
           sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol);
         }
       }
@@ -280,7 +280,9 @@ void sqlite3FinishCoding(Parse *pParse){
 
   /* Get the VDBE program ready for execution
   */
-  if( v && pParse->nErr==0 && !db->mallocFailed ){
+  assert( v!=0 || pParse->nErr );
+  assert( db->mallocFailed==0 || pParse->nErr );
+  if( pParse->nErr==0 ){
     /* A minimum of one cursor is required if autoincrement is used
     *  See ticket [a696379c1f08866] */
     assert( pParse->pAinc==0 || pParse->nTab>0 );
@@ -1289,7 +1291,8 @@ void sqlite3StartTable(
     pTable = sqlite3FindTable(db, zName, zDb);
     if( pTable ){
       if( !noErr ){
-        sqlite3ErrorMsg(pParse, "table %T already exists", pName);
+        sqlite3ErrorMsg(pParse, "%s %T already exists",
+                        (IsView(pTable)? "view" : "table"), pName);
       }else{
         assert( !db->init.busy || CORRUPT_DB );
         sqlite3CodeVerifySchema(pParse, iDb);
@@ -1885,7 +1888,7 @@ void sqlite3AddPrimaryKey(
     pTab->keyConf = (u8)onError;
     assert( autoInc==0 || autoInc==1 );
     pTab->tabFlags |= autoInc*TF_Autoincrement;
-    if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags;
+    if( pList ) pParse->iPkSortOrder = pList->a[0].fg.sortFlags;
     (void)sqlite3HasExplicitNulls(pParse, pList);
   }else if( autoInc ){
 #ifndef SQLITE_OMIT_AUTOINCREMENT
@@ -2379,15 +2382,16 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
     if( IN_RENAME_OBJECT ){
       sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey);
     }
-    pList->a[0].sortFlags = pParse->iPkSortOrder;
+    pList->a[0].fg.sortFlags = pParse->iPkSortOrder;
     assert( pParse->pNewTable==pTab );
     pTab->iPKey = -1;
     sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0,
                        SQLITE_IDXTYPE_PRIMARYKEY);
-    if( db->mallocFailed || pParse->nErr ){
+    if( pParse->nErr ){
       pTab->tabFlags &= ~TF_WithoutRowid;
       return;
     }
+    assert( db->mallocFailed==0 );
     pPk = sqlite3PrimaryKeyIndex(pTab);
     assert( pPk->nKeyCol==1 );
   }else{
@@ -2487,6 +2491,51 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
 
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
+/*
+** Returns non-zero if the module supports at least one of the xShadowName
+** or xShadowName2 methods.
+*/
+static int doesSupportShadowNames(
+  const sqlite3_module *pModule, /* The definition of the module */
+  VTable *pVTab                  /* The virtual table being asked */
+){
+  int iVersion;
+  assert( pModule!=0 );
+  iVersion = pModule->iVersion;
+  if( pVTab!=0 && iVersion>=4 && pModule->xShadowName2!=0 ) return 1;
+  if( iVersion>=3 && pModule->xShadowName!=0 ) return 1;
+  return 0;
+}
+
+/*
+** Attempts to calls one of the module xShadowName methods and returns its
+** result.  The xShadowName2 method will be preferred when supported (i.e.
+** the module is version 4 or later and has an xShadowName2 method).
+*/
+static int isShadowTableOf(
+  sqlite3 *db,      /* Database connection */
+  Table *pTab,      /* The Table that refers to the virtual table */
+  Module *pMod,     /* Definition of the module */
+  VTable *pVTab,    /* The virtual table being asked */
+  const char *zName /* Table name that may refer to a shadow table */
+){
+  int iVersion;
+  assert( pMod!=0 );
+  assert( pMod->pModule!=0 );
+  iVersion = pMod->pModule->iVersion;
+  assert( iVersion>=3 );
+  assert( zName!=0 );
+  assert( sqlite3Strlen30(zName)>0 );
+  if( pVTab!=0 && iVersion>=4 && pMod->pModule->xShadowName2!=0 ){
+    assert( pVTab->pVtab!=0 );
+    return pMod->pModule->xShadowName2(pVTab->pVtab, zName);
+  }
+  if( pMod->pModule->xShadowName!=0 ){
+    return pMod->pModule->xShadowName(zName);
+  }
+  return 0;
+}
+
 /*
 ** Return true if pTab is a virtual table and zName is a shadow table name
 ** for that virtual table.
@@ -2494,6 +2543,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
 int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){
   int nName;                    /* Length of zName */
   Module *pMod;                 /* Module for the virtual table */
+  VTable *pVTab;                /* The virtual table being asked */
 
   if( !IsVirtual(pTab) ) return 0;
   nName = sqlite3Strlen30(pTab->zName);
@@ -2501,9 +2551,10 @@ int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){
   if( zName[nName]!='_' ) return 0;
   pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->u.vtab.azArg[0]);
   if( pMod==0 ) return 0;
-  if( pMod->pModule->iVersion<3 ) return 0;
-  if( pMod->pModule->xShadowName==0 ) return 0;
-  return pMod->pModule->xShadowName(zName+nName+1);
+  if( NEVER(pMod->pModule==0) ) return 0;
+  pVTab = sqlite3GetVTable(db, pTab);
+  if( !doesSupportShadowNames(pMod->pModule, pVTab) ) return 0;
+  return isShadowTableOf(db, pTab, pMod, pVTab, zName+nName+1);
 }
 #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
 
@@ -2517,14 +2568,15 @@ int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){
 void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){
   int nName;                    /* Length of pTab->zName */
   Module *pMod;                 /* Module for the virtual table */
+  VTable *pVTab;                /* Virtual table instance */
   HashElem *k;                  /* For looping through the symbol table */
 
   assert( IsVirtual(pTab) );
   pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->u.vtab.azArg[0]);
   if( pMod==0 ) return;
   if( NEVER(pMod->pModule==0) ) return;
-  if( pMod->pModule->iVersion<3 ) return;
-  if( pMod->pModule->xShadowName==0 ) return;
+  pVTab = sqlite3GetVTable(db, pTab);
+  if( !doesSupportShadowNames(pMod->pModule, pVTab) ) return;
   assert( pTab->zName!=0 );
   nName = sqlite3Strlen30(pTab->zName);
   for(k=sqliteHashFirst(&pTab->pSchema->tblHash); k; k=sqliteHashNext(k)){
@@ -2534,7 +2586,7 @@ void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){
     if( pOther->tabFlags & TF_Shadow ) continue;
     if( sqlite3StrNICmp(pOther->zName, pTab->zName, nName)==0
      && pOther->zName[nName]=='_'
-     && pMod->pModule->xShadowName(pOther->zName+nName+1)
+     && isShadowTableOf(db, pTab, pMod, pVTab, pOther->zName+nName+1)
     ){
       pOther->tabFlags |= TF_Shadow;
     }
@@ -2818,6 +2870,11 @@ void sqlite3EndTable(
       int addrInsLoop;    /* Top of the loop for inserting rows */
       Table *pSelTab;     /* A table that describes the SELECT results */
 
+      if( IN_SPECIAL_PARSE ){
+        pParse->rc = SQLITE_ERROR;
+        pParse->nErr++;
+        return;
+      }
       regYield = ++pParse->nMem;
       regRec = ++pParse->nMem;
       regRowid = ++pParse->nMem;
@@ -3044,7 +3101,6 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
   Table *pSelTab;   /* A fake table from which we get the result set */
   Select *pSel;     /* Copy of the SELECT that implements the view */
   int nErr = 0;     /* Number of errors encountered */
-  int n;            /* Temporarily holds the number of cursors assigned */
   sqlite3 *db = pParse->db;  /* Database connection for malloc errors */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   int rc;
@@ -3102,8 +3158,9 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
   pSel = sqlite3SelectDup(db, pTable->u.view.pSelect, 0);
   if( pSel ){
     u8 eParseMode = pParse->eParseMode;
+    int nTab = pParse->nTab;
+    int nSelect = pParse->nSelect;
     pParse->eParseMode = PARSE_MODE_NORMAL;
-    n = pParse->nTab;
     sqlite3SrcListAssignCursors(pParse, pSel->pSrc);
     pTable->nCol = -1;
     DisableLookaside;
@@ -3115,7 +3172,8 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
 #else
     pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE);
 #endif
-    pParse->nTab = n;
+    pParse->nTab = nTab;
+    pParse->nSelect = nSelect;
     if( pSelTab==0 ){
       pTable->nCol = 0;
       nErr++;
@@ -3128,10 +3186,10 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){
       */
       sqlite3ColumnsFromExprList(pParse, pTable->pCheck, 
                                  &pTable->nCol, &pTable->aCol);
-      if( db->mallocFailed==0 
-       && pParse->nErr==0
+      if( pParse->nErr==0
        && pTable->nCol==pSel->pEList->nExpr
       ){
+        assert( db->mallocFailed==0 );
         sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel,
                                                SQLITE_AFF_NONE);
       }
@@ -3750,7 +3808,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
     tnum = pIndex->tnum;
   }
   pKey = sqlite3KeyInfoOfIndex(pParse, pIndex);
-  assert( pKey!=0 || db->mallocFailed || pParse->nErr );
+  assert( pKey!=0 || pParse->nErr );
 
   /* Open the sorter cursor if we are to use one. */
   iSorter = pParse->nTab++;
@@ -3860,8 +3918,8 @@ int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){
   if( pList ){
     int i;
     for(i=0; i<pList->nExpr; i++){
-      if( pList->a[i].bNulls ){
-        u8 sf = pList->a[i].sortFlags;
+      if( pList->a[i].fg.bNulls ){
+        u8 sf = pList->a[i].fg.sortFlags;
         sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", 
             (sf==0 || sf==3) ? "FIRST" : "LAST"
         );
@@ -3914,9 +3972,11 @@ void sqlite3CreateIndex(
   char *zExtra = 0;                /* Extra space after the Index object */
   Index *pPk = 0;      /* PRIMARY KEY index for WITHOUT ROWID tables */
 
-  if( db->mallocFailed || pParse->nErr>0 ){
+  assert( db->pParse==pParse );
+  if( pParse->nErr ){
     goto exit_create_index;
   }
+  assert( db->mallocFailed==0 );
   if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){
     goto exit_create_index;
   }
@@ -3980,7 +4040,6 @@ void sqlite3CreateIndex(
   pDb = &db->aDb[iDb];
 
   assert( pTab!=0 );
-  assert( pParse->nErr==0 );
   if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 
        && db->init.busy==0
        && pTblName!=0
@@ -4213,7 +4272,7 @@ void sqlite3CreateIndex(
       goto exit_create_index;
     }
     pIndex->azColl[i] = zColl;
-    requestedSortOrder = pListItem->sortFlags & sortOrderMask;
+    requestedSortOrder = pListItem->fg.sortFlags & sortOrderMask;
     pIndex->aSortOrder[i] = (u8)requestedSortOrder;
   }
 
@@ -4544,10 +4603,10 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){
   sqlite3 *db = pParse->db;
   int iDb;
 
-  assert( pParse->nErr==0 );   /* Never called with prior errors */
   if( db->mallocFailed ){
     goto exit_drop_index;
   }
+  assert( pParse->nErr==0 );   /* Never called with prior non-OOM errors */
   assert( pName->nSrc==1 );
   if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){
     goto exit_drop_index;
@@ -4656,18 +4715,17 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){
   if( pList==0 ){
     pList = sqlite3DbMallocZero(db, sizeof(IdList) );
     if( pList==0 ) return 0;
+  }else{
+    IdList *pNew;
+    pNew = sqlite3DbRealloc(db, pList,
+                 sizeof(IdList) + pList->nId*sizeof(pList->a));
+    if( pNew==0 ){
+      sqlite3IdListDelete(db, pList);
+      return 0;
+    }
+    pList = pNew;
   }
-  pList->a = sqlite3ArrayAllocate(
-      db,
-      pList->a,
-      sizeof(pList->a[0]),
-      &pList->nId,
-      &i
-  );
-  if( i<0 ){
-    sqlite3IdListDelete(db, pList);
-    return 0;
-  }
+  i = pList->nId++;
   pList->a[i].zName = sqlite3NameFromToken(db, pToken);
   if( IN_RENAME_OBJECT && pList->a[i].zName ){
     sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken);
@@ -4681,10 +4739,10 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){
 void sqlite3IdListDelete(sqlite3 *db, IdList *pList){
   int i;
   if( pList==0 ) return;
+  assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */
   for(i=0; i<pList->nId; i++){
     sqlite3DbFree(db, pList->a[i].zName);
   }
-  sqlite3DbFree(db, pList->a);
   sqlite3DbFreeNN(db, pList);
 }
 
@@ -4694,7 +4752,7 @@ void sqlite3IdListDelete(sqlite3 *db, IdList *pList){
 */
 int sqlite3IdListIndex(IdList *pList, const char *zName){
   int i;
-  if( pList==0 ) return -1;
+  assert( pList!=0 );
   for(i=0; i<pList->nId; i++){
     if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i;
   }
@@ -4897,8 +4955,11 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){
     if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg);
     sqlite3DeleteTable(db, pItem->pTab);
     if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect);
-    if( pItem->pOn ) sqlite3ExprDelete(db, pItem->pOn);
-    if( pItem->pUsing ) sqlite3IdListDelete(db, pItem->pUsing);
+    if( pItem->fg.isUsing ){
+      sqlite3IdListDelete(db, pItem->u3.pUsing);
+    }else if( pItem->u3.pOn ){
+      sqlite3ExprDelete(db, pItem->u3.pOn);
+    }
   }
   sqlite3DbFreeNN(db, pList);
 }
@@ -4926,14 +4987,13 @@ SrcList *sqlite3SrcListAppendFromTerm(
   Token *pDatabase,       /* Name of the database containing pTable */
   Token *pAlias,          /* The right-hand side of the AS subexpression */
   Select *pSubquery,      /* A subquery used in place of a table name */
-  Expr *pOn,              /* The ON clause of a join */
-  IdList *pUsing          /* The USING clause of a join */
+  OnOrUsing *pOnUsing     /* Either the ON clause or the USING clause */
 ){
   SrcItem *pItem;
   sqlite3 *db = pParse->db;
-  if( !p && (pOn || pUsing) ){
+  if( !p && pOnUsing!=0 && (pOnUsing->pOn || pOnUsing->pUsing) ){
     sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", 
-      (pOn ? "ON" : "USING")
+      (pOnUsing->pOn ? "ON" : "USING")
     );
     goto append_from_error;
   }
@@ -4953,15 +5013,27 @@ SrcList *sqlite3SrcListAppendFromTerm(
   if( pAlias->n ){
     pItem->zAlias = sqlite3NameFromToken(db, pAlias);
   }
-  pItem->pSelect = pSubquery;
-  pItem->pOn = pOn;
-  pItem->pUsing = pUsing;
+  if( pSubquery ){
+    pItem->pSelect = pSubquery;
+    if( pSubquery->selFlags & SF_NestedFrom ){
+      pItem->fg.isNestedFrom = 1;
+    }
+  }
+  assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 );
+  assert( pItem->fg.isUsing==0 );
+  if( pOnUsing==0 ){
+    pItem->u3.pOn = 0;
+  }else if( pOnUsing->pUsing ){
+    pItem->fg.isUsing = 1;
+    pItem->u3.pUsing = pOnUsing->pUsing;
+  }else{
+    pItem->u3.pOn = pOnUsing->pOn;
+  }
   return p;
 
 append_from_error:
   assert( p==0 );
-  sqlite3ExprDelete(db, pOn);
-  sqlite3IdListDelete(db, pUsing);
+  sqlite3ClearOnOrUsing(db, pOnUsing);
   sqlite3SelectDelete(db, pSubquery);
   return 0;
 }
@@ -5006,6 +5078,7 @@ SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){
       p1 = pNew;
       memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem));
       sqlite3DbFree(pParse->db, p2);
+      p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype);
     }
   }
   return p1;
@@ -5042,14 +5115,34 @@ void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){
 ** The operator is "natural cross join".  The A and B operands are stored
 ** in p->a[0] and p->a[1], respectively.  The parser initially stores the
 ** operator with A.  This routine shifts that operator over to B.
+**
+** Additional changes:
+**
+**   *   All tables to the left of the right-most RIGHT JOIN are tagged with
+**       JT_LTORJ (mnemonic: Left Table Of Right Join) so that the
+**       code generator can easily tell that the table is part of
+**       the left operand of at least one RIGHT JOIN.
 */
-void sqlite3SrcListShiftJoinType(SrcList *p){
-  if( p ){
-    int i;
-    for(i=p->nSrc-1; i>0; i--){
-      p->a[i].fg.jointype = p->a[i-1].fg.jointype;
-    }
+void sqlite3SrcListShiftJoinType(Parse *pParse, SrcList *p){
+  (void)pParse;
+  if( p && p->nSrc>1 ){
+    int i = p->nSrc-1;
+    u8 allFlags = 0;
+    do{
+      allFlags |= p->a[i].fg.jointype = p->a[i-1].fg.jointype;
+    }while( (--i)>0 );
     p->a[0].fg.jointype = 0;
+
+    /* All terms to the left of a RIGHT JOIN should be tagged with the
+    ** JT_LTORJ flags */
+    if( allFlags & JT_RIGHT ){
+      for(i=p->nSrc-1; ALWAYS(i>0) && (p->a[i].fg.jointype&JT_RIGHT)==0; i--){}
+      i--;
+      assert( i>=0 );
+      do{
+        p->a[i].fg.jointype |= JT_LTORJ;
+      }while( (--i)>=0 );
+    }
   }
 }
 

src/ctime.c

@@ -307,9 +307,6 @@ static const char * const sqlite3azCompileOpt[] = {
 #ifdef SQLITE_ENABLE_RTREE
   "ENABLE_RTREE",
 #endif
-#ifdef SQLITE_ENABLE_SELECTTRACE
-  "ENABLE_SELECTTRACE",
-#endif
 #ifdef SQLITE_ENABLE_SESSION
   "ENABLE_SESSION",
 #endif
@@ -331,6 +328,9 @@ static const char * const sqlite3azCompileOpt[] = {
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   "ENABLE_STMT_SCANSTATUS",
 #endif
+#ifdef SQLITE_ENABLE_TREETRACE
+  "ENABLE_TREETRACE",
+#endif
 #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
   "ENABLE_UNKNOWN_SQL_FUNCTION",
 #endif

src/date.c

@@ -276,7 +276,7 @@ static void computeJD(DateTime *p){
   p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000);
   p->validJD = 1;
   if( p->validHMS ){
-    p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000);
+    p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5);
     if( p->validTZ ){
       p->iJD -= p->tz*60000;
       p->validYMD = 0;
@@ -503,8 +503,10 @@ static void clearYMD_HMS_TZ(DateTime *p){
 ** is available.  This routine returns 0 on success and
 ** non-zero on any kind of error.
 **
-** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this
-** routine will always fail.
+** If the sqlite3GlobalConfig.bLocaltimeFault variable is non-zero then this
+** routine will always fail.  If bLocaltimeFault is nonzero and
+** sqlite3GlobalConfig.xAltLocaltime is not NULL, then xAltLocaltime() is
+** invoked in place of the OS-defined localtime() function.
 **
 ** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C
 ** library function localtime_r() is used to assist in the calculation of
@@ -520,7 +522,15 @@ static int osLocaltime(time_t *t, struct tm *pTm){
   sqlite3_mutex_enter(mutex);
   pX = localtime(t);
 #ifndef SQLITE_UNTESTABLE
-  if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0;
+  if( sqlite3GlobalConfig.bLocaltimeFault ){
+    if( sqlite3GlobalConfig.xAltLocaltime!=0
+     && 0==sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm)
+    ){
+      pX = pTm;
+    }else{
+      pX = 0;
+    }
+  }
 #endif
   if( pX ) *pTm = *pX;
 #if SQLITE_THREADSAFE>0
@@ -529,7 +539,13 @@ static int osLocaltime(time_t *t, struct tm *pTm){
   rc = pX==0;
 #else
 #ifndef SQLITE_UNTESTABLE
-  if( sqlite3GlobalConfig.bLocaltimeFault ) return 1;
+  if( sqlite3GlobalConfig.bLocaltimeFault ){
+    if( sqlite3GlobalConfig.xAltLocaltime!=0 ){
+      return sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm);
+    }else{
+      return 1;
+    }
+  }
 #endif
 #if HAVE_LOCALTIME_R
   rc = localtime_r(t, pTm)==0;
@@ -544,67 +560,56 @@ static int osLocaltime(time_t *t, struct tm *pTm){
 
 #ifndef SQLITE_OMIT_LOCALTIME
 /*
-** Compute the difference (in milliseconds) between localtime and UTC
-** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs,
-** return this value and set *pRc to SQLITE_OK. 
-**
-** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value
-** is undefined in this case.
+** Assuming the input DateTime is UTC, move it to its localtime equivalent.
 */
-static sqlite3_int64 localtimeOffset(
-  DateTime *p,                    /* Date at which to calculate offset */
-  sqlite3_context *pCtx,          /* Write error here if one occurs */
-  int *pRc                        /* OUT: Error code. SQLITE_OK or ERROR */
+static int toLocaltime(
+  DateTime *p,                   /* Date at which to calculate offset */
+  sqlite3_context *pCtx          /* Write error here if one occurs */
 ){
-  DateTime x, y;
   time_t t;
   struct tm sLocal;
+  int iYearDiff;
 
   /* Initialize the contents of sLocal to avoid a compiler warning. */
   memset(&sLocal, 0, sizeof(sLocal));
 
-  x = *p;
-  computeYMD_HMS(&x);
-  if( x.Y<1971 || x.Y>=2038 ){
+  computeJD(p);
+  if( p->iJD<2108667600*(i64)100000 /* 1970-01-01 */
+   || p->iJD>2130141456*(i64)100000 /* 2038-01-18 */
+  ){
     /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only
     ** works for years between 1970 and 2037. For dates outside this range,
     ** SQLite attempts to map the year into an equivalent year within this
     ** range, do the calculation, then map the year back.
     */
-    x.Y = 2000;
-    x.M = 1;
-    x.D = 1;
-    x.h = 0;
-    x.m = 0;
-    x.s = 0.0;
-  } else {
-    int s = (int)(x.s + 0.5);
-    x.s = s;
+    DateTime x = *p;
+    computeYMD_HMS(&x);
+    iYearDiff = (2000 + x.Y%4) - x.Y;
+    x.Y += iYearDiff;
+    x.validJD = 0;
+    computeJD(&x);
+    t = (time_t)(x.iJD/1000 -  21086676*(i64)10000);
+  }else{
+    iYearDiff = 0;
+    t = (time_t)(p->iJD/1000 -  21086676*(i64)10000);
   }
-  x.tz = 0;
-  x.validJD = 0;
-  computeJD(&x);
-  t = (time_t)(x.iJD/1000 - 21086676*(i64)10000);
   if( osLocaltime(&t, &sLocal) ){
     sqlite3_result_error(pCtx, "local time unavailable", -1);
-    *pRc = SQLITE_ERROR;
-    return 0;
+    return SQLITE_ERROR;
   }
-  y.Y = sLocal.tm_year + 1900;
-  y.M = sLocal.tm_mon + 1;
-  y.D = sLocal.tm_mday;
-  y.h = sLocal.tm_hour;
-  y.m = sLocal.tm_min;
-  y.s = sLocal.tm_sec;
-  y.validYMD = 1;
-  y.validHMS = 1;
-  y.validJD = 0;
-  y.rawS = 0;
-  y.validTZ = 0;
-  y.isError = 0;
-  computeJD(&y);
-  *pRc = SQLITE_OK;
-  return y.iJD - x.iJD;
+  p->Y = sLocal.tm_year + 1900 - iYearDiff;
+  p->M = sLocal.tm_mon + 1;
+  p->D = sLocal.tm_mday;
+  p->h = sLocal.tm_hour;
+  p->m = sLocal.tm_min;
+  p->s = sLocal.tm_sec + (p->iJD%1000)*0.001;
+  p->validYMD = 1;
+  p->validHMS = 1;
+  p->validJD = 0;
+  p->rawS = 0;
+  p->validTZ = 0;
+  p->isError = 0;
+  return SQLITE_OK;
 }
 #endif /* SQLITE_OMIT_LOCALTIME */
 
@@ -676,7 +681,9 @@ static int parseModifier(
         if( !p->rawS || p->validJD ){
           rc = 0;
           p->rawS = 0;
-        }else if( p->s>=-210866760000 && p->s<=253402300799 ){
+        }else if( p->s>=-21086676*(i64)10000        /* -4713-11-24 12:00:00 */
+               && p->s<=(25340230*(i64)10000)+799   /*  9999-12-31 23:59:59 */
+        ){
           r = p->s*1000.0 + 210866760000000.0;
           clearYMD_HMS_TZ(p);
           p->iJD = (sqlite3_int64)(r + 0.5);
@@ -697,7 +704,7 @@ static int parseModifier(
       ** SQLite (0..5373484.5) then the result will be NULL.
       */
       if( sqlite3_stricmp(z, "julianday")==0 ){
-        if( idx>1 ) return 1;
+        if( idx>1 ) return 1;  /* IMP: R-31176-64601 */
         if( p->validJD && p->rawS ){
           rc = 0;
           p->rawS = 0;
@@ -713,9 +720,7 @@ static int parseModifier(
       ** show local time.
       */
       if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){
-        computeJD(p);
-        p->iJD += localtimeOffset(p, pCtx, &rc);
-        clearYMD_HMS_TZ(p);
+        rc = toLocaltime(p, pCtx);
       }
       break;
     }
@@ -728,6 +733,7 @@ static int parseModifier(
       ** seconds since 1970.  Convert to a real julian day number.
       */
       if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){
+        if( idx>1 ) return 1;  /* IMP: R-49255-55373 */
         r = p->s*1000.0 + 210866760000000.0;
         if( r>=0.0 && r<464269060800000.0 ){
           clearYMD_HMS_TZ(p);
@@ -740,18 +746,31 @@ static int parseModifier(
 #ifndef SQLITE_OMIT_LOCALTIME
       else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){
         if( p->tzSet==0 ){
-          sqlite3_int64 c1;
+          i64 iOrigJD;              /* Original localtime */
+          i64 iGuess;               /* Guess at the corresponding utc time */
+          int cnt = 0;              /* Safety to prevent infinite loop */
+          int iErr;                 /* Guess is off by this much */
+
           computeJD(p);
-          c1 = localtimeOffset(p, pCtx, &rc);
-          if( rc==SQLITE_OK ){
-            p->iJD -= c1;
-            clearYMD_HMS_TZ(p);
-            p->iJD += c1 - localtimeOffset(p, pCtx, &rc);
-          }
+          iGuess = iOrigJD = p->iJD;
+          iErr = 0;
+          do{
+            DateTime new;
+            memset(&new, 0, sizeof(new));
+            iGuess -= iErr;
+            new.iJD = iGuess;
+            new.validJD = 1;
+            rc = toLocaltime(&new, pCtx);
+            if( rc ) return rc;
+            computeJD(&new);
+            iErr = new.iJD - iOrigJD;
+          }while( iErr && cnt++<3 );
+          memset(p, 0, sizeof(*p));
+          p->iJD = iGuess;
+          p->validJD = 1;
           p->tzSet = 1;
-        }else{
-          rc = SQLITE_OK;
         }
+        rc = SQLITE_OK;
       }
 #endif
       break;
@@ -1001,11 +1020,38 @@ static void datetimeFunc(
 ){
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
-    char zBuf[100];
+    int Y, s;
+    char zBuf[24];
     computeYMD_HMS(&x);
-    sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d %02d:%02d:%02d",
-                     x.Y, x.M, x.D, x.h, x.m, (int)(x.s));
-    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+    Y = x.Y;
+    if( Y<0 ) Y = -Y;
+    zBuf[1] = '0' + (Y/1000)%10;
+    zBuf[2] = '0' + (Y/100)%10;
+    zBuf[3] = '0' + (Y/10)%10;
+    zBuf[4] = '0' + (Y)%10;
+    zBuf[5] = '-';
+    zBuf[6] = '0' + (x.M/10)%10;
+    zBuf[7] = '0' + (x.M)%10;
+    zBuf[8] = '-';
+    zBuf[9] = '0' + (x.D/10)%10;
+    zBuf[10] = '0' + (x.D)%10;
+    zBuf[11] = ' ';
+    zBuf[12] = '0' + (x.h/10)%10;
+    zBuf[13] = '0' + (x.h)%10;
+    zBuf[14] = ':';
+    zBuf[15] = '0' + (x.m/10)%10;
+    zBuf[16] = '0' + (x.m)%10;
+    zBuf[17] = ':';
+    s = (int)x.s;
+    zBuf[18] = '0' + (s/10)%10;
+    zBuf[19] = '0' + (s)%10;
+    zBuf[20] = 0;
+    if( x.Y<0 ){
+      zBuf[0] = '-';
+      sqlite3_result_text(context, zBuf, 20, SQLITE_TRANSIENT);
+    }else{
+      sqlite3_result_text(context, &zBuf[1], 19, SQLITE_TRANSIENT);
+    }
   }
 }
 
@@ -1021,10 +1067,20 @@ static void timeFunc(
 ){
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
-    char zBuf[100];
+    int s;
+    char zBuf[16];
     computeHMS(&x);
-    sqlite3_snprintf(sizeof(zBuf), zBuf, "%02d:%02d:%02d", x.h, x.m, (int)x.s);
-    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+    zBuf[0] = '0' + (x.h/10)%10;
+    zBuf[1] = '0' + (x.h)%10;
+    zBuf[2] = ':';
+    zBuf[3] = '0' + (x.m/10)%10;
+    zBuf[4] = '0' + (x.m)%10;
+    zBuf[5] = ':';
+    s = (int)x.s;
+    zBuf[6] = '0' + (s/10)%10;
+    zBuf[7] = '0' + (s)%10;
+    zBuf[8] = 0;
+    sqlite3_result_text(context, zBuf, 8, SQLITE_TRANSIENT);
   }
 }
 
@@ -1040,10 +1096,28 @@ static void dateFunc(
 ){
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
-    char zBuf[100];
+    int Y;
+    char zBuf[16];
     computeYMD(&x);
-    sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d", x.Y, x.M, x.D);
-    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
+    Y = x.Y;
+    if( Y<0 ) Y = -Y;
+    zBuf[1] = '0' + (Y/1000)%10;
+    zBuf[2] = '0' + (Y/100)%10;
+    zBuf[3] = '0' + (Y/10)%10;
+    zBuf[4] = '0' + (Y)%10;
+    zBuf[5] = '-';
+    zBuf[6] = '0' + (x.M/10)%10;
+    zBuf[7] = '0' + (x.M)%10;
+    zBuf[8] = '-';
+    zBuf[9] = '0' + (x.D/10)%10;
+    zBuf[10] = '0' + (x.D)%10;
+    zBuf[11] = 0;
+    if( x.Y<0 ){
+      zBuf[0] = '-';
+      sqlite3_result_text(context, zBuf, 11, SQLITE_TRANSIENT);
+    }else{
+      sqlite3_result_text(context, &zBuf[1], 10, SQLITE_TRANSIENT);
+    }
   }
 }
 

src/dbpage.c

@@ -156,6 +156,7 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
   ){
     pIdxInfo->orderByConsumed = 1;
   }
+  sqlite3VtabUsesAllSchemas(pIdxInfo);
   return SQLITE_OK;
 }
 
@@ -333,7 +334,7 @@ static int dbpageUpdate(
     goto update_fail;
   }
   pBt = pTab->db->aDb[iDb].pBt;
-  if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){
+  if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){
     zErr = "bad page number";
     goto update_fail;
   }
@@ -371,11 +372,12 @@ static int dbpageBegin(sqlite3_vtab *pVtab){
   DbpageTable *pTab = (DbpageTable *)pVtab;
   sqlite3 *db = pTab->db;
   int i;
-  for(i=0; i<db->nDb; i++){
+  int rc = SQLITE_OK;
+  for(i=0; rc==SQLITE_OK && i<db->nDb; i++){
     Btree *pBt = db->aDb[i].pBt;
-    if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0);
+    if( pBt ) rc = sqlite3BtreeBeginTrans(pBt, 1, 0);
   }
-  return SQLITE_OK;
+  return rc;
 }
 
 
@@ -407,7 +409,8 @@ int sqlite3DbpageRegister(sqlite3 *db){
     0,                            /* xSavepoint */
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
-    0                             /* xShadowName */
+    0,                            /* xShadowName */
+    0                             /* xShadowName2 */
   };
   return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0);
 }

src/dbstat.c

@@ -891,7 +891,8 @@ int sqlite3DbstatRegister(sqlite3 *db){
     0,                            /* xSavepoint */
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
-    0                             /* xShadowName */
+    0,                            /* xShadowName */
+    0                             /* xShadowName2 */
   };
   return sqlite3_create_module(db, "dbstat", &dbstat_module, 0);
 }

src/delete.c

@@ -128,8 +128,8 @@ void sqlite3MaterializeView(
     assert( pFrom->nSrc==1 );
     pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName);
     pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName);
-    assert( pFrom->a[0].pOn==0 );
-    assert( pFrom->a[0].pUsing==0 );
+    assert( pFrom->a[0].fg.isUsing==0 );
+    assert( pFrom->a[0].u3.pOn==0 );
   }
   pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, 
                           SF_IncludeHidden, pLimit);
@@ -293,12 +293,13 @@ void sqlite3DeleteFrom(
 
   memset(&sContext, 0, sizeof(sContext));
   db = pParse->db;
-  if( pParse->nErr || db->mallocFailed ){
+  assert( db->pParse==pParse );
+  if( pParse->nErr ){
     goto delete_from_cleanup;
   }
+  assert( db->mallocFailed==0 );
   assert( pTabList->nSrc==1 );
 
-
   /* Locate the table which we want to delete.  This table has to be
   ** put in an SrcList structure because some of the subroutines we
   ** will be calling are designed to work with multiple tables and expect
@@ -323,6 +324,14 @@ void sqlite3DeleteFrom(
 # define isView 0
 #endif
 
+#if TREETRACE_ENABLED
+  if( sqlite3TreeTrace & 0x10000 ){
+    sqlite3TreeViewLine(0, "In sqlite3Delete() at %s:%d", __FILE__, __LINE__);
+    sqlite3TreeViewDelete(pParse->pWith, pTabList, pWhere,
+                          pOrderBy, pLimit, pTrigger);
+  }
+#endif
+
 #ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
   if( !isView ){
     pWhere = sqlite3LimitWhere(
@@ -476,7 +485,7 @@ void sqlite3DeleteFrom(
     **  ONEPASS_SINGLE: One-pass approach - at most one row deleted.
     **  ONEPASS_MULTI:  One-pass approach - any number of rows may be deleted.
     */
-    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1);
+    pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0,0,wcf,iTabCur+1);
     if( pWInfo==0 ) goto delete_from_cleanup;
     eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
     assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI );

src/expr.c

@@ -769,9 +769,8 @@ static void heightOfSelect(const Select *pSelect, int *pnHeight){
 ** if appropriate.
 */
 static void exprSetHeight(Expr *p){
-  int nHeight = 0;
-  heightOfExpr(p->pLeft, &nHeight);
-  heightOfExpr(p->pRight, &nHeight);
+  int nHeight = p->pLeft ? p->pLeft->nHeight : 0;
+  if( p->pRight && p->pRight->nHeight>nHeight ) nHeight = p->pRight->nHeight;
   if( ExprUseXSelect(p) ){
     heightOfSelect(p->x.pSelect, &nHeight);
   }else if( p->x.pList ){
@@ -1070,6 +1069,8 @@ Expr *sqlite3ExprFunction(
     sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */
     return 0;
   }
+  assert( !ExprHasProperty(pNew, EP_InnerON|EP_OuterON) );
+  pNew->w.iOfst = (int)(pToken->z - pParse->zTail);
   if( pList 
    && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG]
    && !pParse->nested
@@ -1113,7 +1114,7 @@ void sqlite3ExprFunctionUsable(
       **         SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning
       **         that the schema is possibly tainted).
       */
-      sqlite3ErrorMsg(pParse, "unsafe use of %s()", pDef->zName);
+      sqlite3ErrorMsg(pParse, "unsafe use of %#T()", pExpr);
     }
   }
 }
@@ -1169,6 +1170,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){
       if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){
         sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d",
             db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]);
+        sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr);
         return;
       }
       x = (ynVar)i;
@@ -1196,6 +1198,7 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){
   pExpr->iColumn = x;
   if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){
     sqlite3ErrorMsg(pParse, "too many SQL variables");
+    sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr);
   }
 }
 
@@ -1235,10 +1238,6 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){
 #endif
     }
   }
-  if( ExprHasProperty(p, EP_MemToken) ){
-    assert( !ExprHasProperty(p, EP_IntValue) );
-    sqlite3DbFree(db, p->u.zToken);
-  }
   if( !ExprHasProperty(p, EP_Static) ){
     sqlite3DbFreeNN(db, p);
   }
@@ -1247,6 +1246,18 @@ void sqlite3ExprDelete(sqlite3 *db, Expr *p){
   if( p ) sqlite3ExprDeleteNN(db, p);
 }
 
+/*
+** Clear both elements of an OnOrUsing object
+*/
+void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){
+  if( p==0 ){
+    /* Nothing to clear */
+  }else if( p->pOn ){
+    sqlite3ExprDeleteNN(db, p->pOn);
+  }else if( p->pUsing ){
+    sqlite3IdListDelete(db, p->pUsing);
+  }
+}
 
 /*
 ** Arrange to cause pExpr to be deleted when the pParse is deleted.
@@ -1333,8 +1344,7 @@ static int dupedExprStructSize(const Expr *p, int flags){
     nSize = EXPR_FULLSIZE;
   }else{
     assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) );
-    assert( !ExprHasProperty(p, EP_FromJoin) ); 
-    assert( !ExprHasProperty(p, EP_MemToken) );
+    assert( !ExprHasProperty(p, EP_OuterON) ); 
     assert( !ExprHasVVAProperty(p, EP_NoReduce) );
     if( p->pLeft || p->x.pList ){
       nSize = EXPR_REDUCEDSIZE | EP_Reduced;
@@ -1438,7 +1448,7 @@ static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){
     }
 
     /* Set the EP_Reduced, EP_TokenOnly, and EP_Static flags appropriately. */
-    pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken);
+    pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static);
     pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly);
     pNew->flags |= staticFlag;
     ExprClearVVAProperties(pNew);
@@ -1513,6 +1523,7 @@ With *sqlite3WithDup(sqlite3 *db, With *p){
         pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0);
         pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0);
         pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName);
+        pRet->a[i].eM10d = p->a[i].eM10d;
       }
     }
   }
@@ -1613,11 +1624,8 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){
       }
     }
     pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName);
-    pItem->sortFlags = pOldItem->sortFlags;
-    pItem->eEName = pOldItem->eEName;
-    pItem->done = 0;
-    pItem->bNulls = pOldItem->bNulls;
-    pItem->bSorterRef = pOldItem->bSorterRef;
+    pItem->fg = pOldItem->fg;
+    pItem->fg.done = 0;
     pItem->u = pOldItem->u;
   }
   return pNew;
@@ -1669,8 +1677,12 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){
       pTab->nTabRef++;
     }
     pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags);
-    pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags);
-    pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing);
+    if( pOldItem->fg.isUsing ){
+      assert( pNewItem->fg.isUsing );
+      pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing);
+    }else{
+      pNewItem->u3.pOn = sqlite3ExprDup(db, pOldItem->u3.pOn, flags);
+    }
     pNewItem->colUsed = pOldItem->colUsed;
   }
   return pNew;
@@ -1680,22 +1692,16 @@ IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){
   int i;
   assert( db!=0 );
   if( p==0 ) return 0;
-  pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) );
+  assert( p->eU4!=EU4_EXPR );
+  pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) );
   if( pNew==0 ) return 0;
   pNew->nId = p->nId;
-  pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) );
-  if( pNew->a==0 ){
-    sqlite3DbFreeNN(db, pNew);
-    return 0;
-  }
-  /* Note that because the size of the allocation for p->a[] is not
-  ** necessarily a power of two, sqlite3IdListAppend() may not be called
-  ** on the duplicate created by this function. */
+  pNew->eU4 = p->eU4;
   for(i=0; i<p->nId; i++){
     struct IdList_item *pNewItem = &pNew->a[i];
-    struct IdList_item *pOldItem = &p->a[i];
+    const struct IdList_item *pOldItem = &p->a[i];
     pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
-    pNewItem->idx = pOldItem->idx;
+    pNewItem->u4 = pOldItem->u4;
   }
   return pNew;
 }
@@ -1919,16 +1925,16 @@ void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){
   );
 
   pItem = &p->a[p->nExpr-1];
-  assert( pItem->bNulls==0 );
+  assert( pItem->fg.bNulls==0 );
   if( iSortOrder==SQLITE_SO_UNDEFINED ){
     iSortOrder = SQLITE_SO_ASC;
   }
-  pItem->sortFlags = (u8)iSortOrder;
+  pItem->fg.sortFlags = (u8)iSortOrder;
 
   if( eNulls!=SQLITE_SO_UNDEFINED ){
-    pItem->bNulls = 1;
+    pItem->fg.bNulls = 1;
     if( iSortOrder!=eNulls ){
-      pItem->sortFlags |= KEYINFO_ORDER_BIGNULL;
+      pItem->fg.sortFlags |= KEYINFO_ORDER_BIGNULL;
     }
   }
 }
@@ -1954,7 +1960,7 @@ void sqlite3ExprListSetName(
     assert( pList->nExpr>0 );
     pItem = &pList->a[pList->nExpr-1];
     assert( pItem->zEName==0 );
-    assert( pItem->eEName==ENAME_NAME );
+    assert( pItem->fg.eEName==ENAME_NAME );
     pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n);
     if( dequote ){
       /* If dequote==0, then pName->z does not point to part of a DDL
@@ -1989,7 +1995,7 @@ void sqlite3ExprListSetSpan(
     assert( pList->nExpr>0 );
     if( pItem->zEName==0 ){
       pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd);
-      pItem->eEName = ENAME_SPAN;
+      pItem->fg.eEName = ENAME_SPAN;
     }
   }
 }
@@ -2161,9 +2167,9 @@ Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){
 static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
 
   /* If pWalker->eCode is 2 then any term of the expression that comes from
-  ** the ON or USING clauses of a left join disqualifies the expression
+  ** the ON or USING clauses of an outer join disqualifies the expression
   ** from being considered constant. */
-  if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){
+  if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_OuterON) ){
     pWalker->eCode = 0;
     return WRC_Abort;
   }
@@ -2282,6 +2288,42 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){
   return exprIsConst(p, 3, iCur);
 }
 
+/*
+** Check pExpr to see if it is an invariant constraint on data source pSrc.
+** This is an optimization.  False negatives will perhaps cause slower
+** queries, but false positives will yield incorrect answers.  So when in
+** doubt, return 0.
+**
+** To be an invariant constraint, the following must be true:
+**
+**   (1)  pExpr cannot refer to any table other than pSrc->iCursor.
+**
+**   (2)  pExpr cannot use subqueries or non-deterministic functions.
+**
+**   (3)  pSrc cannot be part of the left operand for a RIGHT JOIN.
+**        (Is there some way to relax this constraint?)
+**
+**   (4)  If pSrc is the right operand of a LEFT JOIN, then...
+**         (4a)  pExpr must come from an ON clause..
+           (4b)  and specifically the ON clause associated with the LEFT JOIN.
+**
+**   (5)  If pSrc is not the right operand of a LEFT JOIN or the left
+**        operand of a RIGHT JOIN, then pExpr must be from the WHERE
+**        clause, not an ON clause.
+*/
+int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc){
+  if( pSrc->fg.jointype & JT_LTORJ ){
+    return 0;  /* rule (3) */
+  }
+  if( pSrc->fg.jointype & JT_LEFT ){
+    if( !ExprHasProperty(pExpr, EP_OuterON) ) return 0;   /* rule (4a) */
+    if( pExpr->w.iJoin!=pSrc->iCursor ) return 0;         /* rule (4b) */
+  }else{
+    if( ExprHasProperty(pExpr, EP_OuterON) ) return 0;    /* rule (5) */
+  }
+  return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */
+}
+
 
 /*
 ** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy().
@@ -2609,7 +2651,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){
 ** all members of the RHS set, skipping duplicates.
 **
 ** A cursor is opened on the b-tree object that is the RHS of the IN operator
-** and pX->iTable is set to the index of that cursor.
+** and the *piTab parameter is set to the index of that cursor.
 **
 ** The returned value of this function indicates the b-tree type, as follows:
 **
@@ -2629,7 +2671,10 @@ static int sqlite3InRhsIsConstant(Expr *pIn){
 ** If the RHS of the IN operator is a list or a more complex subquery, then
 ** an ephemeral table might need to be generated from the RHS and then
 ** pX->iTable made to point to the ephemeral table instead of an
-** existing table.
+** existing table.  In this case, the creation and initialization of the
+** ephmeral table might be put inside of a subroutine, the EP_Subrtn flag
+** will be set on pX and the pX->y.sub fields will be set to show where
+** the subroutine is coded.
 **
 ** The inFlags parameter must contain, at a minimum, one of the bits
 ** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both.  If inFlags contains
@@ -2690,12 +2735,13 @@ int sqlite3FindInIndex(
 ){
   Select *p;                            /* SELECT to the right of IN operator */
   int eType = 0;                        /* Type of RHS table. IN_INDEX_* */
-  int iTab = pParse->nTab++;            /* Cursor of the RHS table */
+  int iTab;                             /* Cursor of the RHS table */
   int mustBeUnique;                     /* True if RHS must be unique */
   Vdbe *v = sqlite3GetVdbe(pParse);     /* Virtual machine being coded */
 
   assert( pX->op==TK_IN );
   mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0;
+  iTab = pParse->nTab++;
 
   /* If the RHS of this IN(...) operator is a SELECT, and if it matters 
   ** whether or not the SELECT result contains NULL values, check whether
@@ -2803,8 +2849,7 @@ int sqlite3FindInIndex(
             CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
             int j;
   
-            assert( pReq!=0 || pRhs->iColumn==XN_ROWID 
-                   || pParse->nErr || db->mallocFailed );
+            assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr );
             for(j=0; j<nExpr; j++){
               if( pIdx->aiColumn[j]!=pRhs->iColumn ) continue;
               assert( pIdx->azColl[j] );
@@ -2862,6 +2907,8 @@ int sqlite3FindInIndex(
    && ExprUseXList(pX)
    && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2)
   ){
+    pParse->nTab--;  /* Back out the allocation of the unused cursor */
+    iTab = -1;       /* Cursor is not allocated */
     eType = IN_INDEX_NOOP;
   }
 
@@ -3028,6 +3075,7 @@ void sqlite3CodeRhsOfIN(
       assert( ExprUseYSub(pExpr) );
       sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn,
                         pExpr->y.sub.iAddr);
+      assert( iTab!=pExpr->iTable );
       sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
       sqlite3VdbeJumpHere(v, addrOnce);
       return;
@@ -3039,8 +3087,7 @@ void sqlite3CodeRhsOfIN(
     assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
     pExpr->y.sub.regReturn = ++pParse->nMem;
     pExpr->y.sub.iAddr =
-      sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1;
-    VdbeComment((v, "return address"));
+      sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;
 
     addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   }
@@ -3142,6 +3189,7 @@ void sqlite3CodeRhsOfIN(
       ** expression we need to rerun this code each time.
       */
       if( addrOnce && !sqlite3ExprIsConstant(pE2) ){
+        sqlite3VdbeChangeToNoop(v, addrOnce-1);
         sqlite3VdbeChangeToNoop(v, addrOnce);
         ExprClearProperty(pExpr, EP_Subrtn);
         addrOnce = 0;
@@ -3162,8 +3210,11 @@ void sqlite3CodeRhsOfIN(
     sqlite3VdbeJumpHere(v, addrOnce);
     /* Subroutine return */
     assert( ExprUseYSub(pExpr) );
-    sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn);
-    sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1);
+    assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn
+            || pParse->nErr );
+    sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn,
+                      pExpr->y.sub.iAddr, 1);
+    VdbeCoverage(v);
     sqlite3ClearTempRegCache(pParse);
   }
 }
@@ -3217,9 +3268,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
   ExprSetProperty(pExpr, EP_Subrtn);
   pExpr->y.sub.regReturn = ++pParse->nMem;
   pExpr->y.sub.iAddr =
-    sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1;
-  VdbeComment((v, "return address"));
-
+    sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;
 
   /* The evaluation of the EXISTS/SELECT must be repeated every time it
   ** is encountered if any of the following is true:
@@ -3271,7 +3320,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
       pLimit = sqlite3PExpr(pParse, TK_NE,
                             sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit);
     }
-    sqlite3ExprDelete(db, pSel->pLimit->pLeft);
+    sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft);
     pSel->pLimit->pLeft = pLimit;
   }else{
     /* If there is no pre-existing limit add a limit of 1 */
@@ -3280,10 +3329,8 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
   }
   pSel->iLimit = 0;
   if( sqlite3Select(pParse, pSel, &dest) ){
-    if( pParse->nErr ){
-      pExpr->op2 = pExpr->op;
-      pExpr->op = TK_ERROR;
-    }
+    pExpr->op2 = pExpr->op;
+    pExpr->op = TK_ERROR;
     return 0;
   }
   pExpr->iTable = rReg = dest.iSDParm;
@@ -3294,8 +3341,11 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){
 
   /* Subroutine return */
   assert( ExprUseYSub(pExpr) );
-  sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn);
-  sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1);
+  assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn
+          || pParse->nErr );
+  sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn,
+                    pExpr->y.sub.iAddr, 1);
+  VdbeCoverage(v);
   sqlite3ClearTempRegCache(pParse);
   return rReg;
 }
@@ -3500,10 +3550,9 @@ static void sqlite3ExprCodeIN(
   }else{
     destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse);
   }
-  if( pParse->nErr ) goto sqlite3ExprCodeIN_finished;
   for(i=0; i<nVector; i++){
     Expr *p = sqlite3VectorFieldSubexpr(pExpr->pLeft, i);
-    if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error;
+    if( pParse->nErr ) goto sqlite3ExprCodeIN_oom_error;
     if( sqlite3ExprCanBeNull(p) ){
       sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2);
       VdbeCoverage(v);
@@ -3641,11 +3690,12 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){
     c = sqlite3DecOrHexToI64(z, &value);
     if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){
 #ifdef SQLITE_OMIT_FLOATING_POINT
-      sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z);
+      sqlite3ErrorMsg(pParse, "oversized integer: %s%#T", negFlag?"-":"",pExpr);
 #else
 #ifndef SQLITE_OMIT_HEX_INTEGER
       if( sqlite3_strnicmp(z,"0x",2)==0 ){
-        sqlite3ErrorMsg(pParse, "hex literal too big: %s%s", negFlag?"-":"",z);
+        sqlite3ErrorMsg(pParse, "hex literal too big: %s%#T",
+                        negFlag?"-":"",pExpr);
       }else
 #endif
       {
@@ -3729,6 +3779,7 @@ void sqlite3ExprCodeGetColumnOfTable(
   }
   if( iCol<0 || iCol==pTab->iPKey ){
     sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut);
+    VdbeComment((v, "%s.rowid", pTab->zName));
   }else{
     int op;
     int x;
@@ -3899,7 +3950,17 @@ static int exprCodeInlineFunction(
       caseExpr.x.pList = pFarg;
       return sqlite3ExprCodeTarget(pParse, &caseExpr, target);
     }
-
+#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
+    case INLINEFUNC_sqlite_offset: {
+      Expr *pArg = pFarg->a[0].pExpr;
+      if( pArg->op==TK_COLUMN && pArg->iTable>=0 ){
+        sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+      }
+      break;
+    }
+#endif
     default: {   
       /* The UNLIKELY() function is a no-op.  The result is the value
       ** of the first argument.
@@ -4146,6 +4207,7 @@ expr_code_doover:
       int n;
       const char *z;
       char *zBlob;
+      if( pParse->nErr ) return target;
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
       assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' );
       assert( pExpr->u.zToken[1]=='\'' );
@@ -4321,7 +4383,7 @@ expr_code_doover:
        || NEVER(pExpr->iAgg>=pInfo->nFunc)
       ){
         assert( !ExprHasProperty(pExpr, EP_IntValue) );
-        sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken);
+        sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr);
       }else{
         return pInfo->aFunc[pExpr->iAgg].iMem;
       }
@@ -4362,7 +4424,7 @@ expr_code_doover:
       }
 #endif
       if( pDef==0 || pDef->xFinalize!=0 ){
-        sqlite3ErrorMsg(pParse, "unknown function: %s()", zId);
+        sqlite3ErrorMsg(pParse, "unknown function: %#T()", pExpr);
         break;
       }
       if( pDef->funcFlags & SQLITE_FUNC_INLINE ){
@@ -4438,20 +4500,8 @@ expr_code_doover:
         if( !pColl ) pColl = db->pDfltColl; 
         sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ);
       }
-#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
-      if( (pDef->funcFlags & SQLITE_FUNC_OFFSET)!=0 && ALWAYS(pFarg!=0) ){
-        Expr *pArg = pFarg->a[0].pExpr;
-        if( pArg->op==TK_COLUMN ){
-          sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target);
-        }else{
-          sqlite3VdbeAddOp2(v, OP_Null, 0, target);
-        }
-      }else
-#endif
-      {
-        sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg,
-                                   pDef, pExpr->op2);
-      }
+      sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg,
+                                 pDef, pExpr->op2);
       if( nFarg ){
         if( constMask==0 ){
           sqlite3ReleaseTempRange(pParse, r1, nFarg);
@@ -4481,16 +4531,18 @@ expr_code_doover:
     }
     case TK_SELECT_COLUMN: {
       int n;
-      if( pExpr->pLeft->iTable==0 ){
-        pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft);
+      Expr *pLeft = pExpr->pLeft;
+      if( pLeft->iTable==0 || pParse->withinRJSubrtn > pLeft->op2 ){
+        pLeft->iTable = sqlite3CodeSubselect(pParse, pLeft);
+        pLeft->op2 = pParse->withinRJSubrtn;
       }
-      assert( pExpr->pLeft->op==TK_SELECT || pExpr->pLeft->op==TK_ERROR );
-      n = sqlite3ExprVectorSize(pExpr->pLeft);
+      assert( pLeft->op==TK_SELECT || pLeft->op==TK_ERROR );
+      n = sqlite3ExprVectorSize(pLeft);
       if( pExpr->iTable!=n ){
         sqlite3ErrorMsg(pParse, "%d columns assigned %d values",
                                 pExpr->iTable, n);
       }
-      return pExpr->pLeft->iTable + pExpr->iColumn;
+      return pLeft->iTable + pExpr->iColumn;
     }
     case TK_IN: {
       int destIfFalse = sqlite3VdbeMakeLabel(pParse);
@@ -4521,8 +4573,24 @@ expr_code_doover:
       exprCodeBetween(pParse, pExpr, target, 0, 0);
       return target;
     }
+    case TK_COLLATE: {
+      if( !ExprHasProperty(pExpr, EP_Collate)
+       && ALWAYS(pExpr->pLeft)
+       && pExpr->pLeft->op==TK_FUNCTION
+      ){
+        inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
+        if( inReg!=target ){
+          sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target);
+          inReg = target;
+        }
+        sqlite3VdbeAddOp1(v, OP_ClrSubtype, inReg);
+        return inReg;
+      }else{
+        pExpr = pExpr->pLeft;
+        goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */
+      }
+    }
     case TK_SPAN:
-    case TK_COLLATE: 
     case TK_UPLUS: {
       pExpr = pExpr->pLeft;
       goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. OSSFuzz. */
@@ -4763,7 +4831,9 @@ int sqlite3ExprCodeRunJustOnce(
     struct ExprList_item *pItem;
     int i;
     for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){
-      if( pItem->reusable && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 ){
+      if( pItem->fg.reusable
+       && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0
+      ){
         return pItem->u.iConstExprReg;
       }
     }
@@ -4786,7 +4856,7 @@ int sqlite3ExprCodeRunJustOnce(
     p = sqlite3ExprListAppend(pParse, p, pExpr);
     if( p ){
        struct ExprList_item *pItem = &p->a[p->nExpr-1];
-       pItem->reusable = regDest<0;
+       pItem->fg.reusable = regDest<0;
        if( regDest<0 ) regDest = ++pParse->nMem;
        pItem->u.iConstExprReg = regDest;
     }
@@ -4920,7 +4990,7 @@ int sqlite3ExprCodeExprList(
   for(pItem=pList->a, i=0; i<n; i++, pItem++){
     Expr *pExpr = pItem->pExpr;
 #ifdef SQLITE_ENABLE_SORTER_REFERENCES
-    if( pItem->bSorterRef ){
+    if( pItem->fg.bSorterRef ){
       i--;
       n--;
     }else
@@ -5014,8 +5084,8 @@ static void exprCodeBetween(
       ** so that the sqlite3ExprCodeTarget() routine will not attempt to move
       ** it into the Parse.pConstExpr list.  We should use a new bit for this,
       ** for clarity, but we are out of bits in the Expr.flags field so we
-      ** have to reuse the EP_FromJoin bit.  Bummer. */
-      pDel->flags |= EP_FromJoin;
+      ** have to reuse the EP_OuterON bit.  Bummer. */
+      pDel->flags |= EP_OuterON;
       sqlite3ExprCodeTarget(pParse, &exprAnd, dest);
     }
     sqlite3ReleaseTempReg(pParse, regFree1);
@@ -5545,7 +5615,7 @@ int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB, int iTab){
     int res;
     Expr *pExprA = pA->a[i].pExpr;
     Expr *pExprB = pB->a[i].pExpr;
-    if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1;
+    if( pA->a[i].fg.sortFlags!=pB->a[i].fg.sortFlags ) return 1;
     if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res;
   }
   return 0;
@@ -5700,7 +5770,7 @@ int sqlite3ExprImpliesExpr(
 static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){
   testcase( pExpr->op==TK_AGG_COLUMN );
   testcase( pExpr->op==TK_AGG_FUNCTION );
-  if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune;
+  if( ExprHasProperty(pExpr, EP_OuterON) ) return WRC_Prune;
   switch( pExpr->op ){
     case TK_ISNOT:
     case TK_ISNULL:
@@ -5797,8 +5867,8 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){
 ** False positives are not allowed, however.  A false positive may result
 ** in an incorrect answer.
 **
-** Terms of p that are marked with EP_FromJoin (and hence that come from
-** the ON or USING clauses of LEFT JOINS) are excluded from the analysis.
+** Terms of p that are marked with EP_OuterON (and hence that come from
+** the ON or USING clauses of OUTER JOINS) are excluded from the analysis.
 **
 ** This routine is used to check if a LEFT JOIN can be converted into
 ** an ordinary JOIN.  The p argument is the WHERE clause.  If the WHERE

src/fkey.c

@@ -390,7 +390,6 @@ static void fkLookupParent(
     }else{
       int nCol = pFKey->nCol;
       int regTemp = sqlite3GetTempRange(pParse, nCol);
-      int regRec = sqlite3GetTempReg(pParse);
   
       sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb);
       sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
@@ -429,12 +428,11 @@ static void fkLookupParent(
         }
         sqlite3VdbeGoto(v, iOk);
       }
-  
-      sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec,
+
+      sqlite3VdbeAddOp4(v, OP_Affinity, regTemp, nCol, 0,
                         sqlite3IndexAffinityStr(pParse->db,pIdx), nCol);
-      sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v);
-  
-      sqlite3ReleaseTempReg(pParse, regRec);
+      sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regTemp, nCol);
+      VdbeCoverage(v);
       sqlite3ReleaseTempRange(pParse, regTemp, nCol);
     }
   }
@@ -536,14 +534,10 @@ static Expr *exprTableColumn(
 **   Operation | FK type   | Action taken
 **   --------------------------------------------------------------------------
 **   DELETE      immediate   Increment the "immediate constraint counter".
-**                           Or, if the ON (UPDATE|DELETE) action is RESTRICT,
-**                           throw a "FOREIGN KEY constraint failed" exception.
 **
 **   INSERT      immediate   Decrement the "immediate constraint counter".
 **
 **   DELETE      deferred    Increment the "deferred constraint counter".
-**                           Or, if the ON (UPDATE|DELETE) action is RESTRICT,
-**                           throw a "FOREIGN KEY constraint failed" exception.
 **
 **   INSERT      deferred    Decrement the "deferred constraint counter".
 **
@@ -651,7 +645,7 @@ static void fkScanChildren(
   ** clause. For each row found, increment either the deferred or immediate
   ** foreign key constraint counter. */
   if( pParse->nErr==0 ){
-    pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0);
+    pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0, 0);
     sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
     if( pWInfo ){
       sqlite3WhereEnd(pWInfo);
@@ -1191,9 +1185,9 @@ int sqlite3FkRequired(
 **
 ** It returns a pointer to a Trigger structure containing a trigger
 ** equivalent to the ON UPDATE or ON DELETE action specified by pFKey.
-** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is
-** returned (these actions require no special handling by the triggers
-** sub-system, code for them is created by fkScanChildren()).
+** If the action is "NO ACTION" then a NULL pointer is returned (these actions
+** require no special handling by the triggers sub-system, code for them is
+** created by fkScanChildren()).
 **
 ** For example, if pFKey is the foreign key and pTab is table "p" in 
 ** the following schema:
@@ -1322,18 +1316,23 @@ static Trigger *fkActionTrigger(
     nFrom = sqlite3Strlen30(zFrom);
 
     if( action==OE_Restrict ){
+      int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
       Token tFrom;
+      Token tDb;
       Expr *pRaise; 
 
       tFrom.z = zFrom;
       tFrom.n = nFrom;
+      tDb.z = db->aDb[iDb].zDbSName;
+      tDb.n = sqlite3Strlen30(tDb.z);
+
       pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed");
       if( pRaise ){
         pRaise->affExpr = OE_Abort;
       }
       pSelect = sqlite3SelectNew(pParse, 
           sqlite3ExprListAppend(pParse, 0, pRaise),
-          sqlite3SrcListAppend(pParse, 0, &tFrom, 0),
+          sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom),
           pWhere,
           0, 0, 0, 0, 0
       );

src/func.c

@@ -106,6 +106,7 @@ static void subtypeFunc(
   int argc,
   sqlite3_value **argv
 ){
+  UNUSED_PARAMETER(argc);
   sqlite3_result_int(context, sqlite3_value_subtype(argv[0]));
 }
 
@@ -1119,8 +1120,9 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){
   sqlite3QuoteValue(&str,argv[0]);
   sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar,
                       SQLITE_DYNAMIC);
-  if( str.accError==SQLITE_NOMEM ){
-    sqlite3_result_error_nomem(context);
+  if( str.accError!=SQLITE_OK ){
+    sqlite3_result_null(context);
+    sqlite3_result_error_code(context, str.accError);
   }
 }
 
@@ -2096,11 +2098,11 @@ static void logFunc(
     switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){
       case 1:
         /* Convert from natural logarithm to log base 10 */
-        ans *= 1.0/M_LN10;
+        ans /= M_LN10;
         break;
       case 2:
         /* Convert from natural logarithm to log base 2 */
-        ans *= 1.0/M_LN2;
+        ans /= M_LN2;
         break;
       default:
         break;
@@ -2239,8 +2241,7 @@ void sqlite3RegisterBuiltinFunctions(void){
     INLINE_FUNC(likelihood,      2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY),
     INLINE_FUNC(likely,          1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY),
 #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC
-    FUNCTION2(sqlite_offset,     1, 0, 0, noopFunc,  SQLITE_FUNC_OFFSET|
-                                                     SQLITE_FUNC_TYPEOF),
+    INLINE_FUNC(sqlite_offset,   1, INLINEFUNC_sqlite_offset, 0 ),
 #endif
     FUNCTION(ltrim,              1, 1, 0, trimFunc         ),
     FUNCTION(ltrim,              2, 1, 0, trimFunc         ),

src/global.c

@@ -286,9 +286,13 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    0,                         /* xTestCallback */
 #endif
    0,                         /* bLocaltimeFault */
+   0,                         /* xAltLocaltime */
    0x7ffffffe,                /* iOnceResetThreshold */
    SQLITE_DEFAULT_SORTERREF_SIZE,   /* szSorterRef */
    0,                         /* iPrngSeed */
+#ifdef SQLITE_DEBUG
+   {0,0,0,0,0,0}              /* aTune */
+#endif
 };
 
 /*
@@ -343,7 +347,7 @@ int sqlite3PendingByte = 0x40000000;
 /*
 ** Tracing flags set by SQLITE_TESTCTRL_TRACEFLAGS.
 */
-u32 sqlite3SelectTrace = 0;
+u32 sqlite3TreeTrace = 0;
 u32 sqlite3WhereTrace = 0;
 
 #include "opcodes.h"

src/insert.c

@@ -181,7 +181,7 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){
     }
 
     for(i=j=0; i<pTab->nCol; i++){
-      assert( pTab->aCol[i].affinity!=0 );
+      assert( pTab->aCol[i].affinity!=0 || sqlite3VdbeParser(v)->nErr>0 );
       if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){
         zColAff[j++] = pTab->aCol[i].affinity;
       }
@@ -715,9 +715,11 @@ void sqlite3Insert(
 #endif
 
   db = pParse->db;
-  if( pParse->nErr || db->mallocFailed ){
+  assert( db->pParse==pParse );
+  if( pParse->nErr ){
     goto insert_cleanup;
   }
+  assert( db->mallocFailed==0 );
   dest.iSDParm = 0;  /* Suppress a harmless compiler warning */
 
   /* If the Select object is really just a simple VALUES() list with a
@@ -763,6 +765,14 @@ void sqlite3Insert(
 #endif
   assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) );
 
+#if TREETRACE_ENABLED
+  if( sqlite3TreeTrace & 0x10000 ){
+    sqlite3TreeViewLine(0, "In sqlite3Insert() at %s:%d", __FILE__, __LINE__);
+    sqlite3TreeViewInsert(pParse->pWith, pTabList, pColumn, pSelect, pList,
+                          onError, pUpsert, pTrigger);
+  }
+#endif
+
   /* If pTab is really a view, make sure it has been initialized.
   ** ViewGetColumnNames() is a no-op if pTab is not a view.
   */
@@ -793,7 +803,11 @@ void sqlite3Insert(
   **
   ** This is the 2nd template.
   */
-  if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){
+  if( pColumn==0 
+   && pSelect!=0
+   && pTrigger==0
+   && xferOptimization(pParse, pTab, pSelect, onError, iDb)
+  ){
     assert( !pTrigger );
     assert( pList==0 );
     goto insert_end;
@@ -837,13 +851,15 @@ void sqlite3Insert(
   */
   bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0;
   if( pColumn ){
+    assert( pColumn->eU4!=EU4_EXPR );
+    pColumn->eU4 = EU4_IDX;
     for(i=0; i<pColumn->nId; i++){
-      pColumn->a[i].idx = -1;
+      pColumn->a[i].u4.idx = -1;
     }
     for(i=0; i<pColumn->nId; i++){
       for(j=0; j<pTab->nCol; j++){
         if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){
-          pColumn->a[i].idx = j;
+          pColumn->a[i].u4.idx = j;
           if( i!=j ) bIdListInOrder = 0;
           if( j==pTab->iPKey ){
             ipkColumn = i;  assert( !withoutRowid );
@@ -893,7 +909,9 @@ void sqlite3Insert(
     dest.nSdst = pTab->nCol;
     rc = sqlite3Select(pParse, pSelect, &dest);
     regFromSelect = dest.iSdst;
-    if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup;
+    assert( db->pParse==pParse );
+    if( rc || pParse->nErr ) goto insert_cleanup;
+    assert( db->mallocFailed==0 );
     sqlite3VdbeEndCoroutine(v, regYield);
     sqlite3VdbeJumpHere(v, addrTop - 1);                       /* label B: */
     assert( pSelect->pEList );
@@ -1143,7 +1161,8 @@ void sqlite3Insert(
       }
     }
     if( pColumn ){
-      for(j=0; j<pColumn->nId && pColumn->a[j].idx!=i; j++){}
+      assert( pColumn->eU4==EU4_IDX );
+      for(j=0; j<pColumn->nId && pColumn->a[j].u4.idx!=i; j++){}
       if( j>=pColumn->nId ){
         /* A column not named in the insert column list gets its
         ** default value */
@@ -2274,7 +2293,7 @@ void sqlite3GenerateConstraintChecks(
         if( isUpdate ){
           /* If currently processing the PRIMARY KEY of a WITHOUT ROWID 
           ** table, only conflict if the new PRIMARY KEY values are actually
-          ** different from the old.
+          ** different from the old.  See TH3 withoutrowid04.test.
           **
           ** For a UNIQUE index, only conflict if the PRIMARY KEY values
           ** of the matched index row are different from the original PRIMARY
@@ -2762,18 +2781,13 @@ static int xferOptimization(
   int destHasUniqueIdx = 0;        /* True if pDest has a UNIQUE index */
   int regData, regRowid;           /* Registers holding data and rowid */
 
-  if( pSelect==0 ){
-    return 0;   /* Must be of the form  INSERT INTO ... SELECT ... */
-  }
+  assert( pSelect!=0 );
   if( pParse->pWith || pSelect->pWith ){
     /* Do not attempt to process this query if there are an WITH clauses
     ** attached to it. Proceeding may generate a false "no such table: xxx"
     ** error if pSelect reads from a CTE named "xxx".  */
     return 0;
   }
-  if( sqlite3TriggerList(pParse, pDest) ){
-    return 0;   /* tab1 must not have triggers */
-  }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   if( IsVirtual(pDest) ){
     return 0;   /* tab1 must not be a virtual table */

src/json.c

@@ -1119,14 +1119,15 @@ static JsonNode *jsonLookupStep(
         *pzErr = zPath;
         return 0;
       }
+      testcase( nKey==0 );
     }else{
       zKey = zPath;
       for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){}
       nKey = i;
-    }
-    if( nKey==0 ){
-      *pzErr = zPath;
-      return 0;
+      if( nKey==0 ){
+        *pzErr = zPath;
+        return 0;
+      }
     }
     j = 1;
     for(;;){
@@ -1625,7 +1626,7 @@ static JsonNode *jsonMergePatch(
   if( pPatch->eType!=JSON_OBJECT ){
     return pPatch;
   }
-  assert( iTarget>=0 && iTarget<pParse->nNode );
+  assert( iTarget<pParse->nNode );
   pTarget = &pParse->aNode[iTarget];
   assert( (pPatch->jnFlags & JNODE_APPEND)==0 );
   if( pTarget->eType!=JSON_OBJECT ){
@@ -2274,6 +2275,33 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){
   return SQLITE_OK;
 }
 
+/* Append an object label to the JSON Path being constructed
+** in pStr.
+*/
+static void jsonAppendObjectPathElement(
+  JsonString *pStr,
+  JsonNode *pNode
+){
+  int jj, nn;
+  const char *z;
+  assert( pNode->eType==JSON_STRING );
+  assert( pNode->jnFlags & JNODE_LABEL );
+  assert( pNode->eU==1 );
+  z = pNode->u.zJContent;
+  nn = pNode->n;
+  assert( nn>=2 );
+  assert( z[0]=='"' );
+  assert( z[nn-1]=='"' );
+  if( nn>2 && sqlite3Isalpha(z[1]) ){
+    for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){}
+    if( jj==nn-1 ){
+      z++;
+      nn -= 2;
+    }
+  }
+  jsonPrintf(nn+2, pStr, ".%.*s", nn, z);
+}
+
 /* Append the name of the path for element i to pStr
 */
 static void jsonEachComputePath(
@@ -2298,10 +2326,7 @@ static void jsonEachComputePath(
   }else{
     assert( pUp->eType==JSON_OBJECT );
     if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--;
-    assert( pNode->eType==JSON_STRING );
-    assert( pNode->jnFlags & JNODE_LABEL );
-    assert( pNode->eU==1 );
-    jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1);
+    jsonAppendObjectPathElement(pStr, pNode);
   }
 }
 
@@ -2372,8 +2397,7 @@ static int jsonEachColumn(
         if( p->eType==JSON_ARRAY ){
           jsonPrintf(30, &x, "[%d]", p->iRowid);
         }else if( p->eType==JSON_OBJECT ){
-          assert( pThis->eU==1 );
-          jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
+          jsonAppendObjectPathElement(&x, pThis);
         }
       }
       jsonResult(&x);
@@ -2582,7 +2606,8 @@ static sqlite3_module jsonEachModule = {
   0,                         /* xSavepoint */
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
-  0                          /* xShadowName */
+  0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 
 /* The methods of the json_tree virtual table. */
@@ -2610,7 +2635,8 @@ static sqlite3_module jsonTreeModule = {
   0,                         /* xSavepoint */
   0,                         /* xRelease */
   0,                         /* xRollbackTo */
-  0                          /* xShadowName */
+  0,                         /* xShadowName */
+  0                          /* xShadowName2 */
 };
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 #endif /* !defined(SQLITE_OMIT_JSON) */
@@ -2666,7 +2692,7 @@ int sqlite3JsonTableFunctions(sqlite3 *db){
     { "json_each",            &jsonEachModule               },
     { "json_tree",            &jsonTreeModule               },
   };
-  int i;
+  unsigned int i;
   for(i=0; i<sizeof(aMod)/sizeof(aMod[0]) && rc==SQLITE_OK; i++){
     rc = sqlite3_create_module(db, aMod[i].zName, aMod[i].pModule, 0);
   }

src/loadext.c

@@ -487,8 +487,28 @@ static const sqlite3_api_routines sqlite3Apis = {
   sqlite3_autovacuum_pages,
   /* Version 3.38.0 and later */
   sqlite3_error_offset,
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   sqlite3_vtab_rhs_value,
   sqlite3_vtab_distinct,
+  sqlite3_vtab_in,
+  sqlite3_vtab_in_first,
+  sqlite3_vtab_in_next,
+#else
+  0,
+  0,
+  0,
+  0,
+  0,
+#endif
+  /* Version 3.39.0 and later */
+#ifndef SQLITE_OMIT_DESERIALIZE
+  sqlite3_deserialize,
+  sqlite3_serialize,
+#else
+  0,
+  0,
+#endif
+  sqlite3_db_name
 };
 
 /* True if x is the directory separator character

src/main.c

@@ -4029,12 +4029,16 @@ int sqlite3_test_control(int op, ...){
     ** sqlite3_test_control().
     */
     case SQLITE_TESTCTRL_FAULT_INSTALL: {
-      /* MSVC is picky about pulling func ptrs from va lists.
-      ** http://support.microsoft.com/kb/47961
+      /* A bug in MSVC prevents it from understanding pointers to functions
+      ** types in the second argument to va_arg().  Work around the problem
+      ** using a typedef.
+      ** http://support.microsoft.com/kb/47961  <-- dead hyperlink
+      ** Search at http://web.archive.org/ to find the 2015-03-16 archive
+      ** of the link above to see the original text.
       ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int));
       */
-      typedef int(*TESTCALLBACKFUNC_t)(int);
-      sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t);
+      typedef int(*sqlite3FaultFuncType)(int);
+      sqlite3GlobalConfig.xTestCallback = va_arg(ap, sqlite3FaultFuncType);
       rc = sqlite3FaultSim(0);
       break;
     }
@@ -4093,6 +4097,28 @@ int sqlite3_test_control(int op, ...){
       volatile int x = 0;
       assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 );
       rc = x;
+#if defined(SQLITE_DEBUG)
+      /* Invoke these debugging routines so that the compiler does not
+      ** issue "defined but not used" warnings. */
+      if( x==9999 ){
+        sqlite3ShowExpr(0);
+        sqlite3ShowExpr(0);
+        sqlite3ShowExprList(0);
+        sqlite3ShowIdList(0);
+        sqlite3ShowSrcList(0);
+        sqlite3ShowWith(0);
+        sqlite3ShowUpsert(0);
+        sqlite3ShowTriggerStep(0);
+        sqlite3ShowTriggerStepList(0);
+        sqlite3ShowTrigger(0);
+        sqlite3ShowTriggerList(0);
+#ifndef SQLITE_OMIT_WINDOWFUNC
+        sqlite3ShowWindow(0);
+        sqlite3ShowWinFunc(0);
+#endif
+        sqlite3ShowSelect(0);
+      }
+#endif
       break;
     }
 
@@ -4161,13 +4187,27 @@ int sqlite3_test_control(int op, ...){
       break;
     }
 
-    /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff);
+    /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt);
     **
-    ** If parameter onoff is non-zero, subsequent calls to localtime()
-    ** and its variants fail. If onoff is zero, undo this setting.
+    ** If parameter onoff is 1, subsequent calls to localtime() fail.
+    ** If 2, then invoke xAlt() instead of localtime().  If 0, normal
+    ** processing.
+    **
+    ** xAlt arguments are void pointers, but they really want to be:
+    **
+    **    int xAlt(const time_t*, struct tm*);
+    **
+    ** xAlt should write results in to struct tm object of its 2nd argument
+    ** and return zero on success, or return non-zero on failure.
     */
     case SQLITE_TESTCTRL_LOCALTIME_FAULT: {
       sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int);
+      if( sqlite3GlobalConfig.bLocaltimeFault==2 ){
+        typedef int(*sqlite3LocaltimeType)(const void*,void*);
+        sqlite3GlobalConfig.xAltLocaltime = va_arg(ap, sqlite3LocaltimeType);
+      }else{
+        sqlite3GlobalConfig.xAltLocaltime = 0;
+      }
       break;
     }
 
@@ -4340,8 +4380,8 @@ int sqlite3_test_control(int op, ...){
     **
     **  "ptr" is a pointer to a u32.  
     **
-    **   op==0       Store the current sqlite3SelectTrace in *ptr
-    **   op==1       Set sqlite3SelectTrace to the value *ptr
+    **   op==0       Store the current sqlite3TreeTrace in *ptr
+    **   op==1       Set sqlite3TreeTrace to the value *ptr
     **   op==3       Store the current sqlite3WhereTrace in *ptr
     **   op==3       Set sqlite3WhereTrace to the value *ptr
     */
@@ -4349,10 +4389,10 @@ int sqlite3_test_control(int op, ...){
        int opTrace = va_arg(ap, int);
        u32 *ptr = va_arg(ap, u32*);
        switch( opTrace ){
-         case 0:   *ptr = sqlite3SelectTrace;      break;
-         case 1:   sqlite3SelectTrace = *ptr;      break;
-         case 2:   *ptr = sqlite3WhereTrace;       break;
-         case 3:   sqlite3WhereTrace = *ptr;       break;
+         case 0:   *ptr = sqlite3TreeTrace;      break;
+         case 1:   sqlite3TreeTrace = *ptr;      break;
+         case 2:   *ptr = sqlite3WhereTrace;     break;
+         case 3:   sqlite3WhereTrace = *ptr;     break;
        }
        break;
     }
@@ -4369,10 +4409,12 @@ int sqlite3_test_control(int op, ...){
     case SQLITE_TESTCTRL_LOGEST: {
       double rIn = va_arg(ap, double);
       LogEst rLogEst = sqlite3LogEstFromDouble(rIn);
-      u64 iInt = sqlite3LogEstToInt(rLogEst);
-      va_arg(ap, int*)[0] = rLogEst;
-      va_arg(ap, u64*)[0] = iInt;
-      va_arg(ap, int*)[0] = sqlite3LogEst(iInt);
+      int *pI1 = va_arg(ap,int*);
+      u64 *pU64 = va_arg(ap,u64*);
+      int *pI2 = va_arg(ap,int*);
+      *pI1 = rLogEst;
+      *pU64 = sqlite3LogEstToInt(rLogEst);
+      *pI2 = sqlite3LogEst(*pU64);
       break;
     }
  
@@ -4587,6 +4629,24 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){
   return iDb<0 ? 0 : db->aDb[iDb].pBt;
 }
 
+/*
+** Return the name of the N-th database schema.  Return NULL if N is out
+** of range.
+*/
+const char *sqlite3_db_name(sqlite3 *db, int N){
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) ){
+    (void)SQLITE_MISUSE_BKPT;
+    return 0;
+  }
+#endif
+  if( N<0 || N>=db->nDb ){
+    return 0;
+  }else{
+    return db->aDb[N].zDbSName;
+  }
+}
+
 /*
 ** Return the filename of the database associated with a database
 ** connection.

src/malloc.c

@@ -750,8 +750,9 @@ char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){
 ** Free any prior content in *pz and replace it with a copy of zNew.
 */
 void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){
+  char *z = sqlite3DbStrDup(db, zNew);
   sqlite3DbFree(db, *pz);
-  *pz = sqlite3DbStrDup(db, zNew);
+  *pz = z;
 }
 
 /*
@@ -759,8 +760,15 @@ void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){
 ** has happened.  This routine will set db->mallocFailed, and also
 ** temporarily disable the lookaside memory allocator and interrupt
 ** any running VDBEs.
+**
+** Always return a NULL pointer so that this routine can be invoked using
+**
+**      return sqlite3OomFault(db);
+**
+** and thereby avoid unnecessary stack frame allocations for the overwhelmingly
+** common case where no OOM occurs.
 */
-void sqlite3OomFault(sqlite3 *db){
+void *sqlite3OomFault(sqlite3 *db){
   if( db->mallocFailed==0 && db->bBenignMalloc==0 ){
     db->mallocFailed = 1;
     if( db->nVdbeExec>0 ){
@@ -768,9 +776,16 @@ void sqlite3OomFault(sqlite3 *db){
     }
     DisableLookaside;
     if( db->pParse ){
+      Parse *pParse;
+      sqlite3ErrorMsg(db->pParse, "out of memory");
       db->pParse->rc = SQLITE_NOMEM_BKPT;
+      for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){
+        pParse->nErr++;
+        pParse->rc = SQLITE_NOMEM;
+      } 
     }
   }
+  return 0;
 }
 
 /*