Add a new OP_BColumn opcode where the cursor is guaranteed to be of type

CURTYPE_BTREE and is thus able to skip some checks and run slightly faster. FossilOrigin-Name: 093e0d10a81adc078dcb3e08e33ce89bdbe53789
Fix over-length source code lines. No logic changes.
2015-11-21 17:53:06 +00:00 · 2015-11-21 17:27:42 +00:00 · 2015-11-21 13:24:46 +00:00 · 2015-11-20 19:22:01 +00:00 · 2015-11-20 13:33:56 +00:00 · 2015-11-20 13:17:29 +00:00
71 changed files with 1966 additions and 1715 deletions
@@ -645,9 +645,9 @@ sqlite3.lo:	sqlite3.c

 # Rules to build the LEMON compiler generator
 #
-lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/src/lempar.c
+lemon$(BEXE):	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o $@ $(TOP)/tool/lemon.c
-	cp $(TOP)/src/lempar.c .
+	cp $(TOP)/tool/lempar.c .

 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
@@ -1310,8 +1310,8 @@ sqlite3.lo:	$(SQLITE3C)

 # Rules to build the LEMON compiler generator
 #
-lempar.c:	$(TOP)\src\lempar.c
-	copy $(TOP)\src\lempar.c .
+lempar.c:	$(TOP)\tool\lempar.c
+	copy $(TOP)\tool\lempar.c .

 lemon.exe:	$(TOP)\tool\lemon.c lempar.c
 	$(BCC) $(NO_WARN) -Daccess=_access \
@@ -6,8 +6,10 @@ libsqlite3_la_SOURCES = sqlite3.c
 libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8

 bin_PROGRAMS = sqlite3
-sqlite3_SOURCES = shell.c sqlite3.c sqlite3.h
-sqlite3_LDADD = @READLINE_LIBS@
+sqlite3_SOURCES = shell.c sqlite3.h
+EXTRA_sqlite3_SOURCES = sqlite3.c
+sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
+sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
 sqlite3_CFLAGS = $(AM_CFLAGS)

 include_HEADERS = sqlite3.h sqlite3ext.h
@@ -4,6 +4,8 @@
 #
 #   --enable-threadsafe
 #   --enable-readline
+#   --enable-editline
+#   --enable-static-shell
 #   --enable-dynamic-extensions
 #

@@ -117,6 +119,21 @@ fi
 AC_SUBST(JSON1_FLAGS)
 #-----------------------------------------------------------------------

+#-----------------------------------------------------------------------
+#   --enable-static-shell
+#
+AC_ARG_ENABLE(static-shell, [AS_HELP_STRING(
+  [--enable-static-shell], 
+  [statically link libsqlite3 into shell tool [default=yes]])], 
+  [], [enable_static_shell=yes])
+if test x"$enable_static_shell" == "xyes"; then
+  EXTRA_SHELL_OBJ=sqlite3.$OBJEXT
+else
+  EXTRA_SHELL_OBJ=libsqlite3.la
+fi
+AC_SUBST(EXTRA_SHELL_OBJ)
+#-----------------------------------------------------------------------
+
 AC_CHECK_FUNCS(posix_fallocate)

 #-----------------------------------------------------------------------
@@ -874,7 +874,6 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){
  int nPhrase;
  int nByte;
  int rc = SQLITE_OK;
-  char *zSql;
  const char *zRank = pCsr->zRank;
  const char *zRankArgs = pCsr->zRankArgs;
  
@@ -58,6 +58,18 @@
 */
 #define yytestcase(X) testcase(X)

+/*
+** Indicate that sqlite3ParserFree() will never be called with a null
+** pointer.
+*/
+#define YYPARSEFREENOTNULL 1
+
+/*
+** Alternative datatype for the argument to the malloc() routine passed
+** into sqlite3ParserAlloc().  The default is size_t.
+*/
+#define YYMALLOCARGTYPE  u64
+
 } // end %include

 %left OR.
@@ -168,7 +180,3 @@ phrase(A) ::= STRING(Y) star_opt(Z). {

 star_opt(A) ::= STAR. { A = 1; }
 star_opt(A) ::= . { A = 0; }
-
-
-
-
@@ -103,7 +103,7 @@ static void ieee754func(
    if( e>0x7ff ) e = 0x7ff;
    a = m & ((((sqlite3_int64)1)<<52)-1);
    a |= e<<52;
-    if( isNeg ) a |= ((sqlite3_int64)1)<<63;
+    if( isNeg ) a |= ((sqlite3_uint64)1)<<63;
    memcpy(&r, &a, sizeof(r));
    sqlite3_result_double(context, r);
  }
@@ -554,9 +554,9 @@ fts3amal.c:	target_source $(TOP)/ext/fts3/mkfts3amal.tcl

 # Rules to build the LEMON compiler generator
 #
-lemon:	$(TOP)/tool/lemon.c $(TOP)/src/lempar.c
+lemon:	$(TOP)/tool/lemon.c $(TOP)/tool/lempar.c
 	$(BCC) -o lemon $(TOP)/tool/lemon.c
-	cp $(TOP)/src/lempar.c .
+	cp $(TOP)/tool/lempar.c .

 # Rules to build individual *.o files from generated *.c files. This
 # applies to:
@@ -1,8 +1,8 @@
-C Merge\sthe\sfixes\sand\stests\sfor\sthe\sieee754\sextension.
-D 2015-11-06T17:59:33.872
-F Makefile.in 3a705bb4bd12e194212ddbdbf068310d17153cdb
+C Add\sa\snew\sOP_BColumn\sopcode\swhere\sthe\scursor\sis\sguaranteed\sto\sbe\sof\stype\nCURTYPE_BTREE\sand\sis\sthus\sable\sto\sskip\ssome\schecks\sand\srun\sslightly\sfaster.
+D 2015-11-21T17:53:06.758
+F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc 702d3e98f3afc6587a78481257f3c4c900efc3a4
+F Makefile.msc e928e68168df69b353300ac87c10105206653a03
 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
 F VERSION 8b9d3ac6f1962f94e06ba05462422a544f9c4e36
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -10,12 +10,12 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2
 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90
 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2
 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903
-F autoconf/Makefile.am bd4a90972aa87f079af6624ddea3df3d58f26d2f
+F autoconf/Makefile.am 089e5ecdb5761e64ea1013ded02feb4d8b29927d
 F autoconf/README 14458f1046c118efa721aadec5f227e876d3cd38
 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7
 F autoconf/config.guess 94cc57e2a3fdb9c235b362ace86d77e89d188cad x
 F autoconf/config.sub 1efb390a8fb4bfafd74783a15a8fb5311c84300e x
-F autoconf/configure.ac bec3caf78cf1bdbaad903dcd5c6029292eda3d2d
+F autoconf/configure.ac 9a65da17e440466f9842288163f16f9b21298129
 F autoconf/depcomp 0b26f101e3bc9fd1ff0be1da9fb4a82371142f92 x
 F autoconf/install-sh 06ee6336e63bb845c8439d777c32eb2eccc4fbf1 x
 F autoconf/ltmain.sh 7a658a24028f02331c1d2446562758083c5eadd1
@@ -110,7 +110,7 @@ F ext/fts5/fts5_config.c 6fc92c0b1bda5244c28a54c9ba740736bd5513d9
 F ext/fts5/fts5_expr.c 28b15c9ae296204bc0a2e5cf7a667d840a9d2900
 F ext/fts5/fts5_hash.c a9d4c1efebc2a91d26ad7ebdfcbf2678ceac405f
 F ext/fts5/fts5_index.c b622a0a70f57a96469e6828da2dd70e0872aeb37
-F ext/fts5/fts5_main.c 0569cd9fc18f3b56f6eeac601836df8ed5fb54cb
+F ext/fts5/fts5_main.c 7581280ee242785477df67402f2853c66f77d45b
 F ext/fts5/fts5_storage.c 9ea3d92178743758b6c54d9fe8836bbbdcc92e3b
 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd
 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf
@@ -118,7 +118,7 @@ F ext/fts5/fts5_tokenize.c 12c5d925286491a71bb3dad7c8924ce9cfd18320
 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
 F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2
-F ext/fts5/fts5parse.y e83dca6028e3309178d05b5bd920e372dc295d35
+F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d
 F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206
@@ -196,7 +196,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767
-F ext/misc/ieee754.c ca4a0a8fd2d4667a7ef10b761312c256bc734039
+F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c 4f45afd9dbcd6feca8c528251efbb7fc09299a09
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
@@ -263,7 +263,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ac9746f24334056c85775dbe6b7279b84a6941d7
+F main.mk ae99be5eb22933b1ecf80f94d41d25a3ea80aaf3
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -280,19 +280,19 @@ F src/analyze.c 4c308880cf53c558070cb8513bdff4ffb1a38a77
 F src/attach.c e944d0052b577703b9b83aac1638452ff42a8395
 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc
-F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
+F src/bitvec.c 1a78d450a17c5016710eec900bedfc5729bf9bdf
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c 233c1536927b12d820cb0a19c657c59837f9334f
-F src/btree.h b512723e4f27d7ba16b4b985cdecdb82c0f6d0c0
+F src/btree.c d3bdd8462a86492e2ebc9aca4a0168429017de25
+F src/btree.h 2d76dee44704c47eed323356a758662724b674a0
 F src/btreeInt.h 3ab435ed27adea54d040584b0bcc488ee7db1e38
-F src/build.c ca574d33ffb1763cfd2979383f4d507095bfbe19
+F src/build.c 19f7585c9747d5043dae64bc82f9cac759f69700
 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0
 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 509ef9c64d1321f42448f111da86400b1799218a
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c ffd63fc8ba7541476ced189b95e95d7f2bc63f78
-F src/delete.c c4c6fb9da78b946fcba2a6aac5b24bc5c15e752a
-F src/expr.c 0080c0f12806eca91e75a23a121a68918e9da357
+F src/delete.c 00af9f08a15ddc5cba5962d3d3e5bf2d67b2e7da
+F src/expr.c dfccb439a2a981d71970ce3d15effeb59b258798
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 31900763094a3736a5fc887469202eb579fef2d0
 F src/func.c ecdd69ec6a1e406f04cc73324be2ebbf6354197f
@@ -300,57 +300,56 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c 419a947f27ce2da18eebf440a5aa80cc825defae
+F src/insert.c e1d20ae8979e25519c2670233718676bedcfedc9
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
-F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012
 F src/loadext.c 18586e45a215325f15096821e9c082035d4fb810
-F src/main.c f393acc6e5d604cbe0054376d62f668a5560b3f1
+F src/main.c 91feb5c7e393ad8f0c434754114a3493c3b0617a
 F src/malloc.c 337bbe9c7d436ef9b7d06b5dd10bbfc8f3025972
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 8768ac94694f31ffaf8b4d0ea5dc08af7010a35a
-F src/mem5.c c1ab1153bd6443bdf6f71e4213c6fb31221b9eb7
+F src/mem5.c 5c267678ba9f745a2ee58102a9f482d64a58577a
 F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85
 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495
 F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c
 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85
 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4
-F src/mutex_unix.c a94b46f3f7beba307dde1b298b0f99f9c3677a93
+F src/mutex_unix.c fc54f25b2a750d53b32512a4a728cec28039ae2a
 F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17
 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7
 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8
 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf
 F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
-F src/os_unix.c cf72e06e15839ebe7121e01d3eebf256c039b0ca
-F src/os_win.c 1716291e5ec2dbfc5a1fe0b32182030f1f7d8acf
+F src/os_unix.c bddde71dc024574ace03ffee859abb99d152fd4a
+F src/os_win.c 386fba30419e8458b13209781c2af5590eab2811
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c ed5cff11793b6a4146582aabb29ed8613a6cf89e
-F src/pager.h 7fc069c07f3120ee466ff3d48a9d376974ebffa7
-F src/parse.y 11078cd8e3af00f030505b6a86a06a4536cfdeaa
-F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef
-F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
+F src/pager.c 18341e2b759b447cbc82fb9215d08d9c5864e92e
+F src/pager.h 87c4118a71ba3965184148b379a6d93179071091
+F src/parse.y 23737e649c26ce327603799e57f5c2ff50e5e6ba
+F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23
+F src/pcache.h 1ff11adce609ba7de139b6abfabaf9a2bac947b5
 F src/pcache1.c 902e1bc7bdaa81b40f8543407c5e2ac8ef4dc035
-F src/pragma.c 8fd4c8a12e25f0d916426f160255ebe25415eba7
-F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a
+F src/pragma.c f3e7147299ca05ef4304a36f1fd6e002729c72c6
+F src/pragma.h 3d94aebbebd2089899fecc01909bf2608b39507d
 F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1
-F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a
+F src/printf.c f8fc8f04e75b1e983ef2793c27ec7a43b287e94a
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
-F src/resolve.c 1954a0f01bf65d78d7d559aea3d5c67f33376d91
+F src/resolve.c f4c897ca76ca6d5e0b3f0499c627392ffe657c8e
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
-F src/select.c 167b4e9058bc8e997d18d6b6b20ecbb0c9c457af
-F src/shell.c d25df04168d6ba5a4fa05bdbf859df667f9eb621
-F src/sqlite.h.in 3cfc86c55e57c63d86b9e1e92869e2bfb162ca8e
+F src/select.c 0495e86f8377026fbd529a1a5bf62046cbb6eec5
+F src/shell.c f0f59ea60ad297f671b7ae0fb957a736ad17c92c
+F src/sqlite.h.in fa62718f73553f06b2f2e362fd09ccb4e1cbb626
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 4b66e3e3435da4b4c8c83696d0349f0c503b3924
-F src/sqliteInt.h b1e72ffe282c91ae30a2bf403319126dbaebd556
+F src/sqliteInt.h 64256d193a16a147d9f6317cc4e095fdd3e0a2e9
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
 F src/tclsqlite.c d9439b6a910985b7fff43ba6756bcef00de22649
-F src/test1.c a719afff3144f7f01c6dc3f7d118ac31d15e7527
+F src/test1.c 4004bcc1b3b361a9137acd1d875599ecbdd6f961
 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b
 F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
@@ -364,7 +363,7 @@ F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803
 F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07
 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f
-F src/test_config.c 426527fbb12fc23669a1e973ecdc8c5e92c2e2cf
+F src/test_config.c f2824de39f59d8d621e2d6ec5cc67006d000b2eb
 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f
@@ -387,7 +386,7 @@ F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d
 F src/test_rtree.c 43fff4c5a01576d6d213f27472598801a247890c
 F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091
 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe
-F src/test_sqllog.c b690c12933f50ff46491e0d56a251f84ae16e914
+F src/test_sqllog.c 0d138a8180a312bf996b37fa66da5c5799d4d57b
 F src/test_superlock.c 06797157176eb7085027d9dd278c0d7a105e3ec9
 F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b
 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
@@ -396,21 +395,21 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0
-F src/tokenize.c 338bc8f7c9dd103188952cda7964696bacac6d22
-F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce
+F src/tokenize.c 5606871a377f390af7040ec3c12e0d183512d785
+F src/treeview.c 78842e90c1f71269e7a73a1d4221b6fe360bab66
 F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f
 F src/update.c 40e51cd0883cb5bfd6abb7d8a7cd8aa47fab2945
 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c
-F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd
+F src/util.c e802e8e311a0d6c48cd1b3e89db164f6f0248d70
 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701
-F src/vdbe.c dfbaae2570172c523bce14299021d352b8508f7f
+F src/vdbe.c 8694c0e6497cd21e19e30ac059f933ac5d9abea2
 F src/vdbe.h efb7a8c1459e31f3ea4377824c6a7e4cb5068637
-F src/vdbeInt.h 33403622c6a8feaaac5f0f3f17f5d1bf6df42286
+F src/vdbeInt.h 75c2e82ee3357e9210c06474f8d9bdf12c81105d
 F src/vdbeapi.c 020681b943e77766b32ae1cddf86d7831b7374ca
-F src/vdbeaux.c 869c95b0fc73026da1ca179e053807f563793e8f
-F src/vdbeblob.c 565fabd302f5fca3bdf3d56cac330483616a39b6
+F src/vdbeaux.c 16e21a6ae0c290f17f6e7395d80686622cafeaa2
+F src/vdbeblob.c fdc4a81605ae7a35ae94a55bd768b66d6be16f15
 F src/vdbemem.c fdd1578e47bea61390d472de53c565781d81e045
-F src/vdbesort.c 8b23930a1289526f6d2a3a9f2e965bcc963e4a68
+F src/vdbesort.c a7ec02da4494c59dfd071126dd3726be5a11459d
 F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0
 F src/vtab.c 2a8b44aa372c33f6154208e7a7f6c44254549806
 F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
@@ -420,7 +419,7 @@ F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
 F src/where.c 6aceb72cc58dc06922a9e1604d559c8ca4c3e728
 F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
 F src/wherecode.c 4c96182e7b25e4be54008dee2da5b9c2f8480b9b
-F src/whereexpr.c e63244ca06c503e5f3c5b7f3c9aea0db826089ed
+F src/whereexpr.c bd4877cd4dd11f6ab551ef0054535ca3c6224950
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -519,7 +518,7 @@ F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
 F test/collate1.test 08c18e7512a5a32c97938854263fa15362eeb846
 F test/collate2.test 9aaa410a00734e48bcb27f3872617d6f69b2a621
 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5
-F test/collate4.test f04d5168685f2eef637ecfa2d4ddf8ec0d600177
+F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd
 F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6
 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907
 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868
@@ -754,7 +753,7 @@ F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1
 F test/fuzz3.test 53fabcd5f0f430f8b221282f6c12c4d0903c21eb
 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b
 F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26
-F test/fuzzcheck.c ee926f1d4090d053ed542899720d4e4d30811bcc
+F test/fuzzcheck.c 7c61352f20a28429b221f406f3854cf9c912f63b
 F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664
 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -763,6 +762,7 @@ F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36
 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536
 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98
 F test/hexlit.test 1d312fa816dfd3650a3bb488093bc09a0c927f67
+F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711
 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f
 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4
 F test/ieee754.test 118b665a97a8df0e8f2fbdb07d113e596f4a6b53
@@ -876,7 +876,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
-F test/misc1.test 3f1c479c5a093a6280f378c0fbff1c2701486660
+F test/misc1.test 48ebfb5b22a6a058f7b7e1df211226dd1d21409c
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
@@ -930,9 +930,9 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0
 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b
 F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442
 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff
-F test/permutations.test 0be620cfc7309d72b0608b3f1cbadef1034417fa
+F test/permutations.test 4ea119731c62d2f7d0aa86dd5b184cbb61ca411b
 F test/pragma.test a44253f911e7d50127d4a08f927f47c861a4c772
-F test/pragma2.test b5e2ce4c892afceb308c6ae6163a9099b2a0d8d7
+F test/pragma2.test 00065068eeab2d15ea55465ec0f1e0a70e2c369e
 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c
 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc
 F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054
@@ -950,7 +950,7 @@ F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736
 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
-F test/releasetest.tcl c12c4122a7b2ddec16b5f8fb9163f7a008886005
+F test/releasetest.tcl 622f2381b217facdf429584a5c292cc1fc47e7c0
 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb
 F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea
 F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14
@@ -1039,6 +1039,7 @@ F test/spellfix.test 0597065ff57042df1f138e6a2611ae19c2698135
 F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3
 F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5
 F test/sqllimits1.test 89b3d5aad05b99f707ee3786bdd4416dccf83304
+F test/sqllog.test a8faa2df39610a037dd372ed872d124260d32953
 F test/stat.test 8de91498c99f5298b303f70f1d1f3b9557af91bf
 F test/statfault.test f525a7bf633e50afd027700e9a486090684b1ac1
 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
@@ -1052,7 +1053,7 @@ F test/symlink.test 2513f7c030df0f435c6415687ba8b739f3d312df
 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85
 F test/syscall.test fba9ebdc6905d05bba6a835e691f20ed9ea2cc88
 F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6
-F test/tabfunc01.test 03c4ad422c6ab596cff6dcaf86dd061a9f039525
+F test/tabfunc01.test cc33684f9480fcf1fd5ce287ac28d22971cad1cc
 F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f
 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
@@ -1060,7 +1061,7 @@ F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054
 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1
-F test/tester.tcl 83cc29c89259490b25d2d7e47535d6f2ed5a57c7
+F test/tester.tcl 9df86ab273a0877ffb4ec26c255166dcdca4c278
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1227,7 +1228,7 @@ F test/trigger3.test aa640bb2bbb03edd5ff69c055117ea088f121945
 F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359
 F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83
 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9
-F test/trigger7.test 200dd51e728c9cdc20c72d99d9e9d45c667248f8
+F test/trigger7.test 799c9d2561facef70340cc9e0dee98aee9b5bdfe
 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4
 F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41
 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332
@@ -1278,7 +1279,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772
 F test/wal.test dbfc482e10c7263298833bb1fc60b3ac9d6340a1
 F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada
-F test/wal3.test 2ab8e490afe0164bfc89b185c8b2572e0d821f23
+F test/wal3.test b1d425f68a1f61d12563f0fa1ee6fca7d5afabf4
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c
 F test/wal6.test 4421cd5a2fa99d29cc91ef12fb23bed171ed3a4c
@@ -1303,10 +1304,10 @@ F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57
 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417
 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
-F test/where.test 66d4c107e82dfe86c01a96277b77e7a8809aff0b
+F test/where.test 9902a3d84e9bc80357a2c54ed0e76c0d6d04a867
 F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23
 F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e
-F test/where4.test 44f506bf1737cf0fa4fc795e340208250f1fcd89
+F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b
 F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8
@@ -1329,8 +1330,9 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
-F test/with1.test a1e8660be88e2eb4648f8860f831d1e38b5b5443
+F test/with1.test 05c8fc7f809f178a8a0519f02c21fe430948c895
 F test/with2.test 2b40da883658eb74ad8ad06afabe11a408e7fb87
+F test/with3.test 511bacdbe41c49cf34f9fd1bd3245fe1575bca98
 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991
 F test/without_rowid1.test 1a7b9bd51b899928d327052df9741d2fe8dbe701
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
@@ -1343,9 +1345,10 @@ F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa
 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac
 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5
 F tool/GetTclKit.bat 8606413d3035c05373a0d7fae82ebf59ae9e16c3
-F tool/addopcodes.tcl 26892c394964c194fe96b9a79b8b9f87347c7151
+F tool/addopcodes.tcl 4ca9c3ef196f08da30add5d07ce0c9458dc8c633
 F tool/build-all-msvc.bat e42141ca3c3812315432f9813ef9eb78aa8d99c9 x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
+F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2
 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1
@@ -1355,15 +1358,15 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 799e73e19a33b8dd7767a7fa34618ed2a9c2397d
-F tool/lempar.c 3617143ddb9b176c3605defe6a9c798793280120
+F tool/lempar.c 3ec1463a034b37d87d782be5f6b8b10a3b1ecbe7
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh 4bdf61548a143e5977bd86ab93d68b694d10c8fa
-F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
+F tool/mkkeywordhash.c 06ec0b78bd4fa68c12d90ef2bdfe76b039133ff8
 F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b
 F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
-F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e
+F tool/mkpragmatab.tcl e94e55d247d4fe3be34f2a4f4edb03fdcd09ce5b
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835
 F tool/mksqlite3c.tcl b66b4170f693602cd6985aed15d9509fe2f18c84
@@ -1376,6 +1379,7 @@ F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b
 F tool/replace.tcl 7727c60a04299b65a92f5e1590896fea0f25b9e0
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
+F tool/run-speed-test.sh 0ae485af4fe9f826e2b494be8c81f8ca9e222a4a
 F tool/showdb.c d4476e000a64eca9f5e2c2f68741e747b9778e8d
 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818
 F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68
@@ -1389,7 +1393,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c b318efc2eaf7a7fac4d281a0ce736193cb2506df
+F tool/sqldiff.c db1232df457fdd4cbf2a919a497fc44bb18fb933
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f
@@ -1400,7 +1404,10 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P a3f7614b2095fbc1eca54801b2da88f85f294ec5 840cbda88675b6012dae2423252bf42d0d563874
-R 281cdf2e28b38f35a9e781fe4ef44557
+P 198d191b2f5ef7d63ac0093c701955c9052fd734
+R 047e676af06478b15f87435c78af3c91
+T *branch * btree-column-opcode
+T *sym-btree-column-opcode *
+T -sym-trunk *
 U drh
-Z 35915d26f010e261a34e33f9bc3e14c7
+Z d7290cc1ec32ea2c40df94d6965d7340
@@ -1 +1 @@
-7a9988d38cd7ab3df79eed97631ff23d5928b4ad
+093e0d10a81adc078dcb3e08e33ce89bdbe53789
@@ -41,7 +41,8 @@

 /* Round the union size down to the nearest pointer boundary, since that's how 
 ** it will be aligned within the Bitvec struct. */
-#define BITVEC_USIZE     (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*))
+#define BITVEC_USIZE \
+    (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*))

 /* Type of the array "element" for the bitmap representation. 
 ** Should be a power of 2, and ideally, evenly divide into BITVEC_USIZE. 
@@ -2556,19 +2556,11 @@ int sqlite3BtreeClose(Btree *p){
 }

 /*
-** Change the limit on the number of pages allowed in the cache.
-**
-** The maximum number of cache pages is set to the absolute
-** value of mxPage.  If mxPage is negative, the pager will
-** operate asynchronously - it will not stop to do fsync()s
-** to insure data is written to the disk surface before
-** continuing.  Transactions still work if synchronous is off,
-** and the database cannot be corrupted if this program
-** crashes.  But if the operating system crashes or there is
-** an abrupt power failure when synchronous is off, the database
-** could be left in an inconsistent and unrecoverable state.
-** Synchronous is on by default so database corruption is not
-** normally a worry.
+** Change the "soft" limit on the number of pages in the cache.
+** Unused and unmodified pages will be recycled when the number of
+** pages in the cache exceeds this soft limit.  But the size of the
+** cache is allowed to grow larger than this limit if it contains
+** dirty pages or pages still in active use.
 */
 int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){
  BtShared *pBt = p->pBt;
@@ -2579,6 +2571,26 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){
  return SQLITE_OK;
 }

+/*
+** Change the "spill" limit on the number of pages in the cache.
+** If the number of pages exceeds this limit during a write transaction,
+** the pager might attempt to "spill" pages to the journal early in
+** order to free up memory.
+**
+** The value returned is the current spill size.  If zero is passed
+** as an argument, no changes are made to the spill size setting, so
+** using mxPage of 0 is a way to query the current spill size.
+*/
+int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){
+  BtShared *pBt = p->pBt;
+  int res;
+  assert( sqlite3_mutex_held(p->db->mutex) );
+  sqlite3BtreeEnter(p);
+  res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage);
+  sqlite3BtreeLeave(p);
+  return res;
+}
+
 #if SQLITE_MAX_MMAP_SIZE>0
 /*
 ** Change the limit on the amount of the database file that may be
@@ -5041,6 +5053,8 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){
 **     *pRes>0      The cursor is left pointing at an entry that
 **                  is larger than intKey/pIdxKey.
 **
+** For index tables, the pIdxKey->eqSeen field is set to 1 if there
+** exists an entry in the table that exactly matches pIdxKey.  
 */
 int sqlite3BtreeMovetoUnpacked(
  BtCursor *pCur,          /* The cursor to be moved */
@@ -9644,7 +9658,6 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
  return rc;
 }

-#ifdef SQLITE_DEBUG
 /*
 ** Return true if the cursor has a hint specified.  This routine is
 ** only used from within assert() statements
@@ -9652,7 +9665,6 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
 int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){
  return (pCsr->hints & mask)!=0;
 }
-#endif

 /*
 ** Return true if the given Btree is read-only.
@@ -63,6 +63,7 @@ int sqlite3BtreeOpen(

 int sqlite3BtreeClose(Btree*);
 int sqlite3BtreeSetCacheSize(Btree*,int);
+int sqlite3BtreeSetSpillSize(Btree*,int);
 #if SQLITE_MAX_MMAP_SIZE>0
  int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64);
 #endif
@@ -257,9 +258,7 @@ int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*);
 void sqlite3BtreeIncrblobCursor(BtCursor *);
 void sqlite3BtreeClearCursor(BtCursor *);
 int sqlite3BtreeSetVersion(Btree *pBt, int iVersion);
-#ifdef SQLITE_DEBUG
 int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask);
-#endif
 int sqlite3BtreeIsReadonly(Btree *pBt);
 int sqlite3HeaderSizeBtree(void);

@@ -1049,18 +1049,19 @@ begin_table_error:
  return;
 }

-/*
-** This macro is used to compare two strings in a case-insensitive manner.
-** It is slightly faster than calling sqlite3StrICmp() directly, but
-** produces larger code.
-**
-** WARNING: This macro is not compatible with the strcmp() family. It
-** returns true if the two strings are equal, otherwise false.
+/* Set properties of a table column based on the (magical)
+** name of the column.
 */
-#define STRICMP(x, y) (\
-sqlite3UpperToLower[*(unsigned char *)(x)]==   \
-sqlite3UpperToLower[*(unsigned char *)(y)]     \
-&& sqlite3StrICmp((x)+1,(y)+1)==0 )
+void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){
+#if SQLITE_ENABLE_HIDDEN_COLUMNS
+  if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){
+    pCol->colFlags |= COLFLAG_HIDDEN;
+  }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){
+    pTab->tabFlags |= TF_OOOHidden;
+  }
+#endif
+}
+

 /*
 ** Add a new column to the table currently being constructed.
@@ -1086,7 +1087,7 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){
  z = sqlite3NameFromToken(db, pName);
  if( z==0 ) return;
  for(i=0; i<p->nCol; i++){
-    if( STRICMP(z, p->aCol[i].zName) ){
+    if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){
      sqlite3ErrorMsg(pParse, "duplicate column name: %s", z);
      sqlite3DbFree(db, z);
      return;
@@ -1104,6 +1105,7 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){
  pCol = &p->aCol[p->nCol];
  memset(pCol, 0, sizeof(p->aCol[0]));
  pCol->zName = z;
+  sqlite3ColumnPropertiesFromName(p, pCol);
 
  /* If there is no type specified, columns have the default affinity
  ** 'BLOB'. If there is a type specified, then sqlite3AddColumnType() will
@@ -3874,7 +3876,7 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
 ** table-valued-function.
 */
 void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){
-  if( p && pList ){
+  if( p ){
    struct SrcList_item *pItem = &p->a[p->nSrc-1];
    assert( pItem->fg.notIndexed==0 );
    assert( pItem->fg.isIndexedBy==0 );
@@ -106,7 +106,8 @@ void sqlite3MaterializeView(
    assert( pFrom->a[0].pOn==0 );
    assert( pFrom->a[0].pUsing==0 );
  }
-  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
+  pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 
+                          SF_IncludeHidden, 0, 0);
  sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
  sqlite3Select(pParse, pSel, &dest);
  sqlite3SelectDelete(db, pSel);
@@ -1889,9 +1889,10 @@ int sqlite3CodeSubselect(

 #ifndef SQLITE_OMIT_EXPLAIN
  if( pParse->explain==2 ){
-    char *zMsg = sqlite3MPrintf(
-        pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ",
-        pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId
+    char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d",
+        jmpIfDynamic>=0?"":"CORRELATED ",
+        pExpr->op==TK_IN?"LIST":"SCALAR",
+        pParse->iNextSelectId
    );
    sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC);
  }
@@ -736,10 +736,8 @@ void sqlite3Insert(
  /* Make sure the number of columns in the source data matches the number
  ** of columns to be inserted into the table.
  */
-  if( IsVirtual(pTab) ){
-    for(i=0; i<pTab->nCol; i++){
-      nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0);
-    }
+  for(i=0; i<pTab->nCol; i++){
+    nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0);
  }
  if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){
    sqlite3ErrorMsg(pParse, 
@@ -835,15 +833,14 @@ void sqlite3Insert(

    /* Create the new column data
    */
-    for(i=0; i<pTab->nCol; i++){
-      if( pColumn==0 ){
-        j = i;
-      }else{
+    for(i=j=0; i<pTab->nCol; i++){
+      if( pColumn ){
        for(j=0; j<pColumn->nId; j++){
          if( pColumn->a[j].idx==i ) break;
        }
      }
-      if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) ){
+      if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId)
+            || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){
        sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1);
      }else if( useTempTable ){
        sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); 
@@ -851,6 +848,7 @@ void sqlite3Insert(
        assert( pSelect==0 ); /* Otherwise useTempTable is true */
        sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1);
      }
+      if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++;
    }

    /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger,
@@ -934,7 +932,6 @@ void sqlite3Insert(
      }
      if( pColumn==0 ){
        if( IsHiddenColumn(&pTab->aCol[i]) ){
-          assert( IsVirtual(pTab) );
          j = -1;
          nHidden++;
        }else{
@@ -1876,7 +1873,7 @@ static int xferOptimization(
    return 0;   /* The result set must have exactly one column */
  }
  assert( pEList->a[0].pExpr );
-  if( pEList->a[0].pExpr->op!=TK_ALL ){
+  if( pEList->a[0].pExpr->op!=TK_ASTERISK ){
    return 0;   /* The result set must be the special operator "*" */
  }

@@ -1912,6 +1909,13 @@ static int xferOptimization(
  for(i=0; i<pDest->nCol; i++){
    Column *pDestCol = &pDest->aCol[i];
    Column *pSrcCol = &pSrc->aCol[i];
+#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
+    if( (db->flags & SQLITE_Vacuum)==0 
+     && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN 
+    ){
+      return 0;    /* Neither table may have __hidden__ columns */
+    }
+#endif
    if( pDestCol->affinity!=pSrcCol->affinity ){
      return 0;    /* Affinity must be the same on all columns */
    }
@@ -1,895 +0,0 @@
-/* Driver template for the LEMON parser generator.
-** The author disclaims copyright to this source code.
-**
-** This version of "lempar.c" is modified, slightly, for use by SQLite.
-** The only modifications are the addition of a couple of NEVER()
-** macros to disable tests that are needed in the case of a general
-** LALR(1) grammar but which are always false in the
-** specific grammar used by SQLite.
-*/
-/* First off, code is included that follows the "include" declaration
-** in the input grammar file. */
-#include <stdio.h>
-%%
-/* Next is all token values, in a form suitable for use by makeheaders.
-** This section will be null unless lemon is run with the -m switch.
-*/
-/* 
-** These constants (all generated automatically by the parser generator)
-** specify the various kinds of tokens (terminals) that the parser
-** understands. 
-**
-** Each symbol here is a terminal symbol in the grammar.
-*/
-%%
-/* Make sure the INTERFACE macro is defined.
-*/
-#ifndef INTERFACE
-# define INTERFACE 1
-#endif
-/* The next thing included is series of defines which control
-** various aspects of the generated parser.
-**    YYCODETYPE         is the data type used for storing terminal
-**                       and nonterminal numbers.  "unsigned char" is
-**                       used if there are fewer than 250 terminals
-**                       and nonterminals.  "int" is used otherwise.
-**    YYNOCODE           is a number of type YYCODETYPE which corresponds
-**                       to no legal terminal or nonterminal number.  This
-**                       number is used to fill in empty slots of the hash 
-**                       table.
-**    YYFALLBACK         If defined, this indicates that one or more tokens
-**                       have fall-back values which should be used if the
-**                       original value of the token will not parse.
-**    YYACTIONTYPE       is the data type used for storing terminal
-**                       and nonterminal numbers.  "unsigned char" is
-**                       used if there are fewer than 250 rules and
-**                       states combined.  "int" is used otherwise.
-**    ParseTOKENTYPE     is the data type used for minor tokens given 
-**                       directly to the parser from the tokenizer.
-**    YYMINORTYPE        is the data type used for all minor tokens.
-**                       This is typically a union of many types, one of
-**                       which is ParseTOKENTYPE.  The entry in the union
-**                       for base tokens is called "yy0".
-**    YYSTACKDEPTH       is the maximum depth of the parser's stack.  If
-**                       zero the stack is dynamically sized using realloc()
-**    ParseARG_SDECL     A static variable declaration for the %extra_argument
-**    ParseARG_PDECL     A parameter declaration for the %extra_argument
-**    ParseARG_STORE     Code to store %extra_argument into yypParser
-**    ParseARG_FETCH     Code to extract %extra_argument from yypParser
-**    YYERRORSYMBOL      is the code number of the error symbol.  If not
-**                       defined, then do no error processing.
-**    YYNSTATE           the combined number of states.
-**    YYNRULE            the number of rules in the grammar
-**    YY_MAX_SHIFT       Maximum value for shift actions
-**    YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
-**    YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
-**    YY_MIN_REDUCE      Maximum value for reduce actions
-**    YY_ERROR_ACTION    The yy_action[] code for syntax error
-**    YY_ACCEPT_ACTION   The yy_action[] code for accept
-**    YY_NO_ACTION       The yy_action[] code for no-op
-*/
-%%
-
-/* The yyzerominor constant is used to initialize instances of
-** YYMINORTYPE objects to zero. */
-static const YYMINORTYPE yyzerominor = { 0 };
-
-/* Define the yytestcase() macro to be a no-op if is not already defined
-** otherwise.
-**
-** Applications can choose to define yytestcase() in the %include section
-** to a macro that can assist in verifying code coverage.  For production
-** code the yytestcase() macro should be turned off.  But it is useful
-** for testing.
-*/
-#ifndef yytestcase
-# define yytestcase(X)
-#endif
-
-
-/* Next are the tables used to determine what action to take based on the
-** current state and lookahead token.  These tables are used to implement
-** functions that take a state number and lookahead value and return an
-** action integer.  
-**
-** Suppose the action integer is N.  Then the action is determined as
-** follows
-**
-**   0 <= N <= YY_MAX_SHIFT             Shift N.  That is, push the lookahead
-**                                      token onto the stack and goto state N.
-**
-**   N between YY_MIN_SHIFTREDUCE       Shift to an arbitrary state then
-**     and YY_MAX_SHIFTREDUCE           reduce by rule N-YY_MIN_SHIFTREDUCE.
-**
-**   N between YY_MIN_REDUCE            Reduce by rule N-YY_MIN_REDUCE
-**     and YY_MAX_REDUCE
-
-**   N == YY_ERROR_ACTION               A syntax error has occurred.
-**
-**   N == YY_ACCEPT_ACTION              The parser accepts its input.
-**
-**   N == YY_NO_ACTION                  No such action.  Denotes unused
-**                                      slots in the yy_action[] table.
-**
-** The action table is constructed as a single large table named yy_action[].
-** Given state S and lookahead X, the action is computed as
-**
-**      yy_action[ yy_shift_ofst[S] + X ]
-**
-** If the index value yy_shift_ofst[S]+X is out of range or if the value
-** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S]
-** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table
-** and that yy_default[S] should be used instead.  
-**
-** The formula above is for computing the action when the lookahead is
-** a terminal symbol.  If the lookahead is a non-terminal (as occurs after
-** a reduce action) then the yy_reduce_ofst[] array is used in place of
-** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
-** YY_SHIFT_USE_DFLT.
-**
-** The following are the tables generated in this section:
-**
-**  yy_action[]        A single table containing all actions.
-**  yy_lookahead[]     A table containing the lookahead for each entry in
-**                     yy_action.  Used to detect hash collisions.
-**  yy_shift_ofst[]    For each state, the offset into yy_action for
-**                     shifting terminals.
-**  yy_reduce_ofst[]   For each state, the offset into yy_action for
-**                     shifting non-terminals after a reduce.
-**  yy_default[]       Default action for each state.
-*/
-%%
-
-/* The next table maps tokens into fallback tokens.  If a construct
-** like the following:
-** 
-**      %fallback ID X Y Z.
-**
-** appears in the grammar, then ID becomes a fallback token for X, Y,
-** and Z.  Whenever one of the tokens X, Y, or Z is input to the parser
-** but it does not parse, the type of the token is changed to ID and
-** the parse is retried before an error is thrown.
-*/
-#ifdef YYFALLBACK
-static const YYCODETYPE yyFallback[] = {
-%%
-};
-#endif /* YYFALLBACK */
-
-/* The following structure represents a single element of the
-** parser's stack.  Information stored includes:
-**
-**   +  The state number for the parser at this level of the stack.
-**
-**   +  The value of the token stored at this level of the stack.
-**      (In other words, the "major" token.)
-**
-**   +  The semantic value stored at this level of the stack.  This is
-**      the information used by the action routines in the grammar.
-**      It is sometimes called the "minor" token.
-**
-** After the "shift" half of a SHIFTREDUCE action, the stateno field
-** actually contains the reduce action for the second half of the
-** SHIFTREDUCE.
-*/
-struct yyStackEntry {
-  YYACTIONTYPE stateno;  /* The state-number, or reduce action in SHIFTREDUCE */
-  YYCODETYPE major;      /* The major token value.  This is the code
-                         ** number for the token at this stack level */
-  YYMINORTYPE minor;     /* The user-supplied minor token value.  This
-                         ** is the value of the token  */
-};
-typedef struct yyStackEntry yyStackEntry;
-
-/* The state of the parser is completely contained in an instance of
-** the following structure */
-struct yyParser {
-  int yyidx;                    /* Index of top element in stack */
-#ifdef YYTRACKMAXSTACKDEPTH
-  int yyidxMax;                 /* Maximum value of yyidx */
-#endif
-  int yyerrcnt;                 /* Shifts left before out of the error */
-  ParseARG_SDECL                /* A place to hold %extra_argument */
-#if YYSTACKDEPTH<=0
-  int yystksz;                  /* Current side of the stack */
-  yyStackEntry *yystack;        /* The parser's stack */
-#else
-  yyStackEntry yystack[YYSTACKDEPTH];  /* The parser's stack */
-#endif
-};
-typedef struct yyParser yyParser;
-
-#ifndef NDEBUG
-#include <stdio.h>
-static FILE *yyTraceFILE = 0;
-static char *yyTracePrompt = 0;
-#endif /* NDEBUG */
-
-#ifndef NDEBUG
-/* 
-** Turn parser tracing on by giving a stream to which to write the trace
-** and a prompt to preface each trace message.  Tracing is turned off
-** by making either argument NULL 
-**
-** Inputs:
-** <ul>
-** <li> A FILE* to which trace output should be written.
-**      If NULL, then tracing is turned off.
-** <li> A prefix string written at the beginning of every
-**      line of trace output.  If NULL, then tracing is
-**      turned off.
-** </ul>
-**
-** Outputs:
-** None.
-*/
-void ParseTrace(FILE *TraceFILE, char *zTracePrompt){
-  yyTraceFILE = TraceFILE;
-  yyTracePrompt = zTracePrompt;
-  if( yyTraceFILE==0 ) yyTracePrompt = 0;
-  else if( yyTracePrompt==0 ) yyTraceFILE = 0;
-}
-#endif /* NDEBUG */
-
-#ifndef NDEBUG
-/* For tracing shifts, the names of all terminals and nonterminals
-** are required.  The following table supplies these names */
-static const char *const yyTokenName[] = { 
-%%
-};
-#endif /* NDEBUG */
-
-#ifndef NDEBUG
-/* For tracing reduce actions, the names of all rules are required.
-*/
-static const char *const yyRuleName[] = {
-%%
-};
-#endif /* NDEBUG */
-
-
-#if YYSTACKDEPTH<=0
-/*
-** Try to increase the size of the parser stack.
-*/
-static void yyGrowStack(yyParser *p){
-  int newSize;
-  yyStackEntry *pNew;
-
-  newSize = p->yystksz*2 + 100;
-  pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
-  if( pNew ){
-    p->yystack = pNew;
-    p->yystksz = newSize;
-#ifndef NDEBUG
-    if( yyTraceFILE ){
-      fprintf(yyTraceFILE,"%sStack grows to %d entries!\n",
-              yyTracePrompt, p->yystksz);
-    }
-#endif
-  }
-}
-#endif
-
-/* 
-** This function allocates a new parser.
-** The only argument is a pointer to a function which works like
-** malloc.
-**
-** Inputs:
-** A pointer to the function used to allocate memory.
-**
-** Outputs:
-** A pointer to a parser.  This pointer is used in subsequent calls
-** to Parse and ParseFree.
-*/
-void *ParseAlloc(void *(*mallocProc)(u64)){
-  yyParser *pParser;
-  pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) );
-  if( pParser ){
-    pParser->yyidx = -1;
-#ifdef YYTRACKMAXSTACKDEPTH
-    pParser->yyidxMax = 0;
-#endif
-#if YYSTACKDEPTH<=0
-    pParser->yystack = NULL;
-    pParser->yystksz = 0;
-    yyGrowStack(pParser);
-#endif
-  }
-  return pParser;
-}
-
-/* The following function deletes the value associated with a
-** symbol.  The symbol can be either a terminal or nonterminal.
-** "yymajor" is the symbol code, and "yypminor" is a pointer to
-** the value.
-*/
-static void yy_destructor(
-  yyParser *yypParser,    /* The parser */
-  YYCODETYPE yymajor,     /* Type code for object to destroy */
-  YYMINORTYPE *yypminor   /* The object to be destroyed */
-){
-  ParseARG_FETCH;
-  switch( yymajor ){
-    /* Here is inserted the actions which take place when a
-    ** terminal or non-terminal is destroyed.  This can happen
-    ** when the symbol is popped from the stack during a
-    ** reduce or during error processing or when a parser is 
-    ** being destroyed before it is finished parsing.
-    **
-    ** Note: during a reduce, the only symbols destroyed are those
-    ** which appear on the RHS of the rule, but which are not used
-    ** inside the C code.
-    */
-%%
-    default:  break;   /* If no destructor action specified: do nothing */
-  }
-}
-
-/*
-** Pop the parser's stack once.
-**
-** If there is a destructor routine associated with the token which
-** is popped from the stack, then call it.
-**
-** Return the major token number for the symbol popped.
-*/
-static int yy_pop_parser_stack(yyParser *pParser){
-  YYCODETYPE yymajor;
-  yyStackEntry *yytos = &pParser->yystack[pParser->yyidx];
-
-  /* There is no mechanism by which the parser stack can be popped below
-  ** empty in SQLite.  */
-  assert( pParser->yyidx>=0 );
-#ifndef NDEBUG
-  if( yyTraceFILE && pParser->yyidx>=0 ){
-    fprintf(yyTraceFILE,"%sPopping %s\n",
-      yyTracePrompt,
-      yyTokenName[yytos->major]);
-  }
-#endif
-  yymajor = yytos->major;
-  yy_destructor(pParser, yymajor, &yytos->minor);
-  pParser->yyidx--;
-  return yymajor;
-}
-
-/* 
-** Deallocate and destroy a parser.  Destructors are all called for
-** all stack elements before shutting the parser down.
-**
-** Inputs:
-** <ul>
-** <li>  A pointer to the parser.  This should be a pointer
-**       obtained from ParseAlloc.
-** <li>  A pointer to a function used to reclaim memory obtained
-**       from malloc.
-** </ul>
-*/
-void ParseFree(
-  void *p,                    /* The parser to be deleted */
-  void (*freeProc)(void*)     /* Function used to reclaim memory */
-){
-  yyParser *pParser = (yyParser*)p;
-  /* In SQLite, we never try to destroy a parser that was not successfully
-  ** created in the first place. */
-  if( NEVER(pParser==0) ) return;
-  while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
-#if YYSTACKDEPTH<=0
-  free(pParser->yystack);
-#endif
-  (*freeProc)((void*)pParser);
-}
-
-/*
-** Return the peak depth of the stack for a parser.
-*/
-#ifdef YYTRACKMAXSTACKDEPTH
-int ParseStackPeak(void *p){
-  yyParser *pParser = (yyParser*)p;
-  return pParser->yyidxMax;
-}
-#endif
-
-/*
-** Find the appropriate action for a parser given the terminal
-** look-ahead token iLookAhead.
-**
-** If the look-ahead token is YYNOCODE, then check to see if the action is
-** independent of the look-ahead.  If it is, return the action, otherwise
-** return YY_NO_ACTION.
-*/
-static int yy_find_shift_action(
-  yyParser *pParser,        /* The parser */
-  YYCODETYPE iLookAhead     /* The look-ahead token */
-){
-  int i;
-  int stateno = pParser->yystack[pParser->yyidx].stateno;
- 
-  if( stateno>=YY_MIN_REDUCE ) return stateno;
-  assert( stateno <= YY_SHIFT_COUNT );
-  i = yy_shift_ofst[stateno];
-  if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
-  assert( iLookAhead!=YYNOCODE );
-  i += iLookAhead;
-  if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
-    if( iLookAhead>0 ){
-#ifdef YYFALLBACK
-      YYCODETYPE iFallback;            /* Fallback token */
-      if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
-             && (iFallback = yyFallback[iLookAhead])!=0 ){
-#ifndef NDEBUG
-        if( yyTraceFILE ){
-          fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
-             yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
-        }
-#endif
-        return yy_find_shift_action(pParser, iFallback);
-      }
-#endif
-#ifdef YYWILDCARD
-      {
-        int j = i - iLookAhead + YYWILDCARD;
-        if( 
-#if YY_SHIFT_MIN+YYWILDCARD<0
-          j>=0 &&
-#endif
-#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
-          j<YY_ACTTAB_COUNT &&
-#endif
-          yy_lookahead[j]==YYWILDCARD
-        ){
-#ifndef NDEBUG
-          if( yyTraceFILE ){
-            fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
-               yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]);
-          }
-#endif /* NDEBUG */
-          return yy_action[j];
-        }
-      }
-#endif /* YYWILDCARD */
-    }
-    return yy_default[stateno];
-  }else{
-    return yy_action[i];
-  }
-}
-
-/*
-** Find the appropriate action for a parser given the non-terminal
-** look-ahead token iLookAhead.
-**
-** If the look-ahead token is YYNOCODE, then check to see if the action is
-** independent of the look-ahead.  If it is, return the action, otherwise
-** return YY_NO_ACTION.
-*/
-static int yy_find_reduce_action(
-  int stateno,              /* Current state number */
-  YYCODETYPE iLookAhead     /* The look-ahead token */
-){
-  int i;
-#ifdef YYERRORSYMBOL
-  if( stateno>YY_REDUCE_COUNT ){
-    return yy_default[stateno];
-  }
-#else
-  assert( stateno<=YY_REDUCE_COUNT );
-#endif
-  i = yy_reduce_ofst[stateno];
-  assert( i!=YY_REDUCE_USE_DFLT );
-  assert( iLookAhead!=YYNOCODE );
-  i += iLookAhead;
-#ifdef YYERRORSYMBOL
-  if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
-    return yy_default[stateno];
-  }
-#else
-  assert( i>=0 && i<YY_ACTTAB_COUNT );
-  assert( yy_lookahead[i]==iLookAhead );
-#endif
-  return yy_action[i];
-}
-
-/*
-** The following routine is called if the stack overflows.
-*/
-static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
-   ParseARG_FETCH;
-   yypParser->yyidx--;
-#ifndef NDEBUG
-   if( yyTraceFILE ){
-     fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
-   }
-#endif
-   while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
-   /* Here code is inserted which will execute if the parser
-   ** stack every overflows */
-%%
-   ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
-}
-
-/*
-** Print tracing information for a SHIFT action
-*/
-#ifndef NDEBUG
-static void yyTraceShift(yyParser *yypParser, int yyNewState){
-  if( yyTraceFILE ){
-    int i;
-    if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
-      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
-      for(i=1; i<=yypParser->yyidx; i++)
-        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
-      fprintf(yyTraceFILE,"\n");
-    }else{
-      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
-    }
-  }
-}
-#else
-# define yyTraceShift(X,Y)
-#endif
-
-/*
-** Perform a shift action.  Return the number of errors.
-*/
-static void yy_shift(
-  yyParser *yypParser,          /* The parser to be shifted */
-  int yyNewState,               /* The new state to shift in */
-  int yyMajor,                  /* The major token to shift in */
-  YYMINORTYPE *yypMinor         /* Pointer to the minor token to shift in */
-){
-  yyStackEntry *yytos;
-  yypParser->yyidx++;
-#ifdef YYTRACKMAXSTACKDEPTH
-  if( yypParser->yyidx>yypParser->yyidxMax ){
-    yypParser->yyidxMax = yypParser->yyidx;
-  }
-#endif
-#if YYSTACKDEPTH>0 
-  if( yypParser->yyidx>=YYSTACKDEPTH ){
-    yyStackOverflow(yypParser, yypMinor);
-    return;
-  }
-#else
-  if( yypParser->yyidx>=yypParser->yystksz ){
-    yyGrowStack(yypParser);
-    if( yypParser->yyidx>=yypParser->yystksz ){
-      yyStackOverflow(yypParser, yypMinor);
-      return;
-    }
-  }
-#endif
-  yytos = &yypParser->yystack[yypParser->yyidx];
-  yytos->stateno = (YYACTIONTYPE)yyNewState;
-  yytos->major = (YYCODETYPE)yyMajor;
-  yytos->minor = *yypMinor;
-  yyTraceShift(yypParser, yyNewState);
-}
-
-/* The following table contains information about every rule that
-** is used during the reduce.
-*/
-static const struct {
-  YYCODETYPE lhs;         /* Symbol on the left-hand side of the rule */
-  unsigned char nrhs;     /* Number of right-hand side symbols in the rule */
-} yyRuleInfo[] = {
-%%
-};
-
-static void yy_accept(yyParser*);  /* Forward Declaration */
-
-/*
-** Perform a reduce action and the shift that must immediately
-** follow the reduce.
-*/
-static void yy_reduce(
-  yyParser *yypParser,         /* The parser */
-  int yyruleno                 /* Number of the rule by which to reduce */
-){
-  int yygoto;                     /* The next state */
-  int yyact;                      /* The next action */
-  YYMINORTYPE yygotominor;        /* The LHS of the rule reduced */
-  yyStackEntry *yymsp;            /* The top of the parser's stack */
-  int yysize;                     /* Amount to pop the stack */
-  ParseARG_FETCH;
-  yymsp = &yypParser->yystack[yypParser->yyidx];
-#ifndef NDEBUG
-  if( yyTraceFILE && yyruleno>=0 
-        && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
-    yysize = yyRuleInfo[yyruleno].nrhs;
-    fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt,
-      yyRuleName[yyruleno], yymsp[-yysize].stateno);
-  }
-#endif /* NDEBUG */
-
-  /* Silence complaints from purify about yygotominor being uninitialized
-  ** in some cases when it is copied into the stack after the following
-  ** switch.  yygotominor is uninitialized when a rule reduces that does
-  ** not set the value of its left-hand side nonterminal.  Leaving the
-  ** value of the nonterminal uninitialized is utterly harmless as long
-  ** as the value is never used.  So really the only thing this code
-  ** accomplishes is to quieten purify.  
-  **
-  ** 2007-01-16:  The wireshark project (www.wireshark.org) reports that
-  ** without this code, their parser segfaults.  I'm not sure what there
-  ** parser is doing to make this happen.  This is the second bug report
-  ** from wireshark this week.  Clearly they are stressing Lemon in ways
-  ** that it has not been previously stressed...  (SQLite ticket #2172)
-  */
-  /*memset(&yygotominor, 0, sizeof(yygotominor));*/
-  yygotominor = yyzerominor;
-
-
-  switch( yyruleno ){
-  /* Beginning here are the reduction cases.  A typical example
-  ** follows:
-  **   case 0:
-  **  #line <lineno> <grammarfile>
-  **     { ... }           // User supplied code
-  **  #line <lineno> <thisfile>
-  **     break;
-  */
-%%
-  };
-  assert( yyruleno>=0 && yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
-  yygoto = yyRuleInfo[yyruleno].lhs;
-  yysize = yyRuleInfo[yyruleno].nrhs;
-  yypParser->yyidx -= yysize;
-  yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
-  if( yyact <= YY_MAX_SHIFTREDUCE ){
-    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-    /* If the reduce action popped at least
-    ** one element off the stack, then we can push the new element back
-    ** onto the stack here, and skip the stack overflow test in yy_shift().
-    ** That gives a significant speed improvement. */
-    if( yysize ){
-      yypParser->yyidx++;
-      yymsp -= yysize-1;
-      yymsp->stateno = (YYACTIONTYPE)yyact;
-      yymsp->major = (YYCODETYPE)yygoto;
-      yymsp->minor = yygotominor;
-      yyTraceShift(yypParser, yyact);
-    }else{
-      yy_shift(yypParser,yyact,yygoto,&yygotominor);
-    }
-  }else{
-    assert( yyact == YY_ACCEPT_ACTION );
-    yy_accept(yypParser);
-  }
-}
-
-/*
-** The following code executes when the parse fails
-*/
-#ifndef YYNOERRORRECOVERY
-static void yy_parse_failed(
-  yyParser *yypParser           /* The parser */
-){
-  ParseARG_FETCH;
-#ifndef NDEBUG
-  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
-  }
-#endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
-  /* Here code is inserted which will be executed whenever the
-  ** parser fails */
-%%
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
-}
-#endif /* YYNOERRORRECOVERY */
-
-/*
-** The following code executes when a syntax error first occurs.
-*/
-static void yy_syntax_error(
-  yyParser *yypParser,           /* The parser */
-  int yymajor,                   /* The major type of the error token */
-  YYMINORTYPE yyminor            /* The minor type of the error token */
-){
-  ParseARG_FETCH;
-#define TOKEN (yyminor.yy0)
-%%
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
-}
-
-/*
-** The following is executed when the parser accepts
-*/
-static void yy_accept(
-  yyParser *yypParser           /* The parser */
-){
-  ParseARG_FETCH;
-#ifndef NDEBUG
-  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
-  }
-#endif
-  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
-  /* Here code is inserted which will be executed whenever the
-  ** parser accepts */
-%%
-  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
-}
-
-/* The main parser program.
-** The first argument is a pointer to a structure obtained from
-** "ParseAlloc" which describes the current state of the parser.
-** The second argument is the major token number.  The third is
-** the minor token.  The fourth optional argument is whatever the
-** user wants (and specified in the grammar) and is available for
-** use by the action routines.
-**
-** Inputs:
-** <ul>
-** <li> A pointer to the parser (an opaque structure.)
-** <li> The major token number.
-** <li> The minor token number.
-** <li> An option argument of a grammar-specified type.
-** </ul>
-**
-** Outputs:
-** None.
-*/
-void Parse(
-  void *yyp,                   /* The parser */
-  int yymajor,                 /* The major token code number */
-  ParseTOKENTYPE yyminor       /* The value for the token */
-  ParseARG_PDECL               /* Optional %extra_argument parameter */
-){
-  YYMINORTYPE yyminorunion;
-  int yyact;            /* The parser action. */
-#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
-  int yyendofinput;     /* True if we are at the end of input */
-#endif
-#ifdef YYERRORSYMBOL
-  int yyerrorhit = 0;   /* True if yymajor has invoked an error */
-#endif
-  yyParser *yypParser;  /* The parser */
-
-  /* (re)initialize the parser, if necessary */
-  yypParser = (yyParser*)yyp;
-  if( yypParser->yyidx<0 ){
-#if YYSTACKDEPTH<=0
-    if( yypParser->yystksz <=0 ){
-      /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/
-      yyminorunion = yyzerominor;
-      yyStackOverflow(yypParser, &yyminorunion);
-      return;
-    }
-#endif
-    yypParser->yyidx = 0;
-    yypParser->yyerrcnt = -1;
-    yypParser->yystack[0].stateno = 0;
-    yypParser->yystack[0].major = 0;
-  }
-  yyminorunion.yy0 = yyminor;
-#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
-  yyendofinput = (yymajor==0);
-#endif
-  ParseARG_STORE;
-
-#ifndef NDEBUG
-  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]);
-  }
-#endif
-
-  do{
-    yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
-    if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
-      yy_shift(yypParser,yyact,yymajor,&yyminorunion);
-      yypParser->yyerrcnt--;
-      yymajor = YYNOCODE;
-    }else if( yyact <= YY_MAX_REDUCE ){
-      yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
-    }else{
-      assert( yyact == YY_ERROR_ACTION );
-#ifdef YYERRORSYMBOL
-      int yymx;
-#endif
-#ifndef NDEBUG
-      if( yyTraceFILE ){
-        fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt);
-      }
-#endif
-#ifdef YYERRORSYMBOL
-      /* A syntax error has occurred.
-      ** The response to an error depends upon whether or not the
-      ** grammar defines an error token "ERROR".  
-      **
-      ** This is what we do if the grammar does define ERROR:
-      **
-      **  * Call the %syntax_error function.
-      **
-      **  * Begin popping the stack until we enter a state where
-      **    it is legal to shift the error symbol, then shift
-      **    the error symbol.
-      **
-      **  * Set the error count to three.
-      **
-      **  * Begin accepting and shifting new tokens.  No new error
-      **    processing will occur until three tokens have been
-      **    shifted successfully.
-      **
-      */
-      if( yypParser->yyerrcnt<0 ){
-        yy_syntax_error(yypParser,yymajor,yyminorunion);
-      }
-      yymx = yypParser->yystack[yypParser->yyidx].major;
-      if( yymx==YYERRORSYMBOL || yyerrorhit ){
-#ifndef NDEBUG
-        if( yyTraceFILE ){
-          fprintf(yyTraceFILE,"%sDiscard input token %s\n",
-             yyTracePrompt,yyTokenName[yymajor]);
-        }
-#endif
-        yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion);
-        yymajor = YYNOCODE;
-      }else{
-         while(
-          yypParser->yyidx >= 0 &&
-          yymx != YYERRORSYMBOL &&
-          (yyact = yy_find_reduce_action(
-                        yypParser->yystack[yypParser->yyidx].stateno,
-                        YYERRORSYMBOL)) >= YY_MIN_REDUCE
-        ){
-          yy_pop_parser_stack(yypParser);
-        }
-        if( yypParser->yyidx < 0 || yymajor==0 ){
-          yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
-          yy_parse_failed(yypParser);
-          yymajor = YYNOCODE;
-        }else if( yymx!=YYERRORSYMBOL ){
-          YYMINORTYPE u2;
-          u2.YYERRSYMDT = 0;
-          yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2);
-        }
-      }
-      yypParser->yyerrcnt = 3;
-      yyerrorhit = 1;
-#elif defined(YYNOERRORRECOVERY)
-      /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to
-      ** do any kind of error recovery.  Instead, simply invoke the syntax
-      ** error routine and continue going as if nothing had happened.
-      **
-      ** Applications can set this macro (for example inside %include) if
-      ** they intend to abandon the parse upon the first syntax error seen.
-      */
-      yy_syntax_error(yypParser,yymajor,yyminorunion);
-      yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
-      yymajor = YYNOCODE;
-      
-#else  /* YYERRORSYMBOL is not defined */
-      /* This is what we do if the grammar does not define ERROR:
-      **
-      **  * Report an error message, and throw away the input token.
-      **
-      **  * If the input token is $, then fail the parse.
-      **
-      ** As before, subsequent error messages are suppressed until
-      ** three input tokens have been successfully shifted.
-      */
-      if( yypParser->yyerrcnt<=0 ){
-        yy_syntax_error(yypParser,yymajor,yyminorunion);
-      }
-      yypParser->yyerrcnt = 3;
-      yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
-      if( yyendofinput ){
-        yy_parse_failed(yypParser);
-      }
-      yymajor = YYNOCODE;
-#endif
-    }
-  }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
-#ifndef NDEBUG
-  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt);
-  }
-#endif
-  return;
-}
@@ -220,6 +220,12 @@ int sqlite3_initialize(void){
  if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){
    FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions);
    sqlite3GlobalConfig.inProgress = 1;
+#ifdef SQLITE_ENABLE_SQLLOG
+    {
+      extern void sqlite3_init_sqllog(void);
+      sqlite3_init_sqllog();
+    }
+#endif
    memset(pHash, 0, sizeof(sqlite3GlobalFunctions));
    sqlite3RegisterGlobalFunctions();
    if( sqlite3GlobalConfig.isPCacheInit==0 ){
@@ -241,7 +241,7 @@ static void *memsys5MallocUnsafe(int nByte){
  }

  /* Round nByte up to the next valid power of two */
-  for(iFullSz=mem5.szAtom, iLogsize=0; iFullSz<nByte; iFullSz *= 2, iLogsize++){}
+  for(iFullSz=mem5.szAtom,iLogsize=0; iFullSz<nByte; iFullSz*=2,iLogsize++){}

  /* Make sure mem5.aiFreelist[iLogsize] contains at least one free
  ** block.  If not, then split a block of the next larger power of
@@ -50,7 +50,7 @@ struct sqlite3_mutex {
 #endif
 };
 #if SQLITE_MUTEX_NREF
-#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0, 0, (pthread_t)0, 0 }
+#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0}
 #else
 #define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
 #endif
@@ -3349,7 +3349,7 @@ static int unixWrite(
  }
 #endif

-#if SQLITE_MAX_MMAP_SIZE>0
+#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0
  /* Deal with as much of this write request as possible by transfering
  ** data from the memory mapping using memcpy().  */
  if( offset<pFile->mmapSize ){
@@ -4774,7 +4774,9 @@ static void unixRemapfile(
  assert( pFd->mmapSizeActual>=pFd->mmapSize );
  assert( MAP_FAILED!=0 );

+#ifdef SQLITE_MMAP_READWRITE
  if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;
+#endif

  if( pOrig ){
 #if HAVE_MREMAP
@@ -6553,7 +6555,7 @@ static int proxyCreateLockPath(const char *lockPath){
    }
    buf[i] = lockPath[i];
  }
-  OSTRACE(("CREATELOCKPATH  proxy lock path=%s pid=%d\n", lockPath, osGetpid(0)));
+  OSTRACE(("CREATELOCKPATH  proxy lock path=%s pid=%d\n",lockPath,osGetpid(0)));
  return 0;
 }

@@ -2602,7 +2602,7 @@ static int winWrite(
           "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile,
           pFile->h, pBuf, amt, offset, pFile->locktype));

-#if SQLITE_MAX_MMAP_SIZE>0
+#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0
  /* Deal with as much of this write request as possible by transfering
  ** data from the memory mapping using memcpy().  */
  if( offset<pFile->mmapSize ){
@@ -3150,7 +3150,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){
    res = 1;
    OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res));
  }else{
-    res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0);
+    res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0);
    if( res ){
      winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0);
    }
@@ -4096,10 +4096,12 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){
    DWORD flags = FILE_MAP_READ;

    winUnmapfile(pFd);
+#ifdef SQLITE_MMAP_READWRITE
    if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){
      protect = PAGE_READWRITE;
      flags |= FILE_MAP_WRITE;
    }
+#endif
 #if SQLITE_OS_WINRT
    pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL);
 #elif defined(SQLITE_WIN32_HAS_WIDE)
@@ -3392,12 +3392,21 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){
 }

 /*
-** Change the maximum number of in-memory pages that are allowed.
+** Change the maximum number of in-memory pages that are allowed
+** before attempting to recycle clean and unused pages.
 */
 void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){
  sqlite3PcacheSetCachesize(pPager->pPCache, mxPage);
 }

+/*
+** Change the maximum number of in-memory pages that are allowed
+** before attempting to spill pages to journal.
+*/
+int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){
+  return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage);
+}
+
 /*
 ** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap.
 */
@@ -123,6 +123,7 @@ void sqlite3PagerAlignReserve(Pager*,Pager*);
 #endif
 int sqlite3PagerMaxPageCount(Pager*, int);
 void sqlite3PagerSetCachesize(Pager*, int);
+int sqlite3PagerSetSpillsize(Pager*, int);
 void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64);
 void sqlite3PagerShrink(Pager*);
 void sqlite3PagerSetFlags(Pager*,unsigned);
@@ -60,6 +60,18 @@
 */
 #define yytestcase(X) testcase(X)

+/*
+** Indicate that sqlite3ParserFree() will never be called with a null
+** pointer.
+*/
+#define YYPARSEFREENEVERNULL 1
+
+/*
+** Alternative datatype for the argument to the malloc() routine passed
+** into sqlite3ParserAlloc().  The default is size_t.
+*/
+#define YYMALLOCARGTYPE  u64
+
 /*
 ** An instance of this structure holds information about the
 ** LIMIT clause of a SELECT statement.
@@ -162,7 +174,7 @@ create_table_args ::= AS select(S). {
  sqlite3EndTable(pParse,0,0,0,S);
  sqlite3SelectDelete(pParse->db, S);
 }
-%type table_options {u8}
+%type table_options {int}
 table_options(A) ::= .    {A = 0;}
 table_options(A) ::= WITHOUT nm(X). {
  if( X.n==5 && sqlite3_strnicmp(X.z,"rowid",5)==0 ){
@@ -364,12 +376,12 @@ defer_subclause_opt(A) ::= defer_subclause(X).  {A = X;}
 // default behavior when there is a constraint conflict.
 //
 %type onconf {int}
-%type orconf {u8}
+%type orconf {int}
 %type resolvetype {int}
 onconf(A) ::= .                              {A = OE_Default;}
 onconf(A) ::= ON CONFLICT resolvetype(X).    {A = X;}
 orconf(A) ::= .                              {A = OE_Default;}
-orconf(A) ::= OR resolvetype(X).             {A = (u8)X;}
+orconf(A) ::= OR resolvetype(X).             {A = X;}
 resolvetype(A) ::= raisetype(X).             {A = X;}
 resolvetype(A) ::= IGNORE.                   {A = OE_Ignore;}
 resolvetype(A) ::= REPLACE.                  {A = OE_Replace;}
@@ -526,7 +538,7 @@ values(A) ::= values(X) COMMA LP exprlist(Y) RP. {
 // The "distinct" nonterminal is true (1) if the DISTINCT keyword is
 // present and false (0) if it is not.
 //
-%type distinct {u16}
+%type distinct {int}
 distinct(A) ::= DISTINCT.   {A = SF_Distinct;}
 distinct(A) ::= ALL.        {A = SF_All;}
 distinct(A) ::= .           {A = 0;}
@@ -534,7 +546,7 @@ distinct(A) ::= .           {A = 0;}
 // selcollist is a list of expressions that are to become the return
 // values of the SELECT statement.  The "*" in statements like
 // "SELECT * FROM ..." is encoded as a special expression with an
-// opcode of TK_ALL.
+// opcode of TK_ASTERISK.
 //
 %type selcollist {ExprList*}
 %destructor selcollist {sqlite3ExprListDelete(pParse->db, $$);}
@@ -548,11 +560,11 @@ selcollist(A) ::= sclp(P) expr(X) as(Y).     {
   sqlite3ExprListSetSpan(pParse,A,&X);
 }
 selcollist(A) ::= sclp(P) STAR. {
-  Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0);
+  Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0);
  A = sqlite3ExprListAppend(pParse, P, p);
 }
 selcollist(A) ::= sclp(P) nm(X) DOT STAR(Y). {
-  Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &Y);
+  Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &Y);
  Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &X);
  Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0);
  A = sqlite3ExprListAppend(pParse,P, pDot);
@@ -639,7 +651,6 @@ dbnm(A) ::= DOT nm(X). {A = X;}
 fullname(A) ::= nm(X) dbnm(Y).  {A = sqlite3SrcListAppend(pParse->db,0,&X,&Y);}

 %type joinop {int}
-%type joinop2 {int}
 joinop(X) ::= COMMA|JOIN.              { X = JT_INNER; }
 joinop(X) ::= JOIN_KW(A) JOIN.         { X = sqlite3JoinType(pParse,&A,0,0); }
 joinop(X) ::= JOIN_KW(A) nm(B) JOIN.   { X = sqlite3JoinType(pParse,&A,&B,0); }
@@ -800,7 +811,7 @@ cmd ::= with(W) insert_cmd(R) INTO fullname(X) idlist_opt(F) DEFAULT VALUES.
  sqlite3Insert(pParse, X, 0, F, R);
 }

-%type insert_cmd {u8}
+%type insert_cmd {int}
 insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
 insert_cmd(A) ::= REPLACE.            {A = OE_Replace;}

@@ -21,6 +21,7 @@ struct PCache {
  PgHdr *pSynced;                     /* Last synced page in dirty page list */
  int nRefSum;                        /* Sum of ref counts over all pages */
  int szCache;                        /* Configured cache size */
+  int szSpill;                        /* Size before spilling occurs */
  int szPage;                         /* Size of every page in this cache */
  int szExtra;                        /* Size of extra space for each page */
  u8 bPurgeable;                      /* True if pages are on backing store */
@@ -110,10 +111,8 @@ static void pcacheUnpin(PgHdr *p){
 }

 /*
-** Compute the number of pages of cache requested.  p->szCache is the
+** Compute the number of pages of cache requested.   p->szCache is the
 ** cache size requested by the "PRAGMA cache_size" statement.
-**
-**
 */
 static int numberOfCachePages(PCache *p){
  if( p->szCache>=0 ){
@@ -176,6 +175,7 @@ int sqlite3PcacheOpen(
  p->xStress = xStress;
  p->pStress = pStress;
  p->szCache = 100;
+  p->szSpill = 1;
  return sqlite3PcacheSetPageSize(p, szPage);
 }

@@ -271,32 +271,33 @@ int sqlite3PcacheFetchStress(
  PgHdr *pPg;
  if( pCache->eCreate==2 ) return 0;

-
-  /* Find a dirty page to write-out and recycle. First try to find a 
-  ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC
-  ** cleared), but if that is not possible settle for any other 
-  ** unreferenced dirty page.
-  */
-  for(pPg=pCache->pSynced; 
-      pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); 
-      pPg=pPg->pDirtyPrev
-  );
-  pCache->pSynced = pPg;
-  if( !pPg ){
-    for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev);
-  }
-  if( pPg ){
-    int rc;
+  if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){
+    /* Find a dirty page to write-out and recycle. First try to find a 
+    ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC
+    ** cleared), but if that is not possible settle for any other 
+    ** unreferenced dirty page.
+    */
+    for(pPg=pCache->pSynced; 
+        pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); 
+        pPg=pPg->pDirtyPrev
+    );
+    pCache->pSynced = pPg;
+    if( !pPg ){
+      for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev);
+    }
+    if( pPg ){
+      int rc;
 #ifdef SQLITE_LOG_CACHE_SPILL
-    sqlite3_log(SQLITE_FULL, 
-                "spill page %d making room for %d - cache used: %d/%d",
-                pPg->pgno, pgno,
-                sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache),
+      sqlite3_log(SQLITE_FULL, 
+                  "spill page %d making room for %d - cache used: %d/%d",
+                  pPg->pgno, pgno,
+                  sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache),
                numberOfCachePages(pCache));
 #endif
-    rc = pCache->xStress(pCache->pStress, pPg);
-    if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
-      return rc;
+      rc = pCache->xStress(pCache->pStress, pPg);
+      if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){
+        return rc;
+      }
    }
  }
  *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2);
@@ -641,6 +642,25 @@ void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){
                                         numberOfCachePages(pCache));
 }

+/*
+** Set the suggested cache-spill value.  Make no changes if if the
+** argument is zero.  Return the effective cache-spill size, which will
+** be the larger of the szSpill and szCache.
+*/
+int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){
+  int res;
+  assert( p->pCache!=0 );
+  if( mxPage ){
+    if( mxPage<0 ){
+      mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra));
+    }
+    p->szSpill = mxPage;
+  }
+  res = numberOfCachePages(p);
+  if( res<p->szSpill ) res = p->szSpill; 
+  return res;
+}
+
 /*
 ** Free up as much memory as possible from the page cache.
 */
@@ -146,6 +146,13 @@ void sqlite3PcacheSetCachesize(PCache *, int);
 int sqlite3PcacheGetCachesize(PCache *);
 #endif

+/* Set or get the suggested spill-size for the specified pager-cache.
+**
+** The spill-size is the minimum number of pages in cache before the cache
+** will attempt to spill dirty pages by calling xStress.
+*/
+int sqlite3PcacheSetSpillsize(PCache *, int);
+
 /* Free up as much memory as possible from the page cache */
 void sqlite3PcacheShrink(PCache*);

@@ -279,7 +279,7 @@ const char *sqlite3JournalModename(int eMode){
 **
 ** Pragmas are of this form:
 **
-**      PRAGMA [database.]id [= value]
+**      PRAGMA [schema.]id [= value]
 **
 ** The identifier might also be a string.  The value is a string, and
 ** identifier, or a number.  If minusFlag is true, then the value is
@@ -291,8 +291,8 @@ const char *sqlite3JournalModename(int eMode){
 */
 void sqlite3Pragma(
  Parse *pParse, 
-  Token *pId1,        /* First part of [database.]id field */
-  Token *pId2,        /* Second part of [database.]id field, or NULL */
+  Token *pId1,        /* First part of [schema.]id field */
+  Token *pId2,        /* Second part of [schema.]id field, or NULL */
  Token *pValue,      /* Token for <value>, or NULL */
  int minusFlag       /* True if a '-' sign preceded <value> */
 ){
@@ -313,7 +313,7 @@ void sqlite3Pragma(
  sqlite3VdbeRunOnlyOnce(v);
  pParse->nMem = 2;

-  /* Interpret the [database.] part of the pragma statement. iDb is the
+  /* Interpret the [schema.] part of the pragma statement. iDb is the
  ** index of the database this pragma is being applied to in db.aDb[]. */
  iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId);
  if( iDb<0 ) return;
@@ -402,8 +402,8 @@ void sqlite3Pragma(
  
 #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED)
  /*
-  **  PRAGMA [database.]default_cache_size
-  **  PRAGMA [database.]default_cache_size=N
+  **  PRAGMA [schema.]default_cache_size
+  **  PRAGMA [schema.]default_cache_size=N
  **
  ** The first form reports the current persistent setting for the
  ** page cache size.  The value returned is the maximum number of
@@ -454,8 +454,8 @@ void sqlite3Pragma(

 #if !defined(SQLITE_OMIT_PAGER_PRAGMAS)
  /*
-  **  PRAGMA [database.]page_size
-  **  PRAGMA [database.]page_size=N
+  **  PRAGMA [schema.]page_size
+  **  PRAGMA [schema.]page_size=N
  **
  ** The first form reports the current setting for the
  ** database page size in bytes.  The second form sets the
@@ -481,8 +481,8 @@ void sqlite3Pragma(
  }

  /*
-  **  PRAGMA [database.]secure_delete
-  **  PRAGMA [database.]secure_delete=ON/OFF
+  **  PRAGMA [schema.]secure_delete
+  **  PRAGMA [schema.]secure_delete=ON/OFF
  **
  ** The first form reports the current setting for the
  ** secure_delete flag.  The second form changes the secure_delete
@@ -507,8 +507,8 @@ void sqlite3Pragma(
  }

  /*
-  **  PRAGMA [database.]max_page_count
-  **  PRAGMA [database.]max_page_count=N
+  **  PRAGMA [schema.]max_page_count
+  **  PRAGMA [schema.]max_page_count=N
  **
  ** The first form reports the current setting for the
  ** maximum number of pages in the database file.  The 
@@ -519,7 +519,7 @@ void sqlite3Pragma(
  ** change.  The only purpose is to provide an easy way to test
  ** the sqlite3AbsInt32() function.
  **
-  **  PRAGMA [database.]page_count
+  **  PRAGMA [schema.]page_count
  **
  ** Return the number of pages in the specified database.
  */
@@ -540,8 +540,8 @@ void sqlite3Pragma(
  }

  /*
-  **  PRAGMA [database.]locking_mode
-  **  PRAGMA [database.]locking_mode = (normal|exclusive)
+  **  PRAGMA [schema.]locking_mode
+  **  PRAGMA [schema.]locking_mode = (normal|exclusive)
  */
  case PragTyp_LOCKING_MODE: {
    const char *zRet = "normal";
@@ -586,8 +586,8 @@ void sqlite3Pragma(
  }

  /*
-  **  PRAGMA [database.]journal_mode
-  **  PRAGMA [database.]journal_mode =
+  **  PRAGMA [schema.]journal_mode
+  **  PRAGMA [schema.]journal_mode =
  **                      (delete|persist|off|truncate|memory|wal|off)
  */
  case PragTyp_JOURNAL_MODE: {
@@ -627,8 +627,8 @@ void sqlite3Pragma(
  }

  /*
-  **  PRAGMA [database.]journal_size_limit
-  **  PRAGMA [database.]journal_size_limit=N
+  **  PRAGMA [schema.]journal_size_limit
+  **  PRAGMA [schema.]journal_size_limit=N
  **
  ** Get or set the size limit on rollback journal files.
  */
@@ -647,8 +647,8 @@ void sqlite3Pragma(
 #endif /* SQLITE_OMIT_PAGER_PRAGMAS */

  /*
-  **  PRAGMA [database.]auto_vacuum
-  **  PRAGMA [database.]auto_vacuum=N
+  **  PRAGMA [schema.]auto_vacuum
+  **  PRAGMA [schema.]auto_vacuum=N
  **
  ** Get or set the value of the database 'auto-vacuum' parameter.
  ** The value is one of:  0 NONE 1 FULL 2 INCREMENTAL
@@ -699,7 +699,7 @@ void sqlite3Pragma(
 #endif

  /*
-  **  PRAGMA [database.]incremental_vacuum(N)
+  **  PRAGMA [schema.]incremental_vacuum(N)
  **
  ** Do N steps of incremental vacuuming on a database.
  */
@@ -722,8 +722,8 @@ void sqlite3Pragma(

 #ifndef SQLITE_OMIT_PAGER_PRAGMAS
  /*
-  **  PRAGMA [database.]cache_size
-  **  PRAGMA [database.]cache_size=N
+  **  PRAGMA [schema.]cache_size
+  **  PRAGMA [schema.]cache_size=N
  **
  ** The first form reports the current local setting for the
  ** page cache size. The second form sets the local
@@ -735,19 +735,60 @@ void sqlite3Pragma(
  case PragTyp_CACHE_SIZE: {
    assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
    if( !zRight ){
-      if( sqlite3ReadSchema(pParse) ) goto pragma_out;
      returnSingleInt(v, "cache_size", pDb->pSchema->cache_size);
    }else{
      int size = sqlite3Atoi(zRight);
      pDb->pSchema->cache_size = size;
      sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
-      if( sqlite3ReadSchema(pParse) ) goto pragma_out;
    }
    break;
  }

  /*
-  **  PRAGMA [database.]mmap_size(N)
+  **  PRAGMA [schema.]cache_spill
+  **  PRAGMA cache_spill=BOOLEAN
+  **  PRAGMA [schema.]cache_spill=N
+  **
+  ** The first form reports the current local setting for the
+  ** page cache spill size. The second form turns cache spill on
+  ** or off.  When turnning cache spill on, the size is set to the
+  ** current cache_size.  The third form sets a spill size that
+  ** may be different form the cache size.
+  ** If N is positive then that is the
+  ** number of pages in the cache.  If N is negative, then the
+  ** number of pages is adjusted so that the cache uses -N kibibytes
+  ** of memory.
+  **
+  ** If the number of cache_spill pages is less then the number of
+  ** cache_size pages, no spilling occurs until the page count exceeds
+  ** the number of cache_size pages.
+  **
+  ** The cache_spill=BOOLEAN setting applies to all attached schemas,
+  ** not just the schema specified.
+  */
+  case PragTyp_CACHE_SPILL: {
+    assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
+    if( !zRight ){
+      returnSingleInt(v, "cache_spill", 
+         (db->flags & SQLITE_CacheSpill)==0 ? 0 : 
+            sqlite3BtreeSetSpillSize(pDb->pBt,0));
+    }else{
+      int size = 1;
+      if( sqlite3GetInt32(zRight, &size) ){
+        sqlite3BtreeSetSpillSize(pDb->pBt, size);
+      }
+      if( sqlite3GetBoolean(zRight, size!=0) ){
+        db->flags |= SQLITE_CacheSpill;
+      }else{
+        db->flags &= ~SQLITE_CacheSpill;
+      }
+      setAllPagerFlags(db);
+    }
+    break;
+  }
+
+  /*
+  **  PRAGMA [schema.]mmap_size(N)
  **
  ** Used to set mapping size limit. The mapping size limit is
  ** used to limit the aggregate size of all memory mapped regions of the
@@ -891,8 +932,8 @@ void sqlite3Pragma(

 #if SQLITE_ENABLE_LOCKING_STYLE
  /*
-  **   PRAGMA [database.]lock_proxy_file
-  **   PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path"
+  **   PRAGMA [schema.]lock_proxy_file
+  **   PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path"
  **
  ** Return or set the value of the lock_proxy_file flag.  Changing
  ** the value sets a specific file to be used for database access locks.
@@ -927,8 +968,8 @@ void sqlite3Pragma(
 #endif /* SQLITE_ENABLE_LOCKING_STYLE */      
    
  /*
-  **   PRAGMA [database.]synchronous
-  **   PRAGMA [database.]synchronous=OFF|ON|NORMAL|FULL
+  **   PRAGMA [schema.]synchronous
+  **   PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL
  **
  ** Return or set the local value of the synchronous flag.  Changing
  ** the local value does not make changes to the disk file and the
@@ -1324,7 +1365,7 @@ void sqlite3Pragma(
  case PragTyp_PARSER_TRACE: {
    if( zRight ){
      if( sqlite3GetBoolean(zRight, 0) ){
-        sqlite3ParserTrace(stderr, "parser: ");
+        sqlite3ParserTrace(stdout, "parser: ");
      }else{
        sqlite3ParserTrace(0, 0);
      }
@@ -1644,16 +1685,16 @@ void sqlite3Pragma(

 #ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS
  /*
-  **   PRAGMA [database.]schema_version
-  **   PRAGMA [database.]schema_version = <integer>
+  **   PRAGMA [schema.]schema_version
+  **   PRAGMA [schema.]schema_version = <integer>
  **
-  **   PRAGMA [database.]user_version
-  **   PRAGMA [database.]user_version = <integer>
+  **   PRAGMA [schema.]user_version
+  **   PRAGMA [schema.]user_version = <integer>
  **
-  **   PRAGMA [database.]freelist_count = <integer>
+  **   PRAGMA [schema.]freelist_count = <integer>
  **
-  **   PRAGMA [database.]application_id
-  **   PRAGMA [database.]application_id = <integer>
+  **   PRAGMA [schema.]application_id
+  **   PRAGMA [schema.]application_id = <integer>
  **
  ** The pragma's schema_version and user_version are used to set or get
  ** the value of the schema-version and user-version, respectively. Both
@@ -1728,7 +1769,7 @@ void sqlite3Pragma(

 #ifndef SQLITE_OMIT_WAL
  /*
-  **   PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate
+  **   PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate
  **
  ** Checkpoint the database.
  */
@@ -8,43 +8,44 @@
 #define PragTyp_FLAG                           2
 #define PragTyp_BUSY_TIMEOUT                   3
 #define PragTyp_CACHE_SIZE                     4
-#define PragTyp_CASE_SENSITIVE_LIKE            5
-#define PragTyp_COLLATION_LIST                 6
-#define PragTyp_COMPILE_OPTIONS                7
-#define PragTyp_DATA_STORE_DIRECTORY           8
-#define PragTyp_DATABASE_LIST                  9
-#define PragTyp_DEFAULT_CACHE_SIZE            10
-#define PragTyp_ENCODING                      11
-#define PragTyp_FOREIGN_KEY_CHECK             12
-#define PragTyp_FOREIGN_KEY_LIST              13
-#define PragTyp_INCREMENTAL_VACUUM            14
-#define PragTyp_INDEX_INFO                    15
-#define PragTyp_INDEX_LIST                    16
-#define PragTyp_INTEGRITY_CHECK               17
-#define PragTyp_JOURNAL_MODE                  18
-#define PragTyp_JOURNAL_SIZE_LIMIT            19
-#define PragTyp_LOCK_PROXY_FILE               20
-#define PragTyp_LOCKING_MODE                  21
-#define PragTyp_PAGE_COUNT                    22
-#define PragTyp_MMAP_SIZE                     23
-#define PragTyp_PAGE_SIZE                     24
-#define PragTyp_SECURE_DELETE                 25
-#define PragTyp_SHRINK_MEMORY                 26
-#define PragTyp_SOFT_HEAP_LIMIT               27
-#define PragTyp_STATS                         28
-#define PragTyp_SYNCHRONOUS                   29
-#define PragTyp_TABLE_INFO                    30
-#define PragTyp_TEMP_STORE                    31
-#define PragTyp_TEMP_STORE_DIRECTORY          32
-#define PragTyp_THREADS                       33
-#define PragTyp_WAL_AUTOCHECKPOINT            34
-#define PragTyp_WAL_CHECKPOINT                35
-#define PragTyp_ACTIVATE_EXTENSIONS           36
-#define PragTyp_HEXKEY                        37
-#define PragTyp_KEY                           38
-#define PragTyp_REKEY                         39
-#define PragTyp_LOCK_STATUS                   40
-#define PragTyp_PARSER_TRACE                  41
+#define PragTyp_CACHE_SPILL                    5
+#define PragTyp_CASE_SENSITIVE_LIKE            6
+#define PragTyp_COLLATION_LIST                 7
+#define PragTyp_COMPILE_OPTIONS                8
+#define PragTyp_DATA_STORE_DIRECTORY           9
+#define PragTyp_DATABASE_LIST                 10
+#define PragTyp_DEFAULT_CACHE_SIZE            11
+#define PragTyp_ENCODING                      12
+#define PragTyp_FOREIGN_KEY_CHECK             13
+#define PragTyp_FOREIGN_KEY_LIST              14
+#define PragTyp_INCREMENTAL_VACUUM            15
+#define PragTyp_INDEX_INFO                    16
+#define PragTyp_INDEX_LIST                    17
+#define PragTyp_INTEGRITY_CHECK               18
+#define PragTyp_JOURNAL_MODE                  19
+#define PragTyp_JOURNAL_SIZE_LIMIT            20
+#define PragTyp_LOCK_PROXY_FILE               21
+#define PragTyp_LOCKING_MODE                  22
+#define PragTyp_PAGE_COUNT                    23
+#define PragTyp_MMAP_SIZE                     24
+#define PragTyp_PAGE_SIZE                     25
+#define PragTyp_SECURE_DELETE                 26
+#define PragTyp_SHRINK_MEMORY                 27
+#define PragTyp_SOFT_HEAP_LIMIT               28
+#define PragTyp_STATS                         29
+#define PragTyp_SYNCHRONOUS                   30
+#define PragTyp_TABLE_INFO                    31
+#define PragTyp_TEMP_STORE                    32
+#define PragTyp_TEMP_STORE_DIRECTORY          33
+#define PragTyp_THREADS                       34
+#define PragTyp_WAL_AUTOCHECKPOINT            35
+#define PragTyp_WAL_CHECKPOINT                36
+#define PragTyp_ACTIVATE_EXTENSIONS           37
+#define PragTyp_HEXKEY                        38
+#define PragTyp_KEY                           39
+#define PragTyp_REKEY                         40
+#define PragTyp_LOCK_STATUS                   41
+#define PragTyp_PARSER_TRACE                  42
 #define PragFlag_NeedSchema           0x01
 #define PragFlag_ReadOnly             0x02
 static const struct sPragmaNames {
@@ -86,14 +87,14 @@ static const struct sPragmaNames {
 #if !defined(SQLITE_OMIT_PAGER_PRAGMAS)
  { /* zName:     */ "cache_size",
    /* ePragTyp:  */ PragTyp_CACHE_SIZE,
-    /* ePragFlag: */ 0,
+    /* ePragFlag: */ PragFlag_NeedSchema,
    /* iArg:      */ 0 },
 #endif
 #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
  { /* zName:     */ "cache_spill",
-    /* ePragTyp:  */ PragTyp_FLAG,
+    /* ePragTyp:  */ PragTyp_CACHE_SPILL,
    /* ePragFlag: */ 0,
-    /* iArg:      */ SQLITE_CacheSpill },
+    /* iArg:      */ 0 },
 #endif
  { /* zName:     */ "case_sensitive_like",
    /* ePragTyp:  */ PragTyp_CASE_SENSITIVE_LIKE,
@@ -726,7 +726,7 @@ void sqlite3VXPrintf(
    if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' ');

    if( zExtra ){
-      sqlite3_free(zExtra);
+      sqlite3DbFree(pAccum->db, zExtra);
      zExtra = 0;
    }
  }/* End for loop over the format string */
@@ -1423,9 +1423,10 @@ int sqlite3ResolveExprListNames(
  ExprList *pList         /* The expression list to be analyzed. */
 ){
  int i;
-  assert( pList!=0 );
-  for(i=0; i<pList->nExpr; i++){
-    if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort;
+  if( pList ){
+    for(i=0; i<pList->nExpr; i++){
+      if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort;
+    }
  }
  return WRC_Continue;
 }
@@ -118,7 +118,7 @@ Select *sqlite3SelectNew(
    memset(pNew, 0, sizeof(*pNew));
  }
  if( pEList==0 ){
-    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ALL,0));
+    pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0));
  }
  pNew->pEList = pEList;
  if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc));
@@ -1596,13 +1596,15 @@ int sqlite3ColumnsFromExprList(
 ){
  sqlite3 *db = pParse->db;   /* Database connection */
  int i, j;                   /* Loop counters */
-  int cnt;                    /* Index added to make the name unique */
+  u32 cnt;                    /* Index added to make the name unique */
  Column *aCol, *pCol;        /* For looping over result columns */
  int nCol;                   /* Number of columns in the result set */
  Expr *p;                    /* Expression for a single result column */
  char *zName;                /* Column name */
  int nName;                  /* Size of name in zName[] */
+  Hash ht;                    /* Hash table of column names */

+  sqlite3HashInit(&ht);
  if( pEList ){
    nCol = pEList->nExpr;
    aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol);
@@ -1614,13 +1616,12 @@ int sqlite3ColumnsFromExprList(
  *pnCol = nCol;
  *paCol = aCol;

-  for(i=0, pCol=aCol; i<nCol; i++, pCol++){
+  for(i=0, pCol=aCol; i<nCol && !db->mallocFailed; i++, pCol++){
    /* Get an appropriate name for the column
    */
    p = sqlite3ExprSkipCollate(pEList->a[i].pExpr);
    if( (zName = pEList->a[i].zName)!=0 ){
      /* If the column contains an "AS <name>" phrase, use <name> as the name */
-      zName = sqlite3DbStrDup(db, zName);
    }else{
      Expr *pColExpr = p;  /* The expression that is the result column name */
      Table *pTab;         /* Table associated with this expression */
@@ -1633,41 +1634,37 @@ int sqlite3ColumnsFromExprList(
        int iCol = pColExpr->iColumn;
        pTab = pColExpr->pTab;
        if( iCol<0 ) iCol = pTab->iPKey;
-        zName = sqlite3MPrintf(db, "%s",
-                 iCol>=0 ? pTab->aCol[iCol].zName : "rowid");
+        zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid";
      }else if( pColExpr->op==TK_ID ){
        assert( !ExprHasProperty(pColExpr, EP_IntValue) );
-        zName = sqlite3MPrintf(db, "%s", pColExpr->u.zToken);
+        zName = pColExpr->u.zToken;
      }else{
        /* Use the original text of the column expression as its name */
-        zName = sqlite3MPrintf(db, "%s", pEList->a[i].zSpan);
+        zName = pEList->a[i].zSpan;
      }
    }
-    if( db->mallocFailed ){
-      sqlite3DbFree(db, zName);
-      break;
-    }
+    zName = sqlite3MPrintf(db, "%s", zName);

    /* Make sure the column name is unique.  If the name is not unique,
    ** append an integer to the name so that it becomes unique.
    */
-    nName = sqlite3Strlen30(zName);
-    for(j=cnt=0; j<i; j++){
-      if( sqlite3StrICmp(aCol[j].zName, zName)==0 ){
-        char *zNewName;
-        int k;
-        for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){}
-        if( k>=0 && zName[k]==':' ) nName = k;
-        zName[nName] = 0;
-        zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt);
-        sqlite3DbFree(db, zName);
-        zName = zNewName;
-        j = -1;
-        if( zName==0 ) break;
+    cnt = 0;
+    while( zName && sqlite3HashFind(&ht, zName)!=0 ){
+      nName = sqlite3Strlen30(zName);
+      if( nName>0 ){
+        for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){}
+        if( zName[j]==':' ) nName = j;
      }
+      zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt);
+      if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt);
    }
    pCol->zName = zName;
+    sqlite3ColumnPropertiesFromName(0, pCol);
+    if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){
+      db->mallocFailed = 1;
+    }
  }
+  sqlite3HashClear(&ht);
  if( db->mallocFailed ){
    for(j=0; j<i; j++){
      sqlite3DbFree(db, aCol[j].zName);
@@ -3626,6 +3623,7 @@ static int flattenSubquery(
    */
    for(i=0; i<nSubSrc; i++){
      sqlite3IdListDelete(db, pSrc->a[i+iFrom].pUsing);
+      assert( pSrc->a[i+iFrom].fg.isTabFunc==0 );
      pSrc->a[i+iFrom] = pSubSrc->a[i];
      memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));
    }
@@ -3941,7 +3939,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
  if( pNewSrc==0 ) return WRC_Abort;
  *pNew = *p;
  p->pSrc = pNewSrc;
-  p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0));
+  p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0));
  p->op = TK_SELECT;
  p->pWhere = 0;
  pNew->pGroupBy = 0;
@@ -3960,6 +3958,19 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
  return WRC_Continue;
 }

+/*
+** Check to see if the FROM clause term pFrom has table-valued function
+** arguments.  If it does, leave an error message in pParse and return
+** non-zero, since pFrom is not allowed to be a table-valued function.
+*/
+static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){
+  if( pFrom->fg.isTabFunc ){
+    sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName);
+    return 1;
+  }
+  return 0;
+}
+
 #ifndef SQLITE_OMIT_CTE
 /*
 ** Argument pWith (which may be NULL) points to a linked list of nested 
@@ -3972,7 +3983,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
 ** object that the returned CTE belongs to.
 */
 static struct Cte *searchWith(
-  With *pWith,                    /* Current outermost WITH clause */
+  With *pWith,                    /* Current innermost WITH clause */
  struct SrcList_item *pItem,     /* FROM clause element to resolve */
  With **ppContext                /* OUT: WITH clause return value belongs to */
 ){
@@ -4003,11 +4014,12 @@ static struct Cte *searchWith(
 ** statement with which it is associated.
 */
 void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
-  assert( bFree==0 || pParse->pWith==0 );
+  assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) );
  if( pWith ){
+    assert( pParse->pWith!=pWith );
    pWith->pOuter = pParse->pWith;
    pParse->pWith = pWith;
-    pParse->bFreeWith = bFree;
+    if( bFree ) pParse->pWithToFree = pWith;
  }
 }

@@ -4054,6 +4066,7 @@ static int withExpand(
      sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName);
      return SQLITE_ERROR;
    }
+    if( cannotBeFunction(pParse, pFrom) ) return SQLITE_ERROR;

    assert( pFrom->pTab==0 );
    pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
@@ -4100,6 +4113,7 @@ static int withExpand(
    pSavedWith = pParse->pWith;
    pParse->pWith = pWith;
    sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel);
+    pParse->pWith = pWith;

    for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
    pEList = pLeft->pEList;
@@ -4246,15 +4260,14 @@ static int selectExpander(Walker *pWalker, Select *p){
        return WRC_Abort;
      }
      pTab->nRef++;
+      if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){
+        return WRC_Abort;
+      }
 #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE)
-      if( pTab->pSelect || IsVirtual(pTab) ){
+      if( IsVirtual(pTab) || pTab->pSelect ){
        i16 nCol;
        if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort;
        assert( pFrom->pSelect==0 );
-        if( pFrom->fg.isTabFunc && !IsVirtual(pTab) ){
-          sqlite3ErrorMsg(pParse, "'%s' is not a function", pTab->zName);
-          return WRC_Abort;
-        }
        pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0);
        sqlite3SelectSetName(pFrom->pSelect, pTab->zName);
        nCol = pTab->nCol;
@@ -4280,19 +4293,20 @@ static int selectExpander(Walker *pWalker, Select *p){
  /* For every "*" that occurs in the column list, insert the names of
  ** all columns in all tables.  And for every TABLE.* insert the names
  ** of all columns in TABLE.  The parser inserted a special expression
-  ** with the TK_ALL operator for each "*" that it found in the column list.
-  ** The following code just has to locate the TK_ALL expressions and expand
-  ** each one to the list of all columns in all tables.
+  ** with the TK_ASTERISK operator for each "*" that it found in the column
+  ** list.  The following code just has to locate the TK_ASTERISK
+  ** expressions and expand each one to the list of all columns in
+  ** all tables.
  **
  ** The first loop just checks to see if there are any "*" operators
  ** that need expanding.
  */
  for(k=0; k<pEList->nExpr; k++){
    pE = pEList->a[k].pExpr;
-    if( pE->op==TK_ALL ) break;
+    if( pE->op==TK_ASTERISK ) break;
    assert( pE->op!=TK_DOT || pE->pRight!=0 );
    assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
-    if( pE->op==TK_DOT && pE->pRight->op==TK_ALL ) break;
+    if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
  }
  if( k<pEList->nExpr ){
    /*
@@ -4310,7 +4324,9 @@ static int selectExpander(Walker *pWalker, Select *p){
      pE = a[k].pExpr;
      pRight = pE->pRight;
      assert( pE->op!=TK_DOT || pRight!=0 );
-      if( pE->op!=TK_ALL && (pE->op!=TK_DOT || pRight->op!=TK_ALL) ){
+      if( pE->op!=TK_ASTERISK
+       && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK)
+      ){
        /* This particular expression does not need to be expanded.
        */
        pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr);
@@ -4362,12 +4378,13 @@ static int selectExpander(Walker *pWalker, Select *p){
              continue;
            }

-            /* If a column is marked as 'hidden' (currently only possible
-            ** for virtual tables), do not include it in the expanded
-            ** result-set list.
+            /* If a column is marked as 'hidden', omit it from the expanded
+            ** result-set list unless the SELECT has the SF_IncludeHidden
+            ** bit set.
            */
-            if( IsHiddenColumn(&pTab->aCol[j]) ){
-              assert(IsVirtual(pTab));
+            if( (p->selFlags & SF_IncludeHidden)==0
+             && IsHiddenColumn(&pTab->aCol[j]) 
+            ){
              continue;
            }
            tableSeen = 1;
@@ -165,7 +165,7 @@ static sqlite3_int64 timeOfDay(void){
  static sqlite3_vfs *clockVfs = 0;
  sqlite3_int64 t;
  if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0);
-  if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){
+  if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){
    clockVfs->xCurrentTimeInt64(clockVfs, &t);
  }else{
    double r;
@@ -2560,7 +2560,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
     { "schema size:",
       "SELECT total(length(sql)) FROM %s" },
  };
-  sqlite3_file *pFile;
+  sqlite3_file *pFile = 0;
  int i;
  char *zSchemaTab;
  char *zDb = nArg>=2 ? azArg[1] : "main";
@@ -4601,10 +4601,10 @@ int SQLITE_CDECL main(int argc, char **argv){
      int n, sz;
      sz = (int)integerValue(cmdline_option_value(argc,argv,++i));
      if( sz>70000 ) sz = 70000;
-      if( sz<800 ) sz = 800;
+      if( sz<0 ) sz = 0;
      n = (int)integerValue(cmdline_option_value(argc,argv,++i));
-      if( n<10 ) n = 10;
-      sqlite3_config(SQLITE_CONFIG_PAGECACHE, malloc(n*sz+1), sz, n);
+      sqlite3_config(SQLITE_CONFIG_PAGECACHE,
+                    (n>0 && sz>0) ? malloc(n*sz) : 0, sz, n);
      data.shellFlgs |= SHFLG_Pagecache;
    }else if( strcmp(z,"-lookaside")==0 ){
      int n, sz;
@@ -1599,29 +1599,34 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_CONFIG_PAGECACHE]] <dt>SQLITE_CONFIG_PAGECACHE</dt>
-** <dd> ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer
+** <dd> ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool
 ** that SQLite can use for the database page cache with the default page
 ** cache implementation.  
-** This configuration should not be used if an application-define page
-** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]
-** configuration option.
+** This configuration option is a no-op if an application-define page
+** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2].
 ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to
-** 8-byte aligned
-** memory, the size of each page buffer (sz), and the number of pages (N).
+** 8-byte aligned memory (pMem), the size of each page cache line (sz),
+** and the number of cache lines (N).
 ** The sz argument should be the size of the largest database page
 ** (a power of two between 512 and 65536) plus some extra bytes for each
 ** page header.  ^The number of extra bytes needed by the page header
-** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option 
-** to [sqlite3_config()].
+** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ].
 ** ^It is harmless, apart from the wasted memory,
-** for the sz parameter to be larger than necessary.  The first
-** argument should pointer to an 8-byte aligned block of memory that
-** is at least sz*N bytes of memory, otherwise subsequent behavior is
-** undefined.
-** ^SQLite will use the memory provided by the first argument to satisfy its
-** memory needs for the first N pages that it adds to cache.  ^If additional
-** page cache memory is needed beyond what is provided by this option, then
-** SQLite goes to [sqlite3_malloc()] for the additional storage space.</dd>
+** for the sz parameter to be larger than necessary.  The pMem
+** argument must be either a NULL pointer or a pointer to an 8-byte
+** aligned block of memory of at least sz*N bytes, otherwise
+** subsequent behavior is undefined.
+** ^When pMem is not NULL, SQLite will strive to use the memory provided
+** to satisfy page cache needs, falling back to [sqlite3_malloc()] if
+** a page cache line is larger than sz bytes or if all of the pMem buffer
+** is exhausted.
+** ^If pMem is NULL and N is non-zero, then each database connection
+** does an initial bulk allocation for page cache memory
+** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or
+** of -1024*N bytes if N is negative, . ^If additional
+** page cache memory is needed beyond what is provided by the initial
+** allocation, then SQLite goes to [sqlite3_malloc()] separately for each
+** additional cache line. </dd>
 **
 ** [[SQLITE_CONFIG_HEAP]] <dt>SQLITE_CONFIG_HEAP</dt>
 ** <dd> ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer 
@@ -527,7 +527,6 @@
 # define SQLITE_DEFAULT_PCACHE_INITSZ 100
 #endif

-
 /*
 ** GCC does not define the offsetof() macro so we'll have to do it
 ** ourselves.
@@ -1383,7 +1382,8 @@ struct FuncDestructor {

 /*
 ** Possible values for FuncDef.flags.  Note that the _LENGTH and _TYPEOF
-** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  There
+** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG.  And
+** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC.  There
 ** are assert() statements in the code to verify this.
 */
 #define SQLITE_FUNC_ENCMASK  0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */
@@ -1674,7 +1674,7 @@ struct Table {
 /*
 ** Allowed values for Table.tabFlags.
 **
-** TF_OOOHidden applies to virtual tables that have hidden columns that are
+** TF_OOOHidden applies to tables or view that have hidden columns that are
 ** followed by non-hidden columns.  Example:  "CREATE VIRTUAL TABLE x USING
 ** vtab1(a HIDDEN, b);".  Since "b" is a non-hidden column but "a" is hidden,
 ** the TF_OOOHidden attribute would apply in this case.  Such tables require
@@ -1697,12 +1697,28 @@ struct Table {
 */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 #  define IsVirtual(X)      (((X)->tabFlags & TF_Virtual)!=0)
-#  define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0)
 #else
 #  define IsVirtual(X)      0
-#  define IsHiddenColumn(X) 0
 #endif

+/*
+** Macros to determine if a column is hidden.  IsOrdinaryHiddenColumn()
+** only works for non-virtual tables (ordinary tables and views) and is
+** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined.  The
+** IsHiddenColumn() macro is general purpose.
+*/
+#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS)
+#  define IsHiddenColumn(X)         (((X)->colFlags & COLFLAG_HIDDEN)!=0)
+#  define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0)
+#elif !defined(SQLITE_OMIT_VIRTUALTABLE)
+#  define IsHiddenColumn(X)         (((X)->colFlags & COLFLAG_HIDDEN)!=0)
+#  define IsOrdinaryHiddenColumn(X) 0
+#else
+#  define IsHiddenColumn(X)         0
+#  define IsOrdinaryHiddenColumn(X) 0
+#endif
+
+
 /* Does the table have a rowid */
 #define HasRowid(X)     (((X)->tabFlags & TF_WithoutRowid)==0)
 #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0)
@@ -1812,9 +1828,8 @@ struct KeyInfo {
 };

 /*
-** An instance of the following structure holds information about a
-** single index record that has already been parsed out into individual
-** values.
+** This object holds a record which has been parsed out into individual
+** fields, for the purposes of doing a comparison.
 **
 ** A record is an object that contains one or more fields of data.
 ** Records are used to store the content of a table row and to store
@@ -1822,20 +1837,40 @@ struct KeyInfo {
 ** the OP_MakeRecord opcode of the VDBE and is disassembled by the
 ** OP_Column opcode.
 **
-** This structure holds a record that has already been disassembled
-** into its constituent fields.
+** An instance of this object serves as a "key" for doing a search on
+** an index b+tree. The goal of the search is to find the entry that
+** is closed to the key described by this object.  This object might hold
+** just a prefix of the key.  The number of fields is given by
+** pKeyInfo->nField.
 **
-** The r1 and r2 member variables are only used by the optimized comparison
-** functions vdbeRecordCompareInt() and vdbeRecordCompareString().
+** The r1 and r2 fields are the values to return if this key is less than
+** or greater than a key in the btree, respectively.  These are normally
+** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree
+** is in DESC order.
+**
+** The key comparison functions actually return default_rc when they find
+** an equals comparison.  default_rc can be -1, 0, or +1.  If there are
+** multiple entries in the b-tree with the same key (when only looking
+** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to 
+** cause the search to find the last match, or +1 to cause the search to
+** find the first match.
+**
+** The key comparison functions will set eqSeen to true if they ever
+** get and equal results when comparing this structure to a b-tree record.
+** When default_rc!=0, the search might end up on the record immediately
+** before the first match or immediately after the last match.  The
+** eqSeen field will indicate whether or not an exact match exists in the
+** b-tree.
 */
 struct UnpackedRecord {
  KeyInfo *pKeyInfo;  /* Collation and sort-order information */
+  Mem *aMem;          /* Values */
  u16 nField;         /* Number of entries in apMem[] */
  i8 default_rc;      /* Comparison result if keys are equal */
  u8 errCode;         /* Error detected by xRecordCompare (CORRUPT or NOMEM) */
-  Mem *aMem;          /* Values */
-  int r1;             /* Value to return if (lhs > rhs) */
-  int r2;             /* Value to return if (rhs < lhs) */
+  i8 r1;              /* Value to return if (lhs > rhs) */
+  i8 r2;              /* Value to return if (rhs < lhs) */
+  u8 eqSeen;          /* True if an equality comparison has been seen */
 };


@@ -2477,6 +2512,7 @@ struct Select {
 #define SF_MinMaxAgg       0x1000  /* Aggregate containing min() or max() */
 #define SF_Recursive       0x2000  /* The recursive part of a recursive CTE */
 #define SF_Converted       0x4000  /* By convertCompoundSelectToSubquery() */
+#define SF_IncludeHidden   0x8000  /* Include hidden columns in output */


 /*
@@ -2735,7 +2771,6 @@ struct Parse {
  int nVar;                 /* Number of '?' variables seen in the SQL so far */
  int nzVar;                /* Number of available slots in azVar[] */
  u8 iPkSortOrder;          /* ASC or DESC for INTEGER PRIMARY KEY */
-  u8 bFreeWith;             /* True if pWith should be freed with parser */
  u8 explain;               /* True if the EXPLAIN flag is found on the query */
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  u8 declareVtab;           /* True if inside sqlite3_declare_vtab() */
@@ -2762,6 +2797,7 @@ struct Parse {
  Table *pZombieTab;        /* List of Table objects to delete after code gen */
  TriggerPrg *pTriggerPrg;  /* Linked list of coded triggers */
  With *pWith;              /* Current WITH clause, or NULL */
+  With *pWithToFree;        /* Free this WITH object at the end of the parse */
 };

 /*
@@ -3252,6 +3288,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list);
  void sqlite3TreeViewExpr(TreeView*, const Expr*, u8);
  void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*);
  void sqlite3TreeViewSelect(TreeView*, const Select*, u8);
+  void sqlite3TreeViewWith(TreeView*, const With*, u8);
 #endif


@@ -3295,6 +3332,7 @@ void sqlite3OpenMasterTable(Parse *, int);
 Index *sqlite3PrimaryKeyIndex(Table*);
 i16 sqlite3ColumnOfIndex(Index*, i16);
 void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int);
+void sqlite3ColumnPropertiesFromName(Table*, Column*);
 void sqlite3AddColumn(Parse*,Token*);
 void sqlite3AddNotNull(Parse*, int);
 void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int);
@@ -2226,6 +2226,49 @@ static int test_stmt_scanstatus_reset(
 }
 #endif

+#ifdef SQLITE_ENABLE_SQLLOG
+/*
+** Usage:  sqlite3_config_sqllog
+**
+** Zero the SQLITE_CONFIG_SQLLOG configuration
+*/
+static int test_config_sqllog(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+  sqlite3_config(SQLITE_CONFIG_SQLLOG, 0, 0);
+  return TCL_OK;
+}
+#endif
+
+/*
+** Usage: vfs_current_time_int64
+**
+** Return the value returned by the default VFS's xCurrentTimeInt64 method.
+*/
+static int vfsCurrentTimeInt64(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  i64 t;
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+  pVfs->xCurrentTimeInt64(pVfs, &t);
+  Tcl_SetObjResult(interp, Tcl_NewWideIntObj(t));
+  return TCL_OK;
+}
+
 /*
 ** Usage:  sqlite3_next_stmt  DB  STMT
 **
@@ -7036,7 +7079,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
     { "sqlite3_stmt_scanstatus",       test_stmt_scanstatus,   0 },
     { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset,   0 },
 #endif
-
+#ifdef SQLITE_ENABLE_SQLLOG
+     { "sqlite3_config_sqllog",         test_config_sqllog,   0 },
+#endif
+     { "vfs_current_time_int64",           vfsCurrentTimeInt64,   0 },
  };
  static int bitmask_size = sizeof(Bitmask)*8;
  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
@@ -125,6 +125,12 @@ static void set_options(Tcl_Interp *interp){
  Tcl_SetVar2(interp, "sqlite_options", "cursorhints", "0", TCL_GLOBAL_ONLY);
 #endif

+#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS
+  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "hiddencolumns", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_MEMSYS3
  Tcl_SetVar2(interp, "sqlite_options", "mem3", "1", TCL_GLOBAL_ONLY);
 #else
@@ -657,6 +663,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
  Tcl_SetVar2(interp, "sqlite_options", "yytrackmaxstackdepth", "0", TCL_GLOBAL_ONLY);
 #endif

+#ifdef SQLITE_ENABLE_SQLLOG
+  Tcl_SetVar2(interp, "sqlite_options", "sqllog", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "sqllog", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #define LINKVAR(x) { \
    static const int cv_ ## x = SQLITE_ ## x; \
    Tcl_LinkVar(interp, "SQLITE_" #x, (char *)&(cv_ ## x), \
@@ -46,6 +46,12 @@
 **   separate copy is taken each time the database file is opened or attached)
 **   by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0.
 **
+**   If the environment variable SQLITE_SQLLOG_CONDITIONAL is defined, then
+**   logging is only done for database connections if a file named
+**   "<database>-sqllog" exists in the same directly as the main database
+**   file when it is first opened ("<database>" is replaced by the actual 
+**   name of the main database file).
+**
 ** OUTPUT:
 **
 **   The SQLITE_SQLLOG_DIR is populated with three types of files:
@@ -88,6 +94,7 @@ static int getProcessId(void){
 /* Names of environment variables to be used */
 #define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR"
 #define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES"
+#define ENVIRONMENT_VARIABLE3_NAME "SQLITE_SQLLOG_CONDITIONAL"

 /* Assume that all database and database file names are shorted than this. */
 #define SQLLOG_NAMESZ 512
@@ -116,6 +123,7 @@ static struct SLGlobal {
  int nConn;                      /* Size of aConn[] array */

  /* Protected by SLGlobal.mutex */
+  int bConditional;               /* Only trace if *-sqllog file is present */
  int bReuse;                     /* True to avoid extra copies of db files */
  char zPrefix[SQLLOG_NAMESZ];    /* Prefix for all created files */
  char zIdx[SQLLOG_NAMESZ];       /* Full path to *.idx file */
@@ -215,7 +223,7 @@ static char *sqllogFindFile(const char *zFile){
 }

 static int sqllogFindAttached(
-  struct SLConn *p,               /* Database connection */
+  sqlite3 *db,                    /* Database connection */
  const char *zSearch,            /* Name to search for (or NULL) */
  char *zName,                    /* OUT: Name of attached database */
  char *zFile                     /* OUT: Name of attached file */
@@ -228,7 +236,7 @@ static int sqllogFindAttached(
  ** described by the last row returned.  */
  assert( sqllogglobal.bRec==0 );
  sqllogglobal.bRec = 1;
-  rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
+  rc = sqlite3_prepare_v2(db, "PRAGMA database_list", -1, &pStmt, 0);
  if( rc==SQLITE_OK ){
    while( SQLITE_ROW==sqlite3_step(pStmt) ){
      const char *zVal1; int nVal1;
@@ -236,7 +244,9 @@ static int sqllogFindAttached(

      zVal1 = (const char*)sqlite3_column_text(pStmt, 1);
      nVal1 = sqlite3_column_bytes(pStmt, 1);
-      memcpy(zName, zVal1, nVal1+1);
+      if( zName ){
+        memcpy(zName, zVal1, nVal1+1);
+      }

      zVal2 = (const char*)sqlite3_column_text(pStmt, 2);
      nVal2 = sqlite3_column_bytes(pStmt, 2);
@@ -285,7 +295,7 @@ static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){
  char *zInit = 0;
  int rc;

-  rc = sqllogFindAttached(p, zSearch, zName, zFile);
+  rc = sqllogFindAttached(p->db, zSearch, zName, zFile);
  if( rc!=SQLITE_OK ) return;

  if( zFile[0]=='\0' ){
@@ -405,6 +415,35 @@ static void testSqllogStmt(struct SLConn *p, const char *zSql){
  }
 }

+/*
+** The database handle passed as the only argument has just been opened.
+** Return true if this module should log initial databases and SQL 
+** statements for this connection, or false otherwise.
+**
+** If an error occurs, sqlite3_log() is invoked to report it to the user
+** and zero returned.
+*/
+static int sqllogTraceDb(sqlite3 *db){
+  int bRet = 1;
+  if( sqllogglobal.bConditional ){
+    char zFile[SQLLOG_NAMESZ];      /* Attached database name */
+    int rc = sqllogFindAttached(db, "main", 0, zFile);
+    if( rc==SQLITE_OK ){
+      int nFile = strlen(zFile);
+      if( (SQLLOG_NAMESZ-nFile)<8 ){
+        sqlite3_log(SQLITE_IOERR, 
+            "sqllogTraceDb(): database name too long (%d bytes)", nFile
+        );
+        bRet = 0;
+      }else{
+        memcpy(&zFile[nFile], "-sqllog", 8);
+        bRet = !access(zFile, F_OK);
+      }
+    }
+  }
+  return bRet;
+}
+
 /*
 ** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog().
 **
@@ -439,15 +478,19 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){
    if( sqllogglobal.mutex==0 ){
      sqllogglobal.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE);
    }
-    p = &sqllogglobal.aConn[sqllogglobal.nConn++];
-    p->fd = 0;
-    p->db = db;
-    p->iLog = sqllogglobal.iNextLog++;
    sqlite3_mutex_leave(master);

-    /* Open the log and take a copy of the main database file */
    sqlite3_mutex_enter(sqllogglobal.mutex);
-    if( sqllogglobal.bRec==0 ){
+    if( sqllogglobal.bRec==0 && sqllogTraceDb(db) ){
+
+      sqlite3_mutex_enter(master);
+      p = &sqllogglobal.aConn[sqllogglobal.nConn++];
+      p->fd = 0;
+      p->db = db;
+      p->iLog = sqllogglobal.iNextLog++;
+      sqlite3_mutex_leave(master);
+
+      /* Open the log and take a copy of the main database file */
      sqllogOpenlog(p);
      if( p->fd ) sqllogCopydb(p, "main", 0);
    }
@@ -461,20 +504,21 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){
      p = &sqllogglobal.aConn[i];
      if( p->db==db ) break;
    }
-    if( i==sqllogglobal.nConn ) return;

    /* A database handle close command */
    if( eType==2 ){
      sqlite3_mutex_enter(master);
-      if( p->fd ) fclose(p->fd);
-      p->db = 0;
-      p->fd = 0;
+      if( i<sqllogglobal.nConn ){
+        if( p->fd ) fclose(p->fd);
+        p->db = 0;
+        p->fd = 0;
+        sqllogglobal.nConn--;
+      }

-      sqllogglobal.nConn--;
      if( sqllogglobal.nConn==0 ){
        sqlite3_mutex_free(sqllogglobal.mutex);
        sqllogglobal.mutex = 0;
-      }else{
+      }else if( i<sqllogglobal.nConn ){
        int nShift = &sqllogglobal.aConn[sqllogglobal.nConn] - p;
        if( nShift>0 ){
          memmove(p, &p[1], nShift*sizeof(struct SLConn));
@@ -483,7 +527,7 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){
      sqlite3_mutex_leave(master);

    /* An ordinary SQL command. */
-    }else if( p->fd ){
+    }else if( i<sqllogglobal.nConn && p->fd ){
      sqlite3_mutex_enter(sqllogglobal.mutex);
      if( sqllogglobal.bRec==0 ){
        testSqllogStmt(p, zSql);
@@ -504,6 +548,9 @@ void sqlite3_init_sqllog(void){
    if( SQLITE_OK==sqlite3_config(SQLITE_CONFIG_SQLLOG, testSqllog, 0) ){
      memset(&sqllogglobal, 0, sizeof(sqllogglobal));
      sqllogglobal.bReuse = 1;
+      if( getenv(ENVIRONMENT_VARIABLE3_NAME) ){
+        sqllogglobal.bConditional = 1;
+      }
    }
  }
 }
@@ -369,8 +369,8 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
        break;
      }
      for(i=1; IdChar(z[i]); i++){}
-      *tokenType = keywordCode((char*)z, i);
-      return i;
+      *tokenType = TK_ID;
+      return keywordCode((char*)z, i, tokenType);
    }
  }
  *tokenType = TK_ILLEGAL;
@@ -416,7 +416,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
  assert( pParse->azVar==0 );
  enableLookaside = db->lookaside.bEnabled;
  if( db->lookaside.pStart ) db->lookaside.bEnabled = 1;
-  while( !db->mallocFailed && zSql[i]!=0 ){
+  while( zSql[i]!=0 ){
    assert( i>=0 );
    pParse->sLastToken.z = &zSql[i];
    pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType);
@@ -425,35 +425,25 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
      pParse->rc = SQLITE_TOOBIG;
      break;
    }
-    switch( tokenType ){
-      case TK_SPACE: {
-        if( db->u1.isInterrupted ){
-          sqlite3ErrorMsg(pParse, "interrupt");
-          pParse->rc = SQLITE_INTERRUPT;
-          goto abort_parse;
-        }
+    if( tokenType>=TK_SPACE ){
+      assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
+      if( db->u1.isInterrupted ){
+        sqlite3ErrorMsg(pParse, "interrupt");
+        pParse->rc = SQLITE_INTERRUPT;
        break;
      }
-      case TK_ILLEGAL: {
+      if( tokenType==TK_ILLEGAL ){
        sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"",
                        &pParse->sLastToken);
-        goto abort_parse;
-      }
-      case TK_SEMI: {
-        pParse->zTail = &zSql[i];
-        /* Fall thru into the default case */
-      }
-      default: {
-        sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
-        lastTokenParsed = tokenType;
-        if( pParse->rc!=SQLITE_OK ){
-          goto abort_parse;
-        }
        break;
      }
+    }else{
+      if( tokenType==TK_SEMI ) pParse->zTail = &zSql[i];
+      sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
+      lastTokenParsed = tokenType;
+      if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
    }
  }
-abort_parse:
  assert( nErr==0 );
  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
    assert( zSql[i]==0 );
@@ -510,7 +500,7 @@ abort_parse:
    sqlite3DeleteTable(db, pParse->pNewTable);
  }

-  if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith);
+  sqlite3WithDelete(db, pParse->pWithToFree);
  sqlite3DeleteTrigger(db, pParse->pNewTrigger);
  for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
  sqlite3DbFree(db, pParse->azVar);
@@ -79,6 +79,45 @@ static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){
  sqlite3TreeViewLine(p, "%s", zLabel);
 }

+/*
+** Generate a human-readable description of a WITH clause.
+*/
+void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){
+  int i;
+  if( pWith==0 ) return;
+  if( pWith->nCte==0 ) return;
+  if( pWith->pOuter ){
+    sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter);
+  }else{
+    sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith);
+  }
+  if( pWith->nCte>0 ){
+    pView = sqlite3TreeViewPush(pView, 1);
+    for(i=0; i<pWith->nCte; i++){
+      StrAccum x;
+      char zLine[1000];
+      const struct Cte *pCte = &pWith->a[i];
+      sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0);
+      sqlite3XPrintf(&x, 0, "%s", pCte->zName);
+      if( pCte->pCols && pCte->pCols->nExpr>0 ){
+        char cSep = '(';
+        int j;
+        for(j=0; j<pCte->pCols->nExpr; j++){
+          sqlite3XPrintf(&x, 0, "%c%s", cSep, pCte->pCols->a[j].zName);
+          cSep = ',';
+        }
+        sqlite3XPrintf(&x, 0, ")");
+      }
+      sqlite3XPrintf(&x, 0, " AS");
+      sqlite3StrAccumFinish(&x);
+      sqlite3TreeViewItem(pView, zLine, i<pWith->nCte-1);
+      sqlite3TreeViewSelect(pView, pCte->pSelect, 0);
+      sqlite3TreeViewPop(pView);
+    }
+    sqlite3TreeViewPop(pView);
+  }
+}
+

 /*
 ** Generate a human-readable description of a the Select object.
@@ -87,6 +126,11 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
  int n = 0;
  int cnt = 0;
  pView = sqlite3TreeViewPush(pView, moreToFollow);
+  if( p->pWith ){
+    sqlite3TreeViewWith(pView, p->pWith, 1);
+    cnt = 1;
+    sqlite3TreeViewPush(pView, 1);
+  }
  do{
    sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x",
      ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
@@ -555,7 +555,8 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){
  testcase( i==18 );
  testcase( i==19 );
  testcase( i==20 );
-  if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum) || i>19*incr || nonNum ){
+  if( (c!=0 && &zNum[i]<zEnd) || (i==0 && zStart==zNum)
+       || i>19*incr || nonNum ){
    /* zNum is empty or contains non-numeric text or is longer
    ** than 19 digits (thus guaranteeing that it is too large) */
    return 1;
@@ -844,7 +845,8 @@ u8 sqlite3GetVarint(const unsigned char *p, u64 *v){
  /* a: p0<<28 | p2<<14 | p4 (unmasked) */
  if (!(a&0x80))
  {
-    /* we can skip these cause they were (effectively) done above in calc'ing s */
+    /* we can skip these cause they were (effectively) done above
+    ** while calculating s */
    /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
    /* b &= (0x7f<<14)|(0x7f); */
    b = b<<7;
@@ -165,7 +165,7 @@ int sqlite3_found_count = 0;
       && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;}

 /* Return true if the cursor was opened using the OP_OpenSorter opcode. */
-#define isSorter(x) ((x)->pSorter!=0)
+#define isSorter(x) ((x)->eCurType==CURTYPE_SORTER)

 /*
 ** Allocate VdbeCursor number iCur.  Return a pointer to it.  Return NULL
@@ -176,7 +176,7 @@ static VdbeCursor *allocateCursor(
  int iCur,             /* Index of the new VdbeCursor */
  int nField,           /* Number of fields in the table or index */
  int iDb,              /* Database the cursor belongs to, or -1 */
-  int isBtreeCursor     /* True for B-Tree.  False for pseudo-table or vtab */
+  u8 eCurType           /* Type of the new cursor */
 ){
  /* Find the memory cell that will be used to store the blob of memory
  ** required for this VdbeCursor structure. It is convenient to use a 
@@ -202,7 +202,7 @@ static VdbeCursor *allocateCursor(
  VdbeCursor *pCx = 0;
  nByte = 
      ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + 
-      (isBtreeCursor?sqlite3BtreeCursorSize():0);
+      (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0);

  assert( iCur<p->nCursor );
  if( p->apCsr[iCur] ){
@@ -212,13 +212,14 @@ static VdbeCursor *allocateCursor(
  if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){
    p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
    memset(pCx, 0, sizeof(VdbeCursor));
+    pCx->eCurType = eCurType;
    pCx->iDb = iDb;
    pCx->nField = nField;
    pCx->aOffset = &pCx->aType[nField];
-    if( isBtreeCursor ){
-      pCx->pCursor = (BtCursor*)
+    if( eCurType==CURTYPE_BTREE ){
+      pCx->uc.pCursor = (BtCursor*)
          &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField];
-      sqlite3BtreeCursorZero(pCx->pCursor);
+      sqlite3BtreeCursorZero(pCx->uc.pCursor);
    }
  }
  return pCx;
@@ -2374,30 +2375,34 @@ case OP_Column: {
  u16 fx;            /* pDest->flags value */
  Mem *pReg;         /* PseudoTable input register */

+  pC = p->apCsr[pOp->p1];
+  if( pC->eCurType==CURTYPE_PSEUDO ) goto skip_moveto;
+  pOp->opcode = OP_BColumn;
+
+case OP_BColumn:
+  pC = p->apCsr[pOp->p1];
+  rc = sqlite3VdbeCursorMoveto(pC);
+  if( rc ) goto abort_due_to_error;
+skip_moveto:
  p2 = pOp->p2;
  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
  pDest = &aMem[pOp->p3];
  memAboutToChange(p, pDest);
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
-  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( p2<pC->nField );
  aOffset = pC->aOffset;
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-  assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */
-#endif
-  pCrsr = pC->pCursor;
-  assert( pCrsr!=0 || pC->pseudoTableReg>0 ); /* pCrsr NULL on PseudoTables */
-  assert( pCrsr!=0 || pC->nullRow );          /* pC->nullRow on PseudoTables */
+  assert( pC->eCurType!=CURTYPE_VTAB );
+  assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow );
+  assert( pC->eCurType!=CURTYPE_SORTER );
+  pCrsr = pC->uc.pCursor;

  /* If the cursor cache is stale, bring it up-to-date */
-  rc = sqlite3VdbeCursorMoveto(pC);
-  if( rc ) goto abort_due_to_error;
  if( pC->cacheStatus!=p->cacheCtr ){
    if( pC->nullRow ){
-      if( pCrsr==0 ){
-        assert( pC->pseudoTableReg>0 );
-        pReg = &aMem[pC->pseudoTableReg];
+      if( pC->eCurType==CURTYPE_PSEUDO ){
+        assert( pC->uc.pseudoTableReg>0 );
+        pReg = &aMem[pC->uc.pseudoTableReg];
        assert( pReg->flags & MEM_Blob );
        assert( memIsValid(pReg) );
        pC->payloadSize = pC->szRow = avail = pReg->n;
@@ -2407,6 +2412,7 @@ case OP_Column: {
        goto op_column_out;
      }
    }else{
+      assert( pC->eCurType==CURTYPE_BTREE );
      assert( pCrsr );
      if( pC->isTable==0 ){
        assert( sqlite3BtreeCursorIsValid(pCrsr) );
@@ -2427,12 +2433,11 @@ case OP_Column: {
      assert( avail<=65536 );  /* Maximum page size is 64KiB */
      if( pC->payloadSize <= (u32)avail ){
        pC->szRow = pC->payloadSize;
+      }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
+        goto too_big;
      }else{
        pC->szRow = avail;
      }
-      if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){
-        goto too_big;
-      }
    }
    pC->cacheStatus = p->cacheCtr;
    pC->iHdrOffset = getVarint32(pC->aRow, offset);
@@ -2796,7 +2801,8 @@ case OP_Count: {         /* out2 */
  i64 nEntry;
  BtCursor *pCrsr;

-  pCrsr = p->apCsr[pOp->p1]->pCursor;
+  assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE );
+  pCrsr = p->apCsr[pOp->p1]->uc.pCursor;
  assert( pCrsr );
  nEntry = 0;  /* Not needed.  Only used to silence a warning. */
  rc = sqlite3BtreeCount(pCrsr, &nEntry);
@@ -3381,12 +3387,12 @@ case OP_OpenWrite:
  assert( pOp->p1>=0 );
  assert( nField>=0 );
  testcase( nField==0 );  /* Table with INTEGER PRIMARY KEY and nothing else */
-  pCur = allocateCursor(p, pOp->p1, nField, iDb, 1);
+  pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE);
  if( pCur==0 ) goto no_mem;
  pCur->nullRow = 1;
  pCur->isOrdered = 1;
  pCur->pgnoRoot = p2;
-  rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor);
+  rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->uc.pCursor);
  pCur->pKeyInfo = pKeyInfo;
  /* Set the VdbeCursor.isTable variable. Previous versions of
  ** SQLite used to check if the root-page flags were sane at this point
@@ -3401,7 +3407,7 @@ open_cursor_set_hints:
 #ifdef SQLITE_ENABLE_CURSOR_HINT
  testcase( pOp->p2 & OPFLAG_SEEKEQ );
 #endif
-  sqlite3BtreeCursorHintFlags(pCur->pCursor,
+  sqlite3BtreeCursorHintFlags(pCur->uc.pCursor,
                               (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ)));
  break;
 }
@@ -3445,7 +3451,7 @@ case OP_OpenEphemeral: {
      SQLITE_OPEN_TRANSIENT_DB;
  assert( pOp->p1>=0 );
  assert( pOp->p2>=0 );
-  pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
+  pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE);
  if( pCx==0 ) goto no_mem;
  pCx->nullRow = 1;
  pCx->isEphemeral = 1;
@@ -3469,11 +3475,13 @@ case OP_OpenEphemeral: {
        assert( pKeyInfo->db==db );
        assert( pKeyInfo->enc==ENC(db) );
        pCx->pKeyInfo = pKeyInfo;
-        rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, pKeyInfo, pCx->pCursor);
+        rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR,
+                                pKeyInfo, pCx->uc.pCursor);
      }
      pCx->isTable = 0;
    }else{
-      rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, 0, pCx->pCursor);
+      rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR,
+                              0, pCx->uc.pCursor);
      pCx->isTable = 1;
    }
  }
@@ -3496,7 +3504,7 @@ case OP_SorterOpen: {

  assert( pOp->p1>=0 );
  assert( pOp->p2>=0 );
-  pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
+  pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER);
  if( pCx==0 ) goto no_mem;
  pCx->pKeyInfo = pOp->p4.pKeyInfo;
  assert( pCx->pKeyInfo->db==db );
@@ -3516,7 +3524,7 @@ case OP_SequenceTest: {
  VdbeCursor *pC;
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
-  assert( pC->pSorter );
+  assert( isSorter(pC) );
  if( (pC->seqCount++)==0 ){
    goto jump_to_p2;
  }
@@ -3544,10 +3552,10 @@ case OP_OpenPseudo: {

  assert( pOp->p1>=0 );
  assert( pOp->p3>=0 );
-  pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, 0);
+  pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO);
  if( pCx==0 ) goto no_mem;
  pCx->nullRow = 1;
-  pCx->pseudoTableReg = pOp->p2;
+  pCx->uc.pseudoTableReg = pOp->p2;
  pCx->isTable = 1;
  assert( pOp->p5==0 );
  break;
@@ -3579,7 +3587,7 @@ case OP_Close: {
 case OP_ColumnsUsed: {
  VdbeCursor *pC;
  pC = p->apCsr[pOp->p1];
-  assert( pC->pCursor );
+  assert( pC->eCurType==CURTYPE_BTREE );
  pC->maskUsed = *(u64*)pOp->p4.pI64;
  break;
 }
@@ -3597,6 +3605,13 @@ case OP_ColumnsUsed: {
 ** is greater than or equal to the key value. If there are no records 
 ** greater than or equal to the key and P2 is not zero, then jump to P2.
 **
+** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this
+** opcode will always land on a record that equally equals the key, or
+** else jump immediately to P2.  When the cursor is OPFLAG_SEEKEQ, this
+** opcode must be followed by an IdxLE opcode with the same arguments.
+** The IdxLE opcode will be skipped if this opcode succeeds, but the
+** IdxLE opcode will be used on subsequent loop iterations.
+**
 ** This opcode leaves the cursor configured to move in forward order,
 ** from the beginning toward the end.  In other words, the cursor is
 ** configured to use Next, not Prev.
@@ -3655,51 +3670,48 @@ case OP_ColumnsUsed: {
 ** from the end toward the beginning.  In other words, the cursor is
 ** configured to use Prev, not Next.
 **
+** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this
+** opcode will always land on a record that equally equals the key, or
+** else jump immediately to P2.  When the cursor is OPFLAG_SEEKEQ, this
+** opcode must be followed by an IdxGE opcode with the same arguments.
+** The IdxGE opcode will be skipped if this opcode succeeds, but the
+** IdxGE opcode will be used on subsequent loop iterations.
+**
 ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt
 */
 case OP_SeekLT:         /* jump, in3 */
 case OP_SeekLE:         /* jump, in3 */
 case OP_SeekGE:         /* jump, in3 */
 case OP_SeekGT: {       /* jump, in3 */
-  int res;
-  int oc;
-  VdbeCursor *pC;
-  UnpackedRecord r;
-  int nField;
-  i64 iKey;      /* The rowid we are to seek to */
+  int res;           /* Comparison result */
+  int oc;            /* Opcode */
+  VdbeCursor *pC;    /* The cursor to seek */
+  UnpackedRecord r;  /* The key to seek for */
+  int nField;        /* Number of columns or fields in the key */
+  i64 iKey;          /* The rowid we are to seek to */
+  int eqOnly;        /* Only interested in == results */

  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  assert( pOp->p2!=0 );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pseudoTableReg==0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
  assert( OP_SeekLE == OP_SeekLT+1 );
  assert( OP_SeekGE == OP_SeekLT+2 );
  assert( OP_SeekGT == OP_SeekLT+3 );
  assert( pC->isOrdered );
-  assert( pC->pCursor!=0 );
+  assert( pC->uc.pCursor!=0 );
  oc = pOp->opcode;
+  eqOnly = 0;
  pC->nullRow = 0;
 #ifdef SQLITE_DEBUG
  pC->seekOp = pOp->opcode;
 #endif

-  /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and
-  ** OP_SeekLE opcodes are allowed, and these must be immediately followed
-  ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key.
-  */
-#ifdef SQLITE_DEBUG
-  if( sqlite3BtreeCursorHasHint(pC->pCursor, BTREE_SEEK_EQ) ){
-    assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE );
-    assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT );
-    assert( pOp[1].p1==pOp[0].p1 );
-    assert( pOp[1].p2==pOp[0].p2 );
-    assert( pOp[1].p3==pOp[0].p3 );
-    assert( pOp[1].p4.i==pOp[0].p4.i );
-  }
-#endif
- 
  if( pC->isTable ){
+    /* The BTREE_SEEK_EQ flag is only set on index cursors */
+    assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 );
+
    /* The input value in P3 might be of any type: integer, real, string,
    ** blob, or NULL.  But it needs to be an integer before we can do
    ** the seek, so convert it. */
@@ -3742,12 +3754,26 @@ case OP_SeekGT: {       /* jump, in3 */
        if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++;
      }
    } 
-    rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res);
+    rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res);
    pC->movetoTarget = iKey;  /* Used by OP_Delete */
    if( rc!=SQLITE_OK ){
      goto abort_due_to_error;
    }
  }else{
+    /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and
+    ** OP_SeekLE opcodes are allowed, and these must be immediately followed
+    ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key.
+    */
+    if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){
+      eqOnly = 1;
+      assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE );
+      assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT );
+      assert( pOp[1].p1==pOp[0].p1 );
+      assert( pOp[1].p2==pOp[0].p2 );
+      assert( pOp[1].p3==pOp[0].p3 );
+      assert( pOp[1].p4.i==pOp[0].p4.i );
+    }
+
    nField = pOp->p4.i;
    assert( pOp->p4type==P4_INT32 );
    assert( nField>0 );
@@ -3772,10 +3798,15 @@ case OP_SeekGT: {       /* jump, in3 */
    { int i; for(i=0; i<r.nField; i++) assert( memIsValid(&r.aMem[i]) ); }
 #endif
    ExpandBlob(r.aMem);
-    rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, &r, 0, 0, &res);
+    r.eqSeen = 0;
+    rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, &r, 0, 0, &res);
    if( rc!=SQLITE_OK ){
      goto abort_due_to_error;
    }
+    if( eqOnly && r.eqSeen==0 ){
+      assert( res!=0 );
+      goto seek_not_found;
+    }
  }
  pC->deferredMoveto = 0;
  pC->cacheStatus = CACHE_STALE;
@@ -3785,7 +3816,7 @@ case OP_SeekGT: {       /* jump, in3 */
  if( oc>=OP_SeekGE ){  assert( oc==OP_SeekGE || oc==OP_SeekGT );
    if( res<0 || (res==0 && oc==OP_SeekGT) ){
      res = 0;
-      rc = sqlite3BtreeNext(pC->pCursor, &res);
+      rc = sqlite3BtreeNext(pC->uc.pCursor, &res);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
    }else{
      res = 0;
@@ -3794,19 +3825,23 @@ case OP_SeekGT: {       /* jump, in3 */
    assert( oc==OP_SeekLT || oc==OP_SeekLE );
    if( res>0 || (res==0 && oc==OP_SeekLT) ){
      res = 0;
-      rc = sqlite3BtreePrevious(pC->pCursor, &res);
+      rc = sqlite3BtreePrevious(pC->uc.pCursor, &res);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
    }else{
      /* res might be negative because the table is empty.  Check to
      ** see if this is the case.
      */
-      res = sqlite3BtreeEof(pC->pCursor);
+      res = sqlite3BtreeEof(pC->uc.pCursor);
    }
  }
+seek_not_found:
  assert( pOp->p2>0 );
  VdbeBranchTaken(res!=0,2);
  if( res ){
    goto jump_to_p2;
+  }else if( eqOnly ){
+    assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT );
+    pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */
  }
  break;
 }
@@ -3827,7 +3862,8 @@ case OP_Seek: {    /* in2 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pCursor!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0 );
  assert( pC->isTable );
  pC->nullRow = 0;
  pIn2 = &aMem[pOp->p2];
@@ -3921,7 +3957,8 @@ case OP_Found: {        /* jump, in3 */
  pC->seekOp = pOp->opcode;
 #endif
  pIn3 = &aMem[pOp->p3];
-  assert( pC->pCursor!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0 );
  assert( pC->isTable==0 );
  pFree = 0;
  if( pOp->p4.i>0 ){
@@ -3958,7 +3995,7 @@ case OP_Found: {        /* jump, in3 */
      }
    }
  }
-  rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res);
+  rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res);
  sqlite3DbFree(db, pFree);
  if( rc!=SQLITE_OK ){
    break;
@@ -4012,8 +4049,8 @@ case OP_NotExists: {        /* jump, in3 */
  pC->seekOp = 0;
 #endif
  assert( pC->isTable );
-  assert( pC->pseudoTableReg==0 );
-  pCrsr = pC->pCursor;
+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCrsr = pC->uc.pCursor;
  assert( pCrsr!=0 );
  res = 0;
  iKey = pIn3->u.i;
@@ -4047,6 +4084,7 @@ case OP_NotExists: {        /* jump, in3 */
 case OP_Sequence: {           /* out2 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  assert( p->apCsr[pOp->p1]!=0 );
+  assert( p->apCsr[pOp->p1]->eCurType!=CURTYPE_VTAB );
  pOut = out2Prerelease(p, pOp);
  pOut->u.i = p->apCsr[pOp->p1]->seqCount++;
  break;
@@ -4082,7 +4120,8 @@ case OP_NewRowid: {           /* out2 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pCursor!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0 );
  {
    /* The next rowid or record number (different terms for the same
    ** thing) is obtained in a two-step algorithm.
@@ -4110,15 +4149,15 @@ case OP_NewRowid: {           /* out2 */
 #endif

    if( !pC->useRandomRowid ){
-      rc = sqlite3BtreeLast(pC->pCursor, &res);
+      rc = sqlite3BtreeLast(pC->uc.pCursor, &res);
      if( rc!=SQLITE_OK ){
        goto abort_due_to_error;
      }
      if( res ){
        v = 1;   /* IMP: R-61914-48074 */
      }else{
-        assert( sqlite3BtreeCursorIsValid(pC->pCursor) );
-        rc = sqlite3BtreeKeySize(pC->pCursor, &v);
+        assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) );
+        rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
        assert( rc==SQLITE_OK );   /* Cannot fail following BtreeLast() */
        if( v>=MAX_ROWID ){
          pC->useRandomRowid = 1;
@@ -4169,7 +4208,7 @@ case OP_NewRowid: {           /* out2 */
      do{
        sqlite3_randomness(sizeof(v), &v);
        v &= (MAX_ROWID>>1); v++;  /* Ensure that v is greater than zero */
-      }while(  ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v,
+      }while(  ((rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)v,
                                                 0, &res))==SQLITE_OK)
            && (res==0)
            && (++cnt<100));
@@ -4249,8 +4288,8 @@ case OP_InsertInt: {
  assert( memIsValid(pData) );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pCursor!=0 );
-  assert( pC->pseudoTableReg==0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0 );
  assert( pC->isTable );
  REGISTER_TRACE(pOp->p2, pData);

@@ -4279,7 +4318,7 @@ case OP_InsertInt: {
  }else{
    nZero = 0;
  }
-  rc = sqlite3BtreeInsert(pC->pCursor, 0, iKey,
+  rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey,
                          pData->z, pData->n, nZero,
                          (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
  );
@@ -4326,12 +4365,13 @@ case OP_Delete: {
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0 );
  assert( pC->deferredMoveto==0 );

  hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable;
  if( pOp->p5 && hasUpdateCallback ){
-    sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget);
+    sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget);
  }

 #ifdef SQLITE_DEBUG
@@ -4340,12 +4380,12 @@ case OP_Delete: {
  ** is being deleted */
  if( pOp->p4.z && pC->isTable && pOp->p5==0 ){
    i64 iKey = 0;
-    sqlite3BtreeKeySize(pC->pCursor, &iKey);
+    sqlite3BtreeKeySize(pC->uc.pCursor, &iKey);
    assert( pC->movetoTarget==iKey ); 
  }
 #endif
 
-  rc = sqlite3BtreeDelete(pC->pCursor, pOp->p5);
+  rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5);
  pC->cacheStatus = CACHE_STALE;

  /* Invoke the update-hook if required. */
@@ -4462,14 +4502,14 @@ case OP_RowData: {
  /* Note that RowKey and RowData are really exactly the same instruction */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
+  assert( pC!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE );
  assert( isSorter(pC)==0 );
  assert( pC->isTable || pOp->opcode!=OP_RowData );
  assert( pC->isTable==0 || pOp->opcode==OP_RowData );
-  assert( pC!=0 );
  assert( pC->nullRow==0 );
-  assert( pC->pseudoTableReg==0 );
-  assert( pC->pCursor!=0 );
-  pCrsr = pC->pCursor;
+  assert( pC->uc.pCursor!=0 );
+  pCrsr = pC->uc.pCursor;

  /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
  ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
@@ -4537,29 +4577,31 @@ case OP_Rowid: {                 /* out2 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  assert( pC->pseudoTableReg==0 || pC->nullRow );
+  assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow );
  if( pC->nullRow ){
    pOut->flags = MEM_Null;
    break;
  }else if( pC->deferredMoveto ){
    v = pC->movetoTarget;
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  }else if( pC->pVtabCursor ){
-    pVtab = pC->pVtabCursor->pVtab;
+  }else if( pC->eCurType==CURTYPE_VTAB ){
+    assert( pC->uc.pVCur!=0 );
+    pVtab = pC->uc.pVCur->pVtab;
    pModule = pVtab->pModule;
    assert( pModule->xRowid );
-    rc = pModule->xRowid(pC->pVtabCursor, &v);
+    rc = pModule->xRowid(pC->uc.pVCur, &v);
    sqlite3VtabImportErrmsg(p, pVtab);
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
  }else{
-    assert( pC->pCursor!=0 );
+    assert( pC->eCurType==CURTYPE_BTREE );
+    assert( pC->uc.pCursor!=0 );
    rc = sqlite3VdbeCursorRestore(pC);
    if( rc ) goto abort_due_to_error;
    if( pC->nullRow ){
      pOut->flags = MEM_Null;
      break;
    }
-    rc = sqlite3BtreeKeySize(pC->pCursor, &v);
+    rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v);
    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
  }
  pOut->u.i = v;
@@ -4580,8 +4622,9 @@ case OP_NullRow: {
  assert( pC!=0 );
  pC->nullRow = 1;
  pC->cacheStatus = CACHE_STALE;
-  if( pC->pCursor ){
-    sqlite3BtreeClearCursor(pC->pCursor);
+  if( pC->eCurType==CURTYPE_BTREE ){
+    assert( pC->uc.pCursor!=0 );
+    sqlite3BtreeClearCursor(pC->uc.pCursor);
  }
  break;
 }
@@ -4606,7 +4649,8 @@ case OP_Last: {        /* jump */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  pCrsr = pC->pCursor;
+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCrsr = pC->uc.pCursor;
  res = 0;
  assert( pCrsr!=0 );
  rc = sqlite3BtreeLast(pCrsr, &res);
@@ -4674,7 +4718,8 @@ case OP_Rewind: {        /* jump */
  if( isSorter(pC) ){
    rc = sqlite3VdbeSorterRewind(pC, &res);
  }else{
-    pCrsr = pC->pCursor;
+    assert( pC->eCurType==CURTYPE_BTREE );
+    pCrsr = pC->uc.pCursor;
    assert( pCrsr );
    rc = sqlite3BtreeFirst(pCrsr, &res);
    pC->deferredMoveto = 0;
@@ -4771,7 +4816,7 @@ case OP_Next:          /* jump */
  res = pOp->p3;
  assert( pC!=0 );
  assert( pC->deferredMoveto==0 );
-  assert( pC->pCursor );
+  assert( pC->eCurType==CURTYPE_BTREE );
  assert( res==0 || (res==1 && pC->isTable==0) );
  testcase( res==1 );
  assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext );
@@ -4788,7 +4833,7 @@ case OP_Next:          /* jump */
       || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE
       || pC->seekOp==OP_Last );

-  rc = pOp->p4.xAdvance(pC->pCursor, &res);
+  rc = pOp->p4.xAdvance(pC->uc.pCursor, &res);
 next_tail:
  pC->cacheStatus = CACHE_STALE;
  VdbeBranchTaken(res==0,2);
@@ -4839,7 +4884,7 @@ case OP_IdxInsert: {        /* in2 */
  pIn2 = &aMem[pOp->p2];
  assert( pIn2->flags & MEM_Blob );
  if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++;
-  assert( pC->pCursor!=0 );
+  assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert );
  assert( pC->isTable==0 );
  rc = ExpandBlob(pIn2);
  if( rc==SQLITE_OK ){
@@ -4848,7 +4893,7 @@ case OP_IdxInsert: {        /* in2 */
    }else{
      nKey = pIn2->n;
      zKey = pIn2->z;
-      rc = sqlite3BtreeInsert(pC->pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
+      rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, 
          ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
          );
      assert( pC->deferredMoveto==0 );
@@ -4876,7 +4921,8 @@ case OP_IdxDelete: {
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  pCrsr = pC->pCursor;
+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCrsr = pC->uc.pCursor;
  assert( pCrsr!=0 );
  assert( pOp->p5==0 );
  r.pKeyInfo = pC->pKeyInfo;
@@ -4913,7 +4959,8 @@ case OP_IdxRowid: {              /* out2 */
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  pCrsr = pC->pCursor;
+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCrsr = pC->uc.pCursor;
  assert( pCrsr!=0 );
  pOut->flags = MEM_Null;
  assert( pC->isTable==0 );
@@ -4994,7 +5041,8 @@ case OP_IdxGE:  {       /* jump */
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( pC->isOrdered );
-  assert( pC->pCursor!=0);
+  assert( pC->eCurType==CURTYPE_BTREE );
+  assert( pC->uc.pCursor!=0);
  assert( pC->deferredMoveto==0 );
  assert( pOp->p5==0 || pOp->p5==1 );
  assert( pOp->p4type==P4_INT32 );
@@ -5127,11 +5175,12 @@ case OP_ResetSorter: {
  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
-  if( pC->pSorter ){
-    sqlite3VdbeSorterReset(db, pC->pSorter);
+  if( isSorter(pC) ){
+    sqlite3VdbeSorterReset(db, pC->uc.pSorter);
  }else{
+    assert( pC->eCurType==CURTYPE_BTREE );
    assert( pC->isEphemeral );
-    rc = sqlite3BtreeClearTableOfCursor(pC->pCursor);
+    rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor);
  }
  break;
 }
@@ -6205,33 +6254,33 @@ case OP_VDestroy: {
 */
 case OP_VOpen: {
  VdbeCursor *pCur;
-  sqlite3_vtab_cursor *pVtabCursor;
+  sqlite3_vtab_cursor *pVCur;
  sqlite3_vtab *pVtab;
  const sqlite3_module *pModule;

  assert( p->bIsReader );
  pCur = 0;
-  pVtabCursor = 0;
+  pVCur = 0;
  pVtab = pOp->p4.pVtab->pVtab;
  if( pVtab==0 || NEVER(pVtab->pModule==0) ){
    rc = SQLITE_LOCKED;
    break;
  }
  pModule = pVtab->pModule;
-  rc = pModule->xOpen(pVtab, &pVtabCursor);
+  rc = pModule->xOpen(pVtab, &pVCur);
  sqlite3VtabImportErrmsg(p, pVtab);
  if( SQLITE_OK==rc ){
    /* Initialize sqlite3_vtab_cursor base class */
-    pVtabCursor->pVtab = pVtab;
+    pVCur->pVtab = pVtab;

    /* Initialize vdbe cursor object */
-    pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
+    pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB);
    if( pCur ){
-      pCur->pVtabCursor = pVtabCursor;
+      pCur->uc.pVCur = pVCur;
      pVtab->nRef++;
    }else{
      assert( db->mallocFailed );
-      pModule->xClose(pVtabCursor);
+      pModule->xClose(pVCur);
      goto no_mem;
    }
  }
@@ -6265,7 +6314,7 @@ case OP_VFilter: {   /* jump */
  const sqlite3_module *pModule;
  Mem *pQuery;
  Mem *pArgc;
-  sqlite3_vtab_cursor *pVtabCursor;
+  sqlite3_vtab_cursor *pVCur;
  sqlite3_vtab *pVtab;
  VdbeCursor *pCur;
  int res;
@@ -6277,9 +6326,9 @@ case OP_VFilter: {   /* jump */
  pCur = p->apCsr[pOp->p1];
  assert( memIsValid(pQuery) );
  REGISTER_TRACE(pOp->p3, pQuery);
-  assert( pCur->pVtabCursor );
-  pVtabCursor = pCur->pVtabCursor;
-  pVtab = pVtabCursor->pVtab;
+  assert( pCur->eCurType==CURTYPE_VTAB );
+  pVCur = pCur->uc.pVCur;
+  pVtab = pVCur->pVtab;
  pModule = pVtab->pModule;

  /* Grab the index number and argc parameters */
@@ -6293,10 +6342,10 @@ case OP_VFilter: {   /* jump */
  for(i = 0; i<nArg; i++){
    apArg[i] = &pArgc[i+1];
  }
-  rc = pModule->xFilter(pVtabCursor, iQuery, pOp->p4.z, nArg, apArg);
+  rc = pModule->xFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg);
  sqlite3VtabImportErrmsg(p, pVtab);
  if( rc==SQLITE_OK ){
-    res = pModule->xEof(pVtabCursor);
+    res = pModule->xEof(pVCur);
  }
  pCur->nullRow = 0;
  VdbeBranchTaken(res!=0,2);
@@ -6320,7 +6369,7 @@ case OP_VColumn: {
  sqlite3_context sContext;

  VdbeCursor *pCur = p->apCsr[pOp->p1];
-  assert( pCur->pVtabCursor );
+  assert( pCur->eCurType==CURTYPE_VTAB );
  assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) );
  pDest = &aMem[pOp->p3];
  memAboutToChange(p, pDest);
@@ -6328,13 +6377,13 @@ case OP_VColumn: {
    sqlite3VdbeMemSetNull(pDest);
    break;
  }
-  pVtab = pCur->pVtabCursor->pVtab;
+  pVtab = pCur->uc.pVCur->pVtab;
  pModule = pVtab->pModule;
  assert( pModule->xColumn );
  memset(&sContext, 0, sizeof(sContext));
  sContext.pOut = pDest;
  MemSetTypeFlag(pDest, MEM_Null);
-  rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2);
+  rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2);
  sqlite3VtabImportErrmsg(p, pVtab);
  if( sContext.isError ){
    rc = sContext.isError;
@@ -6365,11 +6414,11 @@ case OP_VNext: {   /* jump */

  res = 0;
  pCur = p->apCsr[pOp->p1];
-  assert( pCur->pVtabCursor );
+  assert( pCur->eCurType==CURTYPE_VTAB );
  if( pCur->nullRow ){
    break;
  }
-  pVtab = pCur->pVtabCursor->pVtab;
+  pVtab = pCur->uc.pVCur->pVtab;
  pModule = pVtab->pModule;
  assert( pModule->xNext );

@@ -6379,10 +6428,10 @@ case OP_VNext: {   /* jump */
  ** data is available) and the error code returned when xColumn or
  ** some other method is next invoked on the save virtual table cursor.
  */
-  rc = pModule->xNext(pCur->pVtabCursor);
+  rc = pModule->xNext(pCur->uc.pVCur);
  sqlite3VtabImportErrmsg(p, pVtab);
  if( rc==SQLITE_OK ){
-    res = pModule->xEof(pCur->pVtabCursor);
+    res = pModule->xEof(pCur->uc.pVCur);
  }
  VdbeBranchTaken(!res,2);
  if( !res ){
@@ -6606,7 +6655,9 @@ case OP_CursorHint: {
  assert( pOp->p4type==P4_EXPR );
  pC = p->apCsr[pOp->p1];
  if( pC ){
-    sqlite3BtreeCursorHint(pC->pCursor, BTREE_HINT_RANGE, pOp->p4.pExpr, aMem);
+    assert( pC->eCurType==CURTYPE_BTREE );
+    sqlite3BtreeCursorHint(pC->uc.pCursor, BTREE_HINT_RANGE,
+                           pOp->p4.pExpr, aMem);
  }
  break;
 }
@@ -58,42 +58,48 @@ typedef struct Explain Explain;
 /* Elements of the linked list at Vdbe.pAuxData */
 typedef struct AuxData AuxData;

+/* Types of VDBE cursors */
+#define CURTYPE_BTREE       0
+#define CURTYPE_SORTER      1
+#define CURTYPE_VTAB        2
+#define CURTYPE_PSEUDO      3
+
 /*
-** A cursor is a pointer into a single BTree within a database file.
-** The cursor can seek to a BTree entry with a particular key, or
-** loop over all entries of the Btree.  You can also insert new BTree
-** entries or retrieve the key or data from the entry that the cursor
-** is currently pointing to.
+** A VdbeCursor is an superclass (a wrapper) for various cursor objects:
 **
-** Cursors can also point to virtual tables, sorters, or "pseudo-tables".
-** A pseudo-table is a single-row table implemented by registers.
-** 
-** Every cursor that the virtual machine has open is represented by an
-** instance of the following structure.
+**      * A b-tree cursor
+**          -  In the main database or in an ephemeral database
+**          -  On either an index or a table
+**      * A sorter
+**      * A virtual table
+**      * A one-row "pseudotable" stored in a single register
 */
 struct VdbeCursor {
-  BtCursor *pCursor;    /* The cursor structure of the backend */
-  Btree *pBt;           /* Separate file holding temporary table */
-  KeyInfo *pKeyInfo;    /* Info about index keys needed by index cursors */
-  int seekResult;       /* Result of previous sqlite3BtreeMoveto() */
-  int pseudoTableReg;   /* Register holding pseudotable content. */
-  i16 nField;           /* Number of fields in the header */
-  u16 nHdrParsed;       /* Number of header fields parsed so far */
-#ifdef SQLITE_DEBUG
-  u8 seekOp;            /* Most recent seek operation on this cursor */
-#endif
+  u8 eCurType;          /* One of the CURTYPE_* values above */
  i8 iDb;               /* Index of cursor database in db->aDb[] (or -1) */
  u8 nullRow;           /* True if pointing to a row with no data */
  u8 deferredMoveto;    /* A call to sqlite3BtreeMoveto() is needed */
+  u8 isTable;           /* True for rowid tables.  False for indexes */
+#ifdef SQLITE_DEBUG
+  u8 seekOp;            /* Most recent seek operation on this cursor */
+#endif
  Bool isEphemeral:1;   /* True for an ephemeral table */
  Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */
-  Bool isTable:1;       /* True if a table requiring integer keys */
  Bool isOrdered:1;     /* True if the underlying table is BTREE_UNORDERED */
  Pgno pgnoRoot;        /* Root page of the open btree cursor */
-  sqlite3_vtab_cursor *pVtabCursor;  /* The cursor for a virtual table */
+  i16 nField;           /* Number of fields in the header */
+  u16 nHdrParsed;       /* Number of header fields parsed so far */
+  union {
+    BtCursor *pCursor;          /* CURTYPE_BTREE.  Btree cursor */
+    sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB.   Vtab cursor */
+    int pseudoTableReg;         /* CURTYPE_PSEUDO. Reg holding content. */
+    VdbeSorter *pSorter;        /* CURTYPE_SORTER. Sorter object */
+  } uc;
+  Btree *pBt;           /* Separate file holding temporary table */
+  KeyInfo *pKeyInfo;    /* Info about index keys needed by index cursors */
+  int seekResult;       /* Result of previous sqlite3BtreeMoveto() */
  i64 seqCount;         /* Sequence counter */
  i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
-  VdbeSorter *pSorter;  /* Sorter object for OP_SorterOpen cursors */
 #ifdef SQLITE_ENABLE_COLUMN_USED_MASK
  u64 maskUsed;         /* Mask of columns used by this cursor */
 #endif
@@ -157,6 +157,12 @@ static void test_addop_breakpoint(void){
 ** the sqlite3VdbeChangeP4() function to change the value of the P4
 ** operand.
 */
+static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){
+  assert( p->pParse->nOpAlloc<=p->nOp );
+  if( growOpArray(p, 1) ) return 1;
+  assert( p->pParse->nOpAlloc>p->nOp );
+  return sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+}
 int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
  int i;
  VdbeOp *pOp;
@@ -165,9 +171,7 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
  assert( p->magic==VDBE_MAGIC_INIT );
  assert( op>0 && op<0xff );
  if( p->pParse->nOpAlloc<=i ){
-    if( growOpArray(p, 1) ){
-      return 1;
-    }
+    return growOp3(p, op, p1, p2, p3);
  }
  p->nOp++;
  pOp = &p->aOp[i];
@@ -1926,23 +1930,34 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
  if( pCx==0 ){
    return;
  }
-  sqlite3VdbeSorterClose(p->db, pCx);
-  if( pCx->pBt ){
-    sqlite3BtreeClose(pCx->pBt);
-    /* The pCx->pCursor will be close automatically, if it exists, by
-    ** the call above. */
-  }else if( pCx->pCursor ){
-    sqlite3BtreeCloseCursor(pCx->pCursor);
-  }
+  assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE );
+  switch( pCx->eCurType ){
+    case CURTYPE_SORTER: {
+      sqlite3VdbeSorterClose(p->db, pCx);
+      break;
+    }
+    case CURTYPE_BTREE: {
+      if( pCx->pBt ){
+        sqlite3BtreeClose(pCx->pBt);
+        /* The pCx->pCursor will be close automatically, if it exists, by
+        ** the call above. */
+      }else{
+        assert( pCx->uc.pCursor!=0 );
+        sqlite3BtreeCloseCursor(pCx->uc.pCursor);
+      }
+      break;
+    }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  else if( pCx->pVtabCursor ){
-    sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor;
-    const sqlite3_module *pModule = pVtabCursor->pVtab->pModule;
-    assert( pVtabCursor->pVtab->nRef>0 );
-    pVtabCursor->pVtab->nRef--;
-    pModule->xClose(pVtabCursor);
-  }
+    case CURTYPE_VTAB: {
+      sqlite3_vtab_cursor *pVCur = pCx->uc.pVCur;
+      const sqlite3_module *pModule = pVCur->pVtab->pModule;
+      assert( pVCur->pVtab->nRef>0 );
+      pVCur->pVtab->nRef--;
+      pModule->xClose(pVCur);
+      break;
+    }
 #endif
+  }
 }

 /*
@@ -2929,7 +2944,8 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){
 #endif
  assert( p->deferredMoveto );
  assert( p->isTable );
-  rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res);
+  assert( p->eCurType==CURTYPE_BTREE );
+  rc = sqlite3BtreeMovetoUnpacked(p->uc.pCursor, 0, p->movetoTarget, 0, &res);
  if( rc ) return rc;
  if( res!=0 ) return SQLITE_CORRUPT_BKPT;
 #ifdef SQLITE_TEST
@@ -2949,9 +2965,10 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){
 */
 static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){
  int isDifferentRow, rc;
-  assert( p->pCursor!=0 );
-  assert( sqlite3BtreeCursorHasMoved(p->pCursor) );
-  rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow);
+  assert( p->eCurType==CURTYPE_BTREE );
+  assert( p->uc.pCursor!=0 );
+  assert( sqlite3BtreeCursorHasMoved(p->uc.pCursor) );
+  rc = sqlite3BtreeCursorRestore(p->uc.pCursor, &isDifferentRow);
  p->cacheStatus = CACHE_STALE;
  if( isDifferentRow ) p->nullRow = 1;
  return rc;
@@ -2962,7 +2979,8 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){
 ** if need be.  Return any I/O error from the restore operation.
 */
 int sqlite3VdbeCursorRestore(VdbeCursor *p){
-  if( sqlite3BtreeCursorHasMoved(p->pCursor) ){
+  assert( p->eCurType==CURTYPE_BTREE );
+  if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){
    return handleMovedCursor(p);
  }
  return SQLITE_OK;
@@ -2982,10 +3000,11 @@ int sqlite3VdbeCursorRestore(VdbeCursor *p){
 ** not been deleted out from under the cursor, then this routine is a no-op.
 */
 int sqlite3VdbeCursorMoveto(VdbeCursor *p){
+  assert( p->eCurType==CURTYPE_BTREE );
  if( p->deferredMoveto ){
    return handleDeferredMoveto(p);
  }
-  if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){
+  if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){
    return handleMovedCursor(p);
  }
  return SQLITE_OK;
@@ -3319,6 +3338,10 @@ u32 sqlite3VdbeSerialGet(
      /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit
      ** twos-complement integer. */
      pMem->u.i = FOUR_BYTE_INT(buf);
+#ifdef __HP_cc 
+      /* Work around a sign-extension bug in the HP compiler for HP/UX */
+      if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL;
+#endif
      pMem->flags = MEM_Int;
      testcase( pMem->u.i<0 );
      return 4;
@@ -3989,6 +4012,7 @@ int sqlite3VdbeRecordCompareWithSkip(
       || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) 
       || pKeyInfo->db->mallocFailed
  );
+  pPKey2->eqSeen = 1;
  return pPKey2->default_rc;
 }
 int sqlite3VdbeRecordCompare(
@@ -4088,6 +4112,7 @@ static int vdbeRecordCompareInt(
    /* The first fields of the two keys are equal and there are no trailing
    ** fields. Return pPKey2->default_rc in this case. */
    res = pPKey2->default_rc;
+    pPKey2->eqSeen = 1;
  }

  assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) );
@@ -4135,6 +4160,7 @@ static int vdbeRecordCompareString(
          res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1);
        }else{
          res = pPKey2->default_rc;
+          pPKey2->eqSeen = 1;
        }
      }else if( res>0 ){
        res = pPKey2->r2;
@@ -4292,9 +4318,11 @@ int sqlite3VdbeIdxKeyCompare(
 ){
  i64 nCellKey = 0;
  int rc;
-  BtCursor *pCur = pC->pCursor;
+  BtCursor *pCur;
  Mem m;

+  assert( pC->eCurType==CURTYPE_BTREE );
+  pCur = pC->uc.pCursor;
  assert( sqlite3BtreeCursorIsValid(pCur) );
  VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey);
  assert( rc==SQLITE_OK );    /* pCur is always valid so KeySize cannot fail */
@@ -4305,7 +4333,7 @@ int sqlite3VdbeIdxKeyCompare(
    return SQLITE_CORRUPT_BKPT;
  }
  sqlite3VdbeMemInit(&m, db, 0);
-  rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m);
+  rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m);
  if( rc ){
    return rc;
  }
@@ -76,7 +76,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
    }else{
      p->iOffset = pC->aType[p->iCol + pC->nField];
      p->nByte = sqlite3VdbeSerialTypeLen(type);
-      p->pCsr =  pC->pCursor;
+      p->pCsr =  pC->uc.pCursor;
      sqlite3BtreeIncrblobCursor(p->pCsr);
    }
  }
@@ -961,11 +961,12 @@ int sqlite3VdbeSorterInit(
 #endif

  assert( pCsr->pKeyInfo && pCsr->pBt==0 );
+  assert( pCsr->eCurType==CURTYPE_SORTER );
  szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*);
  sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask);

  pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo);
-  pCsr->pSorter = pSorter;
+  pCsr->uc.pSorter = pSorter;
  if( pSorter==0 ){
    rc = SQLITE_NOMEM;
  }else{
@@ -1249,12 +1250,14 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){
 ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines.
 */
 void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
-  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorter *pSorter;
+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
  if( pSorter ){
    sqlite3VdbeSorterReset(db, pSorter);
    sqlite3_free(pSorter->list.aMemory);
    sqlite3DbFree(db, pSorter);
-    pCsr->pSorter = 0;
+    pCsr->uc.pSorter = 0;
  }
 }

@@ -1752,15 +1755,16 @@ int sqlite3VdbeSorterWrite(
  const VdbeCursor *pCsr,         /* Sorter cursor */
  Mem *pVal                       /* Memory cell containing record */
 ){
-  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorter *pSorter;
  int rc = SQLITE_OK;             /* Return Code */
  SorterRecord *pNew;             /* New list element */
-
  int bFlush;                     /* True to flush contents of memory to PMA */
  int nReq;                       /* Bytes of memory required */
  int nPMA;                       /* Bytes of PMA space required */
  int t;                          /* serial type of first record field */

+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
  getVarint32((const u8*)&pVal->z[1], t);
  if( t>0 && t<10 && t!=7 ){
    pSorter->typeMask &= SORTER_TYPE_INTEGER;
@@ -2552,9 +2556,11 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
 ** in sorted order.
 */
 int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){
-  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorter *pSorter;
  int rc = SQLITE_OK;             /* Return code */

+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
  assert( pSorter );

  /* If no data has been written to disk, then do not do so now. Instead,
@@ -2598,9 +2604,11 @@ int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){
 ** Advance to the next element in the sorter.
 */
 int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){
-  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorter *pSorter;
  int rc;                         /* Return code */

+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
  assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) );
  if( pSorter->bUsePMA ){
    assert( pSorter->pReader==0 || pSorter->pMerger==0 );
@@ -2660,9 +2668,11 @@ static void *vdbeSorterRowkey(
 ** Copy the current sorter key into the memory cell pOut.
 */
 int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){
-  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorter *pSorter;
  void *pKey; int nKey;           /* Sorter key to copy into pOut */

+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
  pKey = vdbeSorterRowkey(pSorter, &nKey);
  if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){
    return SQLITE_NOMEM;
@@ -2696,12 +2706,16 @@ int sqlite3VdbeSorterCompare(
  int nKeyCol,                    /* Compare this many columns */
  int *pRes                       /* OUT: Result of comparison */
 ){
-  VdbeSorter *pSorter = pCsr->pSorter;
-  UnpackedRecord *r2 = pSorter->pUnpacked;
-  KeyInfo *pKeyInfo = pCsr->pKeyInfo;
+  VdbeSorter *pSorter;
+  UnpackedRecord *r2;
+  KeyInfo *pKeyInfo;
  int i;
  void *pKey; int nKey;           /* Sorter key to compare pVal with */

+  assert( pCsr->eCurType==CURTYPE_SORTER );
+  pSorter = pCsr->uc.pSorter;
+  r2 = pSorter->pUnpacked;
+  pKeyInfo = pCsr->pKeyInfo;
  if( r2==0 ){
    char *p;
    r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p);
@@ -1222,7 +1222,8 @@ void sqlite3WhereClauseInit(

 /*
 ** Deallocate a WhereClause structure.  The WhereClause structure
-** itself is not freed.  This routine is the inverse of sqlite3WhereClauseInit().
+** itself is not freed.  This routine is the inverse of
+** sqlite3WhereClauseInit().
 */
 void sqlite3WhereClauseClear(WhereClause *pWC){
  int i;
@@ -1316,9 +1317,9 @@ void sqlite3WhereTabFuncArgs(
  pTab = pItem->pTab;
  assert( pTab!=0 );
  pArgs = pItem->u1.pFuncArg;
-  assert( pArgs!=0 );
+  if( pArgs==0 ) return;
  for(j=k=0; j<pArgs->nExpr; j++){
-    while( k<pTab->nCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){ k++; }
+    while( k<pTab->nCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;}
    if( k>=pTab->nCol ){
      sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d",
                      pTab->zName, j);
@@ -352,7 +352,7 @@ do_test collate4-2.1.2 {
  count {
    SELECT * FROM collate4t2, collate4t1 WHERE a = b;
  }
-} {A a A A 5}
+} {A a A A 4}
 do_test collate4-2.1.3 {
  count {
    SELECT * FROM collate4t2, collate4t1 WHERE b = a;
@@ -372,7 +372,7 @@ do_test collate4-2.1.5 {
  count {
    SELECT * FROM collate4t2, collate4t1 WHERE b = a;
  }
-} {A A 4}
+} {A A 3}
 ifcapable subquery {
  do_test collate4-2.1.6 {
    count {
@@ -389,12 +389,12 @@ ifcapable subquery {
      SELECT a FROM collate4t1 WHERE a IN (SELECT * FROM collate4t2)
       ORDER BY rowid
    }
-  } {a A 6}
+  } {a A 5}
  do_test collate4-2.1.8 {
    count {
      SELECT a FROM collate4t1 WHERE a IN ('z', 'a');
    }
-  } {a A 5}
+  } {a A 4}
  do_test collate4-2.1.9 {
    execsql {
      DROP INDEX collate4i1;
@@ -838,6 +838,7 @@ int main(int argc, char **argv){
  int nMem = 0;                /* Memory limit */
  char *zExpDb = 0;            /* Write Databases to files in this directory */
  char *zExpSql = 0;           /* Write SQL to files in this directory */
+  void *pHeap = 0;             /* Heap for use by SQLite */

  iBegin = timeOfDay();
 #ifdef __unix__
@@ -1085,7 +1086,6 @@ int main(int argc, char **argv){

    /* Limit available memory, if requested */
    if( nMem>0 ){
-      void *pHeap;
      sqlite3_shutdown();
      pHeap = malloc(nMem);
      if( pHeap==0 ){
@@ -1184,5 +1184,6 @@ int main(int argc, char **argv){
           sqlite3_libversion(), sqlite3_sourceid());
  }
  free(azSrcDb);
+  free(pHeap);
  return 0;
 }
@@ -0,0 +1,153 @@
+# 2015 November 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test the __hidden__ hack.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix hidden
+
+ifcapable !hiddencolumns {
+  finish_test
+  return
+}
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(__hidden__a, b);
+  INSERT INTO t1 VALUES('1');
+  INSERT INTO t1(__hidden__a, b) VALUES('x', 'y');
+} {}
+
+do_execsql_test 1.2 {
+  SELECT * FROM t1;
+} {1 y}
+
+do_execsql_test 1.3 {
+  SELECT __hidden__a, * FROM t1;
+} {{} 1 x y}
+
+foreach {tn view} {
+  1 { CREATE VIEW v1(a, b, __hidden__c) AS SELECT a, b, c FROM x1 }
+  2 { CREATE VIEW v1 AS SELECT a, b, c AS __hidden__c FROM x1 }
+} {
+  do_execsql_test 2.$tn.1 {
+    DROP TABLE IF EXISTS x1;
+    CREATE TABLE x1(a, b, c);
+    INSERT INTO x1 VALUES(1, 2, 3);
+  }
+
+  catchsql { DROP VIEW v1 }
+  execsql $view
+
+  do_execsql_test 2.$tn.2 {
+    SELECT a, b, __hidden__c FROM v1;
+  } {1 2 3}
+  
+  do_execsql_test 2.$tn.3 {
+    SELECT * FROM v1;
+  } {1 2}
+  
+  do_execsql_test 2.$tn.4 {
+    CREATE TRIGGER tr1 INSTEAD OF INSERT ON v1 BEGIN
+      INSERT INTO x1 VALUES(new.a, new.b, new.__hidden__c);
+    END;
+  
+    INSERT INTO v1 VALUES(4, 5);
+    SELECT * FROM x1;
+  } {1 2 3 4 5 {}}
+  
+  do_execsql_test 2.$tn.5 {
+    INSERT INTO v1(a, b, __hidden__c) VALUES(7, 8, 9);
+    SELECT * FROM x1;
+  } {1 2 3 4 5 {} 7 8 9}
+}
+
+#-------------------------------------------------------------------------
+# Test INSERT INTO ... SELECT ... statements that write to tables with
+# hidden columns.
+#
+do_execsql_test 3.1 {
+  CREATE TABLE t4(a, __hidden__b, c);
+  INSERT INTO t4 SELECT 1, 2;
+  SELECT a, __hidden__b, c FROM t4;
+} {1 {} 2}
+
+do_execsql_test 3.2.1 {
+  CREATE TABLE t5(__hidden__a, b, c);
+  CREATE TABLE t6(__hidden__a, b, c);
+  INSERT INTO t6(__hidden__a, b, c) VALUES(1, 2, 3);
+  INSERT INTO t6(__hidden__a, b, c) VALUES(4, 5, 6);
+  INSERT INTO t6(__hidden__a, b, c) VALUES(7, 8, 9);
+}
+
+do_execsql_test 3.2.2 {
+  INSERT INTO t5 SELECT * FROM t6;
+  SELECT * FROM t5;
+} {2 3   5 6   8 9}
+
+do_execsql_test 3.2.3 {
+  SELECT __hidden__a FROM t5;
+} {{} {} {}}
+
+
+do_execsql_test 3.3.1 {
+  CREATE TABLE t5a(a, b, __hidden__c);
+  CREATE TABLE t6a(a, b, __hidden__c);
+  INSERT INTO t6a(a, b, __hidden__c) VALUES(1, 2, 3);
+  INSERT INTO t6a(a, b, __hidden__c) VALUES(4, 5, 6);
+  INSERT INTO t6a(a, b, __hidden__c) VALUES(7, 8, 9);
+}
+
+do_execsql_test 3.3.2 {
+  INSERT INTO t5a SELECT * FROM t6a;
+  SELECT * FROM t5a;
+} {1 2   4 5   7 8}
+
+do_execsql_test 3.3.3 {
+  SELECT __hidden__c FROM t5a;
+} {{} {} {}}
+
+do_execsql_test 3.4.1 {
+  CREATE TABLE t5b(a, __hidden__b, c);
+  CREATE TABLE t6b(a, b, __hidden__c);
+  INSERT INTO t6b(a, b, __hidden__c) VALUES(1, 2, 3);
+  INSERT INTO t6b(a, b, __hidden__c) VALUES(4, 5, 6);
+  INSERT INTO t6b(a, b, __hidden__c) VALUES(7, 8, 9);
+}
+
+do_execsql_test 3.4.2 {
+  INSERT INTO t5b SELECT * FROM t6b;
+  SELECT * FROM t5b;
+} {1 2   4 5   7 8}
+
+do_execsql_test 3.4.3 {
+  SELECT __hidden__b FROM t5b;
+} {{} {} {}}
+
+#-------------------------------------------------------------------------
+# Test VACUUM
+#
+reset_db
+do_execsql_test 4.1 {
+  CREATE TABLE t1(a, __hidden__b, c UNIQUE);
+  INSERT INTO t1(a, __hidden__b, c) VALUES(1, 2, 3);
+  INSERT INTO t1(a, __hidden__b, c) VALUES(4, 5, 6);
+  INSERT INTO t1(a, __hidden__b, c) VALUES(7, 8, 9);
+  DELETE FROM t1 WHERE __hidden__b = 5;
+  SELECT rowid, a, __hidden__b, c FROM t1;
+} {1 1 2 3   3 7 8 9}
+do_execsql_test 4.2 {
+  VACUUM;
+  SELECT rowid, a, __hidden__b, c FROM t1;
+} {1 1 2 3   3 7 8 9}
+
+finish_test
@@ -701,4 +701,13 @@ do_test misc1-24.0 {
  list [catch { sqlite3_prepare_v2 db ! -1 dummy } msg] $msg
 } {1 {(1) unrecognized token: "!}}

+# The following query (provided by Kostya Serebryany) used to take 25
+# minutes to prepare.  This has been speeded up to about 250 milliseconds.
+#
+do_catchsql_test misc1-25.0 {
+SELECT-1 UNION  SELECT 5 UNION SELECT 0 UNION SElECT*from(SELECT-5) UNION SELECT*from(SELECT-0) UNION  SELECT:SELECT-0 UNION SELECT-1 UNION SELECT 1 UNION SELECT 1 ORDER BY S  in(WITH K AS(WITH K AS(select'CREINDERcharREADEVIRTUL5TABLECONFLICT !1 USIN'' MFtOR(b38q,eWITH K AS(selectCREATe TABLE t0(a,b,c,d,e, PRIMARY KEY(a,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,b,c,d,c,a,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d'CEIl,k'',ab, g, a,b,o11b, i'nEX/charREDE IVT LR!VABLt5SG',N  ,N in rement,l_vacuum,M&U,'te3(''5l' a,bB,b,l*e)SELECT:SELECT, *,*,*from(( SELECT
+$group,:conc ap0,1)fro,(select"",:PBAG,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d, foreign_keysc,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,a,b,d,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,c,a,b,b,c,d,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,d,c,e,d,d,c,a,b,b,c,c,a,b,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,bb,b,E,d,c,d,c,b,c,d,c,d,c,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,a,b,c,e,d,d,c,a,b,b,c,d,d,c,a,b,c,e,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,c,d,c,a,b,d,d,c,a,a,b,d,d,c,a,b,b,c,d,c,a,b,e,e,d,b,c,d,c,a,b,b,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,c,a,b,c,e,d,d,c,a,b,b,c,d,c,d,c,a,b,c,e,c,d,c,a,b,b,c,d,MAato_aecSELEC,+?b," "O,"i","a",""b  ,5 ))KEY)SELECT*FROM((k()reaC,k,K) eA,k '' )t ,K  M);
+} {1 {'k' is not a function}}
+
+
 finish_test
@@ -178,7 +178,7 @@ test_suite "valgrind" -prefix "" -description {
 } -files [
  test_set $allquicktests -exclude *malloc* *ioerr* *fault* wal.test \
              shell*.test crash8.test atof1.test selectG.test \
-              tkt-fc62af4523.test
+              tkt-fc62af4523.test numindex1.test
 ] -initialize {
  set ::G(valgrind) 1
 } -shutdown {
@@ -135,10 +135,12 @@ delete_file test.db test.db-journal
 delete_file test2.db test2.db-journal
 sqlite3 db test.db
 do_execsql_test pragma2-4.1 {
+  PRAGMA main.cache_size=2000;
+  PRAGMA temp.cache_size=2000;
  PRAGMA cache_spill;
  PRAGMA main.cache_spill;
  PRAGMA temp.cache_spill;
-} {1 1 1}
+} {2000 2000 2000}
 do_execsql_test pragma2-4.2 {
  PRAGMA cache_spill=OFF;
  PRAGMA cache_spill;
@@ -178,21 +180,56 @@ do_test pragma2-4.4 {
    PRAGMA lock_status;
  }
 } {main exclusive temp unknown}  ;# EXCLUSIVE lock due to cache spill
-do_test pragma2-4.5 {
+do_test pragma2-4.5.1 {
  db eval {
-    COMMIT;
+    ROLLBACK;
    PRAGMA cache_spill=OFF;
+    PRAGMA Cache_Spill;
    BEGIN;
-    UPDATE t1 SET c=c-1;
+    UPDATE t1 SET c=c+1;
    PRAGMA lock_status;
  }
-} {main reserved temp unknown}   ;# No cache spill, so no exclusive lock
+} {0 main reserved temp unknown}   ;# No cache spill, so no exclusive lock
+do_test pragma2-4.5.2 {
+  db eval {
+    ROLLBACK;
+    PRAGMA cache_spill=100000;
+    PRAGMA cache_spill;
+    BEGIN;
+    UPDATE t1 SET c=c+1;
+    PRAGMA lock_status;
+  }
+} {100000 main reserved temp unknown}   ;# Big spill threshold -> no excl lock
+ifcapable !memorymanage {
+  do_test pragma2-4.5.3 {
+    db eval {
+      ROLLBACK;
+      PRAGMA cache_spill=25;
+      PRAGMA main.cache_spill;
+      BEGIN;
+      UPDATE t1 SET c=c+1;
+      PRAGMA lock_status;
+    }
+  } {50 main exclusive temp unknown}   ;# Small cache spill -> exclusive lock
+  do_test pragma2-4.5.4 {
+    db eval {
+      ROLLBACK;
+      PRAGMA cache_spill(-25);
+      PRAGMA main.cache_spill;
+      BEGIN;
+      UPDATE t1 SET c=c+1;
+      PRAGMA lock_status;
+    }
+  } {50 main exclusive temp unknown}   ;# Small cache spill -> exclusive lock
+}
+

 # Verify that newly attached databases inherit the cache_spill=OFF
 # setting.
 #
 do_execsql_test pragma2-4.6 {
-  COMMIT;
+  ROLLBACK;
+  PRAGMA cache_spill=OFF;
  ATTACH 'test2.db' AS aux1;
  PRAGMA aux1.cache_size=50;
  BEGIN;
@@ -209,6 +246,26 @@ do_execsql_test pragma2-4.8 {
  UPDATE t2 SET c=c-1;
  PRAGMA lock_status;
 } {main unlocked temp unknown aux1 exclusive}
+db close
+forcedelete test.db
+sqlite3 db test.db
+
+breakpoint
+do_execsql_test pragma2-5.1 {
+  PRAGMA page_size=16384;
+  CREATE TABLE t1(x);
+  PRAGMA cache_size=2;
+  PRAGMA cache_spill=YES;
+  PRAGMA cache_spill;
+} {2}
+do_execsql_test pragma2-5.2 {
+  PRAGMA cache_spill=NO;
+  PRAGMA cache_spill;
+} {0}
+do_execsql_test pragma2-5.3 {
+  PRAGMA cache_spill(-51);
+  PRAGMA cache_spill;
+} {3}
   
 test_restore_config_pagecache
 finish_test
@@ -119,6 +119,7 @@ array set ::Configs [strip_comments {
    -DSQLITE_ENABLE_MEMSYS3=1
    -DSQLITE_ENABLE_COLUMN_METADATA=1
    -DSQLITE_ENABLE_STAT4
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
    -DSQLITE_MAX_ATTACHED=125
  }
  "Fast-One" {
@@ -128,6 +129,7 @@ array set ::Configs [strip_comments {
    -DSQLITE_ENABLE_STAT4
    -DSQLITE_ENABLE_RBU
    -DSQLITE_MAX_ATTACHED=125
+    -DLONGDOUBLE_TYPE=double
  }
  "Device-One" {
    -O2
@@ -144,6 +146,7 @@ array set ::Configs [strip_comments {
    -DSQLITE_OMIT_LOAD_EXTENSION=1
    -DSQLITE_OMIT_PROGRESS_CALLBACK=1
    -DSQLITE_OMIT_VIRTUALTABLE=1
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
    -DSQLITE_TEMP_STORE=3
    --enable-json1
  }
@@ -212,6 +215,7 @@ array set ::Configs [strip_comments {
    -DSQLITE_ENABLE_STAT4
    -DSQLITE_ENABLE_FTS4
    -DSQLITE_ENABLE_RTREE
+    -DSQLITE_ENABLE_HIDDEN_COLUMNS
    --enable-json1
  }

@@ -305,22 +309,39 @@ foreach {key value} [array get ::Platforms] {
  }
 }

-# Output log
+# Output log.   Disabled for slave interpreters.
 #
-set LOG [open releasetest-out.txt w]
-proc PUTS {args} {
-  if {[llength $args]==2} {
-    puts [lindex $args 0] [lindex $args 1]
-    puts $::LOG [lindex $args 1]
-  } else {
-    puts [lindex $args 0]
-    puts $::LOG [lindex $args 0]
+if {[lindex $argv end]!="--slave"} {
+  set LOG [open releasetest-out.txt w]
+  proc PUTS {txt} {
+    puts $txt
+    puts $::LOG $txt
+    flush $::LOG
+  }
+  proc PUTSNNL {txt} {
+    puts -nonewline $txt
+    puts -nonewline $::LOG $txt
+    flush $::LOG
+  }
+  proc PUTSERR {txt} {
+    puts stderr $txt
+    puts $::LOG $txt
+    flush $::LOG
+  }
+  puts $LOG "$argv0 $argv"
+  set tm0 [clock format [clock seconds] -format {%Y-%m-%d %H:%M:%S} -gmt 1]
+  puts $LOG "start-time: $tm0 UTC"
+} else {
+  proc PUTS {txt} {
+    puts $txt
+  }
+  proc PUTSNNL {txt} {
+    puts -nonewline $txt
+  }
+  proc PUTSERR {txt} {
+    puts stderr $txt
  }
-  flush $::LOG
 }
-puts $LOG "$argv0 $argv"
-set tm0 [clock format [clock seconds] -format {%Y-%m-%d %H:%M:%S} -gmt 1]
-puts $LOG "start-time: $tm0 UTC"

 # Open the file $logfile and look for a report on the number of errors
 # and the number of test cases run.  Add these values to the global
@@ -421,6 +442,7 @@ proc run_slave_test {} {
  foreach {title dir configOpts testtarget makeOpts cflags opts} $T {}

  # Create and switch to the test directory.
+  set ::env(SQLITE_TMPDIR) [file normalize $dir]
  trace_cmd file mkdir $dir
  trace_cmd cd $dir
  catch {file delete core}
@@ -808,8 +830,8 @@ proc process_options {argv} {
      }

      default {
-        PUTS stderr ""
-        PUTS stderr [string trim $::USAGE_MESSAGE]
+        PUTSERR stderr ""
+        PUTSERR stderr [string trim $::USAGE_MESSAGE]
        exit -1
      }
    }
@@ -817,7 +839,7 @@ proc process_options {argv} {

  if {0==[info exists ::Platforms($platform)]} {
    PUTS "Unknown platform: $platform"
-    PUTS -nonewline "Set the -platform option to "
+    PUTSNNL "Set the -platform option to "
    set print [list]
    foreach p [array names ::Platforms] {
      lappend print "\"$p\""
@@ -843,16 +865,16 @@ proc process_options {argv} {
  }
  PUTS "Running the following test configurations for $platform:"
  PUTS "    [string trim $::CONFIGLIST]"
-  PUTS -nonewline "Flags:"
-  if {$::PROGRESS_MSGS} {PUTS -nonewline " --progress"}
-  if {$::DRYRUN} {PUTS -nonewline " --dryrun"}
-  if {$::BUILDONLY} {PUTS -nonewline " --buildonly"}
-  if {$::MSVC} {PUTS -nonewline " --msvc"}
+  PUTSNNL "Flags:"
+  if {$::PROGRESS_MSGS} {PUTSNNL " --progress"}
+  if {$::DRYRUN} {PUTSNNL " --dryrun"}
+  if {$::BUILDONLY} {PUTSNNL " --buildonly"}
+  if {$::MSVC} {PUTSNNL " --msvc"}
  switch -- $::QUICK {
-     1 {PUTS -nonewline " --quick"}
-     2 {PUTS -nonewline " --veryquick"}
+     1 {PUTSNNL " --quick"}
+     2 {PUTSNNL " --veryquick"}
  }
-  if {$::JOBS>1} {PUTS -nonewline " --jobs $::JOBS"}
+  if {$::JOBS>1} {PUTSNNL " --jobs $::JOBS"}
  PUTS ""
 }

@@ -0,0 +1,116 @@
+# 2015 November 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the test_sqllog.c module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix sqllog
+
+ifcapable !sqllog {
+  finish_test
+  return
+}
+
+proc readfile {f} {
+  set fd [open $f]
+  set txt [read $fd]
+  close $fd
+  set txt
+}
+
+proc delete_all_sqllog_files {} {
+  forcedelete {*}[glob -nocomplain sqllog_*.sql]
+  forcedelete {*}[glob -nocomplain sqllog_*.db]
+  forcedelete {*}[glob -nocomplain sqllog_*.idx]
+}
+
+proc touch {f} {
+  set fd [open $f w+]
+  close $fd
+}
+
+db close
+sqlite3_shutdown
+set ::env(SQLITE_SQLLOG_DIR) [pwd]
+
+delete_all_sqllog_files
+
+sqlite3 db test.db
+set a a
+set b b
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x, y);
+  INSERT INTO t1 VALUES(1, 2);
+  INSERT INTO t1 VALUES($a, $b);
+  SELECT * FROM t1;
+} {1 2 a b}
+db close
+
+do_test 1.1 {
+  readfile [lindex [glob sqllog_*.sql] 0]
+} [string trimleft {
+/-- Main database is '.*/sqllog_.*_0.db'
+CREATE TABLE t1\(x, y\);; -- clock=0
+INSERT INTO t1 VALUES\(1, 2\);; -- clock=1
+INSERT INTO t1 VALUES\('a', 'b'\);; -- clock=2
+SELECT . FROM t1;; -- clock=3
+/}]
+
+do_test 1.2 {
+  file size [lindex [glob sqllog_*_0.db] 0]
+} 1024
+
+#-------------------------------------------------------------------------
+catch { db close }
+sqlite3_shutdown
+delete_all_sqllog_files
+forcedelete test.db-sqllog
+
+set ::env(SQLITE_SQLLOG_CONDITIONAL) 1
+sqlite3 db test.db
+do_execsql_test 2.1 {
+  INSERT INTO t1 VALUES(4, 5);
+  SELECT * FROM t1;
+} {1 2 a b 4 5}
+
+do_test 2.2 {
+  glob -nocomplain sqllog_*
+} {}
+
+db close
+touch test.db-sqllog
+sqlite3 db test.db
+do_execsql_test 2.3 {
+  INSERT INTO t1 VALUES(6, 7);
+  SELECT * FROM t1;
+} {1 2 a b 4 5 6 7}
+db close
+
+do_test 2.4 {
+  readfile [lindex [glob sqllog_*.sql] 0]
+} [string trimleft {
+/-- Main database is '.*/sqllog_.*_0.db'
+INSERT INTO t1 VALUES\(6, 7\);; -- clock=0
+SELECT . FROM t1;; -- clock=1
+/}]
+
+catch { db close }
+sqlite3_shutdown
+unset ::env(SQLITE_SQLLOG_DIR)
+unset ::env(SQLITE_SQLLOG_CONDITIONAL)
+sqlite3_config_sqllog
+sqlite3_initialize
+breakpoint
+finish_test
+
+
@@ -59,16 +59,34 @@ do_execsql_test tabfunc01-1.20 {
  CREATE VIEW v1(a,b) AS VALUES(1,2),(3,4);
  SELECT * FROM v1;
 } {1 2 3 4}
-do_catchsql_test tabfunc01-1.21 {
+do_catchsql_test tabfunc01-1.21.1 {
  SELECT * FROM v1(55);
 } {1 {'v1' is not a function}}
+do_catchsql_test tabfunc01-1.21.2 {
+  SELECT * FROM v1();
+} {1 {'v1' is not a function}}
 do_execsql_test tabfunc01-1.22 {
  CREATE VIEW v2(x) AS SELECT value FROM generate_series(1,5);
  SELECT * FROM v2;
 } {1 2 3 4 5}
-do_catchsql_test tabfunc01-1.23 {
+do_catchsql_test tabfunc01-1.23.1 {
  SELECT * FROM v2(55);
 } {1 {'v2' is not a function}}
+do_catchsql_test tabfunc01-1.23.2 {
+  SELECT * FROM v2();
+} {1 {'v2' is not a function}}
+do_execsql_test tabfunc01-1.24 {
+  CREATE TABLE t0(x);
+  INSERT INTO t0(x) VALUES(123),(456),(789);
+  SELECT * FROM t0 ORDER BY x;
+} {123 456 789}
+do_catchsql_test tabfunc01-1.25 {
+  SELECT * FROM t0(55) ORDER BY x;
+} {1 {'t0' is not a function}}
+do_catchsql_test tabfunc01-1.26 {
+  WITH w0 AS (SELECT * FROM t0)
+  INSERT INTO t0(x) SELECT * FROM w0()
+} {1 {'w0' is not a function}}

 do_execsql_test tabfunc01-2.1 {
  CREATE TABLE t1(x);
@@ -746,8 +746,8 @@ proc do_test {name cmd expected} {
        # if {![info exists ::testprefix] || $::testprefix eq ""} {
        #   error "no test prefix"
        # }
-        output2_if_no_verbose -nonewline $name...
-        output2 "\nExpected: \[$expected\]\n     Got: \[$result\]"
+        output1 ""
+        output2 "! $name expected: \[$expected\]\n! $name got:      \[$result\]"
        fail_test $name
      } else {
        output1 " Ok"
@@ -1031,7 +1031,7 @@ proc finalize_testing {} {
    output2 "$nErr errors out of $nTest tests"
  }
  if {$nErr>$nKnown} {
-    output2 -nonewline "Failures on these tests:"
+    output2 -nonewline "!Failures on these tests:"
    foreach x [set_test_counter fail_list] {
      if {![info exists known_error($x)]} {output2 -nonewline " $x"}
    }
@@ -1047,7 +1047,7 @@ proc finalize_testing {} {
    foreach {rec} [lsort $omitList] {
      if {$rec==$prec} continue
      set prec $rec
-      output2 [format {  %-12s %s} [lindex $rec 0] [lindex $rec 1]]
+      output2 [format {.  %-12s %s} [lindex $rec 0] [lindex $rec 1]]
    }
  }
  if {$nErr>0 && ![working_64bit_int]} {
@@ -21,7 +21,6 @@ ifcapable {!trigger} {
  return
 }

-
 # Error messages resulting from qualified trigger names.
 #
 do_test trigger7-1.1 {
@@ -275,9 +275,9 @@ proc lock_callback {method filename handle lock} {
  if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY }
  return SQLITE_OK
 }
-puts "  Warning: This next test case causes SQLite to call xSleep(1) 100 times."
-puts "  Normally this equates to a 100ms delay, but if SQLite is built on unix"
-puts "  without HAVE_USLEEP defined, it may be 100 seconds."
+puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times."
+puts "# Normally this equates to a 100ms delay, but if SQLite is built on unix"
+puts "# without HAVE_USLEEP defined, it may be 100 seconds."
 do_test wal3-4.3 {
  db close
  set ::locks [list]
@@ -285,7 +285,7 @@ do_test wal3-4.3 {
  catchsql { SELECT * FROM x }
 } {1 {locking protocol}}

-puts "  Warning: Same again!"
+puts "# Warning: Same again!"
 proc lock_callback {method filename handle lock} {
  if {$lock == "0 1 lock exclusive"} { return SQLITE_BUSY }
  return SQLITE_OK
@@ -412,22 +412,22 @@ ifcapable subquery {
    count {
      SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1;
    }
-  } {1 0 4 2 1 9 3 1 16 13}
+  } {1 0 4 2 1 9 3 1 16 12}
  do_test where-5.3b {
    count {
      SELECT * FROM t1 WHERE w IN (3,-1,1,2) order by 1;
    }
-  } {1 0 4 2 1 9 3 1 16 13}
+  } {1 0 4 2 1 9 3 1 16 12}
  do_test where-5.3c {
    count {
      SELECT * FROM t1 WHERE w IN (3,2,-1,1,2) order by 1;
    }
-  } {1 0 4 2 1 9 3 1 16 13}
+  } {1 0 4 2 1 9 3 1 16 12}
  do_test where-5.3d {
    count {
      SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1 DESC;
    }
-  } {3 1 16 2 1 9 1 0 4 12}
+  } {3 1 16 2 1 9 1 0 4 11}
  do_test where-5.4 {
    count {
      SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1;
@@ -465,7 +465,7 @@ ifcapable subquery {
    count {
      SELECT * FROM t1 WHERE x IN (1,7) ORDER BY 1;
    }
-  } {2 1 9 3 1 16 7}
+  } {2 1 9 3 1 16 6}
  do_test where-5.10 {
    count {
      SELECT * FROM t1 WHERE x+0 IN (1,7) ORDER BY 1;
@@ -485,17 +485,17 @@ ifcapable subquery {
    count {
      SELECT * FROM t1 WHERE x IN (1,7) AND y NOT IN (6400,8100) ORDER BY 1;
    }
-  } {2 1 9 3 1 16 7}
+  } {2 1 9 3 1 16 6}
  do_test where-5.14 {
    count {
      SELECT * FROM t1 WHERE x IN (1,7) AND y IN (9,10) ORDER BY 1;
    }
-  } {2 1 9 8}
+  } {2 1 9 5}
  do_test where-5.15 {
    count {
      SELECT * FROM t1 WHERE x IN (1,7) AND y IN (9,16) ORDER BY 1;
    }
-  } {2 1 9 3 1 16 11}
+  } {2 1 9 3 1 16 9}
  do_test where-5.100 {
    db eval {
      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
@@ -91,7 +91,7 @@ do_test where4-1.10 {
 } {6 2}
 do_test where4-1.11 {
  count {SELECT rowid FROM t1 WHERE w=x'78' AND x IS NULL AND y=123}
-} {1}
+} {0}
 do_test where4-1.12 {
  count {SELECT rowid FROM t1 WHERE w=x'78' AND x IS NULL AND y=x'7A'}
 } {6 2}
@@ -865,4 +865,114 @@ do_catchsql_test 16.1 {
  SELECT * FROM i;
 } {1 {recursive aggregate queries not supported}}

+#-------------------------------------------------------------------------
+do_execsql_test 17.1 {
+  WITH x(a) AS (
+    WITH y(b) AS (SELECT 10)
+    SELECT 9 UNION ALL SELECT * FROM y
+  )
+  SELECT * FROM x
+} {9 10}
+
+do_execsql_test 17.2 {
+  WITH x AS (
+    WITH y(b) AS (SELECT 10)
+    SELECT * FROM y UNION ALL SELECT * FROM y
+  )
+  SELECT * FROM x
+} {10 10}
+
+do_test 17.2 {
+  db eval {
+    WITH x AS (
+        WITH y(b) AS (SELECT 10)
+        SELECT * FROM y UNION ALL SELECT * FROM y
+    )
+    SELECT * FROM x
+  } A {
+    # no op
+  }
+  set A(*)
+} {b}
+
+do_catchsql_test 17.3 {
+  WITH i AS (
+    WITH j AS (SELECT 5)
+    SELECT 5 FROM i UNION SELECT 8 FROM i
+  )
+  SELECT * FROM i;
+} {1 {circular reference: i}}
+
+do_catchsql_test 17.4 {
+  WITH i AS (
+    WITH j AS (SELECT 5)
+    SELECT 5 FROM t1 UNION SELECT 8 FROM t11
+  )
+  SELECT * FROM i;
+} {1 {no such table: t11}}
+
+do_execsql_test 17.5 {
+  WITH 
+  x1 AS (SELECT 10),
+  x2 AS (SELECT * FROM x1),
+  x3 AS (
+    WITH x1 AS (SELECT 11)
+    SELECT * FROM x2 UNION ALL SELECT * FROM x2
+  )
+  SELECT * FROM x3;
+} {10 10}
+
+do_execsql_test 17.6 {
+  WITH 
+  x1 AS (SELECT 10),
+  x2 AS (SELECT * FROM x1),
+  x3 AS (
+    WITH x1 AS (SELECT 11)
+    SELECT * FROM x2 UNION ALL SELECT * FROM x1
+  )
+  SELECT * FROM x3;
+} {10 11}
+
+do_execsql_test 17.7 {
+  WITH 
+  x1 AS (SELECT 10),
+  x2 AS (SELECT * FROM x1),
+  x3 AS (
+    WITH 
+      x1 AS ( SELECT 11 ),
+      x4 AS ( SELECT * FROM x2 )
+    SELECT * FROM x4 UNION ALL SELECT * FROM x1
+  )
+  SELECT * FROM x3;
+} {10 11}
+
+do_execsql_test 17.8 {
+  WITH 
+  x1 AS (SELECT 10),
+  x2 AS (SELECT * FROM x1),
+  x3 AS (
+    WITH 
+      x1 AS ( SELECT 11 ),
+      x4 AS ( SELECT * FROM x2 )
+    SELECT * FROM x4 UNION ALL SELECT * FROM x1
+  )
+  SELECT * FROM x3;
+} {10 11}
+
+do_execsql_test 17.9 {
+  WITH 
+  x1 AS (SELECT 10),
+  x2 AS (SELECT 11),
+  x3 AS (
+    SELECT * FROM x1 UNION ALL SELECT * FROM x2
+  ),
+  x4 AS (
+    WITH 
+    x1 AS (SELECT 12),
+    x2 AS (SELECT 13)
+    SELECT * FROM x3
+  )
+  SELECT * FROM x4;
+} {10 11}
+
 finish_test
@@ -0,0 +1,64 @@
+# 2015-11-07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix with3
+
+ifcapable {!cte} {
+  finish_test
+  return
+}
+
+# Test problems found by Kostya Serebryany using 
+# LibFuzzer.  (http://llvm.org/docs/LibFuzzer.html)
+#
+do_catchsql_test 1.0 {
+  WITH i(x) AS (
+    WITH j AS (SELECT 10)
+    SELECT 5 FROM t0 UNION SELECT 8 FROM m
+  )
+  SELECT * FROM i;
+} {1 {no such table: m}}
+
+# Additional test cases that came out of the work to
+# fix for Kostya's problem.
+#
+do_execsql_test 2.0 {
+ WITH
+  x1 AS (SELECT 10),
+  x2 AS (SELECT 11),
+  x3 AS (
+    SELECT * FROM x1 UNION ALL SELECT * FROM x2
+  ),
+  x4 AS (
+    WITH
+    x1 AS (SELECT 12),
+    x2 AS (SELECT 13)
+    SELECT * FROM x3
+  )
+  SELECT * FROM x4;
+
+} {10 11}
+
+do_execsql_test 2.1 {
+  CREATE TABLE t1(x);
+  WITH
+    x1(a) AS (values(100))
+  INSERT INTO t1(x)
+    SELECT * FROM (WITH x2(y) AS (SELECT * FROM x1) SELECT y+a FROM x1, x2);
+  SELECT * FROM t1;
+} {200}
+
+finish_test
@@ -18,7 +18,8 @@ while {![eof $in]} {
 }
 close $in

-# The following are the extra token codes to be added
+# The following are the extra token codes to be added.  SPACE and 
+# ILLEGAL *must* be the last two token codes and they must be in that order.
 #
 set extras {
  TO_TEXT
@@ -28,8 +29,6 @@ set extras {
  TO_REAL
  ISNOT
  END_OF_FILE
-  ILLEGAL
-  SPACE
  UNCLOSED_STRING
  FUNCTION
  COLUMN
@@ -38,6 +37,13 @@ set extras {
  UMINUS
  UPLUS
  REGISTER
+  ASTERISK
+  SPACE
+  ILLEGAL
+}
+if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
+  error "SPACE and ILLEGAL must be the last two token codes and they\
+         must be in that order"
 }
 foreach x $extras {
  incr max
@@ -0,0 +1,24 @@
+#!/usr/bin/tclsh
+#
+# A wrapper around cg_annotate that sets appropriate command-line options
+# and rearranges the output so that annotated files occur in a consistent
+# sorted order.  Used by the run-speed-test.tcl script.
+#
+
+set in [open "|cg_annotate --show=Ir --auto=yes --context=40 $argv" r]
+set dest !
+set out(!) {}
+while {![eof $in]} {
+  set line [string map {\t {        }} [gets $in]]
+  if {[regexp {^-- Auto-annotated source: (.*)} $line all name]} {
+    set dest $name
+  } elseif {[regexp {^-- line \d+ ------} $line]} {
+    set line [lreplace $line 2 2 {#}]
+  } elseif {[regexp {^The following files chosen for } $line]} {
+    set dest !
+  }
+  append out($dest) $line\n
+}
+foreach x [lsort [array names out]] {
+  puts $out($x)
+}
@@ -1,49 +1,67 @@
-/* Driver template for the LEMON parser generator.
-** The author disclaims copyright to this source code.
-*/
-/* First off, code is included that follows the "include" declaration
-** in the input grammar file. */
-#include <stdio.h>
-%%
-/* Next is all token values, in a form suitable for use by makeheaders.
-** This section will be null unless lemon is run with the -m switch.
-*/
-/* 
-** These constants (all generated automatically by the parser generator)
-** specify the various kinds of tokens (terminals) that the parser
-** understands. 
+/*
+** 2000-05-29
 **
-** Each symbol here is a terminal symbol in the grammar.
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Driver template for the LEMON parser generator.
+**
+** The "lemon" program processes an LALR(1) input grammar file, then uses
+** this template to construct a parser.  The "lemon" program inserts text
+** at each "%%" line.  Also, any "P-a-r-s-e" identifer prefix (without the
+** interstitial "-" characters) contained in this template is changed into
+** the value of the %name directive from the grammar.  Otherwise, the content
+** of this template is copied straight through into the generate parser
+** source file.
+**
+** The following is the concatenation of all %include directives from the
+** input grammar file:
 */
+#include <stdio.h>
+/************ Begin %include sections from the grammar ************************/
 %%
-/* Make sure the INTERFACE macro is defined.
-*/
-#ifndef INTERFACE
-# define INTERFACE 1
-#endif
-/* The next thing included is series of defines which control
+/**************** End of %include directives **********************************/
+/* These constants specify the various numeric values for terminal symbols
+** in a format understandable to "makeheaders".  This section is blank unless
+** "lemon" is run with the "-m" command-line option.
+***************** Begin makeheaders token definitions *************************/
+%%
+/**************** End makeheaders token definitions ***************************/
+
+/* The next sections is a series of control #defines.
 ** various aspects of the generated parser.
-**    YYCODETYPE         is the data type used for storing terminal
-**                       and nonterminal numbers.  "unsigned char" is
-**                       used if there are fewer than 250 terminals
-**                       and nonterminals.  "int" is used otherwise.
-**    YYNOCODE           is a number of type YYCODETYPE which corresponds
-**                       to no legal terminal or nonterminal number.  This
-**                       number is used to fill in empty slots of the hash 
-**                       table.
+**    YYCODETYPE         is the data type used to store the integer codes
+**                       that represent terminal and non-terminal symbols.
+**                       "unsigned char" is used if there are fewer than
+**                       256 symbols.  Larger types otherwise.
+**    YYNOCODE           is a number of type YYCODETYPE that is not used for
+**                       any terminal or nonterminal symbol.
 **    YYFALLBACK         If defined, this indicates that one or more tokens
-**                       have fall-back values which should be used if the
-**                       original value of the token will not parse.
-**    YYACTIONTYPE       is the data type used for storing terminal
-**                       and nonterminal numbers.  "unsigned char" is
-**                       used if there are fewer than 250 rules and
-**                       states combined.  "int" is used otherwise.
-**    ParseTOKENTYPE     is the data type used for minor tokens given 
-**                       directly to the parser from the tokenizer.
-**    YYMINORTYPE        is the data type used for all minor tokens.
+**                       (also known as: "terminal symbols") have fall-back
+**                       values which should be used if the original symbol
+**                       would not parse.  This permits keywords to sometimes
+**                       be used as identifiers, for example.
+**    YYACTIONTYPE       is the data type used for "action codes" - numbers
+**                       that indicate what to do in response to the next
+**                       token.
+**    ParseTOKENTYPE     is the data type used for minor type for terminal
+**                       symbols.  Background: A "minor type" is a semantic
+**                       value associated with a terminal or non-terminal
+**                       symbols.  For example, for an "ID" terminal symbol,
+**                       the minor type might be the name of the identifier.
+**                       Each non-terminal can have a different minor type.
+**                       Terminal symbols all have the same minor type, though.
+**                       This macros defines the minor type for terminal 
+**                       symbols.
+**    YYMINORTYPE        is the data type used for all minor types.
 **                       This is typically a union of many types, one of
 **                       which is ParseTOKENTYPE.  The entry in the union
-**                       for base tokens is called "yy0".
+**                       for terminal symbols is called "yy0".
 **    YYSTACKDEPTH       is the maximum depth of the parser's stack.  If
 **                       zero the stack is dynamically sized using realloc()
 **    ParseARG_SDECL     A static variable declaration for the %extra_argument
@@ -62,7 +80,12 @@
 **    YY_ACCEPT_ACTION   The yy_action[] code for accept
 **    YY_NO_ACTION       The yy_action[] code for no-op
 */
+#ifndef INTERFACE
+# define INTERFACE 1
+#endif
+/************* Begin control #defines *****************************************/
 %%
+/************* End control #defines *******************************************/

 /* The yyzerominor constant is used to initialize instances of
 ** YYMINORTYPE objects to zero. */
@@ -131,11 +154,13 @@ static const YYMINORTYPE yyzerominor = { 0 };
 **  yy_reduce_ofst[]   For each state, the offset into yy_action for
 **                     shifting non-terminals after a reduce.
 **  yy_default[]       Default action for each state.
-*/
+**
+*********** Begin parsing tables **********************************************/
 %%
+/********** End of lemon-generated parsing tables *****************************/

-/* The next table maps tokens into fallback tokens.  If a construct
-** like the following:
+/* The next table maps tokens (terminal symbols) into fallback tokens.  
+** If a construct like the following:
 ** 
 **      %fallback ID X Y Z.
 **
@@ -143,6 +168,10 @@ static const YYMINORTYPE yyzerominor = { 0 };
 ** and Z.  Whenever one of the tokens X, Y, or Z is input to the parser
 ** but it does not parse, the type of the token is changed to ID and
 ** the parse is retried before an error is thrown.
+**
+** This feature can be used, for example, to cause some keywords in a language
+** to revert to identifiers if they keyword does not apply in the context where
+** it appears.
 */
 #ifdef YYFALLBACK
 static const YYCODETYPE yyFallback[] = {
@@ -265,6 +294,15 @@ static void yyGrowStack(yyParser *p){
 }
 #endif

+/* Datatype of the argument to the memory allocated passed as the
+** second argument to ParseAlloc() below.  This can be changed by
+** putting an appropriate #define in the %include section of the input
+** grammar.
+*/
+#ifndef YYMALLOCARGTYPE
+# define YYMALLOCARGTYPE size_t
+#endif
+
 /* 
 ** This function allocates a new parser.
 ** The only argument is a pointer to a function which works like
@@ -277,9 +315,9 @@ static void yyGrowStack(yyParser *p){
 ** A pointer to a parser.  This pointer is used in subsequent calls
 ** to Parse and ParseFree.
 */
-void *ParseAlloc(void *(*mallocProc)(size_t)){
+void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
  yyParser *pParser;
-  pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) );
+  pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
  if( pParser ){
    pParser->yyidx = -1;
 #ifdef YYTRACKMAXSTACKDEPTH
@@ -294,10 +332,12 @@ void *ParseAlloc(void *(*mallocProc)(size_t)){
  return pParser;
 }

-/* The following function deletes the value associated with a
-** symbol.  The symbol can be either a terminal or nonterminal.
-** "yymajor" is the symbol code, and "yypminor" is a pointer to
-** the value.
+/* The following function deletes the "minor type" or semantic value
+** associated with a symbol.  The symbol can be either a terminal
+** or nonterminal. "yymajor" is the symbol code, and "yypminor" is
+** a pointer to the value to be deleted.  The code used to do the 
+** deletions is derived from the %destructor and/or %token_destructor
+** directives of the input grammar.
 */
 static void yy_destructor(
  yyParser *yypParser,    /* The parser */
@@ -313,10 +353,12 @@ static void yy_destructor(
    ** being destroyed before it is finished parsing.
    **
    ** Note: during a reduce, the only symbols destroyed are those
-    ** which appear on the RHS of the rule, but which are not used
+    ** which appear on the RHS of the rule, but which are *not* used
    ** inside the C code.
    */
+/********* Begin destructor definitions ***************************************/
 %%
+/********* End destructor definitions *****************************************/
    default:  break;   /* If no destructor action specified: do nothing */
  }
 }
@@ -326,45 +368,37 @@ static void yy_destructor(
 **
 ** If there is a destructor routine associated with the token which
 ** is popped from the stack, then call it.
-**
-** Return the major token number for the symbol popped.
 */
-static int yy_pop_parser_stack(yyParser *pParser){
-  YYCODETYPE yymajor;
-  yyStackEntry *yytos = &pParser->yystack[pParser->yyidx];
-
-  if( pParser->yyidx<0 ) return 0;
+static void yy_pop_parser_stack(yyParser *pParser){
+  yyStackEntry *yytos;
+  assert( pParser->yyidx>=0 );
+  yytos = &pParser->yystack[pParser->yyidx--];
 #ifndef NDEBUG
-  if( yyTraceFILE && pParser->yyidx>=0 ){
+  if( yyTraceFILE ){
    fprintf(yyTraceFILE,"%sPopping %s\n",
      yyTracePrompt,
      yyTokenName[yytos->major]);
  }
 #endif
-  yymajor = yytos->major;
-  yy_destructor(pParser, yymajor, &yytos->minor);
-  pParser->yyidx--;
-  return yymajor;
+  yy_destructor(pParser, yytos->major, &yytos->minor);
 }

 /* 
-** Deallocate and destroy a parser.  Destructors are all called for
+** Deallocate and destroy a parser.  Destructors are called for
 ** all stack elements before shutting the parser down.
 **
-** Inputs:
-** <ul>
-** <li>  A pointer to the parser.  This should be a pointer
-**       obtained from ParseAlloc.
-** <li>  A pointer to a function used to reclaim memory obtained
-**       from malloc.
-** </ul>
+** If the YYPARSEFREENEVERNULL macro exists (for example because it
+** is defined in a %include section of the input grammar) then it is
+** assumed that the input pointer is never NULL.
 */
 void ParseFree(
  void *p,                    /* The parser to be deleted */
  void (*freeProc)(void*)     /* Function used to reclaim memory */
 ){
  yyParser *pParser = (yyParser*)p;
+#ifndef YYPARSEFREENEVERNULL
  if( pParser==0 ) return;
+#endif
  while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
 #if YYSTACKDEPTH<=0
  free(pParser->yystack);
@@ -385,10 +419,6 @@ int ParseStackPeak(void *p){
 /*
 ** Find the appropriate action for a parser given the terminal
 ** look-ahead token iLookAhead.
-**
-** If the look-ahead token is YYNOCODE, then check to see if the action is
-** independent of the look-ahead.  If it is, return the action, otherwise
-** return YY_NO_ACTION.
 */
 static int yy_find_shift_action(
  yyParser *pParser,        /* The parser */
@@ -396,64 +426,65 @@ static int yy_find_shift_action(
 ){
  int i;
  int stateno = pParser->yystack[pParser->yyidx].stateno;
-
-  if( stateno>=YY_MIN_REDUCE ) return stateno; 
+ 
+  if( stateno>=YY_MIN_REDUCE ) return stateno;
  assert( stateno <= YY_SHIFT_COUNT );
-  i = yy_shift_ofst[stateno];
-  if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
-  assert( iLookAhead!=YYNOCODE );
-  i += iLookAhead;
-  if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
-    if( iLookAhead>0 ){
+  do{
+    i = yy_shift_ofst[stateno];
+    if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
+    assert( iLookAhead!=YYNOCODE );
+    i += iLookAhead;
+    if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
+      if( iLookAhead>0 ){
 #ifdef YYFALLBACK
-      YYCODETYPE iFallback;            /* Fallback token */
-      if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
-             && (iFallback = yyFallback[iLookAhead])!=0 ){
-#ifndef NDEBUG
-        if( yyTraceFILE ){
-          fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
-             yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
-        }
-#endif
-        return yy_find_shift_action(pParser, iFallback);
-      }
-#endif
-#ifdef YYWILDCARD
-      {
-        int j = i - iLookAhead + YYWILDCARD;
-        if( 
-#if YY_SHIFT_MIN+YYWILDCARD<0
-          j>=0 &&
-#endif
-#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
-          j<YY_ACTTAB_COUNT &&
-#endif
-          yy_lookahead[j]==YYWILDCARD
-        ){
+        YYCODETYPE iFallback;            /* Fallback token */
+        if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
+               && (iFallback = yyFallback[iLookAhead])!=0 ){
 #ifndef NDEBUG
          if( yyTraceFILE ){
-            fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
-               yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]);
+            fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
+               yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
+          }
+#endif
+          assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
+          iLookAhead = iFallback;
+          continue;
+        }
+#endif
+#ifdef YYWILDCARD
+        {
+          int j = i - iLookAhead + YYWILDCARD;
+          if( 
+#if YY_SHIFT_MIN+YYWILDCARD<0
+            j>=0 &&
+#endif
+#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
+            j<YY_ACTTAB_COUNT &&
+#endif
+            yy_lookahead[j]==YYWILDCARD
+          ){
+#ifndef NDEBUG
+            if( yyTraceFILE ){
+              fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
+                 yyTracePrompt, yyTokenName[iLookAhead],
+                 yyTokenName[YYWILDCARD]);
+            }
+#endif /* NDEBUG */
+            return yy_action[j];
          }
-#endif /* NDEBUG */
-          return yy_action[j];
        }
-      }
 #endif /* YYWILDCARD */
+      }
+      return yy_default[stateno];
+    }else{
+      return yy_action[i];
    }
-    return yy_default[stateno];
-  }else{
-    return yy_action[i];
-  }
+  }while(1);
 }

 /*
 ** Find the appropriate action for a parser given the non-terminal
 ** look-ahead token iLookAhead.
-**
-** If the look-ahead token is YYNOCODE, then check to see if the action is
-** independent of the look-ahead.  If it is, return the action, otherwise
-** return YY_NO_ACTION.
 */
 static int yy_find_reduce_action(
  int stateno,              /* Current state number */
@@ -496,7 +527,9 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
   while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
   /* Here code is inserted which will execute if the parser
   ** stack every overflows */
+/******** Begin %stack_overflow code ******************************************/
 %%
+/******** End %stack_overflow code ********************************************/
   ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
 }

@@ -506,15 +539,13 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
 #ifndef NDEBUG
 static void yyTraceShift(yyParser *yypParser, int yyNewState){
  if( yyTraceFILE ){
-    int i;
    if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
-      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
-      for(i=1; i<=yypParser->yyidx; i++)
-        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
-      fprintf(yyTraceFILE,"\n");
+      fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n",
+         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major],
+         yyNewState);
    }else{
-      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
+      fprintf(yyTraceFILE,"%sShift '%s'\n",
+         yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]);
    }
  }
 }
@@ -523,7 +554,7 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){
 #endif

 /*
-** Perform a shift action.  Return the number of errors.
+** Perform a shift action.
 */
 static void yy_shift(
  yyParser *yypParser,          /* The parser to be shifted */
@@ -590,29 +621,12 @@ static void yy_reduce(
  if( yyTraceFILE && yyruleno>=0 
        && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
    yysize = yyRuleInfo[yyruleno].nrhs;
-    fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt,
+    fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt,
      yyRuleName[yyruleno], yymsp[-yysize].stateno);
  }
 #endif /* NDEBUG */
-
-  /* Silence complaints from purify about yygotominor being uninitialized
-  ** in some cases when it is copied into the stack after the following
-  ** switch.  yygotominor is uninitialized when a rule reduces that does
-  ** not set the value of its left-hand side nonterminal.  Leaving the
-  ** value of the nonterminal uninitialized is utterly harmless as long
-  ** as the value is never used.  So really the only thing this code
-  ** accomplishes is to quieten purify.  
-  **
-  ** 2007-01-16:  The wireshark project (www.wireshark.org) reports that
-  ** without this code, their parser segfaults.  I'm not sure what there
-  ** parser is doing to make this happen.  This is the second bug report
-  ** from wireshark this week.  Clearly they are stressing Lemon in ways
-  ** that it has not been previously stressed...  (SQLite ticket #2172)
-  */
-  /*memset(&yygotominor, 0, sizeof(yygotominor));*/
  yygotominor = yyzerominor;

-
  switch( yyruleno ){
  /* Beginning here are the reduction cases.  A typical example
  ** follows:
@@ -622,8 +636,11 @@ static void yy_reduce(
  **  #line <lineno> <thisfile>
  **     break;
  */
+/********** Begin reduce actions **********************************************/
 %%
+/********** End reduce actions ************************************************/
  };
+  assert( yyruleno>=0 && yyruleno<sizeof(yyRuleInfo)/sizeof(yyRuleInfo[0]) );
  yygoto = yyRuleInfo[yyruleno].lhs;
  yysize = yyRuleInfo[yyruleno].nrhs;
  yypParser->yyidx -= yysize;
@@ -666,7 +683,9 @@ static void yy_parse_failed(
  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
  /* Here code is inserted which will be executed whenever the
  ** parser fails */
+/************ Begin %parse_failure code ***************************************/
 %%
+/************ End %parse_failure code *****************************************/
  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
 }
 #endif /* YYNOERRORRECOVERY */
@@ -681,7 +700,9 @@ static void yy_syntax_error(
 ){
  ParseARG_FETCH;
 #define TOKEN (yyminor.yy0)
+/************ Begin %syntax_error code ****************************************/
 %%
+/************ End %syntax_error code ******************************************/
  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
 }

@@ -700,7 +721,9 @@ static void yy_accept(
  while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
  /* Here code is inserted which will be executed whenever the
  ** parser accepts */
+/*********** Begin %parse_accept code *****************************************/
 %%
+/*********** End %parse_accept code *******************************************/
  ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
 }

@@ -731,7 +754,9 @@ void Parse(
 ){
  YYMINORTYPE yyminorunion;
  int yyact;            /* The parser action. */
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
  int yyendofinput;     /* True if we are at the end of input */
+#endif
 #ifdef YYERRORSYMBOL
  int yyerrorhit = 0;   /* True if yymajor has invoked an error */
 #endif
@@ -752,14 +777,22 @@ void Parse(
    yypParser->yyerrcnt = -1;
    yypParser->yystack[0].stateno = 0;
    yypParser->yystack[0].major = 0;
+#ifndef NDEBUG
+    if( yyTraceFILE ){
+      fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
+              yyTracePrompt);
+    }
+#endif
  }
  yyminorunion.yy0 = yyminor;
+#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
  yyendofinput = (yymajor==0);
+#endif
  ParseARG_STORE;

 #ifndef NDEBUG
  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]);
+    fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]);
  }
 #endif

@@ -873,7 +906,12 @@ void Parse(
  }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
 #ifndef NDEBUG
  if( yyTraceFILE ){
-    fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt);
+    int i;
+    fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
+    for(i=1; i<=yypParser->yyidx; i++)
+      fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', 
+              yyTokenName[yypParser->yystack[i].major]);
+    fprintf(yyTraceFILE,"]\n");
  }
 #endif
  return;
@@ -277,27 +277,8 @@ static Keyword aKeywordTable[] = {
 /* Number of keywords */
 static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0]));

-/* An array to map all upper-case characters into their corresponding
-** lower-case character. 
-*/
-const unsigned char sqlite3UpperToLower[] = {
-      0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
-     18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
-     36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
-     54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
-    104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
-    122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
-    108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
-    126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
-    144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
-    162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
-    180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
-    198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
-    216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
-    234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
-    252,253,254,255
-};
-#define UpperToLower sqlite3UpperToLower
+/* Map all alphabetic characters into the same case */
+#define charMap(X)   (0x20|(X))

 /*
 ** Comparision function for two Keyword records
@@ -372,8 +353,8 @@ int main(int argc, char **argv){
    assert( p->len<sizeof(p->zOrigName) );
    memcpy(p->zOrigName, p->zName, p->len+1);
    totalLen += p->len;
-    p->hash = (UpperToLower[(int)p->zName[0]]*4) ^
-              (UpperToLower[(int)p->zName[p->len-1]]*3) ^ p->len;
+    p->hash = (charMap(p->zName[0])*4) ^
+              (charMap(p->zName[p->len-1])*3) ^ (p->len*1);
    p->id = i+1;
  }

@@ -481,7 +462,7 @@ int main(int argc, char **argv){
  /* Begin generating code */
  printf("%s", zHdr);
  printf("/* Hash score: %d */\n", bestCount);
-  printf("static int keywordCode(const char *z, int n){\n");
+  printf("static int keywordCode(const char *z, int n, int *pType){\n");
  printf("  /* zText[] encodes %d bytes of keywords in %d bytes */\n",
          totalLen + nKeyword, nChar+1 );
  for(i=j=k=0; i<nKeyword; i++){
@@ -585,24 +566,27 @@ int main(int argc, char **argv){
  printf("%s  };\n", j==0 ? "" : "\n");

  printf("  int h, i;\n");
-  printf("  if( n<2 ) return TK_ID;\n");
-  printf("  h = ((charMap(z[0])*4) ^\n"
-         "      (charMap(z[n-1])*3) ^\n"
-         "      n) %% %d;\n", bestSize);
-  printf("  for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n");
-  printf("    if( aLen[i]==n &&"
-                   " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n");
+  printf("  if( n>=2 ){\n");
+  printf("    h = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n",
+          bestSize);
+  printf("    for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n");
+  printf("      if( aLen[i]==n &&"
+                     " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n");
  for(i=0; i<nKeyword; i++){
-    printf("      testcase( i==%d ); /* %s */\n",
+    printf("        testcase( i==%d ); /* %s */\n",
           i, aKeywordTable[i].zOrigName);
  }
-  printf("      return aCode[i];\n");
+  printf("        *pType = aCode[i];\n");
+  printf("        break;\n");
+  printf("      }\n");
  printf("    }\n");
  printf("  }\n");
-  printf("  return TK_ID;\n");
+  printf("  return n;\n");
  printf("}\n");
  printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n");
-  printf("  return keywordCode((char*)z, n);\n");
+  printf("  int id = TK_ID;\n");
+  printf("  keywordCode((char*)z, n, &id);\n");
+  printf("  return id;\n");
  printf("}\n");
  printf("#define SQLITE_N_KEYWORD %d\n", nKeyword);

@@ -47,8 +47,6 @@ set pragma_def {
  IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)

  NAME: cache_spill
-  TYPE: FLAG
-  ARG:  SQLITE_CacheSpill
  IF:   !defined(SQLITE_OMIT_FLAG_PRAGMAS)

  NAME: reverse_unordered_selects
@@ -170,6 +168,7 @@ set pragma_def {
  IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)

  NAME: cache_size
+  FLAG: NeedSchema
  IF:   !defined(SQLITE_OMIT_PAGER_PRAGMAS)

  NAME: mmap_size
@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# This is a template for a script used for day-to-day size and 
+# performance monitoring of SQLite.  Typical usage:
+#
+#     sh run-speed-test.sh trunk  #  Baseline measurement of trunk
+#     sh run-speed-test.sh x1     # Measure some experimental change
+#     fossil test-diff --tk cout-trunk.txt cout-x1.txt   # View chanages
+#
+# There are multiple output files, all with a base name given by
+# the first argument:
+#
+#     summary-$BASE.txt           # Copy of standard output
+#     cout-$BASE.txt              # cachegrind output
+#     explain-$BASE.txt           # EXPLAIN listings
+#
+if test "$1" = ""
+then
+  echo "Usage: $0 OUTPUTFILE [OPTIONS]"
+  exit
+fi
+NAME=$1
+shift
+CC_OPTS="-DSQLITE_ENABLE_RTREE"
+SPEEDTEST_OPTS="--shrink-memory --reprepare"
+SIZE=5
+while test "$1" != ""; do
+  case $1 in
+    --reprepare)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --autovacuum)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --utf16be)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --without-rowid)
+        SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
+        ;;
+    --size)
+        shift; SIZE=$1
+        ;;
+    *)
+        CC_OPTS="$CC_OPTS $1"
+        ;;
+  esac
+  shift
+done
+SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
+echo "NAME           = $NAME" | tee summary-$NAME.txt
+echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
+echo "CC_OPTS        = $CC_OPTS" | tee -a summary-$NAME.txt
+rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
+gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
+size sqlite3.o | tee -a summary-$NAME.txt
+gcc -g -Os -Wall -I. $CC_OPTS \
+   -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
+   ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
+SRC=./speedtest1.c
+gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
+ls -l speedtest1 | tee -a summary-$NAME.txt
+valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
+    $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
+size sqlite3.o | tee -a summary-$NAME.txt
+wc sqlite3.c
+cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
+./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
@@ -558,7 +558,7 @@ static void diff_one_table(const char *zTab, FILE *out){
  az = columnNames("main", zTab, &nPk, 0);
  az2 = columnNames("aux", zTab, &nPk2, 0);
  if( az && az2 ){
-    for(n=0; az[n]; n++){
+    for(n=0; az[n] && az2[n]; n++){
      if( sqlite3_stricmp(az[n],az2[n])!=0 ) break;
    }
  }
@@ -568,13 +568,15 @@ static void diff_one_table(const char *zTab, FILE *out){
   || az[n]
  ){
    /* Schema mismatch */
-    fprintf(out, "DROP TABLE %s;\n", zId);
+    fprintf(out, "DROP TABLE %s; -- due to schema mismatch\n", zId);
    dump_table(zTab, out);
    goto end_diff_one_table;
  }

  /* Build the comparison query */
-  for(n2=n; az[n2]; n2++){}
+  for(n2=n; az2[n2]; n2++){
+    fprintf(out, "ALTER TABLE %s ADD COLUMN %s;\n", zId, safeId(az2[n2]));
+  }
  nQ = nPk2+1+2*(n2-nPk2);
  if( n2>nPk2 ){
    zSep = "SELECT ";
@@ -585,7 +587,12 @@ static void diff_one_table(const char *zTab, FILE *out){
    strPrintf(&sql, ", 1%s -- changed row\n", nPk==n ? "" : ",");
    while( az[i] ){
      strPrintf(&sql, "       A.%s IS NOT B.%s, B.%s%s\n",
-                az[i], az[i], az[i], i==n2-1 ? "" : ",");
+                az[i], az2[i], az2[i], az2[i+1]==0 ? "" : ",");
+      i++;
+    }
+    while( az2[i] ){
+      strPrintf(&sql, "       B.%s IS NOT NULL, B.%s%s\n",
+                az2[i], az2[i], az2[i+1]==0 ? "" : ",");
      i++;
    }
    strPrintf(&sql, "  FROM main.%s A, aux.%s B\n", zId, zId);
@@ -597,7 +604,13 @@ static void diff_one_table(const char *zTab, FILE *out){
    zSep = "\n   AND (";
    while( az[i] ){
      strPrintf(&sql, "%sA.%s IS NOT B.%s%s\n",
-                zSep, az[i], az[i], i==n2-1 ? ")" : "");
+                zSep, az[i], az2[i], az2[i+1]==0 ? ")" : "");
+      zSep = "        OR ";
+      i++;
+    }
+    while( az2[i] ){
+      strPrintf(&sql, "%sB.%s IS NOT NULL%s\n",
+                zSep, az2[i], az2[i+1]==0 ? ")" : "");
      zSep = "        OR ";
      i++;
    }
@@ -609,7 +622,7 @@ static void diff_one_table(const char *zTab, FILE *out){
    zSep = ", ";
  }
  strPrintf(&sql, ", 2%s -- deleted row\n", nPk==n ? "" : ",");
-  while( az[i] ){
+  while( az2[i] ){
    strPrintf(&sql, "       NULL, NULL%s\n", i==n2-1 ? "" : ",");
    i++;
  }
@@ -628,7 +641,7 @@ static void diff_one_table(const char *zTab, FILE *out){
  }
  strPrintf(&sql, ", 3%s -- inserted row\n", nPk==n ? "" : ",");
  while( az2[i] ){
-    strPrintf(&sql, "       1, B.%s%s\n", az[i], i==n2-1 ? "" : ",");
+    strPrintf(&sql, "       1, B.%s%s\n", az2[i], az2[i+1]==0 ? "" : ",");
    i++;
  }
  strPrintf(&sql, "  FROM aux.%s B\n", zId);
@@ -689,7 +702,7 @@ static void diff_one_table(const char *zTab, FILE *out){
        for(i=0; i<nPk; i++){
          fprintf(out, "%s %s=", zSep, az2[i]);
          printQuoted(out, sqlite3_column_value(pStmt,i));
-          zSep = ",";
+          zSep = " AND";
        }
        fprintf(out, ";\n");
      }else{                  /* Insert a row */
@@ -1728,6 +1741,7 @@ static void showHelp(void){
 "  --schema              Show only differences in the schema\n"
 "  --summary             Show only a summary of the differences\n"
 "  --table TAB           Show only differences in table TAB\n"
+"  --transaction         Show SQL output inside a transaction\n"
  );
 }

@@ -1744,6 +1758,8 @@ int main(int argc, char **argv){
  void (*xDiff)(const char*,FILE*) = diff_one_table;
  int nExt = 0;
  char **azExt = 0;
+  int useTransaction = 0;
+  int neverUseTransaction = 0;

  g.zArgv0 = argv[0];
  sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
@@ -1757,6 +1773,7 @@ int main(int argc, char **argv){
        out = fopen(argv[++i], "wb");
        if( out==0 ) cmdlineError("cannot open: %s", argv[i]);
        xDiff = changeset_one_table;
+        neverUseTransaction = 1;
      }else
      if( strcmp(z,"debug")==0 ){
        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
@@ -1790,6 +1807,9 @@ int main(int argc, char **argv){
        if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
        zTab = argv[++i];
      }else
+      if( strcmp(z,"transaction")==0 ){
+        useTransaction = 1;
+      }else
      {
        cmdlineError("unknown option: %s", argv[i]);
      }
@@ -1832,6 +1852,8 @@ int main(int argc, char **argv){
    cmdlineError("\"%s\" does not appear to be a valid SQLite database", zDb2);
  }

+  if( neverUseTransaction ) useTransaction = 0;
+  if( useTransaction ) printf("BEGIN TRANSACTION;\n");
  if( zTab ){
    xDiff(zTab, out);
  }else{
@@ -1849,6 +1871,7 @@ int main(int argc, char **argv){
    }
    sqlite3_finalize(pStmt);
  }
+  if( useTransaction ) printf("COMMIT;\n");

  /* TBD: Handle trigger differences */
  /* TBD: Handle view differences */
Author	SHA1	Message	Date
drh	426504965c	Add a new OP_BColumn opcode where the cursor is guaranteed to be of type CURTYPE_BTREE and is thus able to skip some checks and run slightly faster. FossilOrigin-Name: 093e0d10a81adc078dcb3e08e33ce89bdbe53789	2015-11-21 17:53:06 +00:00
drh	62aaa6ca88	Fix over-length source code lines. No logic changes. FossilOrigin-Name: 198d191b2f5ef7d63ac0093c701955c9052fd734	2015-11-21 17:27:42 +00:00
drh	202923107d	Do not allow table-valued function syntax on ordinary tables and views as those objects are not table-valued functions. Fix for ticket [73eba7ba5c7c0fc]. FossilOrigin-Name: ff5716b89f99d9c4568a39f1f52524528a631623	2015-11-21 13:24:46 +00:00
drh	c960dcbace	Refactor the VdbeCursor object. It is now slightly smaller and faster and is easier to understand. FossilOrigin-Name: 9b1d174d862500a627840008ffac4c8419dc97e2	2015-11-20 19:22:01 +00:00
drh	5f7dacb443	In the OP_Column opcode, only test the btree payload size for exceeding the string length limit if the payload does not fit on a single page. FossilOrigin-Name: 35c7f6cba6febf2480de01fca9d61b8065bf1c12	2015-11-20 13:33:56 +00:00
drh	7977fa3ab2	Enhancements to a comment in sqliteInt.h. No changes to code. FossilOrigin-Name: 5446ae64d7f92444ca40aae2108015d1d77bc03f	2015-11-20 13:17:29 +00:00
drh	18f8e73453	Fix problems with the way the IsHiddenColumn() macro is defined. FossilOrigin-Name: 126b998cf163dcdd5a222634f1e929f04db3c700	2015-11-19 18:11:20 +00:00
dan	aaea31439d	Fix a problem with VACUUM and __hidden__ columns. FossilOrigin-Name: 13995756ad8b80568aa2f74387788a8cab1123ef	2015-11-19 18:09:05 +00:00
drh	80090f9221	When manifesting a view as part of an DELETE or UPDATE, be sure to include the hidden columns in the manifestation. FossilOrigin-Name: 28df5dc4a9569f388af2ee0d1f016afbea132277	2015-11-19 17:55:11 +00:00
dan	ba68f8f3f5	Fix problems with INSERT INTO ... SELECT ... statements that write to tables with __hidden__ columns. FossilOrigin-Name: 59bd0ec7d4327852ee8c0206b2c59d0a12484db8	2015-11-19 16:46:46 +00:00
drh	1a1d3cd2f3	Create the new TK_ASTERISK token to represent the "" in "SELECT ". Formerly that operator was TK_ALL, which was also used for UNION ALL. Less confusion if they operator symbols are distinct. FossilOrigin-Name: 201ac6d449431dadc6b29faecd68b559bd64bc9f	2015-11-19 16:33:31 +00:00
drh	05bdfa4a6f	If compiled with SQLITE_ENABLE_HIDDEN_COLUMNS, then columns in ordinary tables and views that have names beginning with "__hidden__" are omitted from the "*" expansion in SELECT statements and from the automatic list of columns following the table name in an INSERT INTO statement. FossilOrigin-Name: 011904cad2be2ce34e2f37ffae8ff2f1044a2969	2015-11-19 14:11:58 +00:00
drh	03d69a6826	Only support the magic "__hidden__" column name prefix interpretation when compiled with SQLITE_ENABLE_HIDDEN_COLUMNS. FossilOrigin-Name: 5490646b2eb74ea8bd5ab2690f69b9c707a0165f	2015-11-19 13:53:57 +00:00
drh	a6f88ffd3a	Fix a 10-year-old misguided attempt at parser performance improvement that actually made parsing slightly slower. FossilOrigin-Name: 3833cbac0706ecac9a641d70b786d26f7e1a42b1	2015-11-19 13:21:31 +00:00
drh	b1daa3f469	Get the __hidden__ column mechanism working on views with INSTEAD OF triggers. FossilOrigin-Name: 20c1e9ce75cc0b5f7456379f35a4fe24edd98088	2015-11-18 21:22:02 +00:00
dan	ab5a8ee3e4	Add tests for views to hidden.test. FossilOrigin-Name: 27d4b6f5756c7b19c00c95d46a77280c8b6f03a4	2015-11-18 20:59:47 +00:00
drh	6aed239acc	Honor the "__hidden__" prefix on the columns of views. FossilOrigin-Name: 3071ba2bdbda7018d0a285eceb04b8527209ec1e	2015-11-18 20:57:50 +00:00
dan	d1ce65fc76	Add a test for the __hidden__ hack on this branch. FossilOrigin-Name: ebf4bbffec58111a670c46a9eb469bfd2440b0b1	2015-11-18 20:07:12 +00:00
drh	f0c9145a36	If a table column name begins with "__hidden__" then do not include that column in "*" expansions in SELECT statements, nor fill in that column in an INSERT INTO that omits the column list. <b>This branch is a proof-of-concept only and is not intended to ever be merged into trunk.</b> FossilOrigin-Name: 2dbffb3a3b20dba7d7d86c8ad2e34633f616c78a	2015-11-18 18:43:15 +00:00
mistachkin	c88cd13753	When using mmap mode on Win32, use a read-only mapping by default. Write to the database file using WriteFile(). Unless SQLITE_MMAP_READWRITE is defined, in which case use a read/write mapping and write into the file using memcpy(). FossilOrigin-Name: 4f521b5bb36a1117db324e92dcf80abd2f1a5bc1	2015-11-17 21:42:32 +00:00
dan	fe33e39ba6	When using mmap mode on unix, use a read-only mapping by default. Write to the database file using write(). Unless SQLITE_MMAP_READWRITE is defined, in which case use a read/write mapping and write into the file using memcpy(). FossilOrigin-Name: 67c5d3c646c8198c8637069140aca952b802df25	2015-11-17 20:56:06 +00:00
mistachkin	bfe8bd5a5e	Fix uninitialized variable in the command line shell. FossilOrigin-Name: dc2ac024d31a898bbc3e7fc824d806e63e993234	2015-11-17 19:17:14 +00:00
mistachkin	81c7811fe7	Fix harmless compiler warning. FossilOrigin-Name: eea26b6caea37422c9b46a7a3a538b0c9575f392	2015-11-17 19:16:39 +00:00
dan	e4e416e87e	Add testfixture command "vfs_current_time_int64". Returns the value returned by the xCurrentTimeInt64 method of the default VFS. FossilOrigin-Name: f79d5b1853b3d4316ea08de8206ffca33e44c395	2015-11-16 15:28:54 +00:00
dan	3fd415b2bf	In the shell tool, avoid testing if (sqlite3_vfs.xGetCurrentInt64) is NULL for a version 1 VFS. This field is only defined for version 2 and greater. FossilOrigin-Name: ad5fcaa583ef743d143b6c030e0d78019709fe71	2015-11-16 08:54:10 +00:00
drh	f7ee8965a5	Fix the column name uniquifier so that it works with zero-length column names. FossilOrigin-Name: 791761ebac26c82ab67bdf867117ec5b5d8b20b0	2015-11-15 11:13:49 +00:00
drh	96ceaf8680	Code simplification in sqlite3ColumnsFromExprList(). Update the %z format code so that it works with buffers obtained from sqlite3DbMalloc(). Add a testcase for the slow column name uniquifier. FossilOrigin-Name: 9272426057b6cb2d913519ff4c97aa6e211f7d51	2015-11-14 22:04:22 +00:00
drh	0315e3cc14	Use a hash table to improve the preformance of column name uniqueness checking. FossilOrigin-Name: 5b08f29f458c600401860c7d70d8174cf61e69f8	2015-11-14 20:52:43 +00:00
drh	ebed3fa3e1	Use randomness to prevent showness in the generated-column-name uniqueness checking. FossilOrigin-Name: 6266712968a2cdcd6f5a3007d60c2cf1b3faf912	2015-11-14 16:47:23 +00:00
drh	6653fbe91f	Fix a memory leak in the fuzzcheck utility. FossilOrigin-Name: dfd6d9f4fbe902086f9158dfa5f37e781765a683	2015-11-13 20:52:49 +00:00
dan	ce6cbf9ff4	Add the "--enable-static-shell" option to the amalgamation autoconf script. If set (the default) the compiled shell tool is statically linked against sqlite3.o. Otherwise, it is linked against libsqlite3.so. FossilOrigin-Name: 499a02a34316cada9e197ef1d2e77c4cd75c41be	2015-11-13 16:59:00 +00:00
drh	05d4ebf6c3	Add the --transaction option to the sqldiff tool. FossilOrigin-Name: 8882d1f3ef0fc53d7c19918c3af43c5bdd79e88f	2015-11-13 13:15:42 +00:00
drh	d63fbb7103	Fix the releasetest-out.txt log output from releasetest.tcl so that it works when the --jobs option is used. FossilOrigin-Name: 6bb314a5022c8bae63f6cdb48652915e05c06a38	2015-11-13 12:52:34 +00:00
drh	29fbdb731b	Disable two PRAGMA cache_spill tests when MEMORY_MANAGEMENT is enabled. FossilOrigin-Name: ebda77aad4d964593606ded2fdb19259ef9ebb8e	2015-11-13 12:32:01 +00:00
drh	061d35c145	Change the error messages so that they begin can be grepped using '^!' instead of '^ '. There are far too many extraneous outputs that being with a space. FossilOrigin-Name: 367ec0db49607559960d58891771113c205fd951	2015-11-13 00:03:14 +00:00
drh	5e3cefe3d0	Update the SQLLOG logic so that to builds in accordance with the instructions in the header comment. FossilOrigin-Name: ee6f2feec7d777a969ddba6dc00fb571d64e57f8	2015-11-12 23:48:08 +00:00
drh	8d4482da96	Always parse the schema before running "PRAGMA cache_size", not after. FossilOrigin-Name: 12e7d9ec023db4dd837f76006c87defc5a3e9388	2015-11-12 22:15:18 +00:00
drh	d66b2e02ee	Add a space before each line of error output in the test scripts so that all errors can be conveniently located in logs by doing "grep '^ '". FossilOrigin-Name: 9d532fb50d310df629ea65193f21dc9b3594ff5f	2015-11-12 21:42:40 +00:00
dan	d83f7ca114	Add support for the SQLITE_SQLLOG_CONDITIONAL environment variable to the logging code in test_sqllog.c. When defined, logging is only performed if the "<database>-sqllog" file is present in the file system when the main database is opened. FossilOrigin-Name: cab8126be9f63dd596719b12704ba77c128282bc	2015-11-12 20:12:51 +00:00
drh	c2e0dd4ddd	Enhance the "PRAGMA cache_spill" statement to accept an integer argument which is the threshold at which spilling will begin. FossilOrigin-Name: f79d264db24a470d1a4571e15a99cd3d68b5166c	2015-11-12 16:44:40 +00:00
drh	4f9c8ec62d	Test cases. Minor problems fixed. All appears to work now. FossilOrigin-Name: 9a431362dccbc9b8f93375f30a3b8955903cca79	2015-11-12 15:47:48 +00:00
drh	644f4c1bbb	Fix a #define in parse.y that disabled unused code in the generated parser. FossilOrigin-Name: f84e3085c87cfffe4aba0eb4c4a3298b4027db83	2015-11-12 15:04:05 +00:00
drh	9b0cf34f81	First attempt at enhancing the "PRAGMA cache_spill" statement to accept a cache threashold size. FossilOrigin-Name: 549d42be0dac87dc04c3eeccfdc60615c3a6ad3f	2015-11-12 14:57:19 +00:00
mistachkin	24e9895f81	Fix missing word in a documentation comment. No changes to code. FossilOrigin-Name: ed24d302657e8495bef7f5ed698c7cca12717be9	2015-11-11 18:43:49 +00:00
drh	3d38cec99a	Improvements to the SQLITE_CONFIG_PAGECACHE documentation. Enhance the command-line shell to be able to take advantage of the full range of SQLITE_CONFIG_PAGECACHE capabilities, such as setting pMem==NULL and N<0. FossilOrigin-Name: 2518d5c971c4b32d9227b3bb7259162e3e27b00b	2015-11-11 15:28:52 +00:00
drh	0c4105ee19	Improved output formatting for "PRAGMA parser_trace=ON;". FossilOrigin-Name: e43e1171fd7837a08069dc25df4eac14db1c2afe	2015-11-10 14:51:22 +00:00
drh	97e58a2feb	Make the output of "PRAGMA parser_trace" go to stdout instead of stderr. FossilOrigin-Name: 1e4849911e3eed65eabee18fdee63bab4faae57a	2015-11-10 14:27:17 +00:00
drh	3334d08989	Save a few bytes in the parser by using "int" instead of "u8" or "u16" for all small integer types. FossilOrigin-Name: 5dcd212bf6489f4698a0ed0f21497c78379f7c0f	2015-11-10 13:45:21 +00:00
drh	634b242ebe	Change all parsers to use the standard "lempar.c" template in the tool/ folder and remove the customized lempar.c from src/, plus other compiler performance and space enhancements. FossilOrigin-Name: 0e7fb24ad3c010884763a97e6ae7a3fd39d0e5a9	2015-11-10 12:41:03 +00:00
drh	43509768e1	Fix harmless compiler warnings in FTS5. FossilOrigin-Name: 09752e51a18ac1b4c9642965e6ee1b6a32de00df	2015-11-10 12:31:25 +00:00
drh	54bb56d82a	Performance enhancement to the tokenizer. FossilOrigin-Name: 6ea2df86c95c226052f528424e9bee367a2e765a	2015-11-10 03:30:51 +00:00
drh	07b09a9420	Remove an unused non-terminal from the grammar. FossilOrigin-Name: 3c37c522883ea9f2eec4f0ba5c5141912c003425	2015-11-10 00:02:49 +00:00
drh	822a62f84d	Fix a comment typo in the lempar.c template that was missed by the prior check-in. FossilOrigin-Name: c4a7e93fca622fd11a6e16161fbd2f39c2575f00	2015-11-09 19:35:18 +00:00
drh	82415f2d8d	Change the parser to use the standard "lempar.c" template over in the tool/ folder rather than the customized "lempar.c" found in src/. FossilOrigin-Name: 0a72991f4e54548f6c3268c5a9cac1c8d6437d26	2015-11-09 19:33:42 +00:00
drh	a441404f92	Avoid recursion in the yy_find_shift_action() routine of the Lemon-generated parser, so that routine can be inlined, for a size reduction and performance increase. FossilOrigin-Name: 0557a179f932296cc1fd5217f9a0d2f74e34ce1d	2015-11-09 15:06:26 +00:00
drh	3781f01780	Size reduction and performance improvement in the stack-popping logic of the Lemon-generated parser. FossilOrigin-Name: 9748c48a4fbd5c06208bbf80e7bfcb159ec026d9	2015-11-09 14:11:37 +00:00
drh	7450494c50	Fix incorrect WHERE clause in sqldiff, as reported on the mailing list by Youcef Hilem. FossilOrigin-Name: e0ed4c3e376248dfbf903e4b5845f910824fa6c6	2015-11-09 12:47:04 +00:00
drh	f10b1f78ee	Check in the cg_anno.tcl and run-speed-test.sh scripts, as an historical record. FossilOrigin-Name: 836418d3b7cfcd5ec375c4e08c09bd6b78646307	2015-11-09 12:44:19 +00:00
drh	d797035ac7	Avoid unnecessary function prologues in the sqlite3VdbeAddOp3() routine. FossilOrigin-Name: 7c6a19ba9b0bdb1cc0b9a9796b7c1c114944d927	2015-11-09 12:33:39 +00:00
drh	0739e45bcd	Small size reduction and performance increase in the parser. FossilOrigin-Name: d62cd757a69cc49c2d309e27c948610b5868632f	2015-11-09 02:08:09 +00:00
drh	edd2260b40	Enhance the sqldiff utility to deal gracefully with ALTER TABLE ADD COLUMN. FossilOrigin-Name: 7ea036ac37397ed8f6a0fa9f5bfc0994364b53dc	2015-11-07 18:32:17 +00:00
dan	cc03305473	Fix a bug in CTE handling discovered by LibFuzzer that can cause an infinite loop in the query planner. FossilOrigin-Name: 088009efdd56160bb4eee0fbd829a529b141274e	2015-11-07 18:07:15 +00:00
dan	fe88fbfc82	Add test cases for WITH clauses. FossilOrigin-Name: e7e65c7559ed43e7065adc2ec1bd242bf187b7c3	2015-11-07 17:51:39 +00:00
drh	6e7722669e	Proposed fix for an infinite loop bug in the WITH clause semantic analysis logic. FossilOrigin-Name: 028475cb17a4b50baf0e9aba9bd3403d0a5d03b0	2015-11-07 17:48:21 +00:00
drh	2476a6f2cb	Enhance TreeView to show WITH clauses. Add an assert to detect the infinite loop behavior when certain kinds of errors occur on a nested WITH clause. FossilOrigin-Name: 2040d88e877bdb69de125a047cefb4a1558e89e8	2015-11-07 15:19:59 +00:00
drh	d6b7946c32	The OPFLAG_SEEKEQ optimization is only applicable to equality comparisons against an index, not against a rowid table. FossilOrigin-Name: 0f5b147d1fe83c34d0fbeacc7422be94d8441bc1	2015-11-07 01:19:00 +00:00
drh	3901692ede	Fix a harmless sanitizer warning in the ieee754 extension. FossilOrigin-Name: dd9a26ecdeaef7b0b9cbf4ff70448ab13a276b4e	2015-11-07 00:51:15 +00:00
drh	3adf8783b0	Avoid an unnecessary key comparison when doing an indexed lookup against an equality constraint. FossilOrigin-Name: d741e1ccdce96e6142e294fbdd20a0282296dcea	2015-11-06 20:22:25 +00:00
drh	f83075201f	Work around a sign-extension bug in the optimizer on the HP C compiler that comes with HP/UX. FossilOrigin-Name: 46c36b1543d7a8156eaa866c67080526153fb10d	2015-11-06 20:13:48 +00:00
drh	51675f24f3	Test changes: Omit numindex1.test from valgrind, since valgrind uses a none-standard floating-point processor which gives inconsistent answers. Run one releasetest.tcl module using LONGDOUBLE_TYPE=double. FossilOrigin-Name: d421efbf9a5688f5a20b6dfe3aa216299c7b9c86	2015-11-06 20:04:08 +00:00
drh	c750ee999e	Enforce consistent comparison results between integer and floating point values. Fix for ticket [38a97a87a6e4e83]. FossilOrigin-Name: 849e686da2d20fd7f938427475fc792384d2aa7c	2015-11-06 19:53:09 +00:00
drh	c8bb430dfe	A proposed patch for working around a bug in the optimizer for the HP compiler found on HP/UX Itanium. FossilOrigin-Name: 8a171548e4f665f03d00af6dcb46669a4a78fad5	2015-11-06 17:28:00 +00:00
drh	b1d607de25	Improvements and simplifications to the equality seek logic. Tests are adjusted so that they all pass now. FossilOrigin-Name: 997ce6c90b454c03cc2ef6934752ee8dd2e520e3	2015-11-05 22:30:54 +00:00
drh	70528d7868	The top of an index equality loop normally starts with OP_SeekGE and OP_IdxGT. This check-in adds a flag to OP_SeekGE such that it fails immediately if the key is not equal, then jumps over the OP_IdxGT, saving a call to the key comparison functions. Consider this check-in a proof-of-concept. It needs improvement before going on trunk. Some tests fail, but only because they new use fewer key comparisons than expected (which is a good thing!). FossilOrigin-Name: 32e31b9bc8664afcd326a1ff3892d86dc5202474	2015-11-05 20:25:09 +00:00