Add debugging code to count the number of iterations of each loop made as part of statement execution.

FossilOrigin-Name: c6a5b67ae1fb31120a40d2885b848ec2dab15c4e
Take steps to avoid misestimating range query costs based on STAT4 data
2014-10-23 17:26:22 +00:00 · 2014-10-22 15:27:05 +00:00 · 2014-10-21 21:56:06 +00:00 · 2014-10-21 18:16:21 +00:00 · 2014-10-21 16:01:40 +00:00 · 2014-10-21 01:05:09 +00:00
70 changed files with 2718 additions and 1157 deletions
@@ -3116,6 +3116,7 @@ static int fts3FilterMethod(
  /* In case the cursor has been used before, clear it now. */
  sqlite3_finalize(pCsr->pStmt);
  sqlite3_free(pCsr->aDoclist);
+  sqlite3_free(pCsr->aMatchinfo);
  sqlite3Fts3ExprFree(pCsr->pExpr);
  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));

@@ -4426,7 +4427,7 @@ static int fts3EvalIncrPhraseNext(
          bMaxSet = 1;
        }
      }
-      assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 );
+      assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) );
      assert( rc!=SQLITE_OK || bMaxSet );

      /* Keep advancing iterators until they all point to the same document */
@@ -190,7 +190,7 @@ static int getNextToken(
  /* Set variable i to the maximum number of bytes of input to tokenize. */
  for(i=0; i<n; i++){
    if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
-    if( z[i]=='*' || z[i]=='"' ) break;
+    if( z[i]=='"' ) break;
  }

  *pnConsumed = i;
@@ -0,0 +1,103 @@
+/*
+** 2014-09-21
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This SQLite extension adds a debug "authorizer" callback to the database
+** connection.  The callback merely writes the authorization request to
+** standard output and returns SQLITE_OK.
+**
+** This extension can be used (for example) in the command-line shell to
+** trace the operation of the authorizer.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <stdio.h>
+
+/*
+** Display the authorization request
+*/
+static int authCallback(
+  void *pClientData,
+  int op,
+  const char *z1,
+  const char *z2,
+  const char *z3,
+  const char *z4
+){
+  const char *zOp;
+  char zOpSpace[50];
+  switch( op ){
+    case SQLITE_CREATE_INDEX:        zOp = "CREATE_INDEX";        break;
+    case SQLITE_CREATE_TABLE:        zOp = "CREATE_TABLE";        break;
+    case SQLITE_CREATE_TEMP_INDEX:   zOp = "CREATE_TEMP_INDEX";   break;
+    case SQLITE_CREATE_TEMP_TABLE:   zOp = "CREATE_TEMP_TABLE";   break;
+    case SQLITE_CREATE_TEMP_TRIGGER: zOp = "CREATE_TEMP_TRIGGER"; break;
+    case SQLITE_CREATE_TEMP_VIEW:    zOp = "CREATE_TEMP_VIEW";    break;
+    case SQLITE_CREATE_TRIGGER:      zOp = "CREATE_TRIGGER";      break;
+    case SQLITE_CREATE_VIEW:         zOp = "CREATE_VIEW";         break;
+    case SQLITE_DELETE:              zOp = "DELETE";              break;
+    case SQLITE_DROP_INDEX:          zOp = "DROP_INDEX";          break;
+    case SQLITE_DROP_TABLE:          zOp = "DROP_TABLE";          break;
+    case SQLITE_DROP_TEMP_INDEX:     zOp = "DROP_TEMP_INDEX";     break;
+    case SQLITE_DROP_TEMP_TABLE:     zOp = "DROP_TEMP_TABLE";     break;
+    case SQLITE_DROP_TEMP_TRIGGER:   zOp = "DROP_TEMP_TRIGGER";   break;
+    case SQLITE_DROP_TEMP_VIEW:      zOp = "DROP_TEMP_VIEW";      break;
+    case SQLITE_DROP_TRIGGER:        zOp = "DROP_TRIGGER";        break;
+    case SQLITE_DROP_VIEW:           zOp = "DROP_VIEW";           break;
+    case SQLITE_INSERT:              zOp = "INSERT";              break;
+    case SQLITE_PRAGMA:              zOp = "PRAGMA";              break;
+    case SQLITE_READ:                zOp = "READ";                break;
+    case SQLITE_SELECT:              zOp = "SELECT";              break;
+    case SQLITE_TRANSACTION:         zOp = "TRANSACTION";         break;
+    case SQLITE_UPDATE:              zOp = "UPDATE";              break;
+    case SQLITE_ATTACH:              zOp = "ATTACH";              break;
+    case SQLITE_DETACH:              zOp = "DETACH";              break;
+    case SQLITE_ALTER_TABLE:         zOp = "ALTER_TABLE";         break;
+    case SQLITE_REINDEX:             zOp = "REINDEX";             break;
+    case SQLITE_ANALYZE:             zOp = "ANALYZE";             break;
+    case SQLITE_CREATE_VTABLE:       zOp = "CREATE_VTABLE";       break;
+    case SQLITE_DROP_VTABLE:         zOp = "DROP_VTABLE";         break;
+    case SQLITE_FUNCTION:            zOp = "FUNCTION";            break;
+    case SQLITE_SAVEPOINT:           zOp = "SAVEPOINT";           break;
+    case SQLITE_COPY:                zOp = "COPY";                break;
+    case SQLITE_RECURSIVE:           zOp = "RECURSIVE";           break;
+
+
+    default: {
+      sqlite3_snprintf(sizeof(zOpSpace), zOpSpace, "%d", op);
+      zOp = zOpSpace;
+      break;
+    }
+  }
+  if( z1==0 ) z1 = "NULL";
+  if( z2==0 ) z2 = "NULL";
+  if( z3==0 ) z3 = "NULL";
+  if( z4==0 ) z4 = "NULL";
+  printf("AUTH: %s,%s,%s,%s,%s\n", zOp, z1, z2, z3, z4);
+  return SQLITE_OK;
+}
+
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_showauth_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_set_authorizer(db, authCallback, 0);
+  return rc;
+}
@@ -1,5 +1,5 @@
-C Only\senable\sHAVE_STRCHRNUL\sby\sdefault\son\slinux,\sas\sthat\sis\sthe\sonly\splace\nit\sappears\sto\swork\sby\sdefault.
-D 2014-09-20T00:29:20.392
+C Add\sdebugging\scode\sto\scount\sthe\snumber\sof\siterations\sof\seach\sloop\smade\sas\spart\sof\sstatement\sexecution.
+D 2014-10-23T17:26:22.165
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,11 +78,11 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 2f5e925bdb9d6d3e488c5a981af60cad4f9cdfe7
+F ext/fts3/fts3.c 8b6cceb3e0be22da26d83a3cec0e0e337e6b8ec6
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21
 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365
-F ext/fts3/fts3_expr.c 351395fad6fcb16ecfc61db0861008a70101330c
+F ext/fts3/fts3_expr.c 40123785eaa3ebd4c45c9b23407cc44ac0c49905
 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5
@@ -116,6 +116,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63
 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc
 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a
+F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 56739fab8c2ed6a9e2dac5592a88d281a999c43b
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95
@@ -165,26 +166,26 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb
-F src/analyze.c 6290a109be876daaa242cd7216f97240f5401776
+F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447
 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9
 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2
 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e
 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
 F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5
-F src/btree.c 6aa61c0e3d20d1d1acc8fb33d8f0ebd675305d3c
+F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed
 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8
-F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc
-F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60
+F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179
+F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919
 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0
 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14
-F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958
+F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638
 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036
 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f
-F src/expr.c 19392d98e089640c3336e65b4254cc337efef7d1
+F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7
-F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81
-F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937
+F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee
+F src/global.c 01c1f36ecfcf10770db648422a8852c222308bb9
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
@@ -193,11 +194,11 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770
 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994
-F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab
-F src/malloc.c 5bb99ee1e08ad58e457063cf79ce521db0e24195
+F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51
+F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f
-F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f
+F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534
 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb
 F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85
@@ -211,32 +212,32 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace
 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf
 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
-F src/os_unix.c 9096a1b1449182e67e759f59994eee04113bc587
-F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7
+F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7
+F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053
 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21
-F src/pager.c caab007743821d96752597c9cfd7351654697b06
+F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926
 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428
-F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0
+F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45
 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a
 F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a
-F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa
+F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5
 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f
 F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196
-F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1
+F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64
 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
-F src/select.c 0cd6706fd52ae5db229e9041094db6ec27195335
-F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d
-F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6
+F src/select.c 428165951748151e87a15295b7357221433e311b
+F src/shell.c ed2796b3bb5e21c8a7965c846fb5eba87d26aa65
+F src/sqlite.h.in 7b30552eaf9ddd2a820ef39b4d8b656a866abb77
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d
-F src/sqliteInt.h 6fd801cac974d310949ddefd6a6c6982c9778f3f
+F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
-F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
+F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb
 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb
 F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684
-F src/test1.c 523cd70ded28db71af9a30ec184cbe0957de9575
+F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7
 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712
 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c
 F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df
@@ -249,7 +250,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8
 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e
 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f
-F src/test_config.c 6f721f0337b96d58e81ff69bba101113c8168c2b
+F src/test_config.c a4cdebe093474c02eecc5e4008b1a22198edf975
 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f
@@ -261,7 +262,7 @@ F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202
 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64
 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4
 F src/test_malloc.c ba34143f941a9d74b30bbffc8818389bb73a1ca2
-F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76
+F src/test_multiplex.c caadb62cc777268b4f8fb94d5b27b80156c8f7c0
 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3
 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f
 F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25
@@ -281,28 +282,28 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb
 F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
-F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff
-F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c
+F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee
+F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689
 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f
 F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0
 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c
 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8
 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a
-F src/vdbe.c de1af1795bebdad20c23e82bafa2f531e9893198
-F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327
-F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735
-F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d
-F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060
+F src/vdbe.c 5acb0a03f449f6488dbfb12f7fbe5953431e0c0e
+F src/vdbe.h 6bf91795a5775a1950fd01e716906eeb114734c6
+F src/vdbeInt.h 0fdc28cfa597bf8fd763eb085cc84945a7517b88
+F src/vdbeapi.c cbfb0fed9b36cf7db75e7b9fa721ba981b13e8e8
+F src/vdbeaux.c f9954f3100cbf92a2a976f34de0a109c10bfa8ab
 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4
-F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099
-F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef
-F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362
-F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f
+F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f
+F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438
+F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010
+F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de
 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804
-F src/where.c 0888567c0e01a41b6001647e333f8ccfd3ae7d36
-F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c
+F src/where.c c9613a13a066a650675e66515d28fd4ecab5ab17
+F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7
@@ -325,6 +326,8 @@ F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a
 F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a
 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d
 F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93
+F test/analyzeD.test 08f9d0bee4e118a66fff3a32d02dbe0ee0a2b594
+F test/analyzeE.test 8684e8ac5722fb97c251887ad97e5d496a98af1d
 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b
 F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b
 F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7
@@ -340,7 +343,7 @@ F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360
 F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa
 F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7
 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7
-F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61
+F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7
 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de
 F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2
 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74
@@ -418,7 +421,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee
 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4
 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804
 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb
-F test/corruptI.test 0afbba50bfae006094cc548b4605f521c1179502
+F test/corruptI.test 221ad8b7f0a9ac6b80fc577e73b5ad8cdea31243
 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318
 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5
 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62
@@ -437,7 +440,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47
 F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0
 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204
 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2
-F test/default.test 792c3c70836f1901e2a8cb34fa0880ed71e2c1a9
+F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d
 F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701
 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa
 F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab
@@ -447,7 +450,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2
 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e
 F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
-F test/e_createtable.test 181653f6f45e3adde73f8686600ce5ad7515466b
+F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7
 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a
 F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412
 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306
@@ -460,7 +463,7 @@ F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6
 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763
 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f
 F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52
-F test/e_uri.test a2c92d80093a7efdcfbb11093651cbea87097b6b
+F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585
 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9
 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea
 F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473
@@ -468,12 +471,12 @@ F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40
 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
 F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f
 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401
-F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3
+F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c
 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75
 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30
-F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d
+F test/expr.test c4b9bf0cc60b26862475e19999fbd2609ca8259c
 F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9
 F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7
 F test/filectrl.test 14fa712e42c4cb791e09dfd58a6a03efb47ef13a
@@ -559,13 +562,13 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 3401d47b229c4504424caf362cc4ff704cad4162
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f
-F test/fts3expr4.test 0713d94ab951ed88a8c3629a4889a48c55c4067c
+F test/fts3expr4.test e1be1248566f43c252d4404d52914f1fc4bfa065
 F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660
 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
-F test/fts3matchinfo.test ff423e73faab8fc6d7adeefedf74dd8e2b0b14e0
+F test/fts3matchinfo.test 58544fa4d254000fa4e7f494b0a832f7ba61d45e
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1
 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce
@@ -628,7 +631,7 @@ F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee
 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062
 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6
-F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33
+F test/index5.test 25b0b451aceed4ac5f7d49f856f6de7257470b3e
 F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a
 F test/index7.test 917cf1e1c7439bb155abbeabec511b28945e157b
 F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec
@@ -674,7 +677,7 @@ F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4
 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff
 F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00
 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12
-F test/lock5.test 5ad6a1f536036ff1be915cfdd41481aeafda3273
+F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
 F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95
@@ -689,7 +692,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151
 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a
 F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d
 F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e
-F test/mallocA.test c049224adeb0244b8f6eb770c1fa6ac40f9b3518
+F test/mallocA.test 672cd7dedb63771bade3a6f557f851a4ad161d4a
 F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be
 F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6
 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4
@@ -728,6 +731,7 @@ F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3
 F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60
 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a
 F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101
+F test/multiplex4.test d3e8a5a522c51cbf3ed1c5b0bd496be02c29d7b1
 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a
@@ -747,6 +751,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb
 F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859
 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da
 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3
+F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799
 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa
 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71
 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f
@@ -778,11 +783,11 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a
 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a
-F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e
+F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180
 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a
 F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
-F test/rowid.test b78b30afb9537a73788ca1233a23a32190a3bb1f
+F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645
 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798
 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09
 F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313
@@ -835,13 +840,14 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868
 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329
-F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2
+F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff
 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a
 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5
-F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196
+F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2
+F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24
-F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235
+F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d
 F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de
 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2
 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb
@@ -855,13 +861,13 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/speedtest1.c 83f6b3318f7ee60e52b978b5a5e5dd7e83dfb7ee
+F test/speedtest1.c e4e2aa37ff66bad9f414a50a8cb9edfaac65c9e5
 F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49
-F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298
+F test/sqllimits1.test 9014524e7ab16e2a4976b13397db4c29cc29c6d9
 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de
 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
 F test/subquery.test 666fdecceac258f5fd84bed09a64e49d9f37edd9
-F test/subquery2.test 91e1e364072aeff431d1f9689b15147e421d88c7
+F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f
 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4
 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a
 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2
@@ -928,6 +934,7 @@ F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550
 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0
 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3
 F test/tkt-b75a9ca6b0.test 97cc2d5eeaf82799eb42138c0a1ff64370238ce4
+F test/tkt-ba7cbfaedc.test e76d88e572e489ee0d64fe4caf4af18b3d1dc688
 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898
 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d
 F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447
@@ -1193,12 +1200,15 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
 F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f
 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
-F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
+F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c
 F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 35db3e2f350ca2bc6bb9e1a647aec7f93bfb7065
-R 8acbfeff4e0806bab5948d89d293bcf4
-U drh
-Z 6fc488f20a05aa47a5cf4ff95476cc98
+P 3c933bf95f291f7957580d823dce92c981375a5c
+R dd407ae864af5a809eb144aec1696724
+T *branch * debug-loopcounters
+T *sym-debug-loopcounters *
+T -sym-trunk *
+U dan
+Z b7fd2f254655f3286afc23e1478fb538
@@ -1 +1 @@
-0fac2c045f47c7735af4eb68ced81d8b43622a1f
+c6a5b67ae1fb31120a40d2885b848ec2dab15c4e
@@ -1437,7 +1437,7 @@ static void decodeIntArray(
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
  if( z==0 ) z = "";
 #else
-  if( NEVER(z==0) ) z = "";
+  assert( z!=0 );
 #endif
  for(i=0; *z && i<nOut; i++){
    v = 0;
@@ -1446,16 +1446,14 @@ static void decodeIntArray(
      z++;
    }
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
-    if( aOut ){
-      aOut[i] = v;
-    }else
+    if( aOut ) aOut[i] = v;
+    if( aLog ) aLog[i] = sqlite3LogEst(v);
 #else
    assert( aOut==0 );
    UNUSED_PARAMETER(aOut);
+    assert( aLog!=0 );
+    aLog[i] = sqlite3LogEst(v);
 #endif
-    {
-      aLog[i] = sqlite3LogEst(v);
-    }
    if( *z==' ' ) z++;
  }
 #ifndef SQLITE_ENABLE_STAT3_OR_STAT4
@@ -1516,8 +1514,17 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
  z = argv[2];

  if( pIndex ){
+    int nCol = pIndex->nKeyCol+1;
+#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
+    tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(
+        sizeof(tRowcnt) * nCol
+    );
+    if( aiRowEst==0 ) pInfo->db->mallocFailed = 1;
+#else
+    tRowcnt * const aiRowEst = 0;
+#endif
    pIndex->bUnordered = 0;
-    decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex);
+    decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex);
    if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0];
  }else{
    Index fakeIdx;
@@ -1576,25 +1583,39 @@ static void initAvgEq(Index *pIdx){
      pIdx->aAvgEq[nCol] = 1;
    }
    for(iCol=0; iCol<nCol; iCol++){
+      int nSample = pIdx->nSample;
      int i;                    /* Used to iterate through samples */
      tRowcnt sumEq = 0;        /* Sum of the nEq values */
-      tRowcnt nSum = 0;         /* Number of terms contributing to sumEq */
      tRowcnt avgEq = 0;
-      tRowcnt nDLt = pFinal->anDLt[iCol];
+      tRowcnt nRow;             /* Number of rows in index */
+      i64 nSum100 = 0;          /* Number of terms contributing to sumEq */
+      i64 nDist100;             /* Number of distinct values in index */
+
+      if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){
+        nRow = pFinal->anLt[iCol];
+        nDist100 = (i64)100 * pFinal->anDLt[iCol];
+        nSample--;
+      }else{
+        nRow = pIdx->aiRowEst[0];
+        nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1];
+      }
+      pIdx->nRowEst0 = nRow;

      /* Set nSum to the number of distinct (iCol+1) field prefixes that
-      ** occur in the stat4 table for this index before pFinal. Set
-      ** sumEq to the sum of the nEq values for column iCol for the same
-      ** set (adding the value only once where there exist duplicate 
-      ** prefixes).  */
-      for(i=0; i<(pIdx->nSample-1); i++){
-        if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){
+      ** occur in the stat4 table for this index. Set sumEq to the sum of 
+      ** the nEq values for column iCol for the same set (adding the value 
+      ** only once where there exist duplicate prefixes).  */
+      for(i=0; i<nSample; i++){
+        if( i==(pIdx->nSample-1)
+         || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] 
+        ){
          sumEq += aSample[i].anEq[iCol];
-          nSum++;
+          nSum100 += 100;
        }
      }
-      if( nDLt>nSum ){
-        avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum);
+
+      if( nDist100>nSum100 ){
+        avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100);
      }
      if( avgEq==0 ) avgEq = 1;
      pIdx->aAvgEq[iCol] = avgEq;
@@ -1846,6 +1867,11 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
    rc = loadStat4(db, sInfo.zDatabase);
    db->lookaside.bEnabled = lookasideEnabled;
  }
+  for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){
+    Index *pIdx = sqliteHashData(i);
+    sqlite3_free(pIdx->aiRowEst);
+    pIdx->aiRowEst = 0;
+  }
 #endif

  if( rc==SQLITE_NOMEM ){
@@ -273,9 +273,10 @@ typedef struct BtLock BtLock;
 struct MemPage {
  u8 isInit;           /* True if previously initialized. MUST BE FIRST! */
  u8 nOverflow;        /* Number of overflow cell bodies in aCell[] */
-  u8 intKey;           /* True if intkey flag is set */
-  u8 leaf;             /* True if leaf flag is set */
-  u8 hasData;          /* True if this page stores data */
+  u8 intKey;           /* True if table b-trees.  False for index b-trees */
+  u8 intKeyLeaf;       /* True if the leaf of an intKey table */
+  u8 noPayload;        /* True if internal intKey page (thus w/o data) */
+  u8 leaf;             /* True if a leaf page */
  u8 hdrOffset;        /* 100 for page 1.  0 otherwise */
  u8 childPtrSize;     /* 0 if leaf==1.  4 if leaf==0 */
  u8 max1bytePayload;  /* min(maxLocal,127) */
@@ -435,7 +436,7 @@ struct BtShared {
  BtLock *pLock;        /* List of locks held on this shared-btree struct */
  Btree *pWriter;       /* Btree with currently open write transaction */
 #endif
-  u8 *pTmpSpace;        /* BtShared.pageSize bytes of space for tmp use */
+  u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
 };

 /*
@@ -456,12 +457,10 @@ struct BtShared {
 */
 typedef struct CellInfo CellInfo;
 struct CellInfo {
-  i64 nKey;      /* The key for INTKEY tables, or number of bytes in key */
-  u8 *pCell;     /* Pointer to the start of cell content */
-  u32 nData;     /* Number of bytes of data */
-  u32 nPayload;  /* Total amount of payload */
-  u16 nHeader;   /* Size of the cell content header in bytes */
-  u16 nLocal;    /* Amount of payload held locally */
+  i64 nKey;      /* The key for INTKEY tables, or nPayload otherwise */
+  u8 *pPayload;  /* Pointer to the start of payload */
+  u32 nPayload;  /* Bytes of payload */
+  u16 nLocal;    /* Amount of payload held locally, not on overflow */
  u16 iOverflow; /* Offset to overflow page number.  Zero if no overflow */
  u16 nSize;     /* Size of the cell content on the main b-tree page */
 };
@@ -658,6 +657,8 @@ struct IntegrityCk {
  int mxErr;        /* Stop accumulating errors when this reaches zero */
  int nErr;         /* Number of messages written to zErrMsg so far */
  int mallocFailed; /* A memory allocation error has occurred */
+  const char *zPfx; /* Error message prefix */
+  int v1, v2;       /* Values for up to two %d fields in zPfx */
  StrAccum errMsg;  /* Accumulate the error message text here */
 };

@@ -435,6 +435,9 @@ static void freeIndex(sqlite3 *db, Index *p){
  sqlite3ExprDelete(db, p->pPartIdxWhere);
  sqlite3DbFree(db, p->zColAff);
  if( p->isResized ) sqlite3DbFree(db, p->azColl);
+#ifdef SQLITE_ENABLE_STAT3_OR_STAT4
+  sqlite3_free(p->aiRowEst);
+#endif
  sqlite3DbFree(db, p);
 }

@@ -1236,7 +1239,7 @@ void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){
  p = pParse->pNewTable;
  if( p!=0 ){
    pCol = &(p->aCol[p->nCol-1]);
-    if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){
+    if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){
      sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant",
          pCol->zName);
    }else{
@@ -2744,7 +2747,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
  }else{
    addr2 = sqlite3VdbeCurrentAddr(v);
  }
-  sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord);
+  sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx);
  sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1);
  sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
  sqlite3ReleaseTempReg(pParse, regRecord);
@@ -395,7 +395,7 @@ int sqlite3_compileoption_used(const char *zOptName){
  ** linear search is adequate.  No need for a binary search. */
  for(i=0; i<ArraySize(azCompileOpt); i++){
    if( sqlite3StrNICmp(zOptName, azCompileOpt[i], n)==0
-     && sqlite3CtypeMap[(unsigned char)azCompileOpt[i][n]]==0
+     && sqlite3IsIdChar((unsigned char)azCompileOpt[i][n])==0
    ){
      return 1;
    }
@@ -1069,6 +1069,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
  pNew->addrOpenEphm[1] = -1;
  pNew->nSelectRow = p->nSelectRow;
  pNew->pWith = withDup(db, p->pWith);
+  sqlite3SelectSetName(pNew, p->zSelName);
  return pNew;
 }
 #else
@@ -1211,32 +1212,40 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
 /*
 ** These routines are Walker callbacks.  Walker.u.pi is a pointer
 ** to an integer.  These routines are checking an expression to see
-** if it is a constant.  Set *Walker.u.pi to 0 if the expression is
+** if it is a constant.  Set *Walker.u.i to 0 if the expression is
 ** not constant.
 **
 ** These callback routines are used to implement the following:
 **
-**     sqlite3ExprIsConstant()
-**     sqlite3ExprIsConstantNotJoin()
-**     sqlite3ExprIsConstantOrFunction()
+**     sqlite3ExprIsConstant()                  pWalker->u.i==1
+**     sqlite3ExprIsConstantNotJoin()           pWalker->u.i==2
+**     sqlite3ExprIsConstantOrFunction()        pWalker->u.i==3 or 4
 **
+** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions
+** in a CREATE TABLE statement.  The Walker.u.i value is 4 when parsing
+** an existing schema and 3 when processing a new statement.  A bound
+** parameter raises an error for new statements, but is silently converted
+** to NULL for existing schemas.  This allows sqlite_master tables that 
+** contain a bound parameter because they were generated by older versions
+** of SQLite to be parsed by newer versions of SQLite without raising a
+** malformed schema error.
 */
 static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){

-  /* If pWalker->u.i is 3 then any term of the expression that comes from
+  /* If pWalker->u.i is 2 then any term of the expression that comes from
  ** the ON or USING clauses of a join disqualifies the expression
  ** from being considered constant. */
-  if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){
+  if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){
    pWalker->u.i = 0;
    return WRC_Abort;
  }

  switch( pExpr->op ){
    /* Consider functions to be constant if all their arguments are constant
-    ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST
+    ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST
    ** flag. */
    case TK_FUNCTION:
-      if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){
+      if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){
        return WRC_Continue;
      }
      /* Fall through */
@@ -1250,6 +1259,19 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
      testcase( pExpr->op==TK_AGG_COLUMN );
      pWalker->u.i = 0;
      return WRC_Abort;
+    case TK_VARIABLE:
+      if( pWalker->u.i==4 ){
+        /* Silently convert bound parameters that appear inside of CREATE
+        ** statements into a NULL when parsing the CREATE statement text out
+        ** of the sqlite_master table */
+        pExpr->op = TK_NULL;
+      }else if( pWalker->u.i==3 ){
+        /* A bound parameter in a CREATE statement that originates from
+        ** sqlite3_prepare() causes an error */
+        pWalker->u.i = 0;
+        return WRC_Abort;
+      }
+      /* Fall through */
    default:
      testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */
      testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */
@@ -1290,7 +1312,7 @@ int sqlite3ExprIsConstant(Expr *p){
 ** an ON or USING clause.
 */
 int sqlite3ExprIsConstantNotJoin(Expr *p){
-  return exprIsConst(p, 3);
+  return exprIsConst(p, 2);
 }

 /*
@@ -1302,8 +1324,9 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){
 ** is considered a variable but a single-quoted string (ex: 'abc') is
 ** a constant.
 */
-int sqlite3ExprIsConstantOrFunction(Expr *p){
-  return exprIsConst(p, 2);
+int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){
+  assert( isInit==0 || isInit==1 );
+  return exprIsConst(p, 3+isInit);
 }

 /*
@@ -3213,90 +3236,86 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){
  exprToRegister(pExpr, iMem);
 }

-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
+#ifdef SQLITE_DEBUG
 /*
 ** Generate a human-readable explanation of an expression tree.
 */
-void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
-  int op;                   /* The opcode being coded */
+void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){
  const char *zBinOp = 0;   /* Binary operator */
  const char *zUniOp = 0;   /* Unary operator */
+  pView = sqlite3TreeViewPush(pView, moreToFollow);
  if( pExpr==0 ){
-    op = TK_NULL;
-  }else{
-    op = pExpr->op;
+    sqlite3TreeViewLine(pView, "nil");
+    sqlite3TreeViewPop(pView);
+    return;
  }
-  switch( op ){
+  switch( pExpr->op ){
    case TK_AGG_COLUMN: {
-      sqlite3ExplainPrintf(pOut, "AGG{%d:%d}",
+      sqlite3TreeViewLine(pView, "AGG{%d:%d}",
            pExpr->iTable, pExpr->iColumn);
      break;
    }
    case TK_COLUMN: {
      if( pExpr->iTable<0 ){
        /* This only happens when coding check constraints */
-        sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn);
+        sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn);
      }else{
-        sqlite3ExplainPrintf(pOut, "{%d:%d}",
+        sqlite3TreeViewLine(pView, "{%d:%d}",
                             pExpr->iTable, pExpr->iColumn);
      }
      break;
    }
    case TK_INTEGER: {
      if( pExpr->flags & EP_IntValue ){
-        sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue);
+        sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue);
      }else{
-        sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken);
+        sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken);
      }
      break;
    }
 #ifndef SQLITE_OMIT_FLOATING_POINT
    case TK_FLOAT: {
-      sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken);
+      sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken);
      break;
    }
 #endif
    case TK_STRING: {
-      sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken);
+      sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken);
      break;
    }
    case TK_NULL: {
-      sqlite3ExplainPrintf(pOut,"NULL");
+      sqlite3TreeViewLine(pView,"NULL");
      break;
    }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
    case TK_BLOB: {
-      sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken);
+      sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken);
      break;
    }
 #endif
    case TK_VARIABLE: {
-      sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)",
-                           pExpr->u.zToken, pExpr->iColumn);
+      sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)",
+                          pExpr->u.zToken, pExpr->iColumn);
      break;
    }
    case TK_REGISTER: {
-      sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable);
+      sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable);
      break;
    }
    case TK_AS: {
-      sqlite3ExplainExpr(pOut, pExpr->pLeft);
+      sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
+      break;
+    }
+    case TK_ID: {
+      sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken);
      break;
    }
 #ifndef SQLITE_OMIT_CAST
    case TK_CAST: {
      /* Expressions of the form:   CAST(pLeft AS token) */
-      const char *zAff = "unk";
-      switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){
-        case SQLITE_AFF_TEXT:    zAff = "TEXT";     break;
-        case SQLITE_AFF_NONE:    zAff = "NONE";     break;
-        case SQLITE_AFF_NUMERIC: zAff = "NUMERIC";  break;
-        case SQLITE_AFF_INTEGER: zAff = "INTEGER";  break;
-        case SQLITE_AFF_REAL:    zAff = "REAL";     break;
-      }
-      sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff);
-      sqlite3ExplainExpr(pOut, pExpr->pLeft);
-      sqlite3ExplainPrintf(pOut, ")");
+      sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
      break;
    }
 #endif /* SQLITE_OMIT_CAST */
@@ -3320,6 +3339,7 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
    case TK_LSHIFT:  zBinOp = "LSHIFT"; break;
    case TK_RSHIFT:  zBinOp = "RSHIFT"; break;
    case TK_CONCAT:  zBinOp = "CONCAT"; break;
+    case TK_DOT:     zBinOp = "DOT";    break;

    case TK_UMINUS:  zUniOp = "UMINUS"; break;
    case TK_UPLUS:   zUniOp = "UPLUS";  break;
@@ -3329,8 +3349,8 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
    case TK_NOTNULL: zUniOp = "NOTNULL"; break;

    case TK_COLLATE: {
-      sqlite3ExplainExpr(pOut, pExpr->pLeft);
-      sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken);
+      sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken);
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
      break;
    }

@@ -3342,41 +3362,36 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
      }else{
        pFarg = pExpr->x.pList;
      }
-      if( op==TK_AGG_FUNCTION ){
-        sqlite3ExplainPrintf(pOut, "AGG_FUNCTION%d:%s(",
+      if( pExpr->op==TK_AGG_FUNCTION ){
+        sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q",
                             pExpr->op2, pExpr->u.zToken);
      }else{
-        sqlite3ExplainPrintf(pOut, "FUNCTION:%s(", pExpr->u.zToken);
+        sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken);
      }
      if( pFarg ){
-        sqlite3ExplainExprList(pOut, pFarg);
+        sqlite3TreeViewExprList(pView, pFarg, 0, 0);
      }
-      sqlite3ExplainPrintf(pOut, ")");
      break;
    }
 #ifndef SQLITE_OMIT_SUBQUERY
    case TK_EXISTS: {
-      sqlite3ExplainPrintf(pOut, "EXISTS(");
-      sqlite3ExplainSelect(pOut, pExpr->x.pSelect);
-      sqlite3ExplainPrintf(pOut,")");
+      sqlite3TreeViewLine(pView, "EXISTS-expr");
+      sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0);
      break;
    }
    case TK_SELECT: {
-      sqlite3ExplainPrintf(pOut, "(");
-      sqlite3ExplainSelect(pOut, pExpr->x.pSelect);
-      sqlite3ExplainPrintf(pOut, ")");
+      sqlite3TreeViewLine(pView, "SELECT-expr");
+      sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0);
      break;
    }
    case TK_IN: {
-      sqlite3ExplainPrintf(pOut, "IN(");
-      sqlite3ExplainExpr(pOut, pExpr->pLeft);
-      sqlite3ExplainPrintf(pOut, ",");
+      sqlite3TreeViewLine(pView, "IN");
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 1);
      if( ExprHasProperty(pExpr, EP_xIsSelect) ){
-        sqlite3ExplainSelect(pOut, pExpr->x.pSelect);
+        sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0);
      }else{
-        sqlite3ExplainExprList(pOut, pExpr->x.pList);
+        sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0);
      }
-      sqlite3ExplainPrintf(pOut, ")");
      break;
    }
 #endif /* SQLITE_OMIT_SUBQUERY */
@@ -3396,13 +3411,10 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
      Expr *pX = pExpr->pLeft;
      Expr *pY = pExpr->x.pList->a[0].pExpr;
      Expr *pZ = pExpr->x.pList->a[1].pExpr;
-      sqlite3ExplainPrintf(pOut, "BETWEEN(");
-      sqlite3ExplainExpr(pOut, pX);
-      sqlite3ExplainPrintf(pOut, ",");
-      sqlite3ExplainExpr(pOut, pY);
-      sqlite3ExplainPrintf(pOut, ",");
-      sqlite3ExplainExpr(pOut, pZ);
-      sqlite3ExplainPrintf(pOut, ")");
+      sqlite3TreeViewLine(pView, "BETWEEN");
+      sqlite3TreeViewExpr(pView, pX, 1);
+      sqlite3TreeViewExpr(pView, pY, 1);
+      sqlite3TreeViewExpr(pView, pZ, 0);
      break;
    }
    case TK_TRIGGER: {
@@ -3413,15 +3425,14 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
      ** is set to the column of the pseudo-table to read, or to -1 to
      ** read the rowid field.
      */
-      sqlite3ExplainPrintf(pOut, "%s(%d)", 
+      sqlite3TreeViewLine(pView, "%s(%d)", 
          pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn);
      break;
    }
    case TK_CASE: {
-      sqlite3ExplainPrintf(pOut, "CASE(");
-      sqlite3ExplainExpr(pOut, pExpr->pLeft);
-      sqlite3ExplainPrintf(pOut, ",");
-      sqlite3ExplainExprList(pOut, pExpr->x.pList);
+      sqlite3TreeViewLine(pView, "CASE");
+      sqlite3TreeViewExpr(pView, pExpr->pLeft, 1);
+      sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0);
      break;
    }
 #ifndef SQLITE_OMIT_TRIGGER
@@ -3433,55 +3444,57 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
        case OE_Fail:       zType = "fail";      break;
        case OE_Ignore:     zType = "ignore";    break;
      }
-      sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken);
+      sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken);
      break;
    }
 #endif
+    default: {
+      sqlite3TreeViewLine(pView, "op=%d", pExpr->op);
+      break;
+    }
  }
  if( zBinOp ){
-    sqlite3ExplainPrintf(pOut,"%s(", zBinOp);
-    sqlite3ExplainExpr(pOut, pExpr->pLeft);
-    sqlite3ExplainPrintf(pOut,",");
-    sqlite3ExplainExpr(pOut, pExpr->pRight);
-    sqlite3ExplainPrintf(pOut,")");
+    sqlite3TreeViewLine(pView, "%s", zBinOp);
+    sqlite3TreeViewExpr(pView, pExpr->pLeft, 1);
+    sqlite3TreeViewExpr(pView, pExpr->pRight, 0);
  }else if( zUniOp ){
-    sqlite3ExplainPrintf(pOut,"%s(", zUniOp);
-    sqlite3ExplainExpr(pOut, pExpr->pLeft);
-    sqlite3ExplainPrintf(pOut,")");
+    sqlite3TreeViewLine(pView, "%s", zUniOp);
+    sqlite3TreeViewExpr(pView, pExpr->pLeft, 0);
  }
+  sqlite3TreeViewPop(pView);
 }
-#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */
+#endif /* SQLITE_DEBUG */

-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
+#ifdef SQLITE_DEBUG
 /*
 ** Generate a human-readable explanation of an expression list.
 */
-void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){
+void sqlite3TreeViewExprList(
+  TreeView *pView,
+  const ExprList *pList,
+  u8 moreToFollow,
+  const char *zLabel
+){
  int i;
-  if( pList==0 || pList->nExpr==0 ){
-    sqlite3ExplainPrintf(pOut, "(empty-list)");
-    return;
-  }else if( pList->nExpr==1 ){
-    sqlite3ExplainExpr(pOut, pList->a[0].pExpr);
+  pView = sqlite3TreeViewPush(pView, moreToFollow);
+  if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST";
+  if( pList==0 ){
+    sqlite3TreeViewLine(pView, "%s (empty)", zLabel);
  }else{
-    sqlite3ExplainPush(pOut);
+    sqlite3TreeViewLine(pView, "%s", zLabel);
    for(i=0; i<pList->nExpr; i++){
-      sqlite3ExplainPrintf(pOut, "item[%d] = ", i);
-      sqlite3ExplainPush(pOut);
-      sqlite3ExplainExpr(pOut, pList->a[i].pExpr);
-      sqlite3ExplainPop(pOut);
-      if( pList->a[i].zName ){
+      sqlite3TreeViewExpr(pView, pList->a[i].pExpr, i<pList->nExpr-1);
+#if 0
+     if( pList->a[i].zName ){
        sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName);
      }
      if( pList->a[i].bSpanIsTab ){
        sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan);
      }
-      if( i<pList->nExpr-1 ){
-        sqlite3ExplainNL(pOut);
-      }
+#endif
    }
-    sqlite3ExplainPop(pOut);
  }
+  sqlite3TreeViewPop(pView);
 }
 #endif /* SQLITE_DEBUG */

@@ -22,7 +22,10 @@
 ** Return the collating function associated with a function.
 */
 static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){
-  return context->pColl;
+  VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1];
+  assert( pOp->opcode==OP_CollSeq );
+  assert( pOp->p4type==P4_COLLSEQ );
+  return pOp->p4.pColl;
 }

 /*
@@ -567,10 +570,12 @@ struct compareInfo {
 ** whereas only characters less than 0x80 do in ASCII.
 */
 #if defined(SQLITE_EBCDIC)
-# define sqlite3Utf8Read(A)    (*((*A)++))
-# define GlobUpperToLower(A)   A = sqlite3UpperToLower[A]
+# define sqlite3Utf8Read(A)        (*((*A)++))
+# define GlobUpperToLower(A)       A = sqlite3UpperToLower[A]
+# define GlobUpperToLowerAscii(A)  A = sqlite3UpperToLower[A]
 #else
-# define GlobUpperToLower(A)   if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; }
+# define GlobUpperToLower(A)       if( A<=0x7f ){ A = sqlite3UpperToLower[A]; }
+# define GlobUpperToLowerAscii(A)  A = sqlite3UpperToLower[A]
 #endif

 static const struct compareInfo globInfo = { '*', '?', '[', 0 };
@@ -583,7 +588,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_',   0, 0 };

 /*
 ** Compare two UTF-8 strings for equality where the first string can
-** potentially be a "glob" expression.  Return true (1) if they
+** potentially be a "glob" or "like" expression.  Return true (1) if they
 ** are the same and false (0) if they are different.
 **
 ** Globbing rules:
@@ -603,11 +608,18 @@ static const struct compareInfo likeInfoAlt = { '%', '_',   0, 0 };
 ** "[a-z]" matches any single lower-case letter.  To match a '-', make
 ** it the last character in the list.
 **
+** Like matching rules:
+** 
+**      '%'       Matches any sequence of zero or more characters
+**
+***     '_'       Matches any one character
+**
+**      Ec        Where E is the "esc" character and c is any other
+**                character, including '%', '_', and esc, match exactly c.
+**
+** The comments through this routine usually assume glob matching.
+**
 ** This routine is usually quick, but can be N**2 in the worst case.
-**
-** Hints: to match '*' or '?', put them in "[]".  Like this:
-**
-**         abc[*]xyz        Matches "abc*xyz" only
 */
 static int patternCompare(
  const u8 *zPattern,              /* The glob pattern */
@@ -615,17 +627,25 @@ static int patternCompare(
  const struct compareInfo *pInfo, /* Information about how to do the compare */
  u32 esc                          /* The escape character */
 ){
-  u32 c, c2;
-  int invert;
-  int seen;
-  u8 matchOne = pInfo->matchOne;
-  u8 matchAll = pInfo->matchAll;
-  u8 matchSet = pInfo->matchSet;
-  u8 noCase = pInfo->noCase; 
-  int prevEscape = 0;     /* True if the previous character was 'escape' */
+  u32 c, c2;                       /* Next pattern and input string chars */
+  u32 matchOne = pInfo->matchOne;  /* "?" or "_" */
+  u32 matchAll = pInfo->matchAll;  /* "*" or "%" */
+  u32 matchOther;                  /* "[" or the escape character */
+  u8 noCase = pInfo->noCase;       /* True if uppercase==lowercase */
+  const u8 *zEscaped = 0;          /* One past the last escaped input char */
+  
+  /* The GLOB operator does not have an ESCAPE clause.  And LIKE does not
+  ** have the matchSet operator.  So we either have to look for one or
+  ** the other, never both.  Hence the single variable matchOther is used
+  ** to store the one we have to look for.
+  */
+  matchOther = esc ? esc : pInfo->matchSet;

  while( (c = sqlite3Utf8Read(&zPattern))!=0 ){
-    if( c==matchAll && !prevEscape ){
+    if( c==matchAll ){  /* Match "*" */
+      /* Skip over multiple "*" characters in the pattern.  If there
+      ** are also "?" characters, skip those as well, but consume a
+      ** single character of the input string for each "?" skipped */
      while( (c=sqlite3Utf8Read(&zPattern)) == matchAll
               || c == matchOne ){
        if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){
@@ -633,86 +653,98 @@ static int patternCompare(
        }
      }
      if( c==0 ){
-        return 1;
-      }else if( c==esc ){
-        c = sqlite3Utf8Read(&zPattern);
-        if( c==0 ){
-          return 0;
-        }
-      }else if( c==matchSet ){
-        assert( esc==0 );         /* This is GLOB, not LIKE */
-        assert( matchSet<0x80 );  /* '[' is a single-byte character */
-        while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){
-          SQLITE_SKIP_UTF8(zString);
-        }
-        return *zString!=0;
-      }
-      while( (c2 = sqlite3Utf8Read(&zString))!=0 ){
-        if( noCase ){
-          GlobUpperToLower(c2);
-          GlobUpperToLower(c);
-          while( c2 != 0 && c2 != c ){
-            c2 = sqlite3Utf8Read(&zString);
-            GlobUpperToLower(c2);
-          }
+        return 1;   /* "*" at the end of the pattern matches */
+      }else if( c==matchOther ){
+        if( esc ){
+          c = sqlite3Utf8Read(&zPattern);
+          if( c==0 ) return 0;
        }else{
-          while( c2 != 0 && c2 != c ){
-            c2 = sqlite3Utf8Read(&zString);
+          /* "[...]" immediately follows the "*".  We have to do a slow
+          ** recursive search in this case, but it is an unusual case. */
+          assert( matchOther<0x80 );  /* '[' is a single-byte character */
+          while( *zString
+                 && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){
+            SQLITE_SKIP_UTF8(zString);
          }
+          return *zString!=0;
+        }
+      }
+
+      /* At this point variable c contains the first character of the
+      ** pattern string past the "*".  Search in the input string for the
+      ** first matching character and recursively contine the match from
+      ** that point.
+      **
+      ** For a case-insensitive search, set variable cx to be the same as
+      ** c but in the other case and search the input string for either
+      ** c or cx.
+      */
+      if( c<=0x80 ){
+        u32 cx;
+        if( noCase ){
+          cx = sqlite3Toupper(c);
+          c = sqlite3Tolower(c);
+        }else{
+          cx = c;
+        }
+        while( (c2 = *(zString++))!=0 ){
+          if( c2!=c && c2!=cx ) continue;
+          if( patternCompare(zPattern,zString,pInfo,esc) ) return 1;
+        }
+      }else{
+        while( (c2 = sqlite3Utf8Read(&zString))!=0 ){
+          if( c2!=c ) continue;
+          if( patternCompare(zPattern,zString,pInfo,esc) ) return 1;
        }
-        if( c2==0 ) return 0;
-        if( patternCompare(zPattern,zString,pInfo,esc) ) return 1;
      }
      return 0;
-    }else if( c==matchOne && !prevEscape ){
-      if( sqlite3Utf8Read(&zString)==0 ){
-        return 0;
-      }
-    }else if( c==matchSet ){
-      u32 prior_c = 0;
-      assert( esc==0 );    /* This only occurs for GLOB, not LIKE */
-      seen = 0;
-      invert = 0;
-      c = sqlite3Utf8Read(&zString);
-      if( c==0 ) return 0;
-      c2 = sqlite3Utf8Read(&zPattern);
-      if( c2=='^' ){
-        invert = 1;
-        c2 = sqlite3Utf8Read(&zPattern);
-      }
-      if( c2==']' ){
-        if( c==']' ) seen = 1;
-        c2 = sqlite3Utf8Read(&zPattern);
-      }
-      while( c2 && c2!=']' ){
-        if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){
-          c2 = sqlite3Utf8Read(&zPattern);
-          if( c>=prior_c && c<=c2 ) seen = 1;
-          prior_c = 0;
-        }else{
-          if( c==c2 ){
-            seen = 1;
-          }
-          prior_c = c2;
-        }
-        c2 = sqlite3Utf8Read(&zPattern);
-      }
-      if( c2==0 || (seen ^ invert)==0 ){
-        return 0;
-      }
-    }else if( esc==c && !prevEscape ){
-      prevEscape = 1;
-    }else{
-      c2 = sqlite3Utf8Read(&zString);
-      if( noCase ){
-        GlobUpperToLower(c);
-        GlobUpperToLower(c2);
-      }
-      if( c!=c2 ){
-        return 0;
-      }
-      prevEscape = 0;
    }
+    if( c==matchOther ){
+      if( esc ){
+        c = sqlite3Utf8Read(&zPattern);
+        if( c==0 ) return 0;
+        zEscaped = zPattern;
+      }else{
+        u32 prior_c = 0;
+        int seen = 0;
+        int invert = 0;
+        c = sqlite3Utf8Read(&zString);
+        if( c==0 ) return 0;
+        c2 = sqlite3Utf8Read(&zPattern);
+        if( c2=='^' ){
+          invert = 1;
+          c2 = sqlite3Utf8Read(&zPattern);
+        }
+        if( c2==']' ){
+          if( c==']' ) seen = 1;
+          c2 = sqlite3Utf8Read(&zPattern);
+        }
+        while( c2 && c2!=']' ){
+          if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){
+            c2 = sqlite3Utf8Read(&zPattern);
+            if( c>=prior_c && c<=c2 ) seen = 1;
+            prior_c = 0;
+          }else{
+            if( c==c2 ){
+              seen = 1;
+            }
+            prior_c = c2;
+          }
+          c2 = sqlite3Utf8Read(&zPattern);
+        }
+        if( c2==0 || (seen ^ invert)==0 ){
+          return 0;
+        }
+        continue;
+      }
+    }
+    c2 = sqlite3Utf8Read(&zString);
+    if( c==c2 ) continue;
+    if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){
+      continue;
+    }
+    if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue;
+    return 0;
  }
  return *zString==0;
 }
@@ -129,6 +129,13 @@ const unsigned char sqlite3CtypeMap[256] = {
 };
 #endif

+/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards
+** compatibility for legacy applications, the URI filename capability is
+** disabled by default.
+**
+** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled
+** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options.
+*/
 #ifndef SQLITE_USE_URI
 # define  SQLITE_USE_URI 0
 #endif
@@ -476,6 +476,11 @@ int sqlite3_config(int op, ...){
      break;
    }

+    /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames
+    ** can be changed at start-time using the
+    ** sqlite3_config(SQLITE_CONFIG_URI,1) or
+    ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls.
+    */
    case SQLITE_CONFIG_URI: {
      sqlite3GlobalConfig.bOpenUri = va_arg(ap, int);
      break;
@@ -2213,7 +2218,7 @@ int sqlite3ParseUri(
  assert( *pzErrMsg==0 );

  if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) 
-   && nUri>=5 && memcmp(zUri, "file:", 5)==0 
+   && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */
  ){
    char *zOpt;
    int eState;                   /* Parser state when parsing URI */
@@ -2443,7 +2448,9 @@ static int openDatabase(
  testcase( (1<<(flags&7))==0x02 ); /* READONLY */
  testcase( (1<<(flags&7))==0x04 ); /* READWRITE */
  testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */
-  if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT;
+  if( ((1<<(flags&7)) & 0x46)==0 ){
+    return SQLITE_MISUSE_BKPT;  /* IMP: R-65497-44594 */
+  }

  if( sqlite3GlobalConfig.bCoreMutex==0 ){
    isThreadsafe = 0;
@@ -3327,22 +3334,6 @@ int sqlite3_test_control(int op, ...){
      break;
    }

-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
-    /*   sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT,
-    **                        sqlite3_stmt*,const char**);
-    **
-    ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds
-    ** a string that describes the optimized parse tree.  This test-control
-    ** returns a pointer to that string.
-    */
-    case SQLITE_TESTCTRL_EXPLAIN_STMT: {
-      sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*);
-      const char **pzRet = va_arg(ap, const char**);
-      *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt);
-      break;
-    }
-#endif
-
    /*   sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int);
    **
    ** Set or clear a flag that indicates that the database file is always well-
@@ -310,7 +310,7 @@ void *sqlite3Malloc(u64 n){
  }else{
    p = sqlite3GlobalConfig.m.xMalloc((int)n);
  }
-  assert( EIGHT_BYTE_ALIGNMENT(p) );  /* IMP: R-04675-44850 */
+  assert( EIGHT_BYTE_ALIGNMENT(p) );  /* IMP: R-11148-40995 */
  return p;
 }

@@ -447,25 +447,27 @@ static int isLookaside(sqlite3 *db, void *p){
 */
 int sqlite3MallocSize(void *p){
  assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
-  assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) );
  return sqlite3GlobalConfig.m.xSize(p);
 }
 int sqlite3DbMallocSize(sqlite3 *db, void *p){
  if( db==0 ){
+    assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) );
+    assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
    return sqlite3MallocSize(p);
  }else{
    assert( sqlite3_mutex_held(db->mutex) );
    if( isLookaside(db, p) ){
      return db->lookaside.sz;
    }else{
-      assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) );
-      assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) );
-      assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) );
+      assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+      assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
      return sqlite3GlobalConfig.m.xSize(p);
    }
  }
 }
 sqlite3_uint64 sqlite3_msize(void *p){
+  assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) );
+  assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
  return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p);
 }

@@ -474,8 +476,8 @@ sqlite3_uint64 sqlite3_msize(void *p){
 */
 void sqlite3_free(void *p){
  if( p==0 ) return;  /* IMP: R-49053-54554 */
-  assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) );
  assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
+  assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) );
  if( sqlite3GlobalConfig.bMemstat ){
    sqlite3_mutex_enter(mem0.mutex);
    sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p));
@@ -519,8 +521,8 @@ void sqlite3DbFree(sqlite3 *db, void *p){
      return;
    }
  }
-  assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) );
-  assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) );
+  assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+  assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
  assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) );
  sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
  sqlite3_free(p);
@@ -532,11 +534,13 @@ void sqlite3DbFree(sqlite3 *db, void *p){
 void *sqlite3Realloc(void *pOld, u64 nBytes){
  int nOld, nNew, nDiff;
  void *pNew;
+  assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) );
+  assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) );
  if( pOld==0 ){
-    return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */
+    return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */
  }
  if( nBytes==0 ){
-    sqlite3_free(pOld); /* IMP: R-31593-10574 */
+    sqlite3_free(pOld); /* IMP: R-26507-47431 */
    return 0;
  }
  if( nBytes>=0x7fffff00 ){
@@ -558,8 +562,6 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){
          mem0.alarmThreshold-nDiff ){
      sqlite3MallocAlarm(nDiff);
    }
-    assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) );
-    assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) );
    pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
    if( pNew==0 && mem0.alarmCallback ){
      sqlite3MallocAlarm((int)nBytes);
@@ -573,7 +575,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){
  }else{
    pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
  }
-  assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */
+  assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */
  return pNew;
 }

@@ -585,7 +587,7 @@ void *sqlite3_realloc(void *pOld, int n){
 #ifndef SQLITE_OMIT_AUTOINIT
  if( sqlite3_initialize() ) return 0;
 #endif
-  if( n<0 ) n = 0;
+  if( n<0 ) n = 0;  /* IMP: R-26507-47431 */
  return sqlite3Realloc(pOld, n);
 }
 void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){
@@ -672,8 +674,8 @@ void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){
  if( !p && db ){
    db->mallocFailed = 1;
  }
-  sqlite3MemdebugSetType(p, MEMTYPE_DB |
-         ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP));
+  sqlite3MemdebugSetType(p, 
+         (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP);
  return p;
 }

@@ -699,15 +701,14 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){
        sqlite3DbFree(db, p);
      }
    }else{
-      assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) );
-      assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) );
+      assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+      assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
      sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
      pNew = sqlite3_realloc64(p, n);
      if( !pNew ){
-        sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP);
        db->mallocFailed = 1;
      }
-      sqlite3MemdebugSetType(pNew, MEMTYPE_DB | 
+      sqlite3MemdebugSetType(pNew,
            (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP));
    }
  }
@@ -394,7 +394,7 @@ void sqlite3MemdebugSetType(void *p, u8 eType){
 ** This routine is designed for use within an assert() statement, to
 ** verify the type of an allocation.  For example:
 **
-**     assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) );
+**     assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
 */
 int sqlite3MemdebugHasType(void *p, u8 eType){
  int rc = 1;
@@ -416,7 +416,7 @@ int sqlite3MemdebugHasType(void *p, u8 eType){
 ** This routine is designed for use within an assert() statement, to
 ** verify the type of an allocation.  For example:
 **
-**     assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) );
+**     assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) );
 */
 int sqlite3MemdebugNoType(void *p, u8 eType){
  int rc = 1;
@@ -4951,7 +4951,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){
 **   *  An I/O method finder function called FINDER that returns a pointer
 **      to the METHOD object in the previous bullet.
 */
-#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK)      \
+#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \
 static const sqlite3_io_methods METHOD = {                                   \
   VERSION,                    /* iVersion */                                \
   CLOSE,                      /* xClose */                                  \
@@ -4966,7 +4966,7 @@ static const sqlite3_io_methods METHOD = {                                   \
   unixFileControl,            /* xFileControl */                            \
   unixSectorSize,             /* xSectorSize */                             \
   unixDeviceCharacteristics,  /* xDeviceCapabilities */                     \
-   unixShmMap,                 /* xShmMap */                                 \
+   SHMMAP,                     /* xShmMap */                                 \
   unixShmLock,                /* xShmLock */                                \
   unixShmBarrier,             /* xShmBarrier */                             \
   unixShmUnmap,               /* xShmUnmap */                               \
@@ -4992,7 +4992,8 @@ IOMETHODS(
  unixClose,                /* xClose method */
  unixLock,                 /* xLock method */
  unixUnlock,               /* xUnlock method */
-  unixCheckReservedLock     /* xCheckReservedLock method */
+  unixCheckReservedLock,    /* xCheckReservedLock method */
+  unixShmMap                /* xShmMap method */
 )
 IOMETHODS(
  nolockIoFinder,           /* Finder function name */
@@ -5001,7 +5002,8 @@ IOMETHODS(
  nolockClose,              /* xClose method */
  nolockLock,               /* xLock method */
  nolockUnlock,             /* xUnlock method */
-  nolockCheckReservedLock   /* xCheckReservedLock method */
+  nolockCheckReservedLock,  /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )
 IOMETHODS(
  dotlockIoFinder,          /* Finder function name */
@@ -5010,7 +5012,8 @@ IOMETHODS(
  dotlockClose,             /* xClose method */
  dotlockLock,              /* xLock method */
  dotlockUnlock,            /* xUnlock method */
-  dotlockCheckReservedLock  /* xCheckReservedLock method */
+  dotlockCheckReservedLock, /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )

 #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS
@@ -5021,7 +5024,8 @@ IOMETHODS(
  flockClose,               /* xClose method */
  flockLock,                /* xLock method */
  flockUnlock,              /* xUnlock method */
-  flockCheckReservedLock    /* xCheckReservedLock method */
+  flockCheckReservedLock,   /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )
 #endif

@@ -5033,7 +5037,8 @@ IOMETHODS(
  semClose,                 /* xClose method */
  semLock,                  /* xLock method */
  semUnlock,                /* xUnlock method */
-  semCheckReservedLock      /* xCheckReservedLock method */
+  semCheckReservedLock,     /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )
 #endif

@@ -5045,7 +5050,8 @@ IOMETHODS(
  afpClose,                 /* xClose method */
  afpLock,                  /* xLock method */
  afpUnlock,                /* xUnlock method */
-  afpCheckReservedLock      /* xCheckReservedLock method */
+  afpCheckReservedLock,     /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )
 #endif

@@ -5070,7 +5076,8 @@ IOMETHODS(
  proxyClose,               /* xClose method */
  proxyLock,                /* xLock method */
  proxyUnlock,              /* xUnlock method */
-  proxyCheckReservedLock    /* xCheckReservedLock method */
+  proxyCheckReservedLock,   /* xCheckReservedLock method */
+  0                         /* xShmMap method */
 )
 #endif

@@ -5083,7 +5090,8 @@ IOMETHODS(
  unixClose,                 /* xClose method */
  unixLock,                  /* xLock method */
  nfsUnlock,                 /* xUnlock method */
-  unixCheckReservedLock      /* xCheckReservedLock method */
+  unixCheckReservedLock,     /* xCheckReservedLock method */
+  0                          /* xShmMap method */
 )
 #endif

@@ -943,7 +943,11 @@ static struct win_syscall {
 #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \
        DWORD))aSyscall[63].pCurrent)

+#if !SQLITE_OS_WINCE
  { "WaitForSingleObjectEx",   (SYSCALL)WaitForSingleObjectEx,   0 },
+#else
+  { "WaitForSingleObjectEx",   (SYSCALL)0,                       0 },
+#endif

 #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \
        BOOL))aSyscall[64].pCurrent)
@@ -1286,7 +1290,8 @@ void sqlite3_win32_sleep(DWORD milliseconds){
 #endif
 }

-#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0
+#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \
+        SQLITE_THREADSAFE>0
 DWORD sqlite3Win32Wait(HANDLE hObject){
  DWORD rc;
  while( (rc = osWaitForSingleObjectEx(hObject, INFINITE,
@@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
        rc = SQLITE_OK;
      }else{
        rc = sqlite3OsTruncate(pPager->jfd, 0);
+        if( rc==SQLITE_OK && pPager->fullSync ){
+          /* Make sure the new file size is written into the inode right away.
+          ** Otherwise the journal might resurrect following a power loss and
+          ** cause the last transaction to roll back.  See
+          ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773
+          */
+          rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags);
+        }
      }
      pPager->journalOff = 0;
    }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST
@@ -3618,13 +3626,15 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){

    if( rc==SQLITE_OK ){
      pager_reset(pPager);
-      sqlite3PageFree(pPager->pTmpSpace);
-      pPager->pTmpSpace = pNew;
      rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize);
    }
    if( rc==SQLITE_OK ){
+      sqlite3PageFree(pPager->pTmpSpace);
+      pPager->pTmpSpace = pNew;
      pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize);
      pPager->pageSize = pageSize;
+    }else{
+      sqlite3PageFree(pNew);
    }
  }

@@ -399,9 +399,6 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). {
 cmd ::= select(X).  {
  SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0};
  sqlite3Select(pParse, X, &dest);
-  sqlite3ExplainBegin(pParse->pVdbe);
-  sqlite3ExplainSelect(pParse->pVdbe, X);
-  sqlite3ExplainFinish(pParse->pVdbe);
  sqlite3SelectDelete(pParse->db, X);
 }

@@ -459,9 +456,33 @@ multiselect_op(A) ::= UNION(OP).             {A = @OP;}
 multiselect_op(A) ::= UNION ALL.             {A = TK_ALL;}
 multiselect_op(A) ::= EXCEPT|INTERSECT(OP).  {A = @OP;}
 %endif SQLITE_OMIT_COMPOUND_SELECT
-oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y)
+oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y)
                 groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). {
  A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset);
+#if SELECTTRACE_ENABLED
+  /* Populate the Select.zSelName[] string that is used to help with
+  ** query planner debugging, to differentiate between multiple Select
+  ** objects in a complex query.
+  **
+  ** If the SELECT keyword is immediately followed by a C-style comment
+  ** then extract the first few alphanumeric characters from within that
+  ** comment to be the zSelName value.  Otherwise, the label is #N where
+  ** is an integer that is incremented with each SELECT statement seen.
+  */
+  if( A!=0 ){
+    const char *z = S.z+6;
+    int i;
+    sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "#%d",
+                     ++pParse->nSelect);
+    while( z[0]==' ' ) z++;
+    if( z[0]=='/' && z[1]=='*' ){
+      z += 2;
+      while( z[0]==' ' ) z++;
+      for(i=0; sqlite3Isalnum(z[i]); i++){}
+      sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "%.*s", i, z);
+    }
+  }
+#endif /* SELECTRACE_ENABLED */
 }
 oneselect(A) ::= values(X).    {A = X;}

@@ -940,7 +961,7 @@ expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);}
  ** unary TK_ISNULL or TK_NOTNULL expression. */
  static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){
    sqlite3 *db = pParse->db;
-    if( db->mallocFailed==0 && pY->op==TK_NULL ){
+    if( pY && pA && pY->op==TK_NULL ){
      pA->op = (u8)op;
      sqlite3ExprDelete(db, pA->pRight);
      pA->pRight = 0;
@@ -688,7 +688,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2(
  if( createFlag==1 && (
        nPinned>=pGroup->mxPinned
     || nPinned>=pCache->n90pct
-     || pcache1UnderMemoryPressure(pCache)
+     || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclable<nPinned)
  )){
    return 0;
  }
@@ -1056,6 +1056,69 @@ void sqlite3DebugPrintf(const char *zFormat, ...){
 }
 #endif

+#ifdef SQLITE_DEBUG
+/*************************************************************************
+** Routines for implementing the "TreeView" display of hierarchical
+** data structures for debugging.
+**
+** The main entry points (coded elsewhere) are:
+**     sqlite3TreeViewExpr(0, pExpr, 0);
+**     sqlite3TreeViewExprList(0, pList, 0, 0);
+**     sqlite3TreeViewSelect(0, pSelect, 0);
+** Insert calls to those routines while debugging in order to display
+** a diagram of Expr, ExprList, and Select objects.
+**
+*/
+/* Add a new subitem to the tree.  The moreToFollow flag indicates that this
+** is not the last item in the tree. */
+TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){
+  if( p==0 ){
+    p = sqlite3_malloc( sizeof(*p) );
+    if( p==0 ) return 0;
+    memset(p, 0, sizeof(*p));
+  }else{
+    p->iLevel++;
+  }
+  assert( moreToFollow==0 || moreToFollow==1 );
+  if( p->iLevel<sizeof(p->bLine) ) p->bLine[p->iLevel] = moreToFollow;
+  return p;
+}
+/* Finished with one layer of the tree */
+void sqlite3TreeViewPop(TreeView *p){
+  if( p==0 ) return;
+  p->iLevel--;
+  if( p->iLevel<0 ) sqlite3_free(p);
+}
+/* Generate a single line of output for the tree, with a prefix that contains
+** all the appropriate tree lines */
+void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){
+  va_list ap;
+  int i;
+  StrAccum acc;
+  char zBuf[500];
+  sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0);
+  acc.useMalloc = 0;
+  if( p ){
+    for(i=0; i<p->iLevel && i<sizeof(p->bLine)-1; i++){
+      sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|   " : "    ", 4);
+    }
+    sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4);
+  }
+  va_start(ap, zFormat);
+  sqlite3VXPrintf(&acc, 0, zFormat, ap);
+  va_end(ap);
+  if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1);
+  sqlite3StrAccumFinish(&acc);
+  fprintf(stdout,"%s", zBuf);
+  fflush(stdout);
+}
+/* Shorthand for starting a new tree item that consists of a single label */
+void sqlite3TreeViewItem(TreeView *p, const char *zLabel, u8 moreToFollow){
+  p = sqlite3TreeViewPush(p, moreToFollow);
+  sqlite3TreeViewLine(p, "%s", zLabel);
+}
+#endif /* SQLITE_DEBUG */
+
 /*
 ** variable-argument wrapper around sqlite3VXPrintf().
 */
@@ -14,6 +14,20 @@
 */
 #include "sqliteInt.h"

+/*
+** Trace output macros
+*/
+#if SELECTTRACE_ENABLED
+/***/ int sqlite3SelectTrace = 0;
+# define SELECTTRACE(K,P,S,X)  \
+  if(sqlite3SelectTrace&(K))   \
+    sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\
+    sqlite3DebugPrintf X
+#else
+# define SELECTTRACE(K,P,S,X)
+#endif
+
+
 /*
 ** An instance of the following object is used to record information about
 ** how to process the DISTINCT keyword, to simplify passing that information
@@ -126,6 +140,18 @@ Select *sqlite3SelectNew(
  return pNew;
 }

+#if SELECTTRACE_ENABLED
+/*
+** Set the name of a Select object
+*/
+void sqlite3SelectSetName(Select *p, const char *zName){
+  if( p && zName ){
+    sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName);
+  }
+}
+#endif
+
+
 /*
 ** Delete the given Select structure and all of its substructures.
 */
@@ -1155,7 +1181,6 @@ static void generateSortTail(
  int nKey;
  int iSortTab;                   /* Sorter cursor to read from */
  int nSortData;                  /* Trailing values to read from sorter */
-  u8 p5;                          /* p5 parameter for 1st OP_Column */
  int i;
  int bSeq;                       /* True if sorter record includes seq. no. */
 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
@@ -1189,19 +1214,16 @@ static void generateSortTail(
    addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
    VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);
-    sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut);
-    p5 = OPFLAG_CLEARCACHE;
+    sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab);
    bSeq = 0;
  }else{
    addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
    codeOffset(v, p->iOffset, addrContinue);
    iSortTab = iTab;
-    p5 = 0;
    bSeq = 1;
  }
  for(i=0; i<nSortData; i++){
    sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq+i, regRow+i);
-    if( i==0 ) sqlite3VdbeChangeP5(v, p5);
    VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
  }
  switch( eDest ){
@@ -3355,6 +3377,8 @@ static int flattenSubquery(
  }

  /***** If we reach this point, flattening is permitted. *****/
+  SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n",
+                   pSub->zSelName, pSub, iFrom));

  /* Authorize the subquery */
  pParse->zAuthContext = pSubitem->zName;
@@ -3407,6 +3431,7 @@ static int flattenSubquery(
    p->pLimit = 0;
    p->pOffset = 0;
    pNew = sqlite3SelectDup(db, p, 0);
+    sqlite3SelectSetName(pNew, pSub->zSelName);
    p->pOffset = pOffset;
    p->pLimit = pLimit;
    p->pOrderBy = pOrderBy;
@@ -3419,6 +3444,9 @@ static int flattenSubquery(
      if( pPrior ) pPrior->pNext = pNew;
      pNew->pNext = p;
      p->pPrior = pNew;
+      SELECTTRACE(2,pParse,p,
+         ("compound-subquery flattener creates %s.%p as peer\n",
+         pNew->zSelName, pNew));
    }
    if( db->mallocFailed ) return 1;
  }
@@ -3548,8 +3576,23 @@ static int flattenSubquery(
      pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList);
    }
    if( pSub->pOrderBy ){
+      /* At this point, any non-zero iOrderByCol values indicate that the
+      ** ORDER BY column expression is identical to the iOrderByCol'th
+      ** expression returned by SELECT statement pSub. Since these values
+      ** do not necessarily correspond to columns in SELECT statement pParent,
+      ** zero them before transfering the ORDER BY clause.
+      **
+      ** Not doing this may cause an error if a subsequent call to this
+      ** function attempts to flatten a compound sub-query into pParent
+      ** (the only way this can happen is if the compound sub-query is
+      ** currently part of pSub->pSrc). See ticket [d11a6e908f].  */
+      ExprList *pOrderBy = pSub->pOrderBy;
+      for(i=0; i<pOrderBy->nExpr; i++){
+        pOrderBy->a[i].u.x.iOrderByCol = 0;
+      }
      assert( pParent->pOrderBy==0 );
-      pParent->pOrderBy = pSub->pOrderBy;
+      assert( pSub->pPrior==0 );
+      pParent->pOrderBy = pOrderBy;
      pSub->pOrderBy = 0;
    }else if( pParent->pOrderBy ){
      substExprList(db, pParent->pOrderBy, iParent, pSub->pEList);
@@ -3595,6 +3638,13 @@ static int flattenSubquery(
  */
  sqlite3SelectDelete(db, pSub1);

+#if SELECTTRACE_ENABLED
+  if( sqlite3SelectTrace & 0x100 ){
+    sqlite3DebugPrintf("After flattening:\n");
+    sqlite3TreeViewSelect(0, p, 0);
+  }
+#endif
+
  return 1;
 }
 #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
@@ -4065,6 +4115,7 @@ static int selectExpander(Walker *pWalker, Select *p){
        if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort;
        assert( pFrom->pSelect==0 );
        pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0);
+        sqlite3SelectSetName(pFrom->pSelect, pTab->zName);
        sqlite3WalkSelect(pWalker, pFrom->pSelect);
      }
 #endif
@@ -4599,6 +4650,13 @@ int sqlite3Select(
  }
  if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1;
  memset(&sAggInfo, 0, sizeof(sAggInfo));
+#if SELECTTRACE_ENABLED
+  pParse->nSelectIndent++;
+  SELECTTRACE(1,pParse,p, ("begin processing:\n"));
+  if( sqlite3SelectTrace & 0x100 ){
+    sqlite3TreeViewSelect(0, p, 0);
+  }
+#endif

  assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
@@ -4755,6 +4813,10 @@ int sqlite3Select(
  if( p->pPrior ){
    rc = multiSelect(pParse, p, pDest);
    explainSetInteger(pParse->iSelectId, iRestoreSelectId);
+#if SELECTTRACE_ENABLED
+    SELECTTRACE(1,pParse,p,("end compound-select processing\n"));
+    pParse->nSelectIndent--;
+#endif
    return rc;
  }
 #endif
@@ -5090,12 +5152,11 @@ int sqlite3Select(
      addrTopOfLoop = sqlite3VdbeCurrentAddr(v);
      sqlite3ExprCacheClear(pParse);
      if( groupBySort ){
-        sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut);
+        sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut,sortPTab);
      }
      for(j=0; j<pGroupBy->nExpr; j++){
        if( groupBySort ){
          sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j);
-          if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE);
        }else{
          sAggInfo.directMode = 1;
          sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j);
@@ -5354,100 +5415,103 @@ select_end:

  sqlite3DbFree(db, sAggInfo.aCol);
  sqlite3DbFree(db, sAggInfo.aFunc);
+#if SELECTTRACE_ENABLED
+  SELECTTRACE(1,pParse,p,("end processing\n"));
+  pParse->nSelectIndent--;
+#endif
  return rc;
 }

-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
+#ifdef SQLITE_DEBUG
 /*
 ** Generate a human-readable description of a the Select object.
 */
-static void explainOneSelect(Vdbe *pVdbe, Select *p){
-  sqlite3ExplainPrintf(pVdbe, "SELECT ");
-  if( p->selFlags & (SF_Distinct|SF_Aggregate) ){
-    if( p->selFlags & SF_Distinct ){
-      sqlite3ExplainPrintf(pVdbe, "DISTINCT ");
-    }
-    if( p->selFlags & SF_Aggregate ){
-      sqlite3ExplainPrintf(pVdbe, "agg_flag ");
-    }
-    sqlite3ExplainNL(pVdbe);
-    sqlite3ExplainPrintf(pVdbe, "   ");
-  }
-  sqlite3ExplainExprList(pVdbe, p->pEList);
-  sqlite3ExplainNL(pVdbe);
+void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
+  int n = 0;
+  pView = sqlite3TreeViewPush(pView, moreToFollow);
+  sqlite3TreeViewLine(pView, "SELECT%s%s",
+    ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
+    ((p->selFlags & SF_Aggregate) ? " agg_flag" : "")
+  );
+  if( p->pSrc && p->pSrc->nSrc ) n++;
+  if( p->pWhere ) n++;
+  if( p->pGroupBy ) n++;
+  if( p->pHaving ) n++;
+  if( p->pOrderBy ) n++;
+  if( p->pLimit ) n++;
+  if( p->pOffset ) n++;
+  if( p->pPrior ) n++;
+  sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set");
  if( p->pSrc && p->pSrc->nSrc ){
    int i;
-    sqlite3ExplainPrintf(pVdbe, "FROM ");
-    sqlite3ExplainPush(pVdbe);
+    pView = sqlite3TreeViewPush(pView, (n--)>0);
+    sqlite3TreeViewLine(pView, "FROM");
    for(i=0; i<p->pSrc->nSrc; i++){
      struct SrcList_item *pItem = &p->pSrc->a[i];
-      sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor);
-      if( pItem->pSelect ){
-        sqlite3ExplainSelect(pVdbe, pItem->pSelect);
-        if( pItem->pTab ){
-          sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName);
-        }
+      StrAccum x;
+      char zLine[100];
+      sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0);
+      sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor);
+      if( pItem->zDatabase ){
+        sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName);
      }else if( pItem->zName ){
-        sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName);
+        sqlite3XPrintf(&x, 0, " %s", pItem->zName);
+      }
+      if( pItem->pTab ){
+        sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName);
      }
      if( pItem->zAlias ){
-        sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias);
+        sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias);
      }
      if( pItem->jointype & JT_LEFT ){
-        sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN");
+        sqlite3XPrintf(&x, 0, " LEFT-JOIN");
      }
-      sqlite3ExplainNL(pVdbe);
+      sqlite3StrAccumFinish(&x);
+      sqlite3TreeViewItem(pView, zLine, i<p->pSrc->nSrc-1); 
+      if( pItem->pSelect ){
+        sqlite3TreeViewSelect(pView, pItem->pSelect, 0);
+      }
+      sqlite3TreeViewPop(pView);
    }
-    sqlite3ExplainPop(pVdbe);
+    sqlite3TreeViewPop(pView);
  }
  if( p->pWhere ){
-    sqlite3ExplainPrintf(pVdbe, "WHERE ");
-    sqlite3ExplainExpr(pVdbe, p->pWhere);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewItem(pView, "WHERE", (n--)>0);
+    sqlite3TreeViewExpr(pView, p->pWhere, 0);
+    sqlite3TreeViewPop(pView);
  }
  if( p->pGroupBy ){
-    sqlite3ExplainPrintf(pVdbe, "GROUPBY ");
-    sqlite3ExplainExprList(pVdbe, p->pGroupBy);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY");
  }
  if( p->pHaving ){
-    sqlite3ExplainPrintf(pVdbe, "HAVING ");
-    sqlite3ExplainExpr(pVdbe, p->pHaving);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewItem(pView, "HAVING", (n--)>0);
+    sqlite3TreeViewExpr(pView, p->pHaving, 0);
+    sqlite3TreeViewPop(pView);
  }
  if( p->pOrderBy ){
-    sqlite3ExplainPrintf(pVdbe, "ORDERBY ");
-    sqlite3ExplainExprList(pVdbe, p->pOrderBy);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY");
  }
  if( p->pLimit ){
-    sqlite3ExplainPrintf(pVdbe, "LIMIT ");
-    sqlite3ExplainExpr(pVdbe, p->pLimit);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewItem(pView, "LIMIT", (n--)>0);
+    sqlite3TreeViewExpr(pView, p->pLimit, 0);
+    sqlite3TreeViewPop(pView);
  }
  if( p->pOffset ){
-    sqlite3ExplainPrintf(pVdbe, "OFFSET ");
-    sqlite3ExplainExpr(pVdbe, p->pOffset);
-    sqlite3ExplainNL(pVdbe);
+    sqlite3TreeViewItem(pView, "OFFSET", (n--)>0);
+    sqlite3TreeViewExpr(pView, p->pOffset, 0);
+    sqlite3TreeViewPop(pView);
  }
+  if( p->pPrior ){
+    const char *zOp = "UNION";
+    switch( p->op ){
+      case TK_ALL:         zOp = "UNION ALL";  break;
+      case TK_INTERSECT:   zOp = "INTERSECT";  break;
+      case TK_EXCEPT:      zOp = "EXCEPT";     break;
+    }
+    sqlite3TreeViewItem(pView, zOp, (n--)>0);
+    sqlite3TreeViewSelect(pView, p->pPrior, 0);
+    sqlite3TreeViewPop(pView);
+  }
+  sqlite3TreeViewPop(pView);
 }
-void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){
-  if( p==0 ){
-    sqlite3ExplainPrintf(pVdbe, "(null-select)");
-    return;
-  }
-  sqlite3ExplainPush(pVdbe);
-  while( p ){
-    explainOneSelect(pVdbe, p);
-    p = p->pNext;
-    if( p==0 ) break;
-    sqlite3ExplainNL(pVdbe);
-    sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op));
-  }
-  sqlite3ExplainPrintf(pVdbe, "END");
-  sqlite3ExplainPop(pVdbe);
-}
-
-/* End of the structure debug printing code
-*****************************************************************************/
-#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */
+#endif /* SQLITE_DEBUG */
@@ -457,6 +457,7 @@ struct ShellState {
  int echoOn;            /* True to echo input commands */
  int autoEQP;           /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */
  int statsOn;           /* True to display memory stats before each finalize */
+  int loopCountersOn;    /* True to display loop counters */
  int outCount;          /* Revert to stdout when reaching zero */
  int cnt;               /* Number of records displayed so far */
  FILE *out;             /* Write results here */
@@ -882,7 +883,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int
        }
        fprintf(p->out,"%s",p->newline);
      }
-      if( azArg>0 ){
+      if( nArg>0 ){
        for(i=0; i<nArg; i++){
          output_csv(p, azArg[i], i<nArg-1);
        }
@@ -1089,6 +1090,30 @@ static char *save_err_msg(
  return zErrMsg;
 }

+static void display_loop_counters(
+  sqlite3 *db,                /* Database to query */
+  ShellState *pArg            /* Pointer to ShellState */
+){
+#if !defined(SQLITE_DEBUG) || !defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  fprintf(pArg->out, 
+      "No loop counters. "
+      "Rebuild with SQLITE_DEBUG and SQLITE_ENABLE_LOOPCOUNTERS"
+  );
+#else
+  int i;
+  for(i=0; 1; i++){
+    int nTest;
+    int nVisit;
+    int iSub;
+    int iLoop;
+    const char *zLoop;
+    zLoop = sqlite3_stmt_loopcounter(pArg->pStmt, i, &iSub, &iLoop, &nTest, &nVisit);
+    if( zLoop==0 ) break;
+    fprintf(pArg->out, "loop %d.%d: (%d/%d) %s\n", iSub, iLoop, nVisit, nTest, zLoop);
+  }
+#endif
+}
+
 /*
 ** Display memory stats.
 */
@@ -1353,15 +1378,6 @@ static int shell_exec(
        sqlite3_free(zEQP);
      }

-      /* Output TESTCTRL_EXPLAIN text of requested */
-      if( pArg && pArg->mode==MODE_Explain ){
-        const char *zExplain = 0;
-        sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, pStmt, &zExplain);
-        if( zExplain && zExplain[0] ){
-          fprintf(pArg->out, "%s", zExplain);
-        }
-      }
-
      /* If the shell is currently in ".explain" mode, gather the extra
      ** data required to add indents to the output.*/
      if( pArg && pArg->mode==MODE_Explain ){
@@ -1432,6 +1448,11 @@ static int shell_exec(
        display_stats(db, pArg, 0);
      }

+      /* print loop-counters if required */
+      if( pArg && pArg->loopCountersOn ){
+        display_loop_counters(db, pArg);
+      }
+
      /* Finalize the statement just executed. If this fails, save a 
      ** copy of the error message. Otherwise, set zSql to point to the
      ** next statement to execute. */
@@ -2828,6 +2849,16 @@ static int do_meta_command(char *zLine, ShellState *p){
    }
  }else

+  if( c=='l' && strncmp(azArg[0], "loopcounters", n)==0 ){
+    if( nArg==2 ){
+      p->loopCountersOn = booleanValue(azArg[1]);
+    }else{
+      fprintf(stderr, "Usage: .loopcounters on|off\n");
+      rc = 1;
+    }
+  }else
+
+
  if( c=='m' && strncmp(azArg[0], "mode", n)==0 ){
    const char *zMode = nArg>=2 ? azArg[1] : "";
    int n2 = (int)strlen(zMode);
@@ -3101,6 +3132,15 @@ static int do_meta_command(char *zLine, ShellState *p){
    }
  }else

+
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE)
+  if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){
+    extern int sqlite3SelectTrace;
+    sqlite3SelectTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff;
+  }else
+#endif
+
+
 #ifdef SQLITE_DEBUG
  /* Undocumented commands for internal testing.  Subject to change
  ** without notice. */
@@ -3725,6 +3765,7 @@ static int process_input(ShellState *p, FILE *in){
  if( nSql ){
    if( !_all_whitespace(zSql) ){
      fprintf(stderr, "Error: incomplete SQL: %s\n", zSql);
+      errCnt++;
    }
    free(zSql);
  }
@@ -4139,6 +4180,8 @@ int main(int argc, char **argv){
      data.autoEQP = 1;
    }else if( strcmp(z,"-stats")==0 ){
      data.statsOn = 1;
+    }else if( strcmp(z,"-loopcounters")==0 ){
+      data.loopCountersOn = 1;
    }else if( strcmp(z,"-bail")==0 ){
      bail_on_error = 1;
    }else if( strcmp(z,"-version")==0 ){
@@ -2664,9 +2664,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** an English language description of the error following a failure of any
 ** of the sqlite3_open() routines.
 **
-** ^The default encoding for the database will be UTF-8 if
-** sqlite3_open() or sqlite3_open_v2() is called and
-** UTF-16 in the native byte order if sqlite3_open16() is used.
+** ^The default encoding will be UTF-8 for databases created using
+** sqlite3_open() or sqlite3_open_v2().  ^The default encoding for databases
+** created using sqlite3_open16() will be UTF-16 in the native byte order.
 **
 ** Whether or not an error occurs when it is opened, resources
 ** associated with the [database connection] handle should be released by
@@ -2754,13 +2754,14 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** then it is interpreted as an absolute path. ^If the path does not begin 
 ** with a '/' (meaning that the authority section is omitted from the URI)
 ** then the path is interpreted as a relative path. 
-** ^On windows, the first component of an absolute path 
-** is a drive specification (e.g. "C:").
+** ^(On windows, the first component of an absolute path 
+** is a drive specification (e.g. "C:").)^
 **
 ** [[core URI query parameters]]
 ** The query component of a URI may contain parameters that are interpreted
 ** either by SQLite itself, or by a [VFS | custom VFS implementation].
-** SQLite interprets the following three query parameters:
+** SQLite and its built-in [VFSes] interpret the
+** following query parameters:
 **
 ** <ul>
 **   <li> <b>vfs</b>: ^The "vfs" parameter may be used to specify the name of
@@ -2795,11 +2796,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 **     a URI filename, its value overrides any behavior requested by setting
 **     SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag.
 **
-**  <li> <b>psow</b>: ^The psow parameter may be "true" (or "on" or "yes" or
-**     "1") or "false" (or "off" or "no" or "0") to indicate that the
+**  <li> <b>psow</b>: ^The psow parameter indicates whether or not the
 **     [powersafe overwrite] property does or does not apply to the
-**     storage media on which the database file resides.  ^The psow query
-**     parameter only works for the built-in unix and Windows VFSes.
+**     storage media on which the database file resides.
 **
 **  <li> <b>nolock</b>: ^The nolock parameter is a boolean query parameter
 **     which if set disables file locking in rollback journal modes.  This
@@ -3394,11 +3393,10 @@ typedef struct sqlite3_context sqlite3_context;
 ** contain embedded NULs.  The result of expressions involving strings
 ** with embedded NULs is undefined.
 **
-** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and
-** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or
+** ^The fifth argument to the BLOB and string binding interfaces
+** is a destructor used to dispose of the BLOB or
 ** string after SQLite has finished with it.  ^The destructor is called
-** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(),
-** sqlite3_bind_text(), or sqlite3_bind_text16() fails.  
+** to dispose of the BLOB or string even if the call to bind API fails.
 ** ^If the fifth argument is
 ** the special value [SQLITE_STATIC], then SQLite assumes that the
 ** information is in static, unmanaged space and does not need to be freed.
@@ -3409,7 +3407,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** ^The sixth argument to sqlite3_bind_text64() must be one of
 ** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]
 ** to specify the encoding of the text in the third parameter.  If
-** the sixth argument to sqlite3_bind_text64() is not how of the
+** the sixth argument to sqlite3_bind_text64() is not one of the
 ** allowed values shown above, or if the text encoding is different
 ** from the encoding specified by the sixth parameter, then the behavior
 ** is undefined.
@@ -4445,7 +4443,7 @@ typedef void (*sqlite3_destructor_type)(void*);
 ** of the application-defined function to be NULL.
 **
 ** ^The sqlite3_result_text(), sqlite3_result_text16(),
-** sqlite3_result_text16le(), and sqlite3_result_text16be()
+** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces
 ** set the return value of the application-defined function to be
 ** a text string which is represented as UTF-8, UTF-16 native byte order,
 ** UTF-16 little endian, or UTF-16 big endian, respectively.
@@ -6205,7 +6203,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_ISKEYWORD               16
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
-#define SQLITE_TESTCTRL_EXPLAIN_STMT            19
+#define SQLITE_TESTCTRL_EXPLAIN_STMT            19  /* NOT USED */
 #define SQLITE_TESTCTRL_NEVER_CORRUPT           20
 #define SQLITE_TESTCTRL_VDBE_COVERAGE           21
 #define SQLITE_TESTCTRL_BYTEORDER               22
@@ -7408,6 +7406,29 @@ int sqlite3_vtab_on_conflict(sqlite3 *);
 /* #define SQLITE_ABORT 4  // Also an error code */
 #define SQLITE_REPLACE  5

+/*
+** This API function is only available if SQLite is compiled with both 
+** SQLITE_DEBUG and SQLITE_ENABLE_LOOPCOUNTERS defined. It is primarily 
+** useful with SELECT statements.
+**
+** Each call returns information corresponding to a single "loop" run as
+** part of the statement. Each loop corresponds to a single "SEARCH" or "SCAN"
+** row that would be returned by EXPLAIN QUERY PLAN on the same SQL statement.
+** The second parameter - idx - determines which loop information is returned
+** regarding. If idx is less than 0 or greater than or equal to the number of
+** loops in the statement, NULL Is returned and the final values of the four
+** output parameters are undefined. Otherwise, the return value points to a
+** string describing the loop in question - the same string as would appear as
+** the fourth column of EXPLAIN QUERY PLAN output. The first two integer output
+** parameters are set to the values that would be returned as the first two
+** columns of the same row of EQP output.
+**
+** The third output parameter is set to the number of rows that were visited
+** by the loop the last time the statement was run. The fourth output parameter
+** is set to the number of rows visited including those excluded by non-indexed
+** WHERE terms.
+*/
+const char *sqlite3_stmt_loopcounter(sqlite3_stmt*, int idx, int*, int*, int*, int*);


 /*
@@ -159,7 +159,7 @@
 */
 #if defined(__GNUC__)
 #  define SQLITE_NOINLINE  __attribute__((noinline))
-#elif defined(_MSC_VER)
+#elif defined(_MSC_VER) && _MSC_VER>=1310
 #  define SQLITE_NOINLINE  __declspec(noinline)
 #else
 #  define SQLITE_NOINLINE
@@ -469,6 +469,11 @@
 #define MIN(A,B) ((A)<(B)?(A):(B))
 #define MAX(A,B) ((A)>(B)?(A):(B))

+/*
+** Swap two objects of type TYPE.
+*/
+#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
+
 /*
 ** Check to see if this machine uses EBCDIC.  (Yes, believe it or
 ** not, there are still machines out there that use EBCDIC.)
@@ -706,6 +711,16 @@ extern const int sqlite3one;
 # undef SQLITE_ENABLE_STAT3_OR_STAT4
 #endif

+/*
+** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not
+** the Select query generator tracing logic is turned on.
+*/
+#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE)
+# define SELECTTRACE_ENABLED 1
+#else
+# define SELECTTRACE_ENABLED 0
+#endif
+
 /*
 ** An instance of the following structure is used to store the busy-handler
 ** callback for a given sqlite handle. 
@@ -845,6 +860,7 @@ typedef struct StrAccum StrAccum;
 typedef struct Table Table;
 typedef struct TableLock TableLock;
 typedef struct Token Token;
+typedef struct TreeView TreeView;
 typedef struct Trigger Trigger;
 typedef struct TriggerPrg TriggerPrg;
 typedef struct TriggerStep TriggerStep;
@@ -1785,6 +1801,8 @@ struct Index {
  int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
  tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
  IndexSample *aSample;    /* Samples of the left-most key */
+  tRowcnt *aiRowEst;       /* Non-logarithmic stat1 data for this index */
+  tRowcnt nRowEst0;        /* Non-logarithmic number of rows in the index */
 #endif
 };

@@ -2215,7 +2233,7 @@ struct SrcList {
 #define WHERE_OMIT_OPEN_CLOSE  0x0010 /* Table cursors are already open */
 #define WHERE_FORCE_TABLE      0x0020 /* Do not use an index-only search */
 #define WHERE_ONETABLE_ONLY    0x0040 /* Only code the 1st table in pTabList */
-#define WHERE_AND_ONLY         0x0080 /* Don't use indices for OR terms */
+                          /*   0x0080 // not currently used */
 #define WHERE_GROUPBY          0x0100 /* pOrderBy is really a GROUP BY */
 #define WHERE_DISTINCTBY       0x0200 /* pOrderby is really a DISTINCT clause */
 #define WHERE_WANT_DISTINCT    0x0400 /* All output needs to be distinct */
@@ -2300,6 +2318,9 @@ struct Select {
  u8 op;                 /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */
  u16 selFlags;          /* Various SF_* values */
  int iLimit, iOffset;   /* Memory registers holding LIMIT & OFFSET counters */
+#if SELECTTRACE_ENABLED
+  char zSelName[12];     /* Symbolic name of this SELECT use for debugging */
+#endif
  int addrOpenEphm[2];   /* OP_OpenEphem opcodes related to this select */
  u64 nSelectRow;        /* Estimated number of result rows */
  SrcList *pSrc;         /* The FROM clause */
@@ -2558,6 +2579,10 @@ struct Parse {
  int regRowid;        /* Register holding rowid of CREATE TABLE entry */
  int regRoot;         /* Register holding root page number for new objects */
  int nMaxArg;         /* Max args passed to user function by sub-program */
+#if SELECTTRACE_ENABLED
+  int nSelect;         /* Number of SELECT statements seen */
+  int nSelectIndent;   /* How far to indent SELECTTRACE() output */
+#endif
 #ifndef SQLITE_OMIT_SHARED_CACHE
  int nTableLock;        /* Number of locks in aTableLock */
  TableLock *aTableLock; /* Required table locks for shared-cache mode */
@@ -2637,11 +2662,11 @@ struct AuthContext {
 ** Bitfield flags for P5 value in various opcodes.
 */
 #define OPFLAG_NCHANGE       0x01    /* Set to update db->nChange */
+#define OPFLAG_EPHEM         0x01    /* OP_Column: Ephemeral output is ok */
 #define OPFLAG_LASTROWID     0x02    /* Set to update db->lastRowid */
 #define OPFLAG_ISUPDATE      0x04    /* This OP_Insert is an sql UPDATE */
 #define OPFLAG_APPEND        0x08    /* This is likely to be an append */
 #define OPFLAG_USESEEKRESULT 0x10    /* Try to avoid a seek in BtreeInsert() */
-#define OPFLAG_CLEARCACHE    0x20    /* Clear pseudo-table cache in OP_Column */
 #define OPFLAG_LENGTHARG     0x40    /* OP_Column only used for length() */
 #define OPFLAG_TYPEOFARG     0x80    /* OP_Column only used for typeof() */
 #define OPFLAG_BULKCSR       0x01    /* OP_Open** used to open bulk cursor */
@@ -2905,6 +2930,17 @@ struct With {
  } a[1];
 };

+#ifdef SQLITE_DEBUG
+/*
+** An instance of the TreeView object is used for printing the content of
+** data structures on sqlite3DebugPrintf() using a tree-like view.
+*/
+struct TreeView {
+  int iLevel;             /* Which level of the tree we are on */
+  u8  bLine[100];         /* Draw vertical in column i if bLine[i] is true */
+};
+#endif /* SQLITE_DEBUG */
+
 /*
 ** Assuming zIn points to the first byte of a UTF-8 character,
 ** advance zIn to point to the first byte of the next UTF-8 character.
@@ -2970,6 +3006,7 @@ int sqlite3CantopenError(int);
 # define sqlite3Isxdigit(x)  isxdigit((unsigned char)(x))
 # define sqlite3Tolower(x)   tolower((unsigned char)(x))
 #endif
+int sqlite3IsIdChar(u8);

 /*
 ** Internal function prototypes
@@ -3068,25 +3105,14 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...);
  void *sqlite3TestTextToPtr(const char*);
 #endif

-/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */
-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
-  void sqlite3ExplainBegin(Vdbe*);
-  void sqlite3ExplainPrintf(Vdbe*, const char*, ...);
-  void sqlite3ExplainNL(Vdbe*);
-  void sqlite3ExplainPush(Vdbe*);
-  void sqlite3ExplainPop(Vdbe*);
-  void sqlite3ExplainFinish(Vdbe*);
-  void sqlite3ExplainSelect(Vdbe*, Select*);
-  void sqlite3ExplainExpr(Vdbe*, Expr*);
-  void sqlite3ExplainExprList(Vdbe*, ExprList*);
-  const char *sqlite3VdbeExplanation(Vdbe*);
-#else
-# define sqlite3ExplainBegin(X)
-# define sqlite3ExplainSelect(A,B)
-# define sqlite3ExplainExpr(A,B)
-# define sqlite3ExplainExprList(A,B)
-# define sqlite3ExplainFinish(X)
-# define sqlite3VdbeExplanation(X) 0
+#if defined(SQLITE_DEBUG)
+  TreeView *sqlite3TreeViewPush(TreeView*,u8);
+  void sqlite3TreeViewPop(TreeView*);
+  void sqlite3TreeViewLine(TreeView*, const char*, ...);
+  void sqlite3TreeViewItem(TreeView*, const char*, u8);
+  void sqlite3TreeViewExpr(TreeView*, const Expr*, u8);
+  void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*);
+  void sqlite3TreeViewSelect(TreeView*, const Select*, u8);
 #endif


@@ -3268,7 +3294,7 @@ void sqlite3CloseSavepoints(sqlite3 *);
 void sqlite3LeaveMutexAndCloseZombie(sqlite3*);
 int sqlite3ExprIsConstant(Expr*);
 int sqlite3ExprIsConstantNotJoin(Expr*);
-int sqlite3ExprIsConstantOrFunction(Expr*);
+int sqlite3ExprIsConstantOrFunction(Expr*, u8);
 int sqlite3ExprIsInteger(Expr*, int*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
@@ -3292,6 +3318,11 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
 SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
 IdList *sqlite3IdListDup(sqlite3*,IdList*);
 Select *sqlite3SelectDup(sqlite3*,Select*,int);
+#if SELECTTRACE_ENABLED
+void sqlite3SelectSetName(Select*,const char*);
+#else
+# define sqlite3SelectSetName(A,B)
+#endif
 void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*);
 FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8);
 void sqlite3RegisterBuiltinFunctions(sqlite3*);
@@ -3769,10 +3800,9 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...);
 # define sqlite3MemdebugNoType(X,Y)   1
 #endif
 #define MEMTYPE_HEAP       0x01  /* General heap allocations */
-#define MEMTYPE_LOOKASIDE  0x02  /* Might have been lookaside memory */
+#define MEMTYPE_LOOKASIDE  0x02  /* Heap that might have been lookaside */
 #define MEMTYPE_SCRATCH    0x04  /* Scratch allocations */
 #define MEMTYPE_PCACHE     0x08  /* Page cache allocations */
-#define MEMTYPE_DB         0x10  /* Uses sqlite3DbMalloc, not sqlite_malloc */

 /*
 ** Threading interface
@@ -213,7 +213,7 @@ int sqlite3_db_status(
      }
      db->pnBytesFreed = 0;

-      *pHighwater = 0;
+      *pHighwater = 0;  /* IMP: R-64479-57858 */
      *pCurrent = nByte;

      break;
@@ -238,7 +238,9 @@ int sqlite3_db_status(
          sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet);
        }
      }
-      *pHighwater = 0;
+      *pHighwater = 0; /* IMP: R-42420-56072 */
+                       /* IMP: R-54100-20147 */
+                       /* IMP: R-29431-39229 */
      *pCurrent = nRet;
      break;
    }
@@ -248,7 +250,7 @@ int sqlite3_db_status(
    ** have been satisfied.  The *pHighwater is always set to zero.
    */
    case SQLITE_DBSTATUS_DEFERRED_FKS: {
-      *pHighwater = 0;
+      *pHighwater = 0;  /* IMP: R-11967-56545 */
      *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0;
      break;
    }
@@ -5513,10 +5513,11 @@ static int test_limit(
    { "SQLITE_LIMIT_LIKE_PATTERN_LENGTH", SQLITE_LIMIT_LIKE_PATTERN_LENGTH  },
    { "SQLITE_LIMIT_VARIABLE_NUMBER",     SQLITE_LIMIT_VARIABLE_NUMBER      },
    { "SQLITE_LIMIT_TRIGGER_DEPTH",       SQLITE_LIMIT_TRIGGER_DEPTH        },
+    { "SQLITE_LIMIT_WORKER_THREADS",      SQLITE_LIMIT_WORKER_THREADS       },
    
    /* Out of range test cases */
    { "SQLITE_LIMIT_TOOSMALL",            -1,                               },
-    { "SQLITE_LIMIT_TOOBIG",              SQLITE_LIMIT_TRIGGER_DEPTH+1      },
+    { "SQLITE_LIMIT_TOOBIG",              SQLITE_LIMIT_WORKER_THREADS+1     },
  };
  int i, id;
  int val;
@@ -644,6 +644,7 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
  LINKVAR( DEFAULT_FILE_FORMAT );
  LINKVAR( MAX_ATTACHED );
  LINKVAR( MAX_DEFAULT_PAGE_SIZE );
+  LINKVAR( MAX_WORKER_THREADS );

  {
    static const int cv_TEMP_STORE = SQLITE_TEMP_STORE;
@@ -1002,6 +1002,26 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
      /* no-op these */
      rc = SQLITE_OK;
      break;
+    case SQLITE_FCNTL_PRAGMA: {
+      char **aFcntl = (char**)pArg;
+      if( aFcntl[1] && sqlite3_stricmp(aFcntl[1],"multiplex_truncate")==0 ){
+        if( aFcntl[2] && aFcntl[2][0] ){
+          if( sqlite3_stricmp(aFcntl[2], "on")==0
+           || sqlite3_stricmp(aFcntl[2], "1")==0 ){
+            pGroup->bTruncate = 1;
+          }else
+          if( sqlite3_stricmp(aFcntl[2], "off")==0
+           || sqlite3_stricmp(aFcntl[2], "0")==0 ){
+            pGroup->bTruncate = 0;
+          }
+        }
+        aFcntl[0] = sqlite3_mprintf(pGroup->bTruncate ? "on" : "off");
+        rc = SQLITE_OK;
+        break;
+      }
+      /* If the multiplexor does not handle the pragma, pass it through
+      ** into the default case. */
+    }
    default:
      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
      if( pSubOpen ){
@@ -98,14 +98,14 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){


 /********************************* Win32 Threads ****************************/
-#if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0
+#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0

 #define SQLITE_THREADS_IMPLEMENTED 1  /* Prevent the single-thread code below */
 #include <process.h>

 /* A running thread */
 struct SQLiteThread {
-  uintptr_t tid;           /* The thread handle */
+  void *tid;               /* The thread handle */
  unsigned id;             /* The thread identifier */
  void *(*xTask)(void*);   /* The routine to run as a thread */
  void *pIn;               /* Argument to xTask */
@@ -153,7 +153,7 @@ int sqlite3ThreadCreate(
  }else{
    p->xTask = xTask;
    p->pIn = pIn;
-    p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id);
+    p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id);
    if( p->tid==0 ){
      memset(p, 0, sizeof(*p));
    }
@@ -191,7 +191,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){
  return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR;
 }

-#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINRT */
+#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT */
 /******************************** End Win32 Threads *************************/


@@ -102,6 +102,7 @@ const char sqlite3IsEbcdicIdChar[] = {
 };
 #define IdChar(C)  (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
 #endif
+int sqlite3IsIdChar(u8 c){ return IdChar(c); }


 /*
@@ -214,6 +214,7 @@ static VdbeCursor *allocateCursor(
    memset(pCx, 0, sizeof(VdbeCursor));
    pCx->iDb = iDb;
    pCx->nField = nField;
+    pCx->aOffset = &pCx->aType[nField];
    if( isBtreeCursor ){
      pCx->pCursor = (BtCursor*)
          &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField];
@@ -604,6 +605,9 @@ int sqlite3VdbeExec(
    if( db->mallocFailed ) goto no_mem;
 #ifdef VDBE_PROFILE
    start = sqlite3Hwtime();
+#endif
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+    if( p->pFrame==0 ) p->anExec[pc]++;
 #endif
    nVmStep++;
    pOp = &aOp[pc];
@@ -1557,17 +1561,10 @@ case OP_Function: {
  ctx.iOp = pc;
  ctx.pVdbe = p;
  MemSetTypeFlag(ctx.pOut, MEM_Null);
-
  ctx.fErrorOrAux = 0;
-  if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
-    assert( pOp>aOp );
-    assert( pOp[-1].p4type==P4_COLLSEQ );
-    assert( pOp[-1].opcode==OP_CollSeq );
-    ctx.pColl = pOp[-1].p4.pColl;
-  }
  db->lastRowid = lastRowid;
  (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */
-  lastRowid = db->lastRowid;
+  lastRowid = db->lastRowid;  /* Remember rowid changes made by xFunc */

  /* If the function returned an error, throw an exception */
  if( ctx.fErrorOrAux ){
@@ -2283,7 +2280,7 @@ case OP_Column: {
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( p2<pC->nField );
-  aOffset = pC->aType + pC->nField;
+  aOffset = pC->aOffset;
 #ifndef SQLITE_OMIT_VIRTUALTABLE
  assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */
 #endif
@@ -2294,7 +2291,7 @@ case OP_Column: {
  /* If the cursor cache is stale, bring it up-to-date */
  rc = sqlite3VdbeCursorMoveto(pC);
  if( rc ) goto abort_due_to_error;
-  if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){
+  if( pC->cacheStatus!=p->cacheCtr ){
    if( pC->nullRow ){
      if( pCrsr==0 ){
        assert( pC->pseudoTableReg>0 );
@@ -2339,14 +2336,6 @@ case OP_Column: {
    pC->iHdrOffset = getVarint32(pC->aRow, offset);
    pC->nHdrParsed = 0;
    aOffset[0] = offset;
-    if( avail<offset ){
-      /* pC->aRow does not have to hold the entire row, but it does at least
-      ** need to cover the header of the record.  If pC->aRow does not contain
-      ** the complete header, then set it to zero, forcing the header to be
-      ** dynamically allocated. */
-      pC->aRow = 0;
-      pC->szRow = 0;
-    }

    /* Make sure a corrupt database has not given us an oversize header.
    ** Do this now to avoid an oversize memory allocation.
@@ -2361,6 +2350,22 @@ case OP_Column: {
      rc = SQLITE_CORRUPT_BKPT;
      goto op_column_error;
    }
+
+    if( avail<offset ){
+      /* pC->aRow does not have to hold the entire row, but it does at least
+      ** need to cover the header of the record.  If pC->aRow does not contain
+      ** the complete header, then set it to zero, forcing the header to be
+      ** dynamically allocated. */
+      pC->aRow = 0;
+      pC->szRow = 0;
+    }
+
+    /* The following goto is an optimization.  It can be omitted and
+    ** everything will still work.  But OP_Column is measurably faster
+    ** by skipping the subsequent conditional, which is always true.
+    */
+    assert( pC->nHdrParsed<=p2 );         /* Conditional skipped */
+    goto op_column_read_header;
  }

  /* Make sure at least the first p2+1 entries of the header have been
@@ -2370,6 +2375,7 @@ case OP_Column: {
    /* If there is more header available for parsing in the record, try
    ** to extract additional fields up through the p2+1-th field 
    */
+    op_column_read_header:
    if( pC->iHdrOffset<aOffset[0] ){
      /* Make sure zData points to enough of the record to cover the header. */
      if( pC->aRow==0 ){
@@ -2414,15 +2420,16 @@ case OP_Column: {
        sMem.flags = MEM_Null;
      }
  
-      /* If we have read more header data than was contained in the header,
-      ** or if the end of the last field appears to be past the end of the
-      ** record, or if the end of the last field appears to be before the end
-      ** of the record (when all fields present), then we must be dealing 
-      ** with a corrupt database.
+      /* The record is corrupt if any of the following are true:
+      ** (1) the bytes of the header extend past the declared header size
+      **          (zHdr>zEndHdr)
+      ** (2) the entire header was used but not all data was used
+      **          (zHdr==zEndHdr && offset!=pC->payloadSize)
+      ** (3) the end of the data extends beyond the end of the record.
+      **          (offset > pC->payloadSize)
      */
-      if( (zHdr > zEndHdr)
+      if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize))
       || (offset > pC->payloadSize)
-       || (zHdr==zEndHdr && offset!=pC->payloadSize)
      ){
        rc = SQLITE_CORRUPT_BKPT;
        goto op_column_error;
@@ -2613,7 +2620,7 @@ case OP_MakeRecord: {
  pRec = pLast;
  do{
    assert( memIsValid(pRec) );
-    serial_type = sqlite3VdbeSerialType(pRec, file_format);
+    pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format);
    len = sqlite3VdbeSerialTypeLen(serial_type);
    if( pRec->flags & MEM_Zero ){
      if( nData ){
@@ -2662,7 +2669,7 @@ case OP_MakeRecord: {
  assert( pData0<=pLast );
  pRec = pData0;
  do{
-    serial_type = sqlite3VdbeSerialType(pRec, file_format);
+    serial_type = pRec->uTemp;
    i += putVarint32(&zNewRecord[i], serial_type);            /* serial type */
    j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */
  }while( (++pRec)<=pLast );
@@ -3287,10 +3294,6 @@ case OP_OpenWrite: {
  assert( OPFLAG_BULKCSR==BTREE_BULKLOAD );
  sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR));

-  /* Since it performs no memory allocation or IO, the only value that
-  ** sqlite3BtreeCursor() may return is SQLITE_OK. */
-  assert( rc==SQLITE_OK );
-
  /* Set the VdbeCursor.isTable variable. Previous versions of
  ** SQLite used to check if the root-page flags were sane at this point
  ** and report database corruption if they were not, but this check has
@@ -3565,7 +3568,6 @@ case OP_SeekGT: {       /* jump, in3 */
      applyNumericAffinity(pIn3, 0);
    }
    iKey = sqlite3VdbeIntValue(pIn3);
-    pC->rowidIsValid = 0;

    /* If the P3 value could not be converted into an integer without
    ** loss of information, then special processing is required... */
@@ -3601,13 +3603,10 @@ case OP_SeekGT: {       /* jump, in3 */
      }
    } 
    rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res);
+    pC->movetoTarget = iKey;  /* Used by OP_Delete */
    if( rc!=SQLITE_OK ){
      goto abort_due_to_error;
    }
-    if( res==0 ){
-      pC->rowidIsValid = 1;
-      pC->lastRowid = iKey;
-    }
  }else{
    nField = pOp->p4.i;
    assert( pOp->p4type==P4_INT32 );
@@ -3637,7 +3636,6 @@ case OP_SeekGT: {       /* jump, in3 */
    if( rc!=SQLITE_OK ){
      goto abort_due_to_error;
    }
-    pC->rowidIsValid = 0;
  }
  pC->deferredMoveto = 0;
  pC->cacheStatus = CACHE_STALE;
@@ -3649,7 +3647,6 @@ case OP_SeekGT: {       /* jump, in3 */
      res = 0;
      rc = sqlite3BtreeNext(pC->pCursor, &res);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
-      pC->rowidIsValid = 0;
    }else{
      res = 0;
    }
@@ -3659,7 +3656,6 @@ case OP_SeekGT: {       /* jump, in3 */
      res = 0;
      rc = sqlite3BtreePrevious(pC->pCursor, &res);
      if( rc!=SQLITE_OK ) goto abort_due_to_error;
-      pC->rowidIsValid = 0;
    }else{
      /* res might be negative because the table is empty.  Check to
      ** see if this is the case.
@@ -3696,7 +3692,6 @@ case OP_Seek: {    /* in2 */
  pC->nullRow = 0;
  pIn2 = &aMem[pOp->p2];
  pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
-  pC->rowidIsValid = 0;
  pC->deferredMoveto = 1;
  break;
 }
@@ -3882,15 +3877,13 @@ case OP_NotExists: {        /* jump, in3 */
  res = 0;
  iKey = pIn3->u.i;
  rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res);
-  pC->lastRowid = pIn3->u.i;
-  pC->rowidIsValid = res==0 ?1:0;
+  pC->movetoTarget = iKey;  /* Used by OP_Delete */
  pC->nullRow = 0;
  pC->cacheStatus = CACHE_STALE;
  pC->deferredMoveto = 0;
  VdbeBranchTaken(res!=0,2);
  if( res!=0 ){
    pc = pOp->p2 - 1;
-    assert( pC->rowidIsValid==0 );
  }
  pC->seekResult = res;
  break;
@@ -4024,32 +4017,20 @@ case OP_NewRowid: {           /* out2-prerelease */
      ** it finds one that is not previously used. */
      assert( pOp->p3==0 );  /* We cannot be in random rowid mode if this is
                             ** an AUTOINCREMENT table. */
-      /* on the first attempt, simply do one more than previous */
-      v = lastRowid;
-      v &= (MAX_ROWID>>1); /* ensure doesn't go negative */
-      v++; /* ensure non-zero */
      cnt = 0;
-      while(   ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v,
+      do{
+        sqlite3_randomness(sizeof(v), &v);
+        v &= (MAX_ROWID>>1); v++;  /* Ensure that v is greater than zero */
+      }while(  ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v,
                                                 0, &res))==SQLITE_OK)
            && (res==0)
-            && (++cnt<100)){
-        /* collision - try another random rowid */
-        sqlite3_randomness(sizeof(v), &v);
-        if( cnt<5 ){
-          /* try "small" random rowids for the initial attempts */
-          v &= 0xffffff;
-        }else{
-          v &= (MAX_ROWID>>1); /* ensure doesn't go negative */
-        }
-        v++; /* ensure non-zero */
-      }
+            && (++cnt<100));
      if( rc==SQLITE_OK && res==0 ){
        rc = SQLITE_FULL;   /* IMP: R-38219-53002 */
        goto abort_due_to_error;
      }
      assert( v>0 );  /* EV: R-40812-03570 */
    }
-    pC->rowidIsValid = 0;
    pC->deferredMoveto = 0;
    pC->cacheStatus = CACHE_STALE;
  }
@@ -4154,7 +4135,6 @@ case OP_InsertInt: {
                          pData->z, pData->n, nZero,
                          (pOp->p5 & OPFLAG_APPEND)!=0, seekResult
  );
-  pC->rowidIsValid = 0;
  pC->deferredMoveto = 0;
  pC->cacheStatus = CACHE_STALE;

@@ -4191,33 +4171,32 @@ case OP_InsertInt: {
 ** using OP_NotFound prior to invoking this opcode.
 */
 case OP_Delete: {
-  i64 iKey;
  VdbeCursor *pC;

  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
-  iKey = pC->lastRowid;      /* Only used for the update hook */
-
-  /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or
-  ** OP_Column on the same table without any intervening operations that
-  ** might move or invalidate the cursor.  Hence cursor pC is always pointing
-  ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation
-  ** below is always a no-op and cannot fail.  We will run it anyhow, though,
-  ** to guard against future changes to the code generator.
-  **/
  assert( pC->deferredMoveto==0 );
-  rc = sqlite3VdbeCursorMoveto(pC);
-  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;

+#ifdef SQLITE_DEBUG
+  /* The seek operation that positioned the cursor prior to OP_Delete will
+  ** have also set the pC->movetoTarget field to the rowid of the row that
+  ** is being deleted */
+  if( pOp->p4.z && pC->isTable ){
+    i64 iKey = 0;
+    sqlite3BtreeKeySize(pC->pCursor, &iKey);
+    assert( pC->movetoTarget==iKey ); 
+  }
+#endif
+ 
  rc = sqlite3BtreeDelete(pC->pCursor);
  pC->cacheStatus = CACHE_STALE;

  /* Invoke the update-hook if required. */
  if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
    db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
-                        db->aDb[pC->iDb].zName, pOp->p4.z, iKey);
+                        db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
    assert( pC->iDb>=0 );
  }
  if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++;
@@ -4270,10 +4249,17 @@ case OP_SorterCompare: {
  break;
 };

-/* Opcode: SorterData P1 P2 * * *
+/* Opcode: SorterData P1 P2 P3 * *
 ** Synopsis: r[P2]=data
 **
 ** Write into register P2 the current sorter data for sorter cursor P1.
+** Then clear the column header cache on cursor P3.
+**
+** This opcode is normally use to move a record out of the sorter and into
+** a register that is the source for a pseudo-table cursor created using
+** OpenPseudo.  That pseudo-table cursor is the one that is identified by
+** parameter P3.  Clearing the P3 column cache as part of this opcode saves
+** us from having to issue a separate NullRow instruction to clear that cache.
 */
 case OP_SorterData: {
  VdbeCursor *pC;
@@ -4283,6 +4269,8 @@ case OP_SorterData: {
  assert( isSorter(pC) );
  rc = sqlite3VdbeSorterRowkey(pC, pOut);
  assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) );
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE;
  break;
 }

@@ -4329,16 +4317,20 @@ case OP_RowData: {
  assert( pC->pseudoTableReg==0 );
  assert( pC->pCursor!=0 );
  pCrsr = pC->pCursor;
-  assert( sqlite3BtreeCursorIsValid(pCrsr) );

  /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or
  ** OP_Rewind/Op_Next with no intervening instructions that might invalidate
-  ** the cursor.  Hence the following sqlite3VdbeCursorMoveto() call is always
-  ** a no-op and can never fail.  But we leave it in place as a safety.
+  ** the cursor.  If this where not the case, on of the following assert()s
+  ** would fail.  Should this ever change (because of changes in the code
+  ** generator) then the fix would be to insert a call to
+  ** sqlite3VdbeCursorMoveto().
  */
  assert( pC->deferredMoveto==0 );
+  assert( sqlite3BtreeCursorIsValid(pCrsr) );
+#if 0  /* Not required due to the previous to assert() statements */
  rc = sqlite3VdbeCursorMoveto(pC);
-  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
+  if( rc!=SQLITE_OK ) goto abort_due_to_error;
+#endif

  if( pC->isTable==0 ){
    assert( !pC->isTable );
@@ -4355,7 +4347,8 @@ case OP_RowData: {
      goto too_big;
    }
  }
-  if( sqlite3VdbeMemClearAndResize(pOut, n) ){
+  testcase( n==0 );
+  if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){
    goto no_mem;
  }
  pOut->n = n;
@@ -4406,14 +4399,10 @@ case OP_Rowid: {                 /* out2-prerelease */
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
  }else{
    assert( pC->pCursor!=0 );
-    rc = sqlite3VdbeCursorMoveto(pC);
+    rc = sqlite3VdbeCursorRestore(pC);
    if( rc ) goto abort_due_to_error;
-    if( pC->rowidIsValid ){
-      v = pC->lastRowid;
-    }else{
-      rc = sqlite3BtreeKeySize(pC->pCursor, &v);
-      assert( rc==SQLITE_OK );  /* Always so because of CursorMoveto() above */
-    }
+    rc = sqlite3BtreeKeySize(pC->pCursor, &v);
+    assert( rc==SQLITE_OK );  /* Always so because of CursorRestore() above */
  }
  pOut->u.i = v;
  break;
@@ -4432,7 +4421,6 @@ case OP_NullRow: {
  pC = p->apCsr[pOp->p1];
  assert( pC!=0 );
  pC->nullRow = 1;
-  pC->rowidIsValid = 0;
  pC->cacheStatus = CACHE_STALE;
  if( pC->pCursor ){
    sqlite3BtreeClearCursor(pC->pCursor);
@@ -4466,7 +4454,6 @@ case OP_Last: {        /* jump */
  rc = sqlite3BtreeLast(pCrsr, &res);
  pC->nullRow = (u8)res;
  pC->deferredMoveto = 0;
-  pC->rowidIsValid = 0;
  pC->cacheStatus = CACHE_STALE;
 #ifdef SQLITE_DEBUG
  pC->seekOp = OP_Last;
@@ -4533,7 +4520,6 @@ case OP_Rewind: {        /* jump */
    rc = sqlite3BtreeFirst(pCrsr, &res);
    pC->deferredMoveto = 0;
    pC->cacheStatus = CACHE_STALE;
-    pC->rowidIsValid = 0;
  }
  pC->nullRow = (u8)res;
  assert( pOp->p2>0 && pOp->p2<p->nOp );
@@ -4659,7 +4645,6 @@ next_tail:
  }else{
    pC->nullRow = 1;
  }
-  pC->rowidIsValid = 0;
  goto check_for_interrupt;
 }

@@ -4775,10 +4760,16 @@ case OP_IdxRowid: {              /* out2-prerelease */
  pCrsr = pC->pCursor;
  assert( pCrsr!=0 );
  pOut->flags = MEM_Null;
-  rc = sqlite3VdbeCursorMoveto(pC);
-  if( NEVER(rc) ) goto abort_due_to_error;
-  assert( pC->deferredMoveto==0 );
  assert( pC->isTable==0 );
+  assert( pC->deferredMoveto==0 );
+
+  /* sqlite3VbeCursorRestore() can only fail if the record has been deleted
+  ** out from under the cursor.  That will never happend for an IdxRowid
+  ** opcode, hence the NEVER() arround the check of the return value.
+  */
+  rc = sqlite3VdbeCursorRestore(pC);
+  if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error;
+
  if( !pC->nullRow ){
    rowid = 0;  /* Not needed.  Only used to silence a warning. */
    rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid);
@@ -5638,14 +5629,9 @@ case OP_AggStep: {
  sqlite3VdbeMemInit(&t, db, MEM_Null);
  ctx.pOut = &t;
  ctx.isError = 0;
-  ctx.pColl = 0;
+  ctx.pVdbe = p;
+  ctx.iOp = pc;
  ctx.skipFlag = 0;
-  if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
-    assert( pOp>p->aOp );
-    assert( pOp[-1].p4type==P4_COLLSEQ );
-    assert( pOp[-1].opcode==OP_CollSeq );
-    ctx.pColl = pOp[-1].p4.pColl;
-  }
  (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */
  if( ctx.isError ){
    sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t));
@@ -282,4 +282,10 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *);
 # define VDBE_OFFSET_LINENO(x) 0
 #endif

+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+void sqlite3VdbeLoopCounter(Vdbe*, int addrExplain, int addrTest, int addrBody);
+#else
+# define sqlite3VdbeLoopCounter(a,b,c,d)
+#endif
+
 #endif
@@ -73,7 +73,6 @@ struct VdbeCursor {
 #endif
  i8 iDb;               /* Index of cursor database in db->aDb[] (or -1) */
  u8 nullRow;           /* True if pointing to a row with no data */
-  u8 rowidIsValid;      /* True if lastRowid is valid */
  u8 deferredMoveto;    /* A call to sqlite3BtreeMoveto() is needed */
  Bool isEphemeral:1;   /* True for an ephemeral table */
  Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */
@@ -83,7 +82,6 @@ struct VdbeCursor {
  sqlite3_vtab_cursor *pVtabCursor;  /* The cursor for a virtual table */
  i64 seqCount;         /* Sequence counter */
  i64 movetoTarget;     /* Argument to the deferred sqlite3BtreeMoveto() */
-  i64 lastRowid;        /* Rowid being deleted by OP_Delete */
  VdbeSorter *pSorter;  /* Sorter object for OP_SorterOpen cursors */

  /* Cached information about the header for the data record that the
@@ -100,6 +98,7 @@ struct VdbeCursor {
  u32 szRow;            /* Byte available in aRow */
  u32 iHdrOffset;       /* Offset to next unparsed byte of the header */
  const u8 *aRow;       /* Data for the current row, if all on one page */
+  u32 *aOffset;         /* Pointer to aType[nField] */
  u32 aType[1];         /* Type values for all entries in the record */
  /* 2*nField extra array elements allocated for aType[], beyond the one
  ** static element declared in the structure.  nField total array slots for
@@ -176,7 +175,7 @@ struct Mem {
  /* ShallowCopy only needs to copy the information above */
  char *zMalloc;      /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */
  int szMalloc;       /* Size of the zMalloc allocation */
-  int iPadding1;      /* Padding for 8-byte alignment */
+  u32 uTemp;          /* Transient storage for serial_type in OP_MakeRecord */
  sqlite3 *db;        /* The associated database connection */
  void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */
 #ifdef SQLITE_DEBUG
@@ -272,7 +271,6 @@ struct sqlite3_context {
  Mem *pOut;            /* The return value is stored here */
  FuncDef *pFunc;       /* Pointer to function information */
  Mem *pMem;            /* Memory cell used to store aggregate context */
-  CollSeq *pColl;       /* Collating sequence */
  Vdbe *pVdbe;          /* The VM that owns this context */
  int iOp;              /* Instruction number of OP_Function */
  int isError;          /* Error code returned by the function. */
@@ -297,6 +295,13 @@ struct Explain {
 */
 typedef unsigned bft;  /* Bit Field Type */

+typedef struct LoopCounter LoopCounter;
+struct LoopCounter {
+  int addrExplain;                /* OP_Explain for loop */
+  int addrTest;                   /* Address of non-indexed WHERE term tests */
+  int addrBody;                   /* Address of loop body */
+};
+
 /*
 ** An instance of the virtual machine.  This structure contains the complete
 ** state of the virtual machine.
@@ -361,10 +366,6 @@ struct Vdbe {
  i64 nStmtDefImmCons;    /* Number of def. imm constraints when stmt started */
  char *zSql;             /* Text of the SQL statement that generated this */
  void *pFree;            /* Free this when deleting the vdbe */
-#ifdef SQLITE_ENABLE_TREE_EXPLAIN
-  Explain *pExplain;      /* The explainer */
-  char *zExplain;         /* Explanation of data structures */
-#endif
  VdbeFrame *pFrame;      /* Parent frame */
  VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
  int nFrame;             /* Number of frames in pFrame list */
@@ -373,6 +374,11 @@ struct Vdbe {
  int nOnceFlag;          /* Size of array aOnceFlag[] */
  u8 *aOnceFlag;          /* Flags for OP_Once */
  AuxData *pAuxData;      /* Linked list of auxdata allocations */
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  int *anExec;            /* Number of times each op has been executed */
+  int nLoop;              /* Entries in aLoop[] */
+  LoopCounter *aLoop;     /* Loop definitions for sqlite3_stmt_loopcounter() */
+#endif
 };

 /*
@@ -389,6 +395,7 @@ struct Vdbe {
 void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*);
 void sqliteVdbePopStack(Vdbe*,int);
 int sqlite3VdbeCursorMoveto(VdbeCursor*);
+int sqlite3VdbeCursorRestore(VdbeCursor*);
 #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE)
 void sqlite3VdbePrintOp(FILE*, int, Op*);
 #endif
@@ -318,6 +318,7 @@ void sqlite3_result_text64(
 ){
  assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
  assert( xDel!=SQLITE_DYNAMIC );
+  if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE;
  if( n>0x7fffffff ){
    (void)invokeValueDestructor(z, xDel, pCtx);
  }else{
@@ -481,6 +482,9 @@ static int sqlite3Step(Vdbe *p){
      sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime);
    }
 #endif
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+    memset(p->anExec, 0, sizeof(int) * p->nOp);
+#endif

    db->nVdbeActive++;
    if( p->readOnly==0 ) db->nVdbeWrite++;
@@ -1453,3 +1457,19 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){
  if( resetFlag ) pVdbe->aCounter[op] = 0;
  return (int)v;
 }
+
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+const char *sqlite3_stmt_loopcounter(
+    sqlite3_stmt *pStmt, int idx, int *piSub, int *piLoop, int *pnTest, int *pnVisit
+){
+  Vdbe *p = (Vdbe*)pStmt;
+  if( idx>=p->nLoop || idx<0 ) return 0;
+  *piSub = p->aOp[p->aLoop[idx].addrExplain].p1;
+  *piLoop = p->aOp[p->aLoop[idx].addrExplain].p2;
+  *pnTest = p->anExec[ p->aLoop[idx].addrTest ];
+  *pnVisit = p->anExec[ p->aLoop[idx].addrBody ];
+  return p->aOp[p->aLoop[idx].addrExplain].p4.z;
+}
+#endif
+
+
@@ -597,6 +597,29 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){
  return addr;
 }

+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+void sqlite3VdbeLoopCounter(
+  Vdbe *p, 
+  int addrExplain, 
+  int addrTest, 
+  int addrBody
+){
+  int nByte = (p->nLoop+1) * sizeof(LoopCounter);
+  if( addrTest>=0 ){
+    p->aLoop = (LoopCounter*)sqlite3DbReallocOrFree(p->db, p->aLoop, nByte);
+    p->nLoop++;
+  }
+  if( p->aLoop ){
+    LoopCounter *pNew = &p->aLoop[p->nLoop-1];
+    pNew->addrExplain = addrExplain;
+    if( addrTest>=0 ){
+      pNew->addrTest = addrTest;
+      pNew->addrBody = addrBody;
+    }
+  }
+}
+#endif
+
 /*
 ** Change the value of the P1 operand for a specific instruction.
 ** This routine is useful when a large program is loaded from a
@@ -752,7 +775,8 @@ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){
 }

 /*
-** Remove the last opcode inserted
+** If the last opcode is "op" and it is not a jump destination,
+** then remove it.  Return true if and only if an opcode was removed.
 */
 int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){
  if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){
@@ -1701,6 +1725,10 @@ void sqlite3VdbeMakeReady(
    zEnd = &zCsr[nByte];
  }while( nByte && !db->mallocFailed );

+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  p->anExec = (int*)sqlite3DbMallocZero(db, sizeof(int) * p->nOp);
+#endif
+
  p->nCursor = nCursor;
  p->nOnceFlag = nOnce;
  if( p->aVar ){
@@ -1744,7 +1772,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){
    sqlite3BtreeCloseCursor(pCx->pCursor);
  }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
-  if( pCx->pVtabCursor ){
+  else if( pCx->pVtabCursor ){
    sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor;
    const sqlite3_module *pModule = pVtabCursor->pVtab->pModule;
    p->inVtabMethod = 1;
@@ -1787,9 +1815,10 @@ static void closeAllCursors(Vdbe *p){
    VdbeFrame *pFrame;
    for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent);
    sqlite3VdbeFrameRestore(pFrame);
+    p->pFrame = 0;
+    p->nFrame = 0;
  }
-  p->pFrame = 0;
-  p->nFrame = 0;
+  assert( p->nFrame==0 );

  if( p->apCsr ){
    int i;
@@ -1811,7 +1840,7 @@ static void closeAllCursors(Vdbe *p){
  }

  /* Delete any auxdata allocations made by the VM */
-  sqlite3VdbeDeleteAuxData(p, -1, 0);
+  if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0);
  assert( p->pAuxData==0 );
 }

@@ -2677,9 +2706,9 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
  sqlite3DbFree(db, p->aColName);
  sqlite3DbFree(db, p->zSql);
  sqlite3DbFree(db, p->pFree);
-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
-  sqlite3DbFree(db, p->zExplain);
-  sqlite3DbFree(db, p->pExplain);
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  sqlite3DbFree(db, p->aLoop);
+  sqlite3DbFree(db, p->anExec);
 #endif
 }

@@ -2721,9 +2750,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){
  assert( p->isTable );
  rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res);
  if( rc ) return rc;
-  p->lastRowid = p->movetoTarget;
  if( res!=0 ) return SQLITE_CORRUPT_BKPT;
-  p->rowidIsValid = 1;
 #ifdef SQLITE_TEST
  sqlite3_search_count++;
 #endif
@@ -2749,6 +2776,17 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){
  return rc;
 }

+/*
+** Check to ensure that the cursor is valid.  Restore the cursor
+** if need be.  Return any I/O error from the restore operation.
+*/
+int sqlite3VdbeCursorRestore(VdbeCursor *p){
+  if( sqlite3BtreeCursorHasMoved(p->pCursor) ){
+    return handleMovedCursor(p);
+  }
+  return SQLITE_OK;
+}
+
 /*
 ** Make sure the cursor p is ready to read or write the row to which it
 ** was last positioned.  Return an error code if an OOM fault or I/O error
@@ -2766,7 +2804,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){
  if( p->deferredMoveto ){
    return handleDeferredMoveto(p);
  }
-  if( sqlite3BtreeCursorHasMoved(p->pCursor) ){
+  if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){
    return handleMovedCursor(p);
  }
  return SQLITE_OK;
@@ -31,7 +31,10 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){
  */
  assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 );

-  /* MEM_Dyn may only be set if Mem.szMalloc==0 */
+  /* MEM_Dyn may only be set if Mem.szMalloc==0.  In this way we
+  ** ensure that if Mem.szMalloc>0 then it is safe to do
+  ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn.
+  ** That saves a few cycles in inner loops. */
  assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 );

  /* Cannot be both MEM_Int and MEM_Real at the same time */
@@ -140,7 +143,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
    }
  }

-  if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){
+  if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){
    memcpy(pMem->zMalloc, pMem->z, pMem->n);
  }
  if( (pMem->flags&MEM_Dyn)!=0 ){
@@ -167,7 +170,8 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){
 ** if unable to complete the resizing.
 */
 int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){
-  assert( szNew>=0 );
+  assert( szNew>0 );
+  assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 );
  if( pMem->szMalloc<szNew ){
    return sqlite3VdbeMemGrow(pMem, szNew, 0);
  }
@@ -891,7 +895,10 @@ int sqlite3VdbeMemSetStr(
    if( nByte>iLimit ){
      return SQLITE_TOOBIG;
    }
-    if( sqlite3VdbeMemClearAndResize(pMem, nAlloc) ){
+    testcase( nAlloc==0 );
+    testcase( nAlloc==31 );
+    testcase( nAlloc==32 );
+    if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){
      return SQLITE_NOMEM;
    }
    memcpy(pMem->z, z, nAlloc);
@@ -994,7 +1001,7 @@ int sqlite3VdbeMemFromBtree(
 ** Convert it into a string with encoding enc and return a pointer
 ** to a zero-terminated version of that string.
 */
-SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){
+static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){
  assert( pVal!=0 );
  assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) );
  assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) );
@@ -2292,7 +2292,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){
    assert( pSorter->bUseThreads==0 || pSorter->nTask>1 );
    if( pSorter->bUseThreads ){
      int iTask;
-      PmaReader *pReadr;
+      PmaReader *pReadr = 0;
      SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1];
      rc = vdbeSortAllocUnpacked(pLast);
      if( rc==SQLITE_OK ){
@@ -183,118 +183,3 @@ char *sqlite3VdbeExpandSql(
 }

 #endif /* #ifndef SQLITE_OMIT_TRACE */
-
-/*****************************************************************************
-** The following code implements the data-structure explaining logic
-** for the Vdbe.
-*/
-
-#if defined(SQLITE_ENABLE_TREE_EXPLAIN)
-
-/*
-** Allocate a new Explain object
-*/
-void sqlite3ExplainBegin(Vdbe *pVdbe){
-  if( pVdbe ){
-    Explain *p;
-    sqlite3BeginBenignMalloc();
-    p = (Explain *)sqlite3MallocZero( sizeof(Explain) );
-    if( p ){
-      p->pVdbe = pVdbe;
-      sqlite3_free(pVdbe->pExplain);
-      pVdbe->pExplain = p;
-      sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase),
-                          SQLITE_MAX_LENGTH);
-      p->str.useMalloc = 2;
-    }else{
-      sqlite3EndBenignMalloc();
-    }
-  }
-}
-
-/*
-** Return true if the Explain ends with a new-line.
-*/
-static int endsWithNL(Explain *p){
-  return p && p->str.zText && p->str.nChar
-           && p->str.zText[p->str.nChar-1]=='\n';
-}
-    
-/*
-** Append text to the indentation
-*/
-void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){
-  Explain *p;
-  if( pVdbe && (p = pVdbe->pExplain)!=0 ){
-    va_list ap;
-    if( p->nIndent && endsWithNL(p) ){
-      int n = p->nIndent;
-      if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent);
-      sqlite3AppendSpace(&p->str, p->aIndent[n-1]);
-    }   
-    va_start(ap, zFormat);
-    sqlite3VXPrintf(&p->str, SQLITE_PRINTF_INTERNAL, zFormat, ap);
-    va_end(ap);
-  }
-}
-
-/*
-** Append a '\n' if there is not already one.
-*/
-void sqlite3ExplainNL(Vdbe *pVdbe){
-  Explain *p;
-  if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){
-    sqlite3StrAccumAppend(&p->str, "\n", 1);
-  }
-}
-
-/*
-** Push a new indentation level.  Subsequent lines will be indented
-** so that they begin at the current cursor position.
-*/
-void sqlite3ExplainPush(Vdbe *pVdbe){
-  Explain *p;
-  if( pVdbe && (p = pVdbe->pExplain)!=0 ){
-    if( p->str.zText && p->nIndent<ArraySize(p->aIndent) ){
-      const char *z = p->str.zText;
-      int i = p->str.nChar-1;
-      int x;
-      while( i>=0 && z[i]!='\n' ){ i--; }
-      x = (p->str.nChar - 1) - i;
-      if( p->nIndent && x<p->aIndent[p->nIndent-1] ){
-        x = p->aIndent[p->nIndent-1];
-      }
-      p->aIndent[p->nIndent] = x;
-    }
-    p->nIndent++;
-  }
-}
-
-/*
-** Pop the indentation stack by one level.
-*/
-void sqlite3ExplainPop(Vdbe *p){
-  if( p && p->pExplain ) p->pExplain->nIndent--;
-}
-
-/*
-** Free the indentation structure
-*/
-void sqlite3ExplainFinish(Vdbe *pVdbe){
-  if( pVdbe && pVdbe->pExplain ){
-    sqlite3_free(pVdbe->zExplain);
-    sqlite3ExplainNL(pVdbe);
-    pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str);
-    sqlite3_free(pVdbe->pExplain);
-    pVdbe->pExplain = 0;
-    sqlite3EndBenignMalloc();
-  }
-}
-
-/*
-** Return the explanation of a virtual machine.
-*/
-const char *sqlite3VdbeExplanation(Vdbe *pVdbe){
-  return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0;
-}
-#endif /* defined(SQLITE_DEBUG) */
@@ -519,6 +519,7 @@ static int vtabCallConstructor(
  }else if( ALWAYS(pVTable->pVtab) ){
    /* Justification of ALWAYS():  A correct vtab constructor must allocate
    ** the sqlite3_vtab object if successful.  */
+    memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0]));
    pVTable->pVtab->pModule = pMod->pModule;
    pVTable->nRef = 1;
    if( sCtx.pTab ){
@@ -364,11 +364,6 @@ static int allowedOp(int op){
  return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL;
 }

-/*
-** Swap two objects of type TYPE.
-*/
-#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;}
-
 /*
 ** Commute a comparison operator.  Expressions of the form "X op Y"
 ** are converted into "Y op X".
@@ -2203,7 +2198,7 @@ static int whereRangeScanEst(
      /* Determine iLower and iUpper using ($P) only. */
      if( nEq==0 ){
        iLower = 0;
-        iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]);
+        iUpper = p->nRowEst0;
      }else{
        /* Note: this call could be optimized away - since the same values must 
        ** have been requested when testing key $P in whereEqualScanEst().  */
@@ -2212,16 +2207,23 @@ static int whereRangeScanEst(
        iUpper = a[0] + a[1];
      }

+      assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 );
+      assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
+      assert( p->aSortOrder!=0 );
+      if( p->aSortOrder[nEq] ){
+        /* The roles of pLower and pUpper are swapped for a DESC index */
+        SWAP(WhereTerm*, pLower, pUpper);
+      }
+
      /* If possible, improve on the iLower estimate using ($P:$L). */
      if( pLower ){
        int bOk;                    /* True if value is extracted from pExpr */
        Expr *pExpr = pLower->pExpr->pRight;
-        assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 );
        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
        if( rc==SQLITE_OK && bOk ){
          tRowcnt iNew;
          whereKeyStats(pParse, p, pRec, 0, a);
-          iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0);
+          iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
          if( iNew>iLower ) iLower = iNew;
          nOut--;
          pLower = 0;
@@ -2232,12 +2234,11 @@ static int whereRangeScanEst(
      if( pUpper ){
        int bOk;                    /* True if value is extracted from pExpr */
        Expr *pExpr = pUpper->pExpr->pRight;
-        assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
        rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
        if( rc==SQLITE_OK && bOk ){
          tRowcnt iNew;
          whereKeyStats(pParse, p, pRec, 1, a);
-          iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0);
+          iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0);
          if( iNew<iUpper ) iUpper = iNew;
          nOut--;
          pUpper = 0;
@@ -2637,7 +2638,7 @@ static int codeAllEqualityTerms(
  pLoop = pLevel->pWLoop;
  assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 );
  nEq = pLoop->u.btree.nEq;
-  nSkip = pLoop->u.btree.nSkip;
+  nSkip = pLoop->nSkip;
  pIdx = pLoop->u.btree.pIndex;
  assert( pIdx!=0 );

@@ -2736,9 +2737,8 @@ static void explainAppendTerm(

 /*
 ** Argument pLevel describes a strategy for scanning table pTab. This 
-** function returns a pointer to a string buffer containing a description
-** of the subset of table rows scanned by the strategy in the form of an
-** SQL expression. Or, if all rows are scanned, NULL is returned.
+** function appends text to pStr that describes the subset of table
+** rows scanned by the strategy in the form of an SQL expression.
 **
 ** For example, if the query:
 **
@@ -2748,49 +2748,37 @@ static void explainAppendTerm(
 ** string similar to:
 **
 **   "a=? AND b>?"
-**
-** The returned pointer points to memory obtained from sqlite3DbMalloc().
-** It is the responsibility of the caller to free the buffer when it is
-** no longer required.
 */
-static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){
+static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){
  Index *pIndex = pLoop->u.btree.pIndex;
  u16 nEq = pLoop->u.btree.nEq;
-  u16 nSkip = pLoop->u.btree.nSkip;
+  u16 nSkip = pLoop->nSkip;
  int i, j;
  Column *aCol = pTab->aCol;
  i16 *aiColumn = pIndex->aiColumn;
-  StrAccum txt;

-  if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){
-    return 0;
-  }
-  sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH);
-  txt.db = db;
-  sqlite3StrAccumAppend(&txt, " (", 2);
+  if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return;
+  sqlite3StrAccumAppend(pStr, " (", 2);
  for(i=0; i<nEq; i++){
    char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName;
    if( i>=nSkip ){
-      explainAppendTerm(&txt, i, z, "=");
+      explainAppendTerm(pStr, i, z, "=");
    }else{
-      if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5);
-      sqlite3StrAccumAppend(&txt, "ANY(", 4);
-      sqlite3StrAccumAppendAll(&txt, z);
-      sqlite3StrAccumAppend(&txt, ")", 1);
+      if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5);
+      sqlite3XPrintf(pStr, 0, "ANY(%s)", z);
    }
  }

  j = i;
  if( pLoop->wsFlags&WHERE_BTM_LIMIT ){
    char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
-    explainAppendTerm(&txt, i++, z, ">");
+    explainAppendTerm(pStr, i++, z, ">");
  }
  if( pLoop->wsFlags&WHERE_TOP_LIMIT ){
    char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName;
-    explainAppendTerm(&txt, i, z, "<");
+    explainAppendTerm(pStr, i, z, "<");
  }
-  sqlite3StrAccumAppend(&txt, ")", 1);
-  return sqlite3StrAccumFinish(&txt);
+  sqlite3StrAccumAppend(pStr, ")", 1);
 }

 /*
@@ -2814,11 +2802,13 @@ static void explainOneScan(
    struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom];
    Vdbe *v = pParse->pVdbe;      /* VM being constructed */
    sqlite3 *db = pParse->db;     /* Database handle */
-    char *zMsg;                   /* Text to add to EQP output */
    int iId = pParse->iSelectId;  /* Select id (left-most output column) */
    int isSearch;                 /* True for a SEARCH. False for SCAN. */
    WhereLoop *pLoop;             /* The controlling WhereLoop object */
    u32 flags;                    /* Flags that describe this loop */
+    char *zMsg;                   /* Text to add to EQP output */
+    StrAccum str;                 /* EQP output string */
+    char zBuf[100];               /* Initial space for EQP output string */

    pLoop = pLevel->pWLoop;
    flags = pLoop->wsFlags;
@@ -2828,54 +2818,70 @@ static void explainOneScan(
            || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0))
            || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX));

-    zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN");
+    sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
+    str.db = db;
+    sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN");
    if( pItem->pSelect ){
-      zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId);
+      sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId);
    }else{
-      zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName);
+      sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName);
    }

    if( pItem->zAlias ){
-      zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias);
+      sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias);
    }
-    if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0
-     && ALWAYS(pLoop->u.btree.pIndex!=0)
-    ){
-      const char *zFmt;
-      Index *pIdx = pLoop->u.btree.pIndex;
-      char *zWhere = explainIndexRange(db, pLoop, pItem->pTab);
+    if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){
+      const char *zFmt = 0;
+      Index *pIdx;
+
+      assert( pLoop->u.btree.pIndex!=0 );
+      pIdx = pLoop->u.btree.pIndex;
      assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) );
      if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){
-        zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s";
+        if( isSearch ){
+          zFmt = "PRIMARY KEY";
+        }
      }else if( flags & WHERE_AUTO_INDEX ){
-        zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s";
+        zFmt = "AUTOMATIC COVERING INDEX";
      }else if( flags & WHERE_IDX_ONLY ){
-        zFmt = "%s USING COVERING INDEX %s%s";
+        zFmt = "COVERING INDEX %s";
      }else{
-        zFmt = "%s USING INDEX %s%s";
+        zFmt = "INDEX %s";
+      }
+      if( zFmt ){
+        sqlite3StrAccumAppend(&str, " USING ", 7);
+        sqlite3XPrintf(&str, 0, zFmt, pIdx->zName);
+        explainIndexRange(&str, pLoop, pItem->pTab);
      }
-      zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere);
-      sqlite3DbFree(db, zWhere);
    }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){
-      zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg);
-
+      const char *zRange;
      if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){
-        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg);
+        zRange = "(rowid=?)";
      }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){
-        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid<?)", zMsg);
+        zRange = "(rowid>? AND rowid<?)";
      }else if( flags&WHERE_BTM_LIMIT ){
-        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>?)", zMsg);
-      }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){
-        zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid<?)", zMsg);
+        zRange = "(rowid>?)";
+      }else{
+        assert( flags&WHERE_TOP_LIMIT);
+        zRange = "(rowid<?)";
      }
+      sqlite3StrAccumAppendAll(&str, " USING INTEGER PRIMARY KEY ");
+      sqlite3StrAccumAppendAll(&str, zRange);
    }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
    else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
-      zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg,
+      sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s",
                  pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
    }
 #endif
-    zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg);
+#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS
+    if( pLoop->nOut>=10 ){
+      sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut));
+    }else{
+      sqlite3StrAccumAppend(&str, " (~1 row)", 9);
+    }
+#endif
+    zMsg = sqlite3StrAccumFinish(&str);
    sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC);
  }
 }
@@ -2902,17 +2908,19 @@ static Bitmask codeOneLoopStart(
  WhereLoop *pLoop;    /* The WhereLoop object being coded */
  WhereClause *pWC;    /* Decomposition of the entire WHERE clause */
  WhereTerm *pTerm;               /* A WHERE clause term */
-  Parse *pParse;                  /* Parsing context */
+  Parse *pParse = pWInfo->pParse; /* Parsing context */
  sqlite3 *db;                    /* Database connection */
-  Vdbe *v;                        /* The prepared stmt under constructions */
+  Vdbe *v = pParse->pVdbe;        /* The prepared stmt under constructions */
  struct SrcList_item *pTabItem;  /* FROM clause term being coded */
  int addrBrk;                    /* Jump here to break out of the loop */
  int addrCont;                   /* Jump here to continue with next cycle */
  int iRowidReg = 0;        /* Rowid is stored in this register, if not zero */
  int iReleaseReg = 0;      /* Temp register to free before returning */
+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  int addrExplain = sqlite3VdbeCurrentAddr(v)-1;
+  int addrTest;             /* Address of non-indexed WHERE clause test */
+#endif

-  pParse = pWInfo->pParse;
-  v = pParse->pVdbe;
  pWC = &pWInfo->sWC;
  db = pParse->db;
  pLevel = &pWInfo->a[iLevel];
@@ -3183,7 +3191,7 @@ static Bitmask codeOneLoopStart(

    pIdx = pLoop->u.btree.pIndex;
    iIdxCur = pLevel->iIdxCur;
-    assert( nEq>=pLoop->u.btree.nSkip );
+    assert( nEq>=pLoop->nSkip );

    /* If this loop satisfies a sort order (pOrderBy) request that 
    ** was passed to this function to implement a "SELECT min(x) ..." 
@@ -3200,7 +3208,7 @@ static Bitmask codeOneLoopStart(
     && pWInfo->nOBSat>0
     && (pIdx->nKeyCol>nEq)
    ){
-      assert( pLoop->u.btree.nSkip==0 );
+      assert( pLoop->nSkip==0 );
      bSeekPastNull = 1;
      nExtraReg = 1;
    }
@@ -3529,8 +3537,9 @@ static Bitmask codeOneLoopStart(
    ** eliminating duplicates from other WHERE clauses, the action for each
    ** sub-WHERE clause is to to invoke the main loop body as a subroutine.
    */
-    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY |
-                  WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY;
+    wctrlFlags =  WHERE_OMIT_OPEN_CLOSE
+                | WHERE_FORCE_TABLE
+                | WHERE_ONETABLE_ONLY;
    for(ii=0; ii<pOrWc->nTerm; ii++){
      WhereTerm *pOrTerm = &pOrWc->a[ii];
      if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){
@@ -3542,11 +3551,13 @@ static Bitmask codeOneLoopStart(
          pOrExpr = pAndExpr;
        }
        /* Loop through table entries that match term pOrTerm. */
+        WHERETRACE(0xffff, ("Subplan for OR-clause:\n"));
        pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0,
                                      wctrlFlags, iCovCur);
        assert( pSubWInfo || pParse->nErr || db->mallocFailed );
        if( pSubWInfo ){
          WhereLoop *pSubLoop;
+          sqlite3VdbeLoopCounter(v, sqlite3VdbeCurrentAddr(v), -1, -1);
          explainOneScan(
              pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
          );
@@ -3680,6 +3691,10 @@ static Bitmask codeOneLoopStart(
    }
  }

+#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_LOOPCOUNTERS)
+  addrTest = sqlite3VdbeCurrentAddr(v);
+#endif 
+
  /* Insert code to test every subexpression that can be completely
  ** computed using the current set of tables.
  */
@@ -3758,24 +3773,33 @@ static Bitmask codeOneLoopStart(
    }
  }

+  if( pParse->pTriggerTab==0 && 0==(pLoop->wsFlags & WHERE_MULTI_OR) ){
+    sqlite3VdbeLoopCounter(v, addrExplain, addrTest, sqlite3VdbeCurrentAddr(v));
+  }
+
  return pLevel->notReady;
 }

-#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN)
+#ifdef WHERETRACE_ENABLED
 /*
-** Generate "Explanation" text for a WhereTerm.
+** Print the content of a WhereTerm object
 */
-static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){
-  char zType[4];
-  memcpy(zType, "...", 4);
-  if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
-  if( pTerm->eOperator & WO_EQUIV  ) zType[1] = 'E';
-  if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
-  sqlite3ExplainPrintf(v, "%s ", zType);
-  sqlite3ExplainExpr(v, pTerm->pExpr);
+static void whereTermPrint(WhereTerm *pTerm, int iTerm){
+  if( pTerm==0 ){
+    sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm);
+  }else{
+    char zType[4];
+    memcpy(zType, "...", 4);
+    if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V';
+    if( pTerm->eOperator & WO_EQUIV  ) zType[1] = 'E';
+    if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L';
+    sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n",
+                       iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb,
+                       pTerm->eOperator);
+    sqlite3TreeViewExpr(0, pTerm->pExpr, 0);
+  }
 }
-#endif /* WHERETRACE_ENABLED && SQLITE_ENABLE_TREE_EXPLAIN */
-
+#endif

 #ifdef WHERETRACE_ENABLED
 /*
@@ -3814,32 +3838,17 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){
    sqlite3_free(z);
  }
  if( p->wsFlags & WHERE_SKIPSCAN ){
-    sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip);
+    sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip);
  }else{
    sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm);
  }
  sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut);
-#ifdef SQLITE_ENABLE_TREE_EXPLAIN
-  /* If the 0x100 bit of wheretracing is set, then show all of the constraint
-  ** expressions in the WhereLoop.aLTerm[] array.
-  */
-  if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){  /* WHERETRACE 0x100 */
+  if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){
    int i;
-    Vdbe *v = pWInfo->pParse->pVdbe;
-    sqlite3ExplainBegin(v);
    for(i=0; i<p->nLTerm; i++){
-      WhereTerm *pTerm = p->aLTerm[i];
-      if( pTerm==0 ) continue;
-      sqlite3ExplainPrintf(v, "  (%d) #%-2d ", i+1, (int)(pTerm-pWC->a));
-      sqlite3ExplainPush(v);
-      whereExplainTerm(v, pTerm);
-      sqlite3ExplainPop(v);
-      sqlite3ExplainNL(v);
+      whereTermPrint(p->aLTerm[i], i);
    }
-    sqlite3ExplainFinish(v);
-    sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v));
  }
-#endif
 }
 #endif

@@ -3958,12 +3967,15 @@ static int whereLoopCheaperProperSubset(
  const WhereLoop *pY        /* Compare against this WhereLoop */
 ){
  int i, j;
-  if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */
+  if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){
+    return 0; /* X is not a subset of Y */
+  }
  if( pX->rRun >= pY->rRun ){
    if( pX->rRun > pY->rRun ) return 0;    /* X costs more than Y */
    if( pX->nOut > pY->nOut ) return 0;    /* X costs more than Y */
  }
  for(i=pX->nLTerm-1; i>=0; i--){
+    if( pX->aLTerm[i]==0 ) continue;
    for(j=pY->nLTerm-1; j>=0; j--){
      if( pY->aLTerm[j]==pX->aLTerm[i] ) break;
    }
@@ -3985,33 +3997,26 @@ static int whereLoopCheaperProperSubset(
 ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer
 ** WHERE clause terms than Y and that every WHERE clause term used by X is
 ** also used by Y.
-**
-** This adjustment is omitted for SKIPSCAN loops.  In a SKIPSCAN loop, the
-** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE
-** clause terms covered, since some of the first nLTerm entries in aLTerm[]
-** will be NULL (because they are skipped).  That makes it more difficult
-** to compare the loops.  We could add extra code to do the comparison, and
-** perhaps we will someday.  But SKIPSCAN is sufficiently uncommon, and this
-** adjustment is sufficient minor, that it is very difficult to construct
-** a test case where the extra code would improve the query plan.  Better
-** to avoid the added complexity and just omit cost adjustments to SKIPSCAN
-** loops.
 */
 static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){
  if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return;
-  if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return;
  for(; p; p=p->pNextLoop){
    if( p->iTab!=pTemplate->iTab ) continue;
    if( (p->wsFlags & WHERE_INDEXED)==0 ) continue;
-    if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue;
    if( whereLoopCheaperProperSubset(p, pTemplate) ){
      /* Adjust pTemplate cost downward so that it is cheaper than its 
-      ** subset p */
+      ** subset p.  Except, do not adjust the cost estimate downward for
+      ** a loop that skips more columns. */
+      if( pTemplate->nSkip>p->nSkip ) continue;
+      WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
+                       pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1));
      pTemplate->rRun = p->rRun;
      pTemplate->nOut = p->nOut - 1;
    }else if( whereLoopCheaperProperSubset(pTemplate, p) ){
      /* Adjust pTemplate cost upward so that it is costlier than p since
      ** pTemplate is a proper subset of p */
+      WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n",
+                       pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1));
      pTemplate->rRun = p->rRun;
      pTemplate->nOut = p->nOut + 1;
    }
@@ -4152,7 +4157,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
    ** than pTemplate, so just ignore pTemplate */
 #if WHERETRACE_ENABLED /* 0x8 */
    if( sqlite3WhereTrace & 0x8 ){
-      sqlite3DebugPrintf("ins-noop: ");
+      sqlite3DebugPrintf("   skip: ");
      whereLoopPrint(pTemplate, pBuilder->pWC);
    }
 #endif
@@ -4168,10 +4173,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
 #if WHERETRACE_ENABLED /* 0x8 */
  if( sqlite3WhereTrace & 0x8 ){
    if( p!=0 ){
-      sqlite3DebugPrintf("ins-del:  ");
+      sqlite3DebugPrintf("replace: ");
      whereLoopPrint(p, pBuilder->pWC);
    }
-    sqlite3DebugPrintf("ins-new:  ");
+    sqlite3DebugPrintf("    add: ");
    whereLoopPrint(pTemplate, pBuilder->pWC);
  }
 #endif
@@ -4195,7 +4200,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){
      *ppTail = pToDel->pNextLoop;
 #if WHERETRACE_ENABLED /* 0x8 */
      if( sqlite3WhereTrace & 0x8 ){
-        sqlite3DebugPrintf("ins-del:  ");
+        sqlite3DebugPrintf(" delete: ");
        whereLoopPrint(pToDel, pBuilder->pWC);
      }
 #endif
@@ -4297,7 +4302,7 @@ static int whereLoopAddBtreeIndex(
  Bitmask saved_prereq;           /* Original value of pNew->prereq */
  u16 saved_nLTerm;               /* Original value of pNew->nLTerm */
  u16 saved_nEq;                  /* Original value of pNew->u.btree.nEq */
-  u16 saved_nSkip;                /* Original value of pNew->u.btree.nSkip */
+  u16 saved_nSkip;                /* Original value of pNew->nSkip */
  u32 saved_wsFlags;              /* Original value of pNew->wsFlags */
  LogEst saved_nOut;              /* Original value of pNew->nOut */
  int iCol;                       /* Index of the column in the table */
@@ -4326,7 +4331,7 @@ static int whereLoopAddBtreeIndex(
  pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol,
                        opMask, pProbe);
  saved_nEq = pNew->u.btree.nEq;
-  saved_nSkip = pNew->u.btree.nSkip;
+  saved_nSkip = pNew->nSkip;
  saved_nLTerm = pNew->nLTerm;
  saved_wsFlags = pNew->wsFlags;
  saved_prereq = pNew->prereq;
@@ -4334,41 +4339,6 @@ static int whereLoopAddBtreeIndex(
  pNew->rSetup = 0;
  rSize = pProbe->aiRowLogEst[0];
  rLogSize = estLog(rSize);
-
-  /* Consider using a skip-scan if there are no WHERE clause constraints
-  ** available for the left-most terms of the index, and if the average
-  ** number of repeats in the left-most terms is at least 18. 
-  **
-  ** The magic number 18 is selected on the basis that scanning 17 rows
-  ** is almost always quicker than an index seek (even though if the index
-  ** contains fewer than 2^17 rows we assume otherwise in other parts of
-  ** the code). And, even if it is not, it should not be too much slower. 
-  ** On the other hand, the extra seeks could end up being significantly
-  ** more expensive.  */
-  assert( 42==sqlite3LogEst(18) );
-  if( saved_nEq==saved_nSkip
-   && saved_nEq+1<pProbe->nKeyCol
-   && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
-   && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
-  ){
-    LogEst nIter;
-    pNew->u.btree.nEq++;
-    pNew->u.btree.nSkip++;
-    pNew->aLTerm[pNew->nLTerm++] = 0;
-    pNew->wsFlags |= WHERE_SKIPSCAN;
-    nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
-    if( pTerm ){
-      /* TUNING:  When estimating skip-scan for a term that is also indexable,
-      ** increase the cost of the skip-scan by 2x, to make it a little less
-      ** desirable than the regular index lookup. */
-      nIter += 10;  assert( 10==sqlite3LogEst(2) );
-    }
-    pNew->nOut -= nIter;
-    whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
-    pNew->nOut = saved_nOut;
-    pNew->u.btree.nEq = saved_nEq;
-    pNew->u.btree.nSkip = saved_nSkip;
-  }
  for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
    u16 eOp = pTerm->eOperator;   /* Shorthand for pTerm->eOperator */
    LogEst rCostIdx;
@@ -4531,10 +4501,44 @@ static int whereLoopAddBtreeIndex(
  }
  pNew->prereq = saved_prereq;
  pNew->u.btree.nEq = saved_nEq;
-  pNew->u.btree.nSkip = saved_nSkip;
+  pNew->nSkip = saved_nSkip;
  pNew->wsFlags = saved_wsFlags;
  pNew->nOut = saved_nOut;
  pNew->nLTerm = saved_nLTerm;
+
+  /* Consider using a skip-scan if there are no WHERE clause constraints
+  ** available for the left-most terms of the index, and if the average
+  ** number of repeats in the left-most terms is at least 18. 
+  **
+  ** The magic number 18 is selected on the basis that scanning 17 rows
+  ** is almost always quicker than an index seek (even though if the index
+  ** contains fewer than 2^17 rows we assume otherwise in other parts of
+  ** the code). And, even if it is not, it should not be too much slower. 
+  ** On the other hand, the extra seeks could end up being significantly
+  ** more expensive.  */
+  assert( 42==sqlite3LogEst(18) );
+  if( saved_nEq==saved_nSkip
+   && saved_nEq+1<pProbe->nKeyCol
+   && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
+   && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
+  ){
+    LogEst nIter;
+    pNew->u.btree.nEq++;
+    pNew->nSkip++;
+    pNew->aLTerm[pNew->nLTerm++] = 0;
+    pNew->wsFlags |= WHERE_SKIPSCAN;
+    nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1];
+    pNew->nOut -= nIter;
+    /* TUNING:  Because uncertainties in the estimates for skip-scan queries,
+    ** add a 1.375 fudge factor to make skip-scan slightly less likely. */
+    nIter += 5;
+    whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul);
+    pNew->nOut = saved_nOut;
+    pNew->u.btree.nEq = saved_nEq;
+    pNew->nSkip = saved_nSkip;
+    pNew->wsFlags = saved_wsFlags;
+  }
+
  return rc;
 }

@@ -4713,14 +4717,22 @@ static int whereLoopAddBtree(
      if( pTerm->prereqRight & pNew->maskSelf ) continue;
      if( termCanDriveIndex(pTerm, pSrc, 0) ){
        pNew->u.btree.nEq = 1;
-        pNew->u.btree.nSkip = 0;
+        pNew->nSkip = 0;
        pNew->u.btree.pIndex = 0;
        pNew->nLTerm = 1;
        pNew->aLTerm[0] = pTerm;
        /* TUNING: One-time cost for computing the automatic index is
-        ** approximately 7*N*log2(N) where N is the number of rows in
-        ** the table being indexed. */
-        pNew->rSetup = rLogSize + rSize + 28;  assert( 28==sqlite3LogEst(7) );
+        ** estimated to be X*N*log2(N) where N is the number of rows in
+        ** the table being indexed and where X is 7 (LogEst=28) for normal
+        ** tables or 1.375 (LogEst=4) for views and subqueries.  The value
+        ** of X is smaller for views and subqueries so that the query planner
+        ** will be more aggressive about generating automatic indexes for
+        ** those objects, since there is no opportunity to add schema
+        ** indexes on subqueries and views. */
+        pNew->rSetup = rLogSize + rSize + 4;
+        if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){
+          pNew->rSetup += 24;
+        }
        ApplyCostMultiplier(pNew->rSetup, pTab->costMult);
        /* TUNING: Each index lookup yields 20 rows in the table.  This
        ** is more than the usual guess of 10 rows, since we have no way
@@ -4746,7 +4758,7 @@ static int whereLoopAddBtree(
    }
    rSize = pProbe->aiRowLogEst[0];
    pNew->u.btree.nEq = 0;
-    pNew->u.btree.nSkip = 0;
+    pNew->nSkip = 0;
    pNew->nLTerm = 0;
    pNew->iSortIdx = 0;
    pNew->rSetup = 0;
@@ -5008,7 +5020,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
  struct SrcList_item *pItem;
  
  pWC = pBuilder->pWC;
-  if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK;
  pWCEnd = pWC->a + pWC->nTerm;
  pNew = pBuilder->pNew;
  memset(&sSum, 0, sizeof(sSum));
@@ -5029,6 +5040,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
      sSubBuild.pOrderBy = 0;
      sSubBuild.pOrSet = &sCur;

+      WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm));
      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
        if( (pOrTerm->eOperator & WO_AND)!=0 ){
          sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc;
@@ -5043,6 +5055,15 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
          continue;
        }
        sCur.n = 0;
+#ifdef WHERETRACE_ENABLED
+        WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", 
+                   (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm));
+        if( sqlite3WhereTrace & 0x400 ){
+          for(i=0; i<sSubBuild.pWC->nTerm; i++){
+            whereTermPrint(&sSubBuild.pWC->a[i], i);
+          }
+        }
+#endif
 #ifndef SQLITE_OMIT_VIRTUALTABLE
        if( IsVirtual(pItem->pTab) ){
          rc = whereLoopAddVirtual(&sSubBuild, mExtra);
@@ -5051,6 +5072,9 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
        {
          rc = whereLoopAddBtree(&sSubBuild, mExtra);
        }
+        if( rc==SQLITE_OK ){
+          rc = whereLoopAddOr(&sSubBuild, mExtra);
+        }
        assert( rc==SQLITE_OK || sCur.n==0 );
        if( sCur.n==0 ){
          sSum.n = 0;
@@ -5095,6 +5119,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){
        pNew->prereq = sSum.a[i].prereq;
        rc = whereLoopInsert(pBuilder, pNew);
      }
+      WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm));
    }
  }
  return rc;
@@ -5283,7 +5308,7 @@ static i8 wherePathSatisfiesOrderBy(

        /* Skip over == and IS NULL terms */
        if( j<pLoop->u.btree.nEq
-         && pLoop->u.btree.nSkip==0
+         && pLoop->nSkip==0
         && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0
        ){
          if( i & WO_ISNULL ){
@@ -5338,7 +5363,7 @@ static i8 wherePathSatisfiesOrderBy(
          isMatch = 1;
          break;
        }
-        if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){
+        if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){
          /* Make sure the sort order is compatible in an ORDER BY clause.
          ** Sort order is irrelevant for a GROUP BY clause. */
          if( revSet ){
@@ -5737,7 +5762,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
    }

 #ifdef WHERETRACE_ENABLED  /* >=2 */
-    if( sqlite3WhereTrace>=2 ){
+    if( sqlite3WhereTrace & 0x02 ){
      sqlite3DebugPrintf("---- after round %d ----\n", iLoop);
      for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
        sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
@@ -5803,12 +5828,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
    if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)
        && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr
    ){
-      Bitmask notUsed = 0;
+      Bitmask revMask = 0;
      int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, 
-          pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed
+          pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask
      );
      assert( pWInfo->sorted==0 );
-      pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr);
+      if( nOrder==pWInfo->pOrderBy->nExpr ){
+        pWInfo->sorted = 1;
+        pWInfo->revMask = revMask;
+      }
    }
  }

@@ -5853,7 +5881,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
  pWC = &pWInfo->sWC;
  pLoop = pBuilder->pNew;
  pLoop->wsFlags = 0;
-  pLoop->u.btree.nSkip = 0;
+  pLoop->nSkip = 0;
  pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0);
  if( pTerm ){
    pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW;
@@ -5865,7 +5893,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){
  }else{
    for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
      assert( pLoop->aLTermSpace==pLoop->aLTerm );
-      assert( ArraySize(pLoop->aLTermSpace)==4 );
      if( !IsUniqueIndex(pIdx)
       || pIdx->pPartIdxWhere!=0 
       || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) 
@@ -6161,23 +6188,16 @@ WhereInfo *sqlite3WhereBegin(

  /* Construct the WhereLoop objects */
  WHERETRACE(0xffff,("*** Optimizer Start ***\n"));
+#if defined(WHERETRACE_ENABLED)
  /* Display all terms of the WHERE clause */
-#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN)
  if( sqlite3WhereTrace & 0x100 ){
    int i;
-    Vdbe *v = pParse->pVdbe;
-    sqlite3ExplainBegin(v);
    for(i=0; i<sWLB.pWC->nTerm; i++){
-      sqlite3ExplainPrintf(v, "#%-2d ", i);
-      sqlite3ExplainPush(v);
-      whereExplainTerm(v, &sWLB.pWC->a[i]);
-      sqlite3ExplainPop(v);
-      sqlite3ExplainNL(v);
+      whereTermPrint(&sWLB.pWC->a[i], i);
    }
-    sqlite3ExplainFinish(v);
-    sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v));
  }
 #endif
+
  if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
    rc = whereLoopAddAll(&sWLB);
    if( rc ) goto whereBeginError;
@@ -115,7 +115,6 @@ struct WhereLoop {
  union {
    struct {               /* Information for internal btree tables */
      u16 nEq;               /* Number of equality constraints */
-      u16 nSkip;             /* Number of initial index columns to skip */
      Index *pIndex;         /* Index used, or NULL */
    } btree;
    struct {               /* Information for virtual tables */
@@ -128,12 +127,13 @@ struct WhereLoop {
  } u;
  u32 wsFlags;          /* WHERE_* flags describing the plan */
  u16 nLTerm;           /* Number of entries in aLTerm[] */
+  u16 nSkip;            /* Number of NULL aLTerm[] entries */
  /**** whereLoopXfer() copies fields above ***********************/
 # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot)
  u16 nLSlot;           /* Number of slots allocated for aLTerm[] */
  WhereTerm **aLTerm;   /* WhereTerms used */
  WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */
-  WhereTerm *aLTermSpace[4];  /* Initial aLTerm[] space */
+  WhereTerm *aLTermSpace[3];  /* Initial aLTerm[] space */
 };

 /* This object holds the prerequisites and the cost of running a
@@ -0,0 +1,117 @@
+# 2005 July 22
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+# This file implements tests for the ANALYZE command.
+#
+# $Id: analyze.test,v 1.9 2008/08/11 18:44:58 drh Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix analyzeD
+
+ifcapable {!stat4} {
+  finish_test
+  return
+}
+
+
+# Set up a table with the following properties:
+#
+#    * Contains 1000 rows.
+#    * Column a contains even integers between 0 and 18, inclusive (so that
+#      a=? for any such integer matches 100 rows).
+#    * Column b contains integers between 0 and 9, inclusive.
+#    * Column c contains integers between 0 and 199, inclusive (so that
+#      for any such integer, c=? matches 5 rows).
+#    * Then add 7 rows with a new value for "a" - 3001. The stat4 table will
+#      not contain any samples with a=3001.
+#
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, b, c);
+}
+do_test 1.1 {
+  for {set i 1} {$i < 1000} {incr i} {
+    set c [expr $i % 200]
+    execsql { INSERT INTO t1(a, b, c) VALUES( 2*($i/100), $i%10, $c ) }
+  }
+
+  execsql {
+    INSERT INTO t1 VALUES(3001, 3001, 3001);
+    INSERT INTO t1 VALUES(3001, 3001, 3002);
+    INSERT INTO t1 VALUES(3001, 3001, 3003);
+    INSERT INTO t1 VALUES(3001, 3001, 3004);
+    INSERT INTO t1 VALUES(3001, 3001, 3005);
+    INSERT INTO t1 VALUES(3001, 3001, 3006);
+    INSERT INTO t1 VALUES(3001, 3001, 3007);
+
+    CREATE INDEX t1_ab ON t1(a, b);
+    CREATE INDEX t1_c ON t1(c);
+
+    ANALYZE;
+  }
+} {}
+
+# With full ANALYZE data, SQLite sees that c=150 (5 rows) is better than
+# a=3001 (7 rows).
+#
+do_eqp_test 1.2 {
+  SELECT * FROM t1 WHERE a=3001 AND c=150;
+} {
+  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
+}
+
+do_test 1.3 {
+  execsql { DELETE FROM sqlite_stat1 }
+  db close
+  sqlite3 db test.db
+} {}
+
+# Without stat1, because 3001 is larger than all samples in the stat4
+# table, SQLite things that a=3001 matches just 1 row. So it (incorrectly)
+# chooses it over the c=150 index (5 rows). Even with stat1 data, things
+# worked this way before commit [e6f7f97dbc].
+#
+do_eqp_test 1.4 {
+  SELECT * FROM t1 WHERE a=3001 AND c=150;
+} {
+  0 0 0 {SEARCH TABLE t1 USING INDEX t1_ab (a=?)}
+}
+
+do_test 1.5 {
+  execsql { 
+    UPDATE t1 SET a=13 WHERE a = 3001;
+    ANALYZE;
+  }
+} {}
+
+do_eqp_test 1.6 {
+  SELECT * FROM t1 WHERE a=13 AND c=150;
+} {
+  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
+}
+
+do_test 1.7 {
+  execsql { DELETE FROM sqlite_stat1 }
+  db close
+  sqlite3 db test.db
+} {}
+
+# Same test as 1.4, except this time the 7 rows that match the a=? condition 
+# do not feature larger values than all rows in the stat4 table. So SQLite
+# gets this right, even without stat1 data.
+do_eqp_test 1.8 {
+  SELECT * FROM t1 WHERE a=13 AND c=150;
+} {
+  0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)}
+}
+
+finish_test
+
@@ -0,0 +1,242 @@
+# 2014-10-08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for using STAT4 information
+# on a descending index in a range query.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix analyzeE
+
+ifcapable {!stat4} {
+  finish_test
+  return
+}
+
+# Verify that range queries on an ASCENDING index will use the
+# index only if the range covers only a small fraction of the
+# entries.
+#
+do_execsql_test analyzeE-1.0 {
+  CREATE TABLE t1(a,b);
+  WITH RECURSIVE
+    cnt(x) AS (VALUES(1000) UNION ALL SELECT x+1 FROM cnt WHERE x<2000)
+  INSERT INTO t1(a,b) SELECT x, x FROM cnt;
+  CREATE INDEX t1a ON t1(a);
+  ANALYZE;
+} {}
+do_execsql_test analyzeE-1.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500;
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-1.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000;
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.3 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750;
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.4 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1 AND 500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.5 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.6 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.7 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>2500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.8 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1900
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.9 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1100
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-1.10 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1100
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-1.11 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1900
+} {/SCAN TABLE t1/}
+
+# Verify that everything works the same on a DESCENDING index.
+#
+do_execsql_test analyzeE-2.0 {
+  DROP INDEX t1a;
+  CREATE INDEX t1a ON t1(a DESC);
+  ANALYZE;
+} {}
+do_execsql_test analyzeE-2.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500;
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-2.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000;
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.3 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750;
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.4 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1 AND 500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.5 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.6 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.7 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>2500
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.8 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1900
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.9 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1100
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-2.10 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1100
+} {/SEARCH TABLE t1 USING INDEX t1a/}
+do_execsql_test analyzeE-2.11 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1900
+} {/SCAN TABLE t1/}
+
+# Now do a range query on the second term of an ASCENDING index
+# where the first term is constrained by equality.
+#
+do_execsql_test analyzeE-3.0 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a,b,c);
+  WITH RECURSIVE
+    cnt(x) AS (VALUES(1000) UNION ALL SELECT x+1 FROM cnt WHERE x<2000)
+  INSERT INTO t1(a,b,c) SELECT x, x, 123 FROM cnt;
+  CREATE INDEX t1ca ON t1(c,a);
+  ANALYZE;
+} {}
+do_execsql_test analyzeE-3.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123;
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-3.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123;
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.3 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123;
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.4 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.5 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.6 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.7 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>2500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.8 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1900 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.9 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1100 AND c=123
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-3.10 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1100 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-3.11 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1900 AND c=123
+} {/SCAN TABLE t1/}
+
+# Repeat the 3.x tests using a DESCENDING index
+#
+do_execsql_test analyzeE-4.0 {
+  DROP INDEX t1ca;
+  CREATE INDEX t1ca ON t1(c ASC,a DESC);
+  ANALYZE;
+} {}
+do_execsql_test analyzeE-4.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123;
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-4.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123;
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.3 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123;
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.4 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.5 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.6 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.7 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>2500 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.8 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1900 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.9 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a>1100 AND c=123
+} {/SCAN TABLE t1/}
+do_execsql_test analyzeE-4.10 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1100 AND c=123
+} {/SEARCH TABLE t1 USING INDEX t1ca/}
+do_execsql_test analyzeE-4.11 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM t1 WHERE a<1900 AND c=123
+} {/SCAN TABLE t1/}
+
+finish_test
@@ -413,4 +413,101 @@ do_execsql_test autoindex1-801 {
   WHERE mimetypes._id=10 AND data14 IS NOT NULL;
 } {/SEARCH TABLE data .*SEARCH TABLE raw_contacts/}

+# Another test case from an important user of SQLite.  The key feature of
+# this test is that the "aggindex" subquery should make use of an
+# automatic index.  If it does, the query is fast.  If it does not, the
+# query is deathly slow.  It worked OK in 3.7.17 but started going slow
+# with version 3.8.0.  The problem was fixed for 3.8.7 by reducing the
+# cost estimate for automatic indexes on views and subqueries.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_execsql_test autoindex1-900 {
+  CREATE TABLE messages (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, message_id, document_id BLOB, in_reply_to, remote_id INTEGER, sender INTEGER, subject_prefix, subject INTEGER, date_sent INTEGER, date_received INTEGER, date_created INTEGER, date_last_viewed INTEGER, mailbox INTEGER, remote_mailbox INTEGER, original_mailbox INTEGER, flags INTEGER, read, flagged, size INTEGER, color, encoding, type INTEGER, pad, conversation_id INTEGER DEFAULT -1, snippet TEXT DEFAULT NULL, fuzzy_ancestor INTEGER DEFAULT NULL, automated_conversation INTEGER DEFAULT 0, root_status INTEGER DEFAULT -1, conversation_position INTEGER DEFAULT -1);
+  CREATE INDEX date_index ON messages(date_received);
+  CREATE INDEX date_last_viewed_index ON messages(date_last_viewed);
+  CREATE INDEX date_created_index ON messages(date_created);
+  CREATE INDEX message_message_id_mailbox_index ON messages(message_id, mailbox);
+  CREATE INDEX message_document_id_index ON messages(document_id);
+  CREATE INDEX message_read_index ON messages(read);
+  CREATE INDEX message_flagged_index ON messages(flagged);
+  CREATE INDEX message_mailbox_index ON messages(mailbox, date_received);
+  CREATE INDEX message_remote_mailbox_index ON messages(remote_mailbox, remote_id);
+  CREATE INDEX message_type_index ON messages(type);
+  CREATE INDEX message_conversation_id_conversation_position_index ON messages(conversation_id, conversation_position);
+  CREATE INDEX message_fuzzy_ancestor_index ON messages(fuzzy_ancestor);
+  CREATE INDEX message_subject_fuzzy_ancestor_index ON messages(subject, fuzzy_ancestor);
+  CREATE INDEX message_sender_subject_automated_conversation_index ON messages(sender, subject, automated_conversation);
+  CREATE INDEX message_sender_index ON messages(sender);
+  CREATE INDEX message_root_status ON messages(root_status);
+  CREATE TABLE subjects (ROWID INTEGER PRIMARY KEY, subject COLLATE RTRIM, normalized_subject COLLATE RTRIM);
+  CREATE INDEX subject_subject_index ON subjects(subject);
+  CREATE INDEX subject_normalized_subject_index ON subjects(normalized_subject);
+  CREATE TABLE addresses (ROWID INTEGER PRIMARY KEY, address COLLATE NOCASE, comment, UNIQUE(address, comment));
+  CREATE INDEX addresses_address_index ON addresses(address);
+  CREATE TABLE mailboxes (ROWID INTEGER PRIMARY KEY, url UNIQUE, total_count INTEGER DEFAULT 0, unread_count INTEGER DEFAULT 0, unseen_count INTEGER DEFAULT 0, deleted_count INTEGER DEFAULT 0, unread_count_adjusted_for_duplicates INTEGER DEFAULT 0, change_identifier, source INTEGER, alleged_change_identifier);
+  CREATE INDEX mailboxes_source_index ON mailboxes(source);
+  CREATE TABLE labels (ROWID INTEGER PRIMARY KEY, message_id INTEGER NOT NULL, mailbox_id INTEGER NOT NULL, UNIQUE(message_id, mailbox_id));
+  CREATE INDEX labels_message_id_mailbox_id_index ON labels(message_id, mailbox_id);
+  CREATE INDEX labels_mailbox_id_index ON labels(mailbox_id);
+  
+  explain query plan
+  SELECT messages.ROWID,
+         messages.message_id,
+         messages.remote_id,
+         messages.date_received,
+         messages.date_sent,
+         messages.flags,
+         messages.size,
+         messages.color,
+         messages.date_last_viewed,
+         messages.subject_prefix,
+         subjects.subject,
+         sender.comment,
+         sender.address,
+         NULL,
+         messages.mailbox,
+         messages.original_mailbox,
+         NULL,
+         NULL,
+         messages.type,
+         messages.document_id,
+         sender,
+         NULL,
+         messages.conversation_id,
+         messages.conversation_position,
+         agglabels.labels
+   FROM mailboxes AS mailbox
+        JOIN messages ON mailbox.ROWID = messages.mailbox
+        LEFT OUTER JOIN subjects ON messages.subject = subjects.ROWID
+        LEFT OUTER JOIN addresses AS sender ON messages.sender = sender.ROWID
+        LEFT OUTER JOIN (
+               SELECT message_id, group_concat(mailbox_id) as labels
+               FROM labels GROUP BY message_id
+             ) AS agglabels ON messages.ROWID = agglabels.message_id
+  WHERE (mailbox.url = 'imap://email.app@imap.gmail.com/%5BGmail%5D/All%20Mail')
+    AND (messages.ROWID IN (
+            SELECT labels.message_id
+              FROM labels JOIN mailboxes ON labels.mailbox_id = mailboxes.ROWID
+             WHERE mailboxes.url = 'imap://email.app@imap.gmail.com/INBOX'))
+    AND messages.mailbox in (6,12,18,24,30,36,42,1,7,13,19,25,31,37,43,2,8,
+                             14,20,26,32,38,3,9,15,21,27,33,39,4,10,16,22,28,
+                             34,40,5,11,17,23,35,41)
+   ORDER BY date_received DESC;
+} {/agglabels USING AUTOMATIC COVERING INDEX/}
+
+# A test case for VIEWs
+#
+do_execsql_test autoindex1-901 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z);
+  CREATE TABLE t2(a, b);
+  CREATE VIEW agg2 AS SELECT a, sum(b) AS m FROM t2 GROUP BY a;
+  EXPLAIN QUERY PLAN
+  SELECT t1.z, agg2.m
+    FROM t1 JOIN agg2 ON t1.y=agg2.m
+   WHERE t1.x IN (1,2,3);
+} {/USING AUTOMATIC COVERING INDEX/}
+
+
 finish_test
@@ -75,31 +75,34 @@ do_test 2.2 {
  catchsql { SELECT * FROM r WHERE x >= 10 }
 } {1 {database disk image is malformed}}

-reset_db
-
-do_execsql_test 3.1 {
-  PRAGMA page_size = 512;
-  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
-  WITH s(a, b) AS (
-    SELECT 2, 'abcdefghij'
-    UNION ALL
-    SELECT a+2, b FROM s WHERe a < 40
-  )
-  INSERT INTO t1 SELECT * FROM s;
-} {}
-
-do_test 3.2 {
-  hexio_write test.db [expr 512+3] 0054
-  db close
-  sqlite3 db test.db
-  execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') }
-  execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') }
-} {}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 3.2 {
-  INSERT INTO t1 VALUES(9, 'klmnopqrst');
-} {1 {database disk image is malformed}}
-
+if {[db one {SELECT sqlite_compileoption_used('ENABLE_OVERSIZE_CELL_CHECK')}]} {
+  # The following tests only work if OVERSIZE_CELL_CHECK is disabled
+} else {
+  reset_db
+  do_execsql_test 3.1 {
+     PRAGMA auto_vacuum=0;
+     PRAGMA page_size = 512;
+     CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+     WITH s(a, b) AS (
+       SELECT 2, 'abcdefghij'
+       UNION ALL
+       SELECT a+2, b FROM s WHERe a < 40
+     )
+     INSERT INTO t1 SELECT * FROM s;
+   } {}
+   
+   do_test 3.2 {
+     hexio_write test.db [expr 512+3] 0054
+     db close
+     sqlite3 db test.db
+     execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') }
+     execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') }
+   } {}
+   
+   db close
+   sqlite3 db test.db
+   do_catchsql_test 3.3 {
+     INSERT INTO t1 VALUES(9, 'klmnopqrst');
+   } {1 {database disk image is malformed}}
+} ;# end-if !defined(ENABLE_OVERSIZE_CELL_CHECK)
 finish_test
@@ -99,4 +99,32 @@ do_execsql_test default-3.3 {
  SELECT * FROM t300;
 } {2147483647 2147483648 9223372036854775807 -2147483647 -2147483648 -9223372036854775808 9.22337203685478e+18 9223372036854775807}

+# Do now allow bound parameters in new DEFAULT values. 
+# Silently convert bound parameters to NULL in DEFAULT causes
+# in the sqlite_master table, for backwards compatibility.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_execsql_test default-4.0 {
+  CREATE TABLE t1(a TEXT, b TEXT DEFAULT(99));
+  PRAGMA writable_schema=ON;
+  UPDATE sqlite_master SET sql='CREATE TABLE t1(a TEXT, b TEXT DEFAULT(:xyz))';
+} {}
+db close 
+sqlite3 db test.db
+do_execsql_test default-4.1 {
+  INSERT INTO t1(a) VALUES('xyzzy');
+  SELECT a, quote(b) FROM t1;
+} {xyzzy NULL}
+do_catchsql_test default-4.2 {
+  CREATE TABLE t2(a TEXT, b TEXT DEFAULT(:xyz));
+} {1 {default value of column [b] is not constant}}
+do_catchsql_test default-4.3 {
+  CREATE TABLE t2(a TEXT, b TEXT DEFAULT(abs(:xyz)));
+} {1 {default value of column [b] is not constant}}
+do_catchsql_test default-4.4 {
+  CREATE TABLE t2(a TEXT, b TEXT DEFAULT(98+coalesce(5,:xyz)));
+} {1 {default value of column [b] is not constant}}
+
 finish_test
@@ -862,11 +862,11 @@ do_createtable_tests 3.2.3 -query {
  3   "INSERT INTO t1 DEFAULT VALUES"               {NULL NULL NULL}
 }

-# EVIDENCE-OF: R-62940-43005 An explicit DEFAULT clause may specify that
+# EVIDENCE-OF: R-07343-35026 An explicit DEFAULT clause may specify that
 # the default value is NULL, a string constant, a blob constant, a
-# signed-number, or any constant expression enclosed in parentheses. An
-# explicit default value may also be one of the special case-independent
-# keywords CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP.
+# signed-number, or any constant expression enclosed in parentheses. A
+# default value may also be one of the special case-independent keywords
+# CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP.
 #
 do_execsql_test e_createtable-3.3.1 {
  CREATE TABLE t4(
@@ -884,9 +884,9 @@ do_execsql_test e_createtable-3.3.1 {
  );
 } {}

-# EVIDENCE-OF: R-36381-62919 For the purposes of the DEFAULT clause, an
-# expression is considered constant provided that it does not contain
-# any sub-queries, column or table references, or string literals
+# EVIDENCE-OF: R-18415-27776 For the purposes of the DEFAULT clause, an
+# expression is considered constant if it does contains no sub-queries,
+# column or table references, bound parameters, or string literals
 # enclosed in double-quotes instead of single-quotes.
 #
 do_createtable_tests 3.4.1 -error {
@@ -896,6 +896,7 @@ do_createtable_tests 3.4.1 -error {
  2   {CREATE TABLE t5(x DEFAULT ( "abc" ))}  {}
  3   {CREATE TABLE t5(x DEFAULT ( 1 IN (SELECT 1) ))}  {}
  4   {CREATE TABLE t5(x DEFAULT ( EXISTS (SELECT 1) ))}  {}
+  5   {CREATE TABLE t5(x DEFAULT ( x!=?1 ))}  {}
 }
 do_createtable_tests 3.4.2 -repair {
  catchsql { DROP TABLE t5 }
@@ -125,6 +125,9 @@ if {$tcl_platform(platform) == "unix"} {
 sqlite3_shutdown
 sqlite3_config_uri 1

+# EVIDENCE-OF: R-06842-00595 If the URI contains an authority, then it
+# must be either an empty string or the string "localhost".
+#
 # EVIDENCE-OF: R-17482-00398 If the authority is not an empty string or
 # "localhost", an error is returned to the caller.
 #
@@ -58,6 +58,18 @@ do_test eval-2.2 {
    SELECT * FROM t2
  }
 } {}
+do_test eval-2.3 {
+  execsql {
+    INSERT INTO t2 SELECT x, x+1 FROM t1 WHERE x<5;
+    SELECT x, test_eval('DELETE FROM t2 WHERE x='||x), y FROM t2
+     ORDER BY rowid DESC;
+  }
+} {4 {} {} 3 {} {} 2 {} {} 1 {} {}}
+do_test eval-2.4 {
+  execsql {
+    SELECT * FROM t2
+  }
+} {}

 # Modify a row while it is being read.
 #
@@ -205,6 +205,10 @@ test_expr expr-1.125 {i1=6, i2=NULL} \
 test_expr expr-1.126 {i1=8, i2=8} \
  {CASE WHEN i1 IS NOT i2 THEN 'yes' ELSE 'no' END} no

+do_catchsql_test expr-1.127 {
+  SELECT 1 IS #1;
+} {1 {near "#1": syntax error}}
+
 ifcapable floatingpoint {if {[working_64bit_int]} {
  test_expr expr-1.200\
      {i1=9223372036854775806, i2=1} {i1+i2}      9223372036854775807
@@ -24,12 +24,16 @@ ifcapable !fts3||!icu {

 set sqlite_fts3_enable_parentheses 1

-proc test_icu_fts3expr {expr} {
-  db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')}
+proc test_fts3expr {tokenizer expr} {
+  db one {SELECT fts3_exprtest($tokenizer, $expr, 'a', 'b', 'c')}
 }

 proc do_icu_expr_test {tn expr res} {
-  uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res]
+  uplevel [list do_test $tn [list test_fts3expr icu $expr] [list {*}$res]]
+}
+
+proc do_simple_expr_test {tn expr res} {
+  uplevel [list do_test $tn [list test_fts3expr simple $expr] [list {*}$res]]
 }

 #-------------------------------------------------------------------------
@@ -53,5 +57,26 @@ do_icu_expr_test 2.1 {
  f (e NEAR/2 a)
 } {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}

+#-------------------------------------------------------------------------
+#
+do_simple_expr_test 3.1 {*lOl* *h4h*} {
+  AND {PHRASE 3 0 lol+} {PHRASE 3 0 h4h+}
+}
+
+do_icu_expr_test 3.2 {*lOl* *h4h*} {
+  AND {AND {AND {PHRASE 3 0 *} {PHRASE 3 0 lol+}} {PHRASE 3 0 *}} {PHRASE 3 0 h4h+}
+}
+
+do_simple_expr_test 3.3 { * }    { }
+do_simple_expr_test 3.4 { *a }   { PHRASE 3 0 a }
+do_simple_expr_test 3.5 { a*b }  { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
+do_simple_expr_test 3.6 { *a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
+do_simple_expr_test 3.7 { *"abc" } { PHRASE 3 0 abc }
+do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc }
+do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c }
+
+do_icu_expr_test    3.9 { "ab*c" } { PHRASE 3 0 ab+ * c }
+do_icu_expr_test    3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}}
+
 finish_test

@@ -433,4 +433,21 @@ do_execsql_test 9.1 {
  SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c';
 } {{[c]} {[c]}}

+#---------------------------------------------------------------------------
+# Test for a memory leak
+#
+do_execsql_test 10.1 {
+  DROP TABLE t10;
+  CREATE VIRTUAL TABLE t10 USING fts4(idx, value);
+  INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
+  SELECT docId, t10.*
+    FROM t10
+    JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
+   WHERE t10 MATCH x.idx
+     AND matchinfo(t10) not null
+   GROUP BY docId
+   ORDER BY 1;
+} {1 1 one 2 2 two 3 3 three}
+  
+
 finish_test
@@ -16,6 +16,9 @@ source $testdir/tester.tcl
 set ::testprefix index5

 do_test 1.1 {
+  if {[permutation]=="memsubsys1"} {
+    execsql { PRAGMA auto_vacuum = 0; }
+  }
  execsql {
    PRAGMA page_size = 1024;
    CREATE TABLE t1(x);
@@ -38,7 +41,7 @@ tvfs filter xWrite
 tvfs script write_cb
 proc write_cb {xCall file handle iOfst args} {
  if {[file tail $file]=="test.db"} {
-    lappend ::write_list [expr $iOfst/1024]
+    lappend ::write_list [expr $iOfst/1024 + 1]
  }
 }

@@ -154,6 +154,7 @@ do_test lock5-flock.8 {
 do_test lock5-none.1 {
  sqlite3 db test.db -vfs unix-none
  sqlite3 db2 test.db -vfs unix-none
+  execsql { PRAGMA mmap_size = 0 } db2
  execsql {
    BEGIN;
    INSERT INTO t1 VALUES(3, 4);
@@ -162,8 +163,8 @@ do_test lock5-none.1 {
 do_test lock5-none.2 {
  execsql { SELECT * FROM t1 }
 } {1 2 3 4}
-do_test lock5-flock.3 {
-  execsql { SELECT * FROM t1 } db2
+do_test lock5-none.3 {
+  execsql { SELECT * FROM t1; } db2
 } {1 2}
 do_test lock5-none.4 {
  execsql { 
@@ -183,7 +184,7 @@ ifcapable memorymanage {
  } {1 2 3 4}
 }

-do_test lock5-flock.X {
+do_test lock5-none.X {
  db close
  db2 close
 } {}
@@ -119,8 +119,6 @@ do_execsql_test 7.0 {
  PRAGMA cache_size = 5;
 }
 do_faultsim_test 7 -faults oom-trans* -prep {
-  if {$iFail < 500} { set iFail 2000 }
-  if {$iFail > 1215} { set iFail 2000 }
 } -body {
  execsql {
    WITH r(x,y) AS (
@@ -0,0 +1,114 @@
+# 2014-09-25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for the "truncate" option in the multiplexor.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix multiplex4
+
+db close
+sqlite3_shutdown
+sqlite3_multiplex_initialize {} 0
+
+# delete all filesl with the base name of $basename
+#
+proc multiplex_delete_db {basename} {
+  foreach file [glob -nocomplain $basename.*] {
+    forcedelete $file
+  }
+}
+
+# Return a sorted list of all files with the base name of $basename.
+# Except, delete all text from the end of $basename through the NNN
+# suffix on the end of the filename.
+#
+proc multiplex_file_list {basename} {
+  set x {}
+  foreach file [glob -nocomplain $basename.*] {
+    regsub "^$basename\\..*(\\d\\d\\d)\$" $file $basename.\\1 file
+    lappend x $file
+  }
+  return [lsort $x]
+}
+
+do_test multiplex4-1.0 {
+  multiplex_delete_db mx4test
+  sqlite3 db {file:mx4test.db?chunksize=10&truncate=1} -uri 1 -vfs multiplex
+  db eval {
+    CREATE TABLE t1(x);
+    INSERT INTO t1(x) VALUES(randomblob(250000));
+  }
+  multiplex_file_list mx4test
+} {mx4test.001 mx4test.db}
+
+do_test multiplex4-1.1 {
+  db eval {
+    DELETE FROM t1;
+    VACUUM;
+  }
+  multiplex_file_list mx4test
+} {mx4test.db}
+
+do_test multiplex4-1.2 {
+  db eval {PRAGMA multiplex_truncate}
+} {on}
+do_test multiplex4-1.3 {
+  db eval {PRAGMA multiplex_truncate=off}
+} {off}
+do_test multiplex4-1.4 {
+  db eval {PRAGMA multiplex_truncate}
+} {off}
+do_test multiplex4-1.5 {
+  db eval {PRAGMA multiplex_truncate=on}
+} {on}
+do_test multiplex4-1.6 {
+  db eval {PRAGMA multiplex_truncate}
+} {on}
+do_test multiplex4-1.7 {
+  db eval {PRAGMA multiplex_truncate=0}
+} {off}
+do_test multiplex4-1.8 {
+  db eval {PRAGMA multiplex_truncate=1}
+} {on}
+do_test multiplex4-1.9 {
+  db eval {PRAGMA multiplex_truncate=0}
+} {off}
+
+do_test multiplex4-1.10 {
+  db eval {
+    INSERT INTO t1(x) VALUES(randomblob(250000));
+  }
+  multiplex_file_list mx4test
+} {mx4test.001 mx4test.db}
+
+do_test multiplex4-1.11 {
+  db eval {
+    DELETE FROM t1;
+    VACUUM;
+  }
+  multiplex_file_list mx4test
+} {mx4test.001 mx4test.db}
+
+do_test multiplex4-1.12 {
+  db eval {
+    PRAGMA multiplex_truncate=ON;
+    DROP TABLE t1;
+    VACUUM;
+  }
+  multiplex_file_list mx4test
+} {mx4test.db}
+
+catch { db close }
+forcedelete mx4test.db
+sqlite3_multiplex_shutdown
+finish_test
@@ -0,0 +1,49 @@
+# 2014 October 01
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the SQLITE_DIRECT_OVERFLOW_READ logic.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix ovfl
+
+# Populate table t2:
+#
+#   CREATE TABLE t1(c1 TEXT, c2 TEXT);
+#
+# with 2000 rows. In each row, c2 spans multiple overflow pages. The text
+# value of c1 ranges in size from 1 to 2000 bytes. The idea is to create
+# at least one row where the first byte of c2 is also the first byte of
+# an overflow page. This was at one point exposing an obscure bug in the
+# SQLITE_DIRECT_OVERFLOW_READ logic.
+#
+do_test 1.1 {
+  set c2 [string repeat abcdefghij 200]
+  execsql {
+    PRAGMA cache_size = 10;
+    CREATE TABLE t1(c1 TEXT, c2 TEXT);
+    BEGIN;
+  }
+  for {set i 1} {$i <= 2000} {incr i} {
+    set c1 [string repeat . $i]
+    execsql { INSERT INTO t1 VALUES($c1, $c2) }
+  }
+  execsql COMMIT
+} {}
+
+do_execsql_test 1.2 {
+  SELECT sum(length(c2)) FROM t1;
+} [expr 2000 * 2000]
+
+finish_test
+
+
@@ -13,6 +13,7 @@ optional) are:
    -makefile PATH-TO-MAKEFILE           (default "releasetest.mk")
    -platform PLATFORM                   (see below)
    -quick    BOOLEAN                    (default "0")
+    -config   CONFIGNAME                 (Run only CONFIGNAME)

 The default value for -makefile is "./releasetest.mk".

@@ -195,7 +196,8 @@ array set ::Platforms {
    "Device-Two"              test
    "Ftrapv"                  test
    "No-lookaside"            test
-    "Default"                 "threadtest test"
+    "Devkit"                  test
+    "Default"                 "threadtest fulltest"
    "Device-One"              fulltest
  }
  Linux-i686 {
@@ -292,6 +294,7 @@ proc run_test_suite {name testtarget config} {
 proc process_options {argv} {
  set ::MAKEFILE releasetest.mk                       ;# Default value
  set ::QUICK    0                                    ;# Default value
+  set config {}
  set platform $::tcl_platform(os)-$::tcl_platform(machine)

  for {set i 0} {$i < [llength $argv]} {incr i} {
@@ -310,6 +313,11 @@ proc process_options {argv} {
        incr i
        set ::QUICK [lindex $argv $i]
      }
+
+      -config {
+        incr i
+        set config [lindex $argv $i]
+      }
  
      default {
        puts stderr ""
@@ -333,7 +341,12 @@ proc process_options {argv} {
    exit
  }

-  set ::CONFIGLIST $::Platforms($platform)
+  if {$config!=""} {
+    if {[llength $config]==1} {lappend config fulltest}
+    set ::CONFIGLIST $config
+  } else {
+    set ::CONFIGLIST $::Platforms($platform)
+  }
  puts "Running the following configurations for $platform:"
  puts "    [string trim $::CONFIGLIST]"
 }
@@ -679,9 +679,9 @@ do_test rowid-12.2 {
  save_prng_state
  execsql {
    INSERT INTO t7 VALUES(NULL,'b');
-    SELECT x, y FROM t7;
+    SELECT x, y FROM t7 ORDER BY x;
  }
-} {1 b 9223372036854775807 a}
+} {/\d+ b 9223372036854775807 a/}
 execsql {INSERT INTO t7 VALUES(2,'y');}
 for {set i 1} {$i<100} {incr i} {
  do_test rowid-12.3.$i {
@@ -701,5 +701,19 @@ do_test rowid-12.4 {
  }
 } {1 {database or disk is full}}

+# INSERTs that happen inside of nested function calls are recorded
+# by last_insert_rowid.
+#
+proc rowid_addrow_func {n} {
+  db eval {INSERT INTO t13(rowid,x) VALUES($n,$n*$n)}
+  return [db last_insert_rowid]
+}
+db function addrow rowid_addrow_func
+do_execsql_test rowid-13.1 {
+  CREATE TABLE t13(x);
+  INSERT INTO t13(rowid,x) VALUES(1234,5);
+  SELECT rowid, x, addrow(rowid+1000), '|' FROM t13 LIMIT 3;
+  SELECT last_insert_rowid();
+} {1234 5 2234 | 2234 4990756 3234 | 3234 10458756 4234 | 4234}

 finish_test
@@ -245,6 +245,32 @@ do_execsql_test skipscan1-5.3 {
    SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N';
 } {/.*COVERING INDEX t5i1 .*/}

-
+# The column used by the skip-scan needs to be sufficiently selective.
+# See the private email from Adi Zaimi to drh@sqlite.org on 2014-09-22.
+#
+db close
+forcedelete test.db
+sqlite3 db test.db
+do_execsql_test skipscan1-6.1 {
+  CREATE TABLE t1(a,b,c,d,e,f,g,h varchar(300));
+  CREATE INDEX t1ab ON t1(a,b);
+  ANALYZE sqlite_master;
+  -- Only two distinct values for the skip-scan column.  Skip-scan is not used.
+  INSERT INTO sqlite_stat1 VALUES('t1','t1ab','500000 250000 125000');
+  ANALYZE sqlite_master;
+  EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1;
+} {~/ANY/}
+do_execsql_test skipscan1-6.2 {
+  -- Four distinct values for the skip-scan column.  Skip-scan is used.
+  UPDATE sqlite_stat1 SET stat='500000 250000 62500';
+  ANALYZE sqlite_master;
+  EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1;
+} {/ANY.a. AND b=/}
+do_execsql_test skipscan1-6.3 {
+  -- Two distinct values for the skip-scan column again.  Skip-scan is not used.
+  UPDATE sqlite_stat1 SET stat='500000 125000 62500';
+  ANALYZE sqlite_master;
+  EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1;
+} {~/ANY/}

 finish_test
@@ -108,7 +108,7 @@ foreach {tn dbenc coll} {
    3 { c > 'q' }                     {/*ANY(a) AND ANY(b) AND c>?*/}
    4 { c > 'e' }                     {/*SCAN TABLE t2*/}
    5 { c < 'q' }                     {/*SCAN TABLE t2*/}
-    4 { c < 'e' }                     {/*ANY(a) AND ANY(b) AND c<?*/}
+    6 { c < 'c' }                     {/*ANY(a) AND ANY(b) AND c<?*/}
  } {
    set sql "EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE $q" 
    do_execsql_test 2.$tn.$tn2 $sql $res
@@ -180,7 +180,3 @@ foreach {tn q res} {
 }

 finish_test
-
-
-
-
@@ -0,0 +1,145 @@
+# 2014-10-21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements tests of the "skip-scan" query strategy. In 
+# particular, this file verifies that use of all columns of an index
+# is always preferred over the use of a skip-scan on some columns of
+# the same index.  Because of difficulties in scoring a skip-scan,
+# the skip-scan can sometimes come out with a lower raw score when
+# using STAT4.  But the query planner should detect this and use the
+# full index rather than the skip-scan.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix skipscan6
+
+ifcapable !stat4 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.1 {
+  CREATE TABLE t1(
+    aa int,
+    bb int,
+    cc int,
+    dd int,
+    ee int
+  );
+  CREATE INDEX ix on t1(aa, bb, cc,  dd DESC);
+  ANALYZE sqlite_master;
+  INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5');
+  INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35');
+  ANALYZE sqlite_master;
+} {}
+do_execsql_test 1.2 {
+  EXPLAIN QUERY PLAN
+  SELECT COUNT(*)
+    FROM t1
+   WHERE bb=21
+     AND aa=1
+     AND dd BETWEEN 1413833728 and 1413837331;
+} {/INDEX ix .aa=. AND bb=../}
+
+do_execsql_test 2.1 {
+  DROP INDEX ix;
+  CREATE INDEX good on t1(bb, aa, dd DESC);
+  CREATE INDEX bad on t1(aa, bb, cc,  dd DESC);
+  DELETE FROM sqlite_stat1;
+  DELETE FROM sqlite_stat4;
+  INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2');
+  INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F');
+  INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5');
+  INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35');
+  ANALYZE sqlite_master;
+} {}
+do_execsql_test 2.2 {
+  EXPLAIN QUERY PLAN
+  SELECT COUNT(*)
+    FROM t1
+   WHERE bb=21
+     AND aa=1
+     AND dd BETWEEN 1413833728 and 1413837331;
+} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../}
+
+
+
+finish_test
@@ -15,6 +15,7 @@

 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
+set testprefix sort

 # Create a bunch of data to sort against
 #
@@ -934,6 +934,7 @@ void testset_cte(void){

 }

+#ifdef SQLITE_ENABLE_RTREE
 /* Generate two numbers between 1 and mx.  The first number is less than
 ** the second.  Usually the numbers are near each other but can sometimes
 ** be far apart.
@@ -954,7 +955,9 @@ static void twoCoords(
  *pX0 = x0;
  *pX1 = x1;
 }
+#endif

+#ifdef SQLITE_ENABLE_RTREE
 /* The following routine is an R-Tree geometry callback.  It returns
 ** true if the object overlaps a slice on the Y coordinate between the
 ** two values given as arguments.  In other words
@@ -974,7 +977,9 @@ static int xsliceGeometryCallback(
  *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1];
  return SQLITE_OK;
 }
+#endif /* SQLITE_ENABLE_RTREE */

+#ifdef SQLITE_ENABLE_RTREE
 /*
 ** A testset for the R-Tree virtual table
 */
@@ -1110,6 +1115,7 @@ void testset_rtree(int p1, int p2){
  }
  speedtest1_end_test();
 }
+#endif /* SQLITE_ENABLE_RTREE */

 /*
 ** A testset used for debugging speedtest1 itself.
@@ -1329,7 +1335,12 @@ int main(int argc, char **argv){
  }else if( strcmp(zTSet,"cte")==0 ){
    testset_cte();
  }else if( strcmp(zTSet,"rtree")==0 ){
+#ifdef SQLITE_ENABLE_RTREE
    testset_rtree(6, 147);
+#else
+    fatal_error("compile with -DSQLITE_ENABLE_RTREE to enable "
+                "the R-Tree tests\n");
+#endif
  }else{
    fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n",
                 zTSet);
@@ -51,6 +51,13 @@ do_test sqllimits1-1.9 {
 do_test sqllimits1-1.10 {
  sqlite3_limit db SQLITE_LIMIT_VARIABLE_NUMBER -1
 } $SQLITE_MAX_VARIABLE_NUMBER
+do_test sqllimits1-1.11 {
+  sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH -1
+} $SQLITE_MAX_TRIGGER_DEPTH
+do_test sqllimits1-1.12 {
+  sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 99999
+  sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1
+} $SQLITE_MAX_WORKER_THREADS

 # Limit parameters out of range.
 #
@@ -103,5 +103,50 @@ do_execsql_test 2.2 {
  LIMIT (SELECT a FROM t5)
 } {2 3   3 6   4 10}

+############################################################################
+# Ticket http://www.sqlite.org/src/info/d11a6e908f (2014-09-20)
+# Query planner fault on three-way nested join with compound inner SELECT 
+#
+do_execsql_test 3.0 {
+  DROP TABLE IF EXISTS t1;
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t1 (id INTEGER PRIMARY KEY, data TEXT);
+  INSERT INTO t1(id,data) VALUES(9,'nine-a');
+  INSERT INTO t1(id,data) VALUES(10,'ten-a');
+  INSERT INTO t1(id,data) VALUES(11,'eleven-a');
+  CREATE TABLE t2 (id INTEGER PRIMARY KEY, data TEXT);
+  INSERT INTO t2(id,data) VALUES(9,'nine-b');
+  INSERT INTO t2(id,data) VALUES(10,'ten-b');
+  INSERT INTO t2(id,data) VALUES(11,'eleven-b');
+  
+  SELECT id FROM (
+    SELECT id,data FROM (
+       SELECT * FROM t1 UNION ALL SELECT * FROM t2
+    )
+    WHERE id=10 ORDER BY data
+  );
+} {10 10}
+do_execsql_test 3.1 {
+  SELECT data FROM (
+     SELECT 'dummy', data FROM (
+       SELECT data FROM t1 UNION ALL SELECT data FROM t1
+     ) ORDER BY data
+  );
+} {eleven-a eleven-a nine-a nine-a ten-a ten-a}
+do_execsql_test 3.2 {
+  DROP TABLE IF EXISTS t3;
+  DROP TABLE IF EXISTS t4;
+  CREATE TABLE t3(id INTEGER, data TEXT);
+  CREATE TABLE t4(id INTEGER, data TEXT);
+  INSERT INTO t3 VALUES(4, 'a'),(2,'c');
+  INSERT INTO t4 VALUES(3, 'b'),(1,'d');
+
+  SELECT data, id FROM (
+    SELECT id, data FROM (
+       SELECT * FROM t3 UNION ALL SELECT * FROM t4
+    ) ORDER BY data
+  );
+} {a 4 b 3 c 2 d 1}
+

 finish_test
@@ -0,0 +1,65 @@
+# 2014-10-11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Test that ticket [ba7cbfaedc] has been fixed.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix tkt-ba7cbfaedc
+
+do_execsql_test 1 {
+  CREATE TABLE t1 (x, y);
+  INSERT INTO t1 VALUES (3, 'a');
+  INSERT INTO t1 VALUES (1, 'a'); 
+  INSERT INTO t1 VALUES (2, 'b');
+  INSERT INTO t1 VALUES (2, 'a');
+  INSERT INTO t1 VALUES (3, 'b');
+  INSERT INTO t1 VALUES (1, 'b'); 
+}
+
+do_execsql_test 1.1 {
+  CREATE INDEX i1 ON t1(x, y);
+}
+
+foreach {n idx} {
+  1 { CREATE INDEX i1 ON t1(x, y) }
+  2 { CREATE INDEX i1 ON t1(x DESC, y) }
+  3 { CREATE INDEX i1 ON t1(x, y DESC) }
+  4 { CREATE INDEX i1 ON t1(x DESC, y DESC) }
+} {
+  catchsql { DROP INDEX i1 }
+  execsql $idx
+  foreach {tn q res} {
+    1 "GROUP BY x, y ORDER BY x, y"            {1 a 1 b   2 a 2 b   3 a 3 b}
+    2 "GROUP BY x, y ORDER BY x DESC, y"       {3 a 3 b   2 a 2 b   1 a 1 b}
+    3 "GROUP BY x, y ORDER BY x, y DESC"       {1 b 1 a   2 b 2 a   3 b 3 a}
+    4 "GROUP BY x, y ORDER BY x DESC, y DESC"  {3 b 3 a   2 b 2 a   1 b 1 a}
+  } {
+    do_execsql_test 1.$n.$tn "SELECT * FROM t1 $q" $res
+  }
+}
+
+do_execsql_test 2.0 {
+  drop table if exists t1;
+  create table t1(id int);
+  insert into t1(id) values(1),(2),(3),(4),(5);
+  create index t1_idx_id on t1(id asc);
+  select * from t1 group by id order by id;
+  select * from t1 group by id order by id asc;
+  select * from t1 group by id order by id desc;
+} {
+  1 2 3 4 5   1 2 3 4 5   5 4 3 2 1
+}
+
+finish_test
+
+
@@ -110,6 +110,11 @@ while {![eof stdin]} {
    foreach v $vlist {
      regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line
      regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line
+
+      # The expressions above fail to catch instance of variable "abc" in
+      # expressions like (32>abc). The following expression makes those
+      # substitutions.
+      regsub -all "(\[^-\])>${v}(\\W)" $line "\\1>u.$sname.$v\\2" line
    }
    append afterUnion [string trimright $line]\n
  } elseif {$line=="" && [eof stdin]} {