Prevent Tcl file encoding issues in the tests for the unicode() and char() functions.

FossilOrigin-Name: d2e7dfca5a92074a7984032deb6a4e3681389c72
Remove extra use of the sqlite3_value_int64() function.
2013-02-26 05:44:33 +00:00 · 2013-02-26 05:42:30 +00:00 · 2013-02-25 14:39:47 +00:00 · 2013-02-25 13:55:59 +00:00 · 2013-02-25 13:44:56 +00:00 · 2013-02-25 13:31:30 +00:00
28 changed files with 870 additions and 309 deletions
@@ -1,5 +1,5 @@
-C Improve\smemory\sallocation\serror\shandling\son\sWinCE.
-D 2013-02-12T09:46:48.810
+C Prevent\sTcl\sfile\sencoding\sissues\sin\sthe\stests\sfor\sthe\sunicode()\sand\schar()\sfunctions.
+D 2013-02-26T05:44:33.955
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -118,12 +118,12 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168
 F src/analyze.c 7553068d21e32a57fc33ab6b2393fc8c1ba41410
 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1
 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
-F src/backup.c 32e35a3a4ea55b45c0e5f74eeb793aec71491517
+F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68
 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1
 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 7a80e4a67f32a2494c383a28a495bf3bd71cc230
+F src/btree.c cbad71970cfadfa342fc137ca5e319f98b2d0da1
 F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd
-F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621
+F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2
 F src/build.c 73ca65f32938e4e0d94e831b61b5749b211b79be
 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
@@ -133,7 +133,7 @@ F src/delete.c 9b8d308979114991e5dc7cee958316e07186941d
 F src/expr.c f6c20285bd36e87ec47f4d840e90a32755e2a90c
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179
-F src/func.c 8147799b048065a1590805be464d05b4913e652c
+F src/func.c cac45cca7bbe29bbefef46116174e89e1284763b
 F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a
 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4
 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970
@@ -143,7 +143,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
 F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d
-F src/main.c 448564ee3795b13c373bbc5c8590df8fbfe1d64c
+F src/main.c 8d204866d1abf5100503dcd54d3187b88f6846b7
 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa
@@ -161,25 +161,25 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c
 F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57
 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04
 F src/os_unix.c dfdc04b126f7b05dcb2e2cc5c1262f98acbb49d9
-F src/os_win.c e988c2de4266fbb3b8c58e4b5550a2e17f541327
-F src/pager.c 4092c907222cfd451c74fe6bd2fd64b342f7190f
+F src/os_win.c eabd00b813577d36bd66271cb08dd64ea0589dac
+F src/pager.c 0dbf5ff5d5d7d3a21fcab82e9e4d129b6fe6314f
 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0
 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c
 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9
-F src/pragma.c eb9d39e30f00e3ba51892749ddf9d7b86541c43e
+F src/pragma.c bdb484d0283965c431d4153f28c30f836a1f16b1
 F src/prepare.c 931ad0d852a0df48f79adcba6ce79ca5f475625c
 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f
 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
 F src/resolve.c 652ae6dc0f185b01b4536bb2fa7d878f13f0f1df
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 741c623c70c09b5fbe55d8ae6413d9215c1dedbf
-F src/shell.c 266791241d7add796ccce2317977ae6c3c67d77f
+F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9
+F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca
 F src/sqlite.h.in 6296506a8fba279d8fa31f4abf01ab0cc92738a6
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
-F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477
-F src/sqliteInt.h 8e01aa31d5337ca0c0d0000745994f63762ec1bb
+F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75
+F src/sqliteInt.h 601c887f6d9c92e75551873c0a34711fff745bed
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -222,14 +222,14 @@ F src/test_regexp.c 58e0349f155bc307dfa209df4b03add0a7749866
 F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9
 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0
 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f
-F src/test_spellfix.c 860eb723100d4e3cff846ba5d25e02815b2a5cac
+F src/test_spellfix.c 83abe9d8c364cdd5f93bc06eaf40a349ebbf6c5c
 F src/test_sqllog.c 8acb843ddb9928dea8962e31bb09f421a72ffccb
 F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935
 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd
 F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae
 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
 F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4
-F src/test_vfs.c c6260ef238c1142c8f8bd402db02216afd182ae3
+F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275
 F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2
 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
@@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2
 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6
 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b
-F src/where.c 427c6ec07c2150e6d2bd28aefd98ab234cc374c3
+F src/where.c 43e05406f0e05960a62d4719ed77f551f8204d3f
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
-F test/func.test 0d89043dab9a8853358d14c68e028ee0093bf066
+F test/func.test b058483c17952eff7797b837bbb61e27e6b05606
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74
@@ -534,12 +534,13 @@ F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597
 F test/incrblobfault.test 917c0292224c64a56ef7215fd633a3a82f805be0
 F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32
 F test/incrvacuum2.test 379eeb8740b0ef60c372c439ad4cbea20b34bb9b
-F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637
+F test/incrvacuum3.test 1159ec2b3e7bafbde5718867bc7328ba58863313
+F test/incrvacuum_ioerr.test 293f2714571255539c8c789da2f7de4ec3f7101e
 F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097
 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7
 F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026
-F test/index5.test edc8c64ca78bee140c21ce3836820fadf47906bb
+F test/index5.test 5616266b2387003bf83c39df626c7292622d58de
 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -656,7 +657,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/permutations.test 360b92859c0af814b3fe10b68746936389606501
-F test/pragma.test 5ce333cae37d86cfe9b3add840906e375e2272b0
+F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a
 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301
@@ -734,7 +735,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523
 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
-F test/spellfix.test 2953e9da0e46dab5f83059ef6bfdebca66e13418
+F test/spellfix.test 52ae2680b1247c52b9e2b2116de3fd26a78e6bd2
 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298
 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce
 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
@@ -752,7 +753,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43
 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
 F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2
 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d
-F test/tester.tcl 0560b09498876da7714fff680c5d892b9228862f
+F test/tester.tcl 2918ebca150b67ca25b1682f8ecd857af77fab05
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -871,7 +872,7 @@ F test/tkt3718.test 3b59dcb5c4e7754dacd91e7fd353a61492cc402a
 F test/tkt3731.test 0c5f4cbffe102d43c3b2188af91a9e36348f974b
 F test/tkt3757.test 10cd679a88675c880533083fc79ac04324525595
 F test/tkt3761.test b95ea9c98f21cf91325f18a984887e62caceab33
-F test/tkt3762.test 2a9f3b03df44ec49ec0cfa8d5da6574c2a7853df
+F test/tkt3762.test 4d439ff7abdc8d9323150269d182c37c2d514576
 F test/tkt3773.test 7bca904d2a647a6a4a291bd86d7fd7c73855b789
 F test/tkt3791.test a6624b9a80b216a26cf473607f42f3e51898c267
 F test/tkt3793.test d90ffd75c52413908d15e1c44fc2ea9c80fcc449
@@ -970,8 +971,8 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1
 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761
 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
-F test/where.test 9714e6f292d70c22e78e1cecb3d896457186b9b7
-F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554
+F test/where.test 15ac8611c9439a2c5f4a6ac10cfe4c1ec9854c24
+F test/where2.test 399b3178289925a0aa976b3d60ef139740540ecd
 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006
 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
@@ -1014,7 +1015,7 @@ F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
 F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a
 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5
-F tool/showdb.c aca2644aa4de7c0cad5821e50bbd55397e0974b8
+F tool/showdb.c 16960a5ce59d8b1d70dc3bdc51e2c0fe7bb91359
 F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02
 F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
@@ -1034,10 +1035,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 939d8282748b00a7032cdd72e5d3bf3086ebfc97
-R c401d0f64d91c49092b5d167ad9b4b56
-T *branch * wince
-T *sym-wince *
+P 6d7973524a7d3bf3158fdac58975945da7f51740
+R 49ff1f2b0a98496405857ab66db0f4ad
+T *branch * funcTestEncoding
+T *sym-funcTestEncoding *
 T -sym-trunk *
 U mistachkin
-Z 7666d75ce0c62ebd1a3a9ee46fc0b14d
+Z 6ef7a794e2bcbed68706686e53ccd98f
@@ -1 +1 @@
-09dfc0c915ec2f0c5f633a3485d47cad15eec4dc
+d2e7dfca5a92074a7984032deb6a4e3681389c72
@@ -462,7 +462,6 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
          nDestTruncate = nSrcPage * (pgszSrc/pgszDest);
        }
        assert( nDestTruncate>0 );
-        sqlite3PagerTruncateImage(pDestPager, nDestTruncate);

        if( pgszSrc<pgszDest ){
          /* If the source page-size is smaller than the destination page-size,
@@ -476,6 +475,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
          */
          const i64 iSize = (i64)pgszSrc * (i64)nSrcPage;
          sqlite3_file * const pFile = sqlite3PagerFile(pDestPager);
+          Pgno iPg;
+          int nDstPage;
          i64 iOff;
          i64 iEnd;

@@ -486,13 +487,26 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
             && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest
          ));

-          /* This call ensures that all data required to recreate the original
+          /* This block ensures that all data required to recreate the original
          ** database has been stored in the journal for pDestPager and the
          ** journal synced to disk. So at this point we may safely modify
          ** the database file in any way, knowing that if a power failure
          ** occurs, the original database will be reconstructed from the 
          ** journal file.  */
-          rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1);
+          sqlite3PagerPagecount(pDestPager, &nDstPage);
+          for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){
+            if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){
+              DbPage *pPg;
+              rc = sqlite3PagerGet(pDestPager, iPg, &pPg);
+              if( rc==SQLITE_OK ){
+                rc = sqlite3PagerWrite(pPg);
+                sqlite3PagerUnref(pPg);
+              }
+            }
+          }
+          if( rc==SQLITE_OK ){
+            rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1);
+          }

          /* Write the extra pages and truncate the database file as required */
          iEnd = MIN(PENDING_BYTE + pgszDest, iSize);
@@ -519,6 +533,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
            rc = sqlite3PagerSync(pDestPager);
          }
        }else{
+          sqlite3PagerTruncateImage(pDestPager, nDestTruncate);
          rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0);
        }
    
@@ -2595,6 +2595,7 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){
  if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){
    goto trans_begun;
  }
+  assert( pBt->bDoTruncate==0 );

  /* Write transactions are not possible on a read-only database */
  if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){
@@ -2909,26 +2910,28 @@ static int relocatePage(

 /* Forward declaration required by incrVacuumStep(). */
 static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8);
+#define BTALLOC_ANY   0           /* Allocate any page */
+#define BTALLOC_EXACT 1           /* Allocate exact page if possible */
+#define BTALLOC_LE    2           /* Allocate any page <= the parameter */

 /*
-** Perform a single step of an incremental-vacuum. If successful,
-** return SQLITE_OK. If there is no work to do (and therefore no
-** point in calling this function again), return SQLITE_DONE.
+** Perform a single step of an incremental-vacuum. If successful, return
+** SQLITE_OK. If there is no work to do (and therefore no point in 
+** calling this function again), return SQLITE_DONE. Or, if an error 
+** occurs, return some other error code.
 **
-** More specificly, this function attempts to re-organize the 
-** database so that the last page of the file currently in use
-** is no longer in use.
+** More specificly, this function attempts to re-organize the database so 
+** that the last page of the file currently in use is no longer in use.
 **
-** If the nFin parameter is non-zero, this function assumes
-** that the caller will keep calling incrVacuumStep() until
-** it returns SQLITE_DONE or an error, and that nFin is the
-** number of pages the database file will contain after this 
-** process is complete.  If nFin is zero, it is assumed that
-** incrVacuumStep() will be called a finite amount of times
-** which may or may not empty the freelist.  A full autovacuum
-** has nFin>0.  A "PRAGMA incremental_vacuum" has nFin==0.
+** Parameter nFin is the number of pages that this database would contain
+** were this function called until it returns SQLITE_DONE.
+**
+** If the bCommit parameter is non-zero, this function assumes that the 
+** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE 
+** or an error. bCommit is passed true for an auto-vacuum-on-commmit 
+** operation, or false for an incremental vacuum.
 */
-static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
+static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){
  Pgno nFreeList;           /* Number of pages still on the free-list */
  int rc;

@@ -2953,15 +2956,15 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
    }

    if( eType==PTRMAP_FREEPAGE ){
-      if( nFin==0 ){
+      if( bCommit==0 ){
        /* Remove the page from the files free-list. This is not required
-        ** if nFin is non-zero. In that case, the free-list will be
+        ** if bCommit is non-zero. In that case, the free-list will be
        ** truncated to zero after this function returns, so it doesn't 
        ** matter if it still contains some garbage entries.
        */
        Pgno iFreePg;
        MemPage *pFreePg;
-        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, 1);
+        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT);
        if( rc!=SQLITE_OK ){
          return rc;
        }
@@ -2971,34 +2974,37 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
    } else {
      Pgno iFreePg;             /* Index of free page to move pLastPg to */
      MemPage *pLastPg;
+      u8 eMode = BTALLOC_ANY;   /* Mode parameter for allocateBtreePage() */
+      Pgno iNear = 0;           /* nearby parameter for allocateBtreePage() */

      rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0);
      if( rc!=SQLITE_OK ){
        return rc;
      }

-      /* If nFin is zero, this loop runs exactly once and page pLastPg
+      /* If bCommit is zero, this loop runs exactly once and page pLastPg
      ** is swapped with the first free page pulled off the free list.
      **
-      ** On the other hand, if nFin is greater than zero, then keep
+      ** On the other hand, if bCommit is greater than zero, then keep
      ** looping until a free-page located within the first nFin pages
      ** of the file is found.
      */
+      if( bCommit==0 ){
+        eMode = BTALLOC_LE;
+        iNear = nFin;
+      }
      do {
        MemPage *pFreePg;
-        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, 0, 0);
+        rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode);
        if( rc!=SQLITE_OK ){
          releasePage(pLastPg);
          return rc;
        }
        releasePage(pFreePg);
-      }while( nFin!=0 && iFreePg>nFin );
+      }while( bCommit && iFreePg>nFin );
      assert( iFreePg<iLastPg );
      
-      rc = sqlite3PagerWrite(pLastPg->pDbPage);
-      if( rc==SQLITE_OK ){
-        rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0);
-      }
+      rc = relocatePage(pBt, pLastPg, eType, iPtrPage, iFreePg, nFin!=0);
      releasePage(pLastPg);
      if( rc!=SQLITE_OK ){
        return rc;
@@ -3006,29 +3012,39 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg){
    }
  }

-  if( nFin==0 ){
-    iLastPg--;
-    while( iLastPg==PENDING_BYTE_PAGE(pBt)||PTRMAP_ISPAGE(pBt, iLastPg) ){
-      if( PTRMAP_ISPAGE(pBt, iLastPg) ){
-        MemPage *pPg;
-        rc = btreeGetPage(pBt, iLastPg, &pPg, 0);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-        rc = sqlite3PagerWrite(pPg->pDbPage);
-        releasePage(pPg);
-        if( rc!=SQLITE_OK ){
-          return rc;
-        }
-      }
+  if( bCommit==0 ){
+    do {
      iLastPg--;
-    }
-    sqlite3PagerTruncateImage(pBt->pPager, iLastPg);
+    }while( iLastPg==PENDING_BYTE_PAGE(pBt) || PTRMAP_ISPAGE(pBt, iLastPg) );
+    pBt->bDoTruncate = 1;
    pBt->nPage = iLastPg;
  }
  return SQLITE_OK;
 }

+/*
+** The database opened by the first argument is an auto-vacuum database
+** nOrig pages in size containing nFree free pages. Return the expected 
+** size of the database in pages following an auto-vacuum operation.
+*/
+static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){
+  int nEntry;                     /* Number of entries on one ptrmap page */
+  Pgno nPtrmap;                   /* Number of PtrMap pages to be freed */
+  Pgno nFin;                      /* Return value */
+
+  nEntry = pBt->usableSize/5;
+  nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry;
+  nFin = nOrig - nFree - nPtrmap;
+  if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){
+    nFin--;
+  }
+  while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
+    nFin--;
+  }
+
+  return nFin;
+}
+
 /*
 ** A write-transaction must be opened before calling this function.
 ** It performs a single unit of work towards an incremental vacuum.
@@ -3046,11 +3062,21 @@ int sqlite3BtreeIncrVacuum(Btree *p){
  if( !pBt->autoVacuum ){
    rc = SQLITE_DONE;
  }else{
-    invalidateAllOverflowCache(pBt);
-    rc = incrVacuumStep(pBt, 0, btreePagecount(pBt));
-    if( rc==SQLITE_OK ){
-      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
-      put4byte(&pBt->pPage1->aData[28], pBt->nPage);
+    Pgno nOrig = btreePagecount(pBt);
+    Pgno nFree = get4byte(&pBt->pPage1->aData[36]);
+    Pgno nFin = finalDbSize(pBt, nOrig, nFree);
+
+    if( nOrig<nFin ){
+      rc = SQLITE_CORRUPT_BKPT;
+    }else if( nFree>0 ){
+      invalidateAllOverflowCache(pBt);
+      rc = incrVacuumStep(pBt, nFin, nOrig, 0);
+      if( rc==SQLITE_OK ){
+        rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
+        put4byte(&pBt->pPage1->aData[28], pBt->nPage);
+      }
+    }else{
+      rc = SQLITE_DONE;
    }
  }
  sqlite3BtreeLeave(p);
@@ -3077,9 +3103,7 @@ static int autoVacuumCommit(BtShared *pBt){
  if( !pBt->incrVacuum ){
    Pgno nFin;         /* Number of pages in database after autovacuuming */
    Pgno nFree;        /* Number of pages on the freelist initially */
-    Pgno nPtrmap;      /* Number of PtrMap pages to be freed */
    Pgno iFree;        /* The next page to be freed */
-    int nEntry;        /* Number of entries on one ptrmap page */
    Pgno nOrig;        /* Database size before freeing */

    nOrig = btreePagecount(pBt);
@@ -3092,26 +3116,18 @@ static int autoVacuumCommit(BtShared *pBt){
    }

    nFree = get4byte(&pBt->pPage1->aData[36]);
-    nEntry = pBt->usableSize/5;
-    nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry;
-    nFin = nOrig - nFree - nPtrmap;
-    if( nOrig>PENDING_BYTE_PAGE(pBt) && nFin<PENDING_BYTE_PAGE(pBt) ){
-      nFin--;
-    }
-    while( PTRMAP_ISPAGE(pBt, nFin) || nFin==PENDING_BYTE_PAGE(pBt) ){
-      nFin--;
-    }
+    nFin = finalDbSize(pBt, nOrig, nFree);
    if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT;

    for(iFree=nOrig; iFree>nFin && rc==SQLITE_OK; iFree--){
-      rc = incrVacuumStep(pBt, nFin, iFree);
+      rc = incrVacuumStep(pBt, nFin, iFree, 1);
    }
    if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){
      rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
      put4byte(&pBt->pPage1->aData[32], 0);
      put4byte(&pBt->pPage1->aData[36], 0);
      put4byte(&pBt->pPage1->aData[28], nFin);
-      sqlite3PagerTruncateImage(pBt->pPager, nFin);
+      pBt->bDoTruncate = 1;
      pBt->nPage = nFin;
    }
    if( rc!=SQLITE_OK ){
@@ -3166,6 +3182,9 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){
        return rc;
      }
    }
+    if( pBt->bDoTruncate ){
+      sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage);
+    }
 #endif
    rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0);
    sqlite3BtreeLeave(p);
@@ -3181,6 +3200,9 @@ static void btreeEndTransaction(Btree *p){
  BtShared *pBt = p->pBt;
  assert( sqlite3BtreeHoldsMutex(p) );

+#ifndef SQLITE_OMIT_AUTOVACUUM
+  pBt->bDoTruncate = 0;
+#endif
  btreeClearHasContent(pBt);
  if( p->inTrans>TRANS_NONE && p->db->activeVdbeCnt>1 ){
    /* If there are other active statements that belong to this database
@@ -4867,7 +4889,7 @@ static int allocateBtreePage(
  MemPage **ppPage, 
  Pgno *pPgno, 
  Pgno nearby,
-  u8 exact
+  u8 eMode
 ){
  MemPage *pPage1;
  int rc;
@@ -4895,16 +4917,19 @@ static int allocateBtreePage(
    ** the entire-list will be searched for that page.
    */
 #ifndef SQLITE_OMIT_AUTOVACUUM
-    if( exact && nearby<=mxPage ){
-      u8 eType;
-      assert( nearby>0 );
-      assert( pBt->autoVacuum );
-      rc = ptrmapGet(pBt, nearby, &eType, 0);
-      if( rc ) return rc;
-      if( eType==PTRMAP_FREEPAGE ){
-        searchList = 1;
+    if( eMode==BTALLOC_EXACT ){
+      if( nearby<=mxPage ){
+        u8 eType;
+        assert( nearby>0 );
+        assert( pBt->autoVacuum );
+        rc = ptrmapGet(pBt, nearby, &eType, 0);
+        if( rc ) return rc;
+        if( eType==PTRMAP_FREEPAGE ){
+          searchList = 1;
+        }
      }
-      *pPgno = nearby;
+    }else if( eMode==BTALLOC_LE ){
+      searchList = 1;
    }
 #endif

@@ -4959,11 +4984,13 @@ static int allocateBtreePage(
        rc = SQLITE_CORRUPT_BKPT;
        goto end_allocate_page;
 #ifndef SQLITE_OMIT_AUTOVACUUM
-      }else if( searchList && nearby==iTrunk ){
+      }else if( searchList 
+            && (nearby==iTrunk || (iTrunk<nearby && eMode==BTALLOC_LE)) 
+      ){
        /* The list is being searched and this trunk page is the page
        ** to allocate, regardless of whether it has leaves.
        */
-        assert( *pPgno==iTrunk );
+        *pPgno = iTrunk;
        *ppPage = pTrunk;
        searchList = 0;
        rc = sqlite3PagerWrite(pTrunk->pDbPage);
@@ -5026,14 +5053,24 @@ static int allocateBtreePage(
        unsigned char *aData = pTrunk->aData;
        if( nearby>0 ){
          u32 i;
-          int dist;
          closest = 0;
-          dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby);
-          for(i=1; i<k; i++){
-            int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby);
-            if( d2<dist ){
-              closest = i;
-              dist = d2;
+          if( eMode==BTALLOC_LE ){
+            for(i=0; i<k; i++){
+              iPage = get4byte(&aData[8+i*4]);
+              if( iPage<=nearby ){
+                closest = i;
+                break;
+              }
+            }
+          }else{
+            int dist;
+            dist = sqlite3AbsInt32(get4byte(&aData[8]) - nearby);
+            for(i=1; i<k; i++){
+              int d2 = sqlite3AbsInt32(get4byte(&aData[8+i*4]) - nearby);
+              if( d2<dist ){
+                closest = i;
+                dist = d2;
+              }
            }
          }
        }else{
@@ -5047,7 +5084,9 @@ static int allocateBtreePage(
          goto end_allocate_page;
        }
        testcase( iPage==mxPage );
-        if( !searchList || iPage==nearby ){
+        if( !searchList 
+         || (iPage==nearby || (iPage<nearby && eMode==BTALLOC_LE)) 
+        ){
          int noContent;
          *pPgno = iPage;
          TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d"
@@ -5074,8 +5113,26 @@ static int allocateBtreePage(
      pPrevTrunk = 0;
    }while( searchList );
  }else{
-    /* There are no pages on the freelist, so create a new page at the
-    ** end of the file */
+    /* There are no pages on the freelist, so append a new page to the
+    ** database image.
+    **
+    ** Normally, new pages allocated by this block can be requested from the
+    ** pager layer with the 'no-content' flag set. This prevents the pager
+    ** from trying to read the pages content from disk. However, if the
+    ** current transaction has already run one or more incremental-vacuum
+    ** steps, then the page we are about to allocate may contain content
+    ** that is required in the event of a rollback. In this case, do
+    ** not set the no-content flag. This causes the pager to load and journal
+    ** the current page content before overwriting it.
+    **
+    ** Note that the pager will not actually attempt to load or journal 
+    ** content for any page that really does lie past the end of the database
+    ** file on disk. So the effects of disabling the no-content optimization
+    ** here are confined to those pages that lie between the end of the
+    ** database image and the end of the database file.
+    */
+    int bNoContent = (0==pBt->bDoTruncate);
+
    rc = sqlite3PagerWrite(pBt->pPage1->pDbPage);
    if( rc ) return rc;
    pBt->nPage++;
@@ -5090,7 +5147,7 @@ static int allocateBtreePage(
      MemPage *pPg = 0;
      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage));
      assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) );
-      rc = btreeGetPage(pBt, pBt->nPage, &pPg, 1);
+      rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent);
      if( rc==SQLITE_OK ){
        rc = sqlite3PagerWrite(pPg->pDbPage);
        releasePage(pPg);
@@ -5104,7 +5161,7 @@ static int allocateBtreePage(
    *pPgno = pBt->nPage;

    assert( *pPgno!=PENDING_BYTE_PAGE(pBt) );
-    rc = btreeGetPage(pBt, *pPgno, ppPage, 1);
+    rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent);
    if( rc ) return rc;
    rc = sqlite3PagerWrite((*ppPage)->pDbPage);
    if( rc!=SQLITE_OK ){
@@ -7119,7 +7176,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){
    ** be moved to the allocated page (unless the allocated page happens
    ** to reside at pgnoRoot).
    */
-    rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, 1);
+    rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT);
    if( rc!=SQLITE_OK ){
      return rc;
    }
@@ -411,6 +411,7 @@ struct BtShared {
 #ifndef SQLITE_OMIT_AUTOVACUUM
  u8 autoVacuum;        /* True if auto-vacuum is enabled */
  u8 incrVacuum;        /* True if incr-vacuum is enabled */
+  u8 bDoTruncate;       /* True to truncate db on commit */
 #endif
  u8 inTransaction;     /* Transaction state */
  u8 max1bytePayload;   /* Maximum first byte of cell for a 1-byte payload */
@@ -962,6 +962,56 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){
  }
 }

+/*
+** The unicode() function.  Return the integer unicode code-point value
+** for the first character of the input string. 
+*/
+static void unicodeFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const unsigned char *z = sqlite3_value_text(argv[0]);
+  if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z));
+}
+
+/*
+** The char() function takes zero or more arguments, each of which is
+** an integer.  It constructs a string where each character of the string
+** is the unicode character for the corresponding integer argument.
+*/
+static void charFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  unsigned char *z, *zOut;
+  int i;
+  zOut = z = sqlite3_malloc( argc*4 );
+  if( z==0 ){
+    sqlite3_result_error_nomem(context);
+    return;
+  }
+  for(i=0; i<argc; i++){
+    sqlite3_int64 x;
+    unsigned c;
+    x = sqlite3_value_int64(argv[i]);
+    if( x<0 || x>0x10ffff ) x = 0xfffd;
+    c = (unsigned)(x & 0x1fffff);
+    if( c<=0xFFFF ){
+      *zOut++ = (u8)(c&0x00FF);
+      *zOut++ = (u8)((c>>8)&0x00FF);
+    }else{
+      if( c>=0xd800 && c<=0xdbff ) c = 0xfffd;
+      *zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0));
+      *zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03));
+      *zOut++ = (u8)(c&0x00FF);
+      *zOut++ = (u8)(0x00DC + ((c>>8)&0x03));
+    }
+  }
+  sqlite3_result_text16le(context, (char*)z, (int)(zOut-z), sqlite3_free);
+}
+
 /*
 ** The hex() function.  Interpret the argument as a blob.  Return
 ** a hexadecimal rendering as text.
@@ -1589,6 +1639,8 @@ void sqlite3RegisterGlobalFunctions(void){
    FUNCTION(instr,              2, 0, 0, instrFunc        ),
    FUNCTION(substr,             2, 0, 0, substrFunc       ),
    FUNCTION(substr,             3, 0, 0, substrFunc       ),
+    FUNCTION(unicode,            1, 0, 0, unicodeFunc      ),
+    FUNCTION(char,              -1, 0, 0, charFunc         ),
    FUNCTION(abs,                1, 0, 0, absFunc          ),
 #ifndef SQLITE_OMIT_FLOATING_POINT
    FUNCTION(round,              1, 0, 0, roundFunc        ),
@@ -1002,7 +1002,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){
  sqlite3VtabRollback(db);
  sqlite3EndBenignMalloc();

-  if( db->flags&SQLITE_InternChanges ){
+  if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){
    sqlite3ExpirePreparedStatements(db);
    sqlite3ResetAllSchemasOfConnection(db);
  }
@@ -2076,6 +2076,7 @@ static int winClose(sqlite3_file *id){
  assert( pFile->pShm==0 );
 #endif
  OSTRACE(("CLOSE %d\n", pFile->h));
+  assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE );
  do{
    rc = osCloseHandle(pFile->h);
    /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */
@@ -2992,7 +2993,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
                 (int)osGetCurrentProcessId(), i,
                 bRc ? "ok" : "failed"));
      }
-      if( p->hFile.h != INVALID_HANDLE_VALUE ){
+      if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){
        SimulateIOErrorBenign(1);
        winClose((sqlite3_file *)&p->hFile);
        SimulateIOErrorBenign(0);
@@ -3687,8 +3688,9 @@ static int winOpen(
       || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL
  );

-  assert( id!=0 );
-  UNUSED_PARAMETER(pVfs);
+  assert( pFile!=0 );
+  memset(pFile, 0, sizeof(winFile));
+  pFile->h = INVALID_HANDLE_VALUE;

 #if SQLITE_OS_WINRT
  if( !sqlite3_temp_directory ){
@@ -3697,8 +3699,6 @@ static int winOpen(
  }
 #endif

-  pFile->h = INVALID_HANDLE_VALUE;
-
  /* If the second argument to this function is NULL, generate a 
  ** temporary file name to use 
  */
@@ -3841,19 +3841,6 @@ static int winOpen(
    }
  }

-  memset(pFile, 0, sizeof(*pFile));
-  pFile->pMethod = &winIoMethod;
-  pFile->h = h;
-  pFile->lastErrno = NO_ERROR;
-  pFile->pVfs = pVfs;
-#ifndef SQLITE_OMIT_WAL
-  pFile->pShm = 0;
-#endif
-  pFile->zPath = zName;
-  if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){
-    pFile->ctrlFlags |= WINFILE_PSOW;
-  }
-
 #if SQLITE_OS_WINCE
  if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB
       && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK
@@ -3870,6 +3857,15 @@ static int winOpen(
    sqlite3_free(zConverted);
  }

+  pFile->pMethod = &winIoMethod;
+  pFile->pVfs = pVfs;
+  pFile->h = h;
+  if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){
+    pFile->ctrlFlags |= WINFILE_PSOW;
+  }
+  pFile->lastErrno = NO_ERROR;
+  pFile->zPath = zName;
+
  OpenCounter(+1);
  return rc;
 }
@@ -1838,6 +1838,8 @@ static int pager_error(Pager *pPager, int rc){
  return rc;
 }

+static int pager_truncate(Pager *pPager, Pgno nPage);
+
 /*
 ** This routine ends a transaction. A transaction is usually ended by 
 ** either a COMMIT or a ROLLBACK operation. This routine may be called 
@@ -1891,7 +1893,7 @@ static int pager_error(Pager *pPager, int rc){
 ** to the first error encountered (the journal finalization one) is
 ** returned.
 */
-static int pager_end_transaction(Pager *pPager, int hasMaster){
+static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
  int rc = SQLITE_OK;      /* Error code from journal finalization operation */
  int rc2 = SQLITE_OK;     /* Error code from db file unlock operation */

@@ -1977,7 +1979,17 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){
    */
    rc2 = sqlite3WalEndWriteTransaction(pPager->pWal);
    assert( rc2==SQLITE_OK );
+  }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){
+    /* This branch is taken when committing a transaction in rollback-journal
+    ** mode if the database file on disk is larger than the database image.
+    ** At this point the journal has been finalized and the transaction 
+    ** successfully committed, but the EXCLUSIVE lock is still held on the
+    ** file. So it is safe to truncate the database file to its minimum
+    ** required size.  */
+    assert( pPager->eLock==EXCLUSIVE_LOCK );
+    rc = pager_truncate(pPager, pPager->dbSize);
  }
+
  if( !pPager->exclusiveMode 
   && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0))
  ){
@@ -2016,7 +2028,7 @@ static void pagerUnlockAndRollback(Pager *pPager){
      sqlite3EndBenignMalloc();
    }else if( !pPager->exclusiveMode ){
      assert( pPager->eState==PAGER_READER );
-      pager_end_transaction(pPager, 0);
+      pager_end_transaction(pPager, 0, 0);
    }
  }
  pager_unlock(pPager);
@@ -2791,7 +2803,7 @@ end_playback:
    rc = sqlite3PagerSync(pPager);
  }
  if( rc==SQLITE_OK ){
-    rc = pager_end_transaction(pPager, zMaster[0]!='\0');
+    rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0);
    testcase( rc!=SQLITE_OK );
  }
  if( rc==SQLITE_OK && zMaster[0] && res ){
@@ -5885,36 +5897,6 @@ int sqlite3PagerCommitPhaseOne(
  #endif
      if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
  
-      /* If this transaction has made the database smaller, then all pages
-      ** being discarded by the truncation must be written to the journal
-      ** file.
-      **
-      ** Before reading the pages with page numbers larger than the 
-      ** current value of Pager.dbSize, set dbSize back to the value
-      ** that it took at the start of the transaction. Otherwise, the
-      ** calls to sqlite3PagerGet() return zeroed pages instead of 
-      ** reading data from the database file.
-      */
-      if( pPager->dbSize<pPager->dbOrigSize 
-       && pPager->journalMode!=PAGER_JOURNALMODE_OFF
-      ){
-        Pgno i;                                   /* Iterator variable */
-        const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */
-        const Pgno dbSize = pPager->dbSize;       /* Database image size */ 
-        pPager->dbSize = pPager->dbOrigSize;
-        for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){
-          if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){
-            PgHdr *pPage;             /* Page to journal */
-            rc = sqlite3PagerGet(pPager, i, &pPage);
-            if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
-            rc = sqlite3PagerWrite(pPage);
-            sqlite3PagerUnref(pPage);
-            if( rc!=SQLITE_OK ) goto commit_phase_one_exit;
-          }
-        }
-        pPager->dbSize = dbSize;
-      } 
-  
      /* Write the master journal name into the journal file. If a master 
      ** journal file name has already been written to the journal file, 
      ** or if zMaster is NULL (no master journal), then this call is a no-op.
@@ -5942,11 +5924,14 @@ int sqlite3PagerCommitPhaseOne(
        goto commit_phase_one_exit;
      }
      sqlite3PcacheCleanAll(pPager->pPCache);
-  
-      /* If the file on disk is not the same size as the database image,
-      ** then use pager_truncate to grow or shrink the file here.
-      */
-      if( pPager->dbSize!=pPager->dbFileSize ){
+
+      /* If the file on disk is smaller than the database image, use 
+      ** pager_truncate to grow the file here. This can happen if the database
+      ** image was extended as part of the current transaction and then the
+      ** last page in the db image moved to the free-list. In this case the
+      ** last page is never written out to disk, leaving the database file
+      ** undersized. Fix this now if it is the case.  */
+      if( pPager->dbSize>pPager->dbFileSize ){
        Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager));
        assert( pPager->eState==PAGER_WRITER_DBMOD );
        rc = pager_truncate(pPager, nNew);
@@ -6019,7 +6004,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){
  }

  PAGERTRACE(("COMMIT %d\n", PAGERID(pPager)));
-  rc = pager_end_transaction(pPager, pPager->setMaster);
+  rc = pager_end_transaction(pPager, pPager->setMaster, 1);
  return pager_error(pPager, rc);
 }

@@ -6064,11 +6049,11 @@ int sqlite3PagerRollback(Pager *pPager){
  if( pagerUseWal(pPager) ){
    int rc2;
    rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1);
-    rc2 = pager_end_transaction(pPager, pPager->setMaster);
+    rc2 = pager_end_transaction(pPager, pPager->setMaster, 0);
    if( rc==SQLITE_OK ) rc = rc2;
  }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){
    int eState = pPager->eState;
-    rc = pager_end_transaction(pPager, 0);
+    rc = pager_end_transaction(pPager, 0, 0);
    if( !MEMDB && eState>PAGER_WRITER_LOCKED ){
      /* This can happen using journal_mode=off. Move the pager to the error 
      ** state to indicate that the contents of the cache may not be trusted.
@@ -958,6 +958,7 @@ void sqlite3Pragma(
      for(pPk=pTab->pIndex; pPk && pPk->autoIndex!=2; pPk=pPk->pNext){}
      sqlite3VdbeSetNumCols(v, 6);
      pParse->nMem = 6;
+      sqlite3CodeVerifySchema(pParse, iDb);
      sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "cid", SQLITE_STATIC);
      sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC);
      sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "type", SQLITE_STATIC);
@@ -1003,6 +1004,7 @@ void sqlite3Pragma(
      pTab = pIdx->pTable;
      sqlite3VdbeSetNumCols(v, 3);
      pParse->nMem = 3;
+      sqlite3CodeVerifySchema(pParse, iDb);
      sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seqno", SQLITE_STATIC);
      sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "cid", SQLITE_STATIC);
      sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "name", SQLITE_STATIC);
@@ -1029,6 +1031,7 @@ void sqlite3Pragma(
        int i = 0; 
        sqlite3VdbeSetNumCols(v, 3);
        pParse->nMem = 3;
+        sqlite3CodeVerifySchema(pParse, iDb);
        sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC);
        sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC);
        sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC);
@@ -1092,6 +1095,7 @@ void sqlite3Pragma(
        int i = 0; 
        sqlite3VdbeSetNumCols(v, 8);
        pParse->nMem = 8;
+        sqlite3CodeVerifySchema(pParse, iDb);
        sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "id", SQLITE_STATIC);
        sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "seq", SQLITE_STATIC);
        sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "table", SQLITE_STATIC);
@@ -2366,7 +2366,8 @@ static int multiSelectOrderBy(
  }else{
    int nExpr = p->pEList->nExpr;
    assert( nOrderBy>=nExpr || db->mallocFailed );
-    regPrev = sqlite3GetTempRange(pParse, nExpr+1);
+    regPrev = pParse->nMem+1;
+    pParse->nMem += nExpr+1;
    sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev);
    pKeyDup = sqlite3DbMallocZero(db,
                  sizeof(*pKeyDup) + nExpr*(sizeof(CollSeq*)+1) );
@@ -2548,12 +2549,6 @@ static int multiSelectOrderBy(
  sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE);
  sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB);

-  /* Release temporary registers
-  */
-  if( regPrev ){
-    sqlite3ReleaseTempRange(pParse, regPrev, nOrderBy+1);
-  }
-
  /* Jump to the this point in order to terminate the query.
  */
  sqlite3VdbeResolveLabel(v, labelEnd);
@@ -3298,14 +3293,15 @@ static int selectExpander(Walker *pWalker, Select *p){
  struct SrcList_item *pFrom;
  sqlite3 *db = pParse->db;
  Expr *pE, *pRight, *pExpr;
+  u16 selFlags = p->selFlags;

+  p->selFlags |= SF_Expanded;
  if( db->mallocFailed  ){
    return WRC_Abort;
  }
-  if( NEVER(p->pSrc==0) || (p->selFlags & SF_Expanded)!=0 ){
+  if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){
    return WRC_Prune;
  }
-  p->selFlags |= SF_Expanded;
  pTabList = p->pSrc;
  pEList = p->pEList;

@@ -3666,6 +3662,7 @@ void sqlite3SelectPrep(
  sqlite3 *db;
  if( NEVER(p==0) ) return;
  db = pParse->db;
+  if( db->mallocFailed ) return;
  if( p->selFlags & SF_HasTypeInfo ) return;
  sqlite3SelectExpand(pParse, p);
  if( pParse->nErr || db->mallocFailed ) return;
@@ -90,7 +90,8 @@ static int enableTimer = 0;
 #define IsDigit(X)  isdigit((unsigned char)X)
 #define ToLower(X)  (char)tolower((unsigned char)X)

-#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL)
+#if !defined(_WIN32) && !defined(WIN32) && !defined(_WRS_KERNEL) \
+ && !defined(__minux)
 #include <sys/time.h>
 #include <sys/resource.h>

@@ -236,6 +236,20 @@ struct sqlite3_api_routines {
  int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
  int (*vtab_config)(sqlite3*,int op,...);
  int (*vtab_on_conflict)(sqlite3*);
+  /* Version 3.7.16 and later */
+  int (*close_v2)(sqlite3*);
+  const char *(*db_filename)(sqlite3*,const char*);
+  int (*db_readonly)(sqlite3*,const char*);
+  int (*db_release_memory)(sqlite3*);
+  const char *(*errstr)(int);
+  int (*stmt_busy)(sqlite3_stmt*);
+  int (*stmt_readonly)(sqlite3_stmt*);
+  int (*stricmp)(const char*,const char*);
+  int (*uri_boolean)(const char*,const char*,int);
+  sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
+  const char *(*uri_parameter)(const char*,const char*);
+  char *(*vsnprintf)(int,char*,const char*,va_list);
+  int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
 };

 /*
@@ -439,6 +453,20 @@ struct sqlite3_api_routines {
 #define sqlite3_blob_reopen            sqlite3_api->blob_reopen
 #define sqlite3_vtab_config            sqlite3_api->vtab_config
 #define sqlite3_vtab_on_conflict       sqlite3_api->vtab_on_conflict
+/* Version 3.7.16 and later */
+#define sqlite3_close_v2               sqlite3_api->close_v2
+#define sqlite3_db_filename            sqlite3_api->db_filename
+#define sqlite3_db_readonly            sqlite3_api->db_readonly
+#define sqlite3_db_release_memory      sqlite3_api->db_release_memory
+#define sqlite3_errstr                 sqlite3_api->errstr
+#define sqlite3_stmt_busy              sqlite3_api->stmt_busy
+#define sqlite3_stmt_readonly          sqlite3_api->stmt_readonly
+#define sqlite3_stricmp                sqlite3_api->stricmp
+#define sqlite3_uri_boolean            sqlite3_api->uri_boolean
+#define sqlite3_uri_int64              sqlite3_api->uri_int64
+#define sqlite3_uri_parameter          sqlite3_api->uri_parameter
+#define sqlite3_uri_vsnprintf          sqlite3_api->vsnprintf
+#define sqlite3_wal_checkpoint_v2      sqlite3_api->wal_checkpoint_v2
 #endif /* SQLITE_CORE */

 #define SQLITE_EXTENSION_INIT1     const sqlite3_api_routines *sqlite3_api = 0;
@@ -200,7 +200,8 @@
 **
 ** See also ticket #2741.
 */
-#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) && SQLITE_THREADSAFE
+#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) \
+ && !defined(__APPLE__) && SQLITE_THREADSAFE
 #  define _XOPEN_SOURCE 500  /* Needed to enable pthread recursive mutexes */
 #endif

@@ -1965,6 +1966,7 @@ struct WhereLevel {
      struct InLoop {
        int iCur;              /* The VDBE cursor used by this IN operator */
        int addrInTop;         /* Top of the IN loop */
+        u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
      } *aInLoop;           /* Information about each nested IN operator */
    } in;                 /* Used when plan.wsFlags&WHERE_IN_ABLE */
    Index *pCovidx;       /* Possible covering index for WHERE_MULTI_OR */
@@ -2836,7 +2838,7 @@ Table *sqlite3SrcListLookup(Parse*, SrcList*);
 int sqlite3IsReadOnly(Parse*, Table*, int);
 void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
-Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, char *);
+Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*);
 #endif
 void sqlite3DeleteFrom(Parse*, SrcList*, Expr*);
 void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int);
@@ -3017,8 +3019,11 @@ int sqlite3VarintLen(u64 v);
 **     x = putVarint32( A, B );
 **
 */
-#define getVarint32(A,B)  (u8)((*(A)<(u8)0x80) ? ((B) = (u32)*(A)),1 : sqlite3GetVarint32((A), (u32 *)&(B)))
-#define putVarint32(A,B)  (u8)(((u32)(B)<(u32)0x80) ? (*(A) = (unsigned char)(B)),1 : sqlite3PutVarint32((A), (B)))
+#define getVarint32(A,B)  \
+  (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B)))
+#define putVarint32(A,B)  \
+  (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\
+  sqlite3PutVarint32((A),(B)))
 #define getVarint    sqlite3GetVarint
 #define putVarint    sqlite3PutVarint

@@ -2681,6 +2681,18 @@ static int spellfix1Update(
        p->pConfig3 = 0;
        return SQLITE_OK;
      }
+      if( memcmp(zCmd,"edit_cost_table=",16)==0 ){
+        editDist3ConfigDelete(p->pConfig3);
+        p->pConfig3 = 0;
+        sqlite3_free(p->zCostTable);
+        p->zCostTable = spellfix1Dequote(zCmd+16);
+        if( p->zCostTable==0 ) return SQLITE_NOMEM;
+        if( p->zCostTable[0]==0 || sqlite3_stricmp(p->zCostTable,"null")==0 ){
+          sqlite3_free(p->zCostTable);
+          p->zCostTable = 0;
+        }
+        return SQLITE_OK;
+      }
      pVTab->zErrMsg = sqlite3_mprintf("unknown value for %s.command: \"%w\"",
                                       p->zTableName, zCmd);
      return SQLITE_ERROR;
@@ -265,7 +265,8 @@ static void tvfsExecTcl(
  const char *zMethod,
  Tcl_Obj *arg1,
  Tcl_Obj *arg2,
-  Tcl_Obj *arg3
+  Tcl_Obj *arg3,
+  Tcl_Obj *arg4
 ){
  int rc;                         /* Return code from Tcl_EvalObj() */
  Tcl_Obj *pEval;
@@ -282,6 +283,7 @@ static void tvfsExecTcl(
  if( arg1 ) Tcl_ListObjAppendElement(p->interp, pEval, arg1);
  if( arg2 ) Tcl_ListObjAppendElement(p->interp, pEval, arg2);
  if( arg3 ) Tcl_ListObjAppendElement(p->interp, pEval, arg3);
+  if( arg4 ) Tcl_ListObjAppendElement(p->interp, pEval, arg4);

  rc = Tcl_EvalObjEx(p->interp, pEval, TCL_EVAL_GLOBAL);
  if( rc!=TCL_OK ){
@@ -302,7 +304,7 @@ static int tvfsClose(sqlite3_file *pFile){

  if( p->pScript && p->mask&TESTVFS_CLOSE_MASK ){
    tvfsExecTcl(p, "xClose", 
-        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0
+        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0
    );
  }

@@ -333,7 +335,7 @@ static int tvfsRead(
  Testvfs *p = (Testvfs *)pFd->pVfs->pAppData;
  if( p->pScript && p->mask&TESTVFS_READ_MASK ){
    tvfsExecTcl(p, "xRead", 
-        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0
+        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -362,7 +364,7 @@ static int tvfsWrite(
  if( p->pScript && p->mask&TESTVFS_WRITE_MASK ){
    tvfsExecTcl(p, "xWrite", 
        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 
-        Tcl_NewWideIntObj(iOfst)
+        Tcl_NewWideIntObj(iOfst), Tcl_NewIntObj(iAmt)
    );
    tvfsResultCode(p, &rc);
  }
@@ -390,7 +392,7 @@ static int tvfsTruncate(sqlite3_file *pFile, sqlite_int64 size){

  if( p->pScript && p->mask&TESTVFS_TRUNCATE_MASK ){
    tvfsExecTcl(p, "xTruncate", 
-        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0
+        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0, 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -431,7 +433,7 @@ static int tvfsSync(sqlite3_file *pFile, int flags){

    tvfsExecTcl(p, "xSync", 
        Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId,
-        Tcl_NewStringObj(zFlags, -1)
+        Tcl_NewStringObj(zFlags, -1), 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -578,7 +580,7 @@ static int tvfsOpen(
        z += strlen(z) + 1;
      }
    }
-    tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0);
+    tvfsExecTcl(p, "xOpen", Tcl_NewStringObj(pFd->zFilename, -1), pArg, 0, 0);
    Tcl_DecrRefCount(pArg);
    if( tvfsResultCode(p, &rc) ){
      if( rc!=SQLITE_OK ) return rc;
@@ -635,7 +637,7 @@ static int tvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){

  if( p->pScript && p->mask&TESTVFS_DELETE_MASK ){
    tvfsExecTcl(p, "xDelete", 
-        Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0
+        Tcl_NewStringObj(zPath, -1), Tcl_NewIntObj(dirSync), 0, 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -663,7 +665,7 @@ static int tvfsAccess(
    if( flags==SQLITE_ACCESS_READWRITE ) zArg = "SQLITE_ACCESS_READWRITE";
    if( flags==SQLITE_ACCESS_READ ) zArg = "SQLITE_ACCESS_READ";
    tvfsExecTcl(p, "xAccess", 
-        Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0
+        Tcl_NewStringObj(zPath, -1), Tcl_NewStringObj(zArg, -1), 0, 0
    );
    if( tvfsResultCode(p, &rc) ){
      if( rc!=SQLITE_OK ) return rc;
@@ -691,7 +693,7 @@ static int tvfsFullPathname(
  Testvfs *p = (Testvfs *)pVfs->pAppData;
  if( p->pScript && p->mask&TESTVFS_FULLPATHNAME_MASK ){
    int rc;
-    tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0);
+    tvfsExecTcl(p, "xFullPathname", Tcl_NewStringObj(zPath, -1), 0, 0, 0);
    if( tvfsResultCode(p, &rc) ){
      if( rc!=SQLITE_OK ) return rc;
    }
@@ -771,7 +773,7 @@ static int tvfsShmOpen(sqlite3_file *pFile){
  */
  Tcl_ResetResult(p->interp);
  if( p->pScript && p->mask&TESTVFS_SHMOPEN_MASK ){
-    tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0);
+    tvfsExecTcl(p, "xShmOpen", Tcl_NewStringObj(pFd->zFilename, -1), 0, 0, 0);
    if( tvfsResultCode(p, &rc) ){
      if( rc!=SQLITE_OK ) return rc;
    }
@@ -841,7 +843,7 @@ static int tvfsShmMap(
    Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz));
    Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite));
    tvfsExecTcl(p, "xShmMap", 
-        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg
+        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg, 0
    );
    tvfsResultCode(p, &rc);
    Tcl_DecrRefCount(pArg);
@@ -891,7 +893,7 @@ static int tvfsShmLock(
    }
    tvfsExecTcl(p, "xShmLock", 
        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId,
-        Tcl_NewStringObj(zLock, -1)
+        Tcl_NewStringObj(zLock, -1), 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -937,7 +939,7 @@ static void tvfsShmBarrier(sqlite3_file *pFile){

  if( p->pScript && p->mask&TESTVFS_SHMBARRIER_MASK ){
    tvfsExecTcl(p, "xShmBarrier", 
-        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0
+        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0
    );
  }
 }
@@ -961,7 +963,7 @@ static int tvfsShmUnmap(

  if( p->pScript && p->mask&TESTVFS_SHMCLOSE_MASK ){
    tvfsExecTcl(p, "xShmUnmap", 
-        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0
+        Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0, 0
    );
    tvfsResultCode(p, &rc);
  }
@@ -140,7 +140,6 @@ struct WhereTerm {
 struct WhereClause {
  Parse *pParse;           /* The parser context */
  WhereMaskSet *pMaskSet;  /* Mapping of table cursor numbers to bitmasks */
-  Bitmask vmask;           /* Bitmask identifying virtual table cursors */
  WhereClause *pOuter;     /* Outer conjunction */
  u8 op;                   /* Split operator.  TK_AND or TK_OR */
  u16 wctrlFlags;          /* Might include WHERE_AND_ONLY */
@@ -317,7 +316,6 @@ static void whereClauseInit(
  pWC->nTerm = 0;
  pWC->nSlot = ArraySize(pWC->aStatic);
  pWC->a = pWC->aStatic;
-  pWC->vmask = 0;
  pWC->wctrlFlags = wctrlFlags;
 }

@@ -917,7 +915,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){
 **
 ** CASE 1:
 **
-** If all subterms are of the form T.C=expr for some single column of C
+** If all subterms are of the form T.C=expr for some single column of C and
 ** a single table T (as shown in example B above) then create a new virtual
 ** term that is an equivalent IN expression.  In other words, if the term
 ** being analyzed is:
@@ -1005,7 +1003,7 @@ static void exprAnalyzeOrTerm(
  ** Compute the set of tables that might satisfy cases 1 or 2.
  */
  indexable = ~(Bitmask)0;
-  chngToIN = ~(pWC->vmask);
+  chngToIN = ~(Bitmask)0;
  for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){
    if( (pOrTerm->eOperator & WO_SINGLE)==0 ){
      WhereAndInfo *pAndInfo;
@@ -2272,8 +2270,9 @@ static void bestVirtualIndex(WhereBestIdx *p){
  struct sqlite3_index_constraint *pIdxCons;
  struct sqlite3_index_constraint_usage *pUsage;
  WhereTerm *pTerm;
-  int i, j;
+  int i, j, k;
  int nOrderBy;
+  int sortOrder;                  /* Sort order for IN clauses */
  int bAllowIN;                   /* Allow IN optimizations */
  double rCost;

@@ -2372,18 +2371,27 @@ static void bestVirtualIndex(WhereBestIdx *p){
      return;
    }
  
+    sortOrder = SQLITE_SO_ASC;
    pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
    for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
      if( pUsage[i].argvIndex>0 ){
        j = pIdxCons->iTermOffset;
        pTerm = &pWC->a[j];
        p->cost.used |= pTerm->prereqRight;
-        if( (pTerm->eOperator & WO_IN)!=0 && pUsage[i].omit==0 ){
-          /* Do not attempt to use an IN constraint if the virtual table
-          ** says that the equivalent EQ constraint cannot be safely omitted.
-          ** If we do attempt to use such a constraint, some rows might be
-          ** repeated in the output. */
-          break;
+        if( (pTerm->eOperator & WO_IN)!=0 ){
+          if( pUsage[i].omit==0 ){
+            /* Do not attempt to use an IN constraint if the virtual table
+            ** says that the equivalent EQ constraint cannot be safely omitted.
+            ** If we do attempt to use such a constraint, some rows might be
+            ** repeated in the output. */
+            break;
+          }
+          for(k=0; k<pIdxInfo->nOrderBy; k++){
+            if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){
+              sortOrder = pIdxInfo->aOrderBy[k].desc;
+              break;
+            }
+          }
        }
      }
    }
@@ -2413,7 +2421,8 @@ static void bestVirtualIndex(WhereBestIdx *p){
  }
  p->cost.plan.u.pVtabIdx = pIdxInfo;
  if( pIdxInfo->orderByConsumed ){
-    p->cost.plan.wsFlags |= WHERE_ORDERED;
+    assert( sortOrder==0 || sortOrder==1 );
+    p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE;
    p->cost.plan.nOBSat = nOrderBy;
  }else{
    p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0;
@@ -3010,10 +3019,7 @@ static int isSortingIndex(
    if( pConstraint==0 ){
      isEq = 0;
    }else if( (pConstraint->eOperator & WO_IN)!=0 ){
-      /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY
-      ** because we do not know in what order the values on the RHS of the IN
-      ** operator will occur. */
-      break;
+      isEq = 0;
    }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){
      uniqueNotNull = 0;
      isEq = 1;  /* "X IS NULL" means X has only a single value */
@@ -3317,8 +3323,8 @@ static void bestBtreeIndex(WhereBestIdx *p){
    ** indicate this to the caller.
    **
    ** Otherwise, if the search may find more than one row, test to see if
-    ** there is a range constraint on indexed column (pc.plan.nEq+1) that can be 
-    ** optimized using the index. 
+    ** there is a range constraint on indexed column (pc.plan.nEq+1) that
+    ** can be optimized using the index. 
    */
    if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){
      testcase( pc.plan.wsFlags & WHERE_COLUMN_IN );
@@ -3659,7 +3665,8 @@ static void bestIndex(WhereBestIdx *p){
    sqlite3_index_info *pIdxInfo = 0;
    p->ppIdxInfo = &pIdxInfo;
    bestVirtualIndex(p);
-    if( pIdxInfo->needToFreeIdxStr ){
+    assert( pIdxInfo!=0 || p->pParse->db->mallocFailed );
+    if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){
      sqlite3_free(pIdxInfo->idxStr);
    }
    sqlite3DbFree(p->pParse->db, pIdxInfo);
@@ -3783,12 +3790,13 @@ static int codeEqualityTerm(
    int eType;
    int iTab;
    struct InLoop *pIn;
+    u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;

    assert( pX->op==TK_IN );
    iReg = iTarget;
    eType = sqlite3FindInIndex(pParse, pX, 0);
    iTab = pX->iTable;
-    sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
+    sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0);
    assert( pLevel->plan.wsFlags & WHERE_IN_ABLE );
    if( pLevel->u.in.nIn==0 ){
      pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
@@ -3806,6 +3814,7 @@ static int codeEqualityTerm(
      }else{
        pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg);
      }
+      pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next;
      sqlite3VdbeAddOp1(v, OP_IsNull, iReg);
    }else{
      pLevel->u.in.nIn = 0;
@@ -5057,24 +5066,13 @@ WhereInfo *sqlite3WhereBegin(
  ** bitmask for all tables to the left of the join.  Knowing the bitmask
  ** for all tables to the left of a left join is important.  Ticket #3015.
  **
-  ** Configure the WhereClause.vmask variable so that bits that correspond
-  ** to virtual table cursors are set. This is used to selectively disable 
-  ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful 
-  ** with virtual tables.
-  **
  ** Note that bitmasks are created for all pTabList->nSrc tables in
  ** pTabList, not just the first nTabList tables.  nTabList is normally
  ** equal to pTabList->nSrc but might be shortened to 1 if the
  ** WHERE_ONETABLE_ONLY flag is set.
  */
-  assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 );
  for(ii=0; ii<pTabList->nSrc; ii++){
    createMask(pMaskSet, pTabList->a[ii].iCursor);
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-    if( ALWAYS(pTabList->a[ii].pTab) && IsVirtual(pTabList->a[ii].pTab) ){
-      sWBI.pWC->vmask |= ((Bitmask)1 << ii);
-    }
-#endif
  }
 #ifndef NDEBUG
  {
@@ -5558,7 +5556,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
      sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
      for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
        sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
-        sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop);
+        sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
        sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
      }
      sqlite3DbFree(db, pLevel->u.in.aInLoop);
@@ -1289,6 +1289,21 @@ do_test func-29.6 {
  if {$x<5} {set x 1}
  set x
 } {1}
-  
+
+do_execsql_test func-30.1 {SELECT unicode('$');} 36
+do_execsql_test func-30.2 [subst {SELECT unicode('\u00A2');}] 162
+do_execsql_test func-30.3 [subst {SELECT unicode('\u20AC');}] 8364
+do_execsql_test func-30.4 {SELECT char(36,162,8364);} [subst {$\u00A2\u20AC}]
+
+for {set i 1} {$i<0xd800} {incr i 13} {
+  do_execsql_test func-30.5.$i {SELECT unicode(char($i))} $i
+}
+for {set i 57344} {$i<=0xfffd} {incr i 17} {
+  if {$i==0xfeff} continue
+  do_execsql_test func-30.5.$i {SELECT unicode(char($i))} $i
+}
+for {set i 65536} {$i<=0x10ffff} {incr i 139} {
+  do_execsql_test func-30.5.$i {SELECT unicode(char($i))} $i
+}

 finish_test
@@ -0,0 +1,137 @@
+# 2013 Feb 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for the SQLite library, focusing
+# on the incremental vacuum feature.
+#
+# The tests in this file were added at the same time as optimizations 
+# were made to:
+#
+#   * Truncate the database after a rollback mode commit, and
+#
+#   * Avoid moving pages to locations from which they may need to be moved
+#     a second time if an incremental-vacuum proccess is allowed to vacuum
+#     the entire database.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix incrvacuum3
+
+# If this build of the library does not support auto-vacuum, omit this
+# whole file.
+ifcapable {!autovacuum || !pragma} {
+  finish_test
+  return
+}
+
+proc check_on_disk {} {
+
+  # Copy the files for database "test.db" to "test2.db".
+  forcedelete test2.db test2.db-journal test2.db-wal
+  forcecopy test.db test2.db
+  if {[file exists test.db-journal]} { 
+    forcecopy test.db-journal test2.db-journal 
+  }
+  if {[file exists test.db-wal]} { 
+    forcecopy test.db-wal test2.db-wal 
+  }
+
+  # Open "test2.db" and check it is Ok.
+  sqlite3 dbcheck test2.db
+  set ret [dbcheck eval { PRAGMA integrity_check }]
+  dbcheck close
+  set ret
+}
+
+# Run these tests once in rollback journal mode, and once in wal mode.
+#
+foreach {T jrnl_mode} {
+  1 delete
+  2 wal
+} {
+  catch { db close }
+  forcedelete test.db test.db-journal test.db-wal
+  sqlite3 db test.db
+  db eval {
+    PRAGMA cache_size = 5;
+    PRAGMA page_size = 1024;
+    PRAGMA auto_vacuum = 2;
+  }
+  db eval "PRAGMA journal_mode = $jrnl_mode"
+  
+  foreach {tn sql} {
+    1 {
+      CREATE TABLE t1(x UNIQUE);
+      INSERT INTO t1 VALUES(randomblob(400));
+      INSERT INTO t1 VALUES(randomblob(400));
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    --   4
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    --   8
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  16
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  32
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  64
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 128
+      INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 256
+    }
+  
+    2 {
+      DELETE FROM t1 WHERE rowid%8;
+    }
+  
+    3 { 
+      BEGIN;
+        PRAGMA incremental_vacuum = 100;
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  64
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 128
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 256
+      ROLLBACK;
+    }
+  
+    4 { 
+      BEGIN;
+        SAVEPOINT one;
+          PRAGMA incremental_vacuum = 100;
+          SAVEPOINT two;
+            INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  64
+            INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 128
+            INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 256
+    }
+  
+    5 {   ROLLBACK to two }
+  
+    6 { ROLLBACK to one }
+  
+    7 { 
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  64
+        PRAGMA incremental_vacuum = 1000;
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 128
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    -- 256
+      ROLLBACK;
+    }
+  
+    8 { 
+      BEGIN;
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  64
+        PRAGMA incremental_vacuum = 1000;
+        INSERT INTO t1 SELECT randomblob(400) FROM t1;    --  128
+      COMMIT;
+    }
+  } {
+    do_execsql_test $T.1.$tn.1 $sql
+    do_execsql_test $T.1.$tn.2 {PRAGMA integrity_check} ok
+    do_test         $T.1.$tn.3 { check_on_disk }        ok
+  }
+
+  do_execsql_test $T.1.x.1 { PRAGMA freelist_count   } 0
+  do_execsql_test $T.1.x.2 { SELECT count(*) FROM t1 } 128
+}
+
+finish_test
+
@@ -139,8 +139,8 @@ ifcapable shared_cache {
    # Figure out how big the database is and how many free pages it
    # has before running incremental-vacuum.
    #
-    set nPage [expr {[file size test.db]/1024}]
    set nFree [execsql {pragma freelist_count} db1]
+    set nPage [execsql {pragma page_count} db1]
  
    # Now run incremental-vacuum to vacuum 5 pages from the db file.
    # The iTest'th I/O call is set to fail.
@@ -158,7 +158,7 @@ ifcapable shared_cache {
    set ::sqlite_io_error_hardhit 0
  
    set nFree2 [execsql {pragma freelist_count} db1]
-    set nPage2 [expr {[file size test.db]/1024}]
+    set nPage2 [execsql {pragma page_count} db1]
  
    do_test incrvacuum-ioerr-4.$iTest.2 {
      set shrink [expr {$nPage-$nPage2}]
@@ -36,11 +36,10 @@ db close
 testvfs tvfs
 tvfs filter xWrite
 tvfs script write_cb
-proc write_cb {xCall file handle iOfst} {
+proc write_cb {xCall file handle iOfst args} {
  if {[file tail $file]=="test.db"} {
    lappend ::write_list [expr $iOfst/1024]
  }
-  puts "$xCall $file $args"
 }

 do_test 1.2 {
@@ -65,11 +64,12 @@ do_test 1.3 {
    }
    set iPrev $iNext
  }
+  puts -nonewline \
+      " (forward=$nForward, back=$nBackward, noncontiguous=$nNoncont)"

-  expr {$nForward > $nBackward}
+  expr {$nForward > 25*($nBackward + $nNoncont)}
 } {1}
 db close
 tvfs delete

 finish_test
-
@@ -1626,4 +1626,48 @@ do_test 22.4.3 {
  execsql { PRAGMA aux.integrity_check; }
 } {ok}

+db close
+forcedelete test.db test.db-wal test.db-journal
+sqlite3 db test.db
+sqlite3 db2 test.db
+do_test 23.1 {
+  db eval {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d);
+    CREATE INDEX i1 ON t1(b,c);
+    CREATE INDEX i2 ON t1(c,d);
+    CREATE TABLE t2(x INTEGER REFERENCES t1);
+  }
+  db2 eval {SELECT name FROM sqlite_master}
+} {t1 i1 i2 t2}
+do_test 23.2 {
+  db eval {
+    DROP INDEX i2;
+    CREATE INDEX i2 ON t1(c,d,b);
+  }
+  db2 eval {PRAGMA index_info(i2)}
+} {0 2 c 1 3 d 2 1 b}
+do_test 23.3 {
+  db eval {
+    CREATE INDEX i3 ON t1(d,b,c);
+  }
+  db2 eval {PRAGMA index_list(t1)}
+} {0 i3 0 1 i2 0 2 i1 0}
+do_test 23.4 {
+  db eval {
+    ALTER TABLE t1 ADD COLUMN e;
+  }
+  db2 eval {
+    PRAGMA table_info(t1);
+  }
+} {/4 e {} 0 {} 0/}
+do_test 23.5 {
+  db eval {
+    DROP TABLE t2;
+    CREATE TABLE t2(x, y INTEGER REFERENCES t1);
+  }
+  db2 eval {
+    PRAGMA foreign_key_list(t2);
+  }
+} {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE}
+
 finish_test
@@ -136,16 +136,52 @@ do_test 3.2 {
  }
 } {}

-breakpoint
 foreach {tn word res} {
  1   kos*     {kosher 3 kiosk 4 kudo 2 kiss 3 kissed 3}
  2   kellj*   {killjoy 5 kill 4 killed 4 killer 4 killers 4}
  3   kellj    {kill 4 kills 5 killjoy 7 keel 4 killed 6}
 } {
-  do_execsql_test 1.2.$tn {
+  do_execsql_test 3.2.$tn {
    SELECT word, matchlen FROM t3 WHERE word MATCH $word
     ORDER BY score, word LIMIT 5
  } $res
-} 
+}
+
+do_execsql_test 4.0 {
+  INSERT INTO t3(command) VALUES('edit_cost_table=NULL');
+}
+foreach {tn word res} {
+  1   kosher     {kosher 0 kisser 51 kissers 76 kissed 126 kisses 126}
+  2   kellj      {keels 60 killjoy 68 kills 80 keel 120 kill 125}
+  3   kashar     {kosher 80 kisser 91 kissers 116 kissed 166 kisses 166}
+} {
+  do_execsql_test 4.1.$tn {
+    SELECT word, distance FROM t3 WHERE word MATCH $word
+     ORDER BY score, word LIMIT 5
+  } $res
+}
+do_execsql_test 5.0 {
+  CREATE TABLE costs2(iLang, cFrom, cTo, iCost);
+  INSERT INTO costs2 VALUES(0, 'a', 'o', 1);
+  INSERT INTO costs2 VALUES(0, 'e', 'o', 4);
+  INSERT INTO costs2 VALUES(0, 'i', 'o', 8);
+  INSERT INTO costs2 VALUES(0, 'u', 'o', 16);
+  INSERT INTO t3(command) VALUES('edit_cost_table="costs2"');
+}
+
+foreach {tn word res} {
+  1   kasher     {kosher 1}
+  2   kesher     {kosher 4}
+  3   kisher     {kosher 8}
+  4   kosher     {kosher 0}
+  5   kusher     {kosher 16}
+} {
+  do_execsql_test 5.1.$tn {
+    SELECT word, distance FROM t3 WHERE word MATCH $word
+     ORDER BY score, word LIMIT 1
+  } $res
+}
+
+

 finish_test
@@ -1118,6 +1118,25 @@ proc crashsql {args} {
  lappend r $msg
 }

+proc run_ioerr_prep {} {
+  set ::sqlite_io_error_pending 0
+  catch {db close}
+  catch {db2 close}
+  catch {forcedelete test.db}
+  catch {forcedelete test.db-journal}
+  catch {forcedelete test2.db}
+  catch {forcedelete test2.db-journal}
+  set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db]
+  sqlite3_extended_result_codes $::DB $::ioerropts(-erc)
+  if {[info exists ::ioerropts(-tclprep)]} {
+    eval $::ioerropts(-tclprep)
+  }
+  if {[info exists ::ioerropts(-sqlprep)]} {
+    execsql $::ioerropts(-sqlprep)
+  }
+  expr 0
+}
+
 # Usage: do_ioerr_test <test number> <options...>
 #
 # This proc is used to implement test cases that check that IO errors
@@ -1150,10 +1169,26 @@ proc do_ioerr_test {testname args} {
  # TEMPORARY: For 3.5.9, disable testing of extended result codes. There are
  # a couple of obscure IO errors that do not return them.
  set ::ioerropts(-erc) 0
+  
+  # Create a single TCL script from the TCL and SQL specified
+  # as the body of the test.
+  set ::ioerrorbody {}
+  if {[info exists ::ioerropts(-tclbody)]} {
+    append ::ioerrorbody "$::ioerropts(-tclbody)\n"
+  }
+  if {[info exists ::ioerropts(-sqlbody)]} {
+    append ::ioerrorbody "db eval {$::ioerropts(-sqlbody)}"
+  }
+
+  save_prng_state
+  if {$::ioerropts(-cksum)} {
+    run_ioerr_prep
+    eval $::ioerrorbody
+    set ::goodcksum [cksum]
+  }

  set ::go 1
  #reset_prng_state
-  save_prng_state
  for {set n $::ioerropts(-start)} {$::go} {incr n} {
    set ::TN $n
    incr ::ioerropts(-count) -1
@@ -1170,27 +1205,12 @@ proc do_ioerr_test {testname args} {
    # Delete the files test.db and test2.db, then execute the TCL and 
    # SQL (in that order) to prepare for the test case.
    do_test $testname.$n.1 {
-      set ::sqlite_io_error_pending 0
-      catch {db close}
-      catch {db2 close}
-      catch {forcedelete test.db}
-      catch {forcedelete test.db-journal}
-      catch {forcedelete test2.db}
-      catch {forcedelete test2.db-journal}
-      set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db]
-      sqlite3_extended_result_codes $::DB $::ioerropts(-erc)
-      if {[info exists ::ioerropts(-tclprep)]} {
-        eval $::ioerropts(-tclprep)
-      }
-      if {[info exists ::ioerropts(-sqlprep)]} {
-        execsql $::ioerropts(-sqlprep)
-      }
-      expr 0
+      run_ioerr_prep
    } {0}

    # Read the 'checksum' of the database.
    if {$::ioerropts(-cksum)} {
-      set checksum [cksum]
+      set ::checksum [cksum]
    }

    # Set the Nth IO error to fail.
@@ -1198,20 +1218,10 @@ proc do_ioerr_test {testname args} {
      set ::sqlite_io_error_persist $::ioerropts(-persist)
      set ::sqlite_io_error_pending $n
    }] $n
-  
-    # Create a single TCL script from the TCL and SQL specified
-    # as the body of the test.
-    set ::ioerrorbody {}
-    if {[info exists ::ioerropts(-tclbody)]} {
-      append ::ioerrorbody "$::ioerropts(-tclbody)\n"
-    }
-    if {[info exists ::ioerropts(-sqlbody)]} {
-      append ::ioerrorbody "db eval {$::ioerropts(-sqlbody)}"
-    }

-    # Execute the TCL Script created in the above block. If
-    # there are at least N IO operations performed by SQLite as
-    # a result of the script, the Nth will fail.
+    # Execute the TCL script created for the body of this test. If
+    # at least N IO operations performed by SQLite as a result of 
+    # the script, the Nth will fail.
    do_test $testname.$n.3 {
      set ::sqlite_io_error_hit 0
      set ::sqlite_io_error_hardhit 0
@@ -1315,8 +1325,15 @@ proc do_ioerr_test {testname args} {
        catch {db close}
        catch {db2 close}
        set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db]
-        cksum
-      } $checksum
+        set nowcksum [cksum]
+        set res [expr {$nowcksum==$::checksum || $nowcksum==$::goodcksum}]
+        if {$res==0} {
+          puts "now=$nowcksum"
+          puts "the=$::checksum"
+          puts "fwd=$::goodcksum"
+        }
+        set res
+      } 1
    }

    set ::sqlite_io_error_hardhit 0
@@ -10,8 +10,8 @@
 #***********************************************************************
 #
 # Ticket #3762:  Make sure that an incremental vacuum that reduces the
-# size of the database file such that a pointer-map page is elemented
-# can be correctly rolled back.
+# size of the database file such that if a pointer-map page is eliminated
+# it can be correctly rolled back.
 #
 # That ticket #3762 has been fixed has already been verified by the
 # savepoint6.test test script.  But this script is simplier and a
@@ -379,11 +379,26 @@ ifcapable subquery {
      SELECT * FROM t1 WHERE rowid+0 IN (1,2,3,1234) order by 1;
    }
  } {1 0 4 2 1 9 3 1 16 102}
-  do_test where-5.3 {
+  do_test where-5.3a {
    count {
      SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1;
    }
-  } {1 0 4 2 1 9 3 1 16 14}
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3b {
+    count {
+      SELECT * FROM t1 WHERE w IN (3,-1,1,2) order by 1;
+    }
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3c {
+    count {
+      SELECT * FROM t1 WHERE w IN (3,2,-1,1,2) order by 1;
+    }
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3d {
+    count {
+      SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1 DESC;
+    }
+  } {3 1 16 2 1 9 1 0 4 12}
  do_test where-5.4 {
    count {
      SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1;
@@ -452,6 +467,30 @@ ifcapable subquery {
      SELECT * FROM t1 WHERE x IN (1,7) AND y IN (9,16) ORDER BY 1;
    }
  } {2 1 9 3 1 16 11}
+  do_test where-5.100 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x, y
+    }
+  } {2 1 9 54 5 3025 62 5 3969}
+  do_test where-5.101 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x DESC, y DESC
+    }
+  } {62 5 3969 54 5 3025 2 1 9}
+  do_test where-5.102 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x DESC, y
+    }
+  } {54 5 3025 62 5 3969 2 1 9}
+  do_test where-5.103 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x, y DESC
+    }
+  } {2 1 9 62 5 3969 54 5 3025}
 }

 # This procedure executes the SQL.  Then it checks to see if the OP_Sort
@@ -511,11 +550,16 @@ do_test where-6.7 {
  }
 } {1 100 4 2 99 9 3 98 16 nosort}
 ifcapable subquery {
-  do_test where-6.8 {
+  do_test where-6.8a {
    cksort {
      SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a LIMIT 3
    }
-  } {1 100 4 2 99 9 3 98 16 sort}
+  } {1 100 4 2 99 9 3 98 16 nosort}
+  do_test where-6.8b {
+    cksort {
+      SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a DESC LIMIT 3
+    }
+  } {9 92 100 7 94 64 5 96 36 nosort}
 }
 do_test where-6.9.1 {
  cksort {
@@ -167,24 +167,54 @@ ifcapable subquery {
      }
    } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx}
  }
-  do_test where2-4.6 {
+  do_test where2-4.6a {
    queryplan {
      SELECT * FROM t1
       WHERE x IN (1,2,3,4,5,6,7,8)
         AND y IN (10000,10001,10002,10003,10004,10005)
-       ORDER BY 2
+       ORDER BY x
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6b {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x DESC
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6c {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x, y
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6d {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x, y DESC
    }
  } {99 6 10000 10006 sort t1 i1xy}

  # Duplicate entires on the RHS of an IN operator do not cause duplicate
  # output rows.
  #
-  do_test where2-4.6 {
+  do_test where2-4.6x {
    queryplan {
      SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207)
      ORDER BY w
    }
  } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx}
+  do_test where2-4.6y {
+    queryplan {
+      SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207)
+      ORDER BY w DESC
+    }
+  } {100 6 10201 10207 99 6 10000 10006 sort t1 i1zyx}
  ifcapable compound {
    do_test where2-4.7 {
      queryplan {
@@ -207,11 +237,16 @@ do_test where2-5.1 {
 } {99 6 10000 10006 nosort t1 i1w}

 ifcapable subquery {
-  do_test where2-5.2 {
+  do_test where2-5.2a {
    queryplan {
      SELECT * FROM t1 WHERE w IN (99) ORDER BY w
    }
-  } {99 6 10000 10006 sort t1 i1w}
+  } {99 6 10000 10006 nosort t1 i1w}
+  do_test where2-5.2b {
+    queryplan {
+      SELECT * FROM t1 WHERE w IN (99) ORDER BY w DESC
+    }
+  } {99 6 10000 10006 nosort t1 i1w}
 }

 # Verify that OR clauses get translated into IN operators.
@@ -479,7 +479,7 @@ static void page_usage_msg(int pgno, const char *zFormat, ...){
  if( zPageUse[pgno]!=0 ){
    printf("ERROR: page %d used multiple times:\n", pgno);
    printf("ERROR:    previous: %s\n", zPageUse[pgno]);
-    printf("ERROR:    current:  %s\n", zPageUse[pgno]);
+    printf("ERROR:    current:  %s\n", zMsg);
    sqlite3_free(zPageUse[pgno]);
  }
  zPageUse[pgno] = zMsg;
@@ -615,15 +615,32 @@ static void page_usage_freelist(int pgno){
  }
 }

+/*
+** Determine pages used as PTRMAP pages
+*/
+static void page_usage_ptrmap(unsigned char *a){
+  if( a[55] ){
+    int usable = pagesize - a[20];
+    int pgno = 2;
+    int perPage = usable/5;
+    while( pgno<=mxPage ){
+      page_usage_msg(pgno, "PTRMAP page covering %d..%d",
+                           pgno+1, pgno+perPage);
+      pgno += perPage + 1;
+    }
+  }
+}
+
 /*
 ** Try to figure out how every page in the database file is being used.
 */
 static void page_usage_report(const char *zDbName){
-  int i;
+  int i, j;
  int rc;
  sqlite3 *db;
  sqlite3_stmt *pStmt;
  unsigned char *a;
+  char zQuery[200];

  /* Avoid the pathological case */
  if( mxPage<1 ){
@@ -648,20 +665,26 @@ static void page_usage_report(const char *zDbName){
  /* Discover the usage of each page */
  a = getContent(0, 100);
  page_usage_freelist(decodeInt32(a+32));
+  page_usage_ptrmap(a);
  free(a);
  page_usage_btree(1, 0, 0, "sqlite_master");
-  rc = sqlite3_prepare_v2(db,
-           "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage",
-           -1, &pStmt, 0);
-  if( rc==SQLITE_OK ){
-    while( sqlite3_step(pStmt)==SQLITE_ROW ){
-      int pgno = sqlite3_column_int(pStmt, 2);
-      page_usage_btree(pgno, 0, 0, sqlite3_column_text(pStmt, 1));
+  sqlite3_exec(db, "PRAGMA writable_schema=ON", 0, 0, 0);
+  for(j=0; j<2; j++){
+    sqlite3_snprintf(sizeof(zQuery), zQuery,
+             "SELECT type, name, rootpage FROM SQLITE_MASTER WHERE rootpage"
+             " ORDER BY rowid %s", j?"DESC":"");
+    rc = sqlite3_prepare_v2(db, zQuery, -1, &pStmt, 0);
+    if( rc==SQLITE_OK ){
+      while( sqlite3_step(pStmt)==SQLITE_ROW ){
+        int pgno = sqlite3_column_int(pStmt, 2);
+        page_usage_btree(pgno, 0, 0, sqlite3_column_text(pStmt, 1));
+      }
+    }else{
+      printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db));
    }
-  }else{
-    printf("ERROR: cannot query database: %s\n", sqlite3_errmsg(db));
+    rc = sqlite3_finalize(pStmt);
+    if( rc==SQLITE_OK ) break;
  }
-  sqlite3_finalize(pStmt);
  sqlite3_close(db);

  /* Print the report and free memory used */
@@ -673,6 +696,53 @@ static void page_usage_report(const char *zDbName){
  zPageUse = 0;
 }

+/*
+** Try to figure out how every page in the database file is being used.
+*/
+static void ptrmap_coverage_report(const char *zDbName){
+  unsigned int pgno;
+  unsigned char *aHdr;
+  unsigned char *a;
+  int usable;
+  int perPage;
+  unsigned int i;
+
+  /* Avoid the pathological case */
+  if( mxPage<1 ){
+    printf("empty database\n");
+    return;
+  }
+
+  /* Make sure PTRMAPs are used in this database */
+  aHdr = getContent(0, 100);
+  if( aHdr[55]==0 ){
+    printf("database does not use PTRMAP pages\n");
+    return;
+  }
+  usable = pagesize - aHdr[20];
+  perPage = usable/5;
+  free(aHdr);
+  printf("%5d: root of sqlite_master\n", 1);
+  for(pgno=2; pgno<=mxPage; pgno += perPage+1){
+    printf("%5d: PTRMAP page covering %d..%d\n", pgno,
+           pgno+1, pgno+perPage);
+    a = getContent((pgno-1)*pagesize, usable);
+    for(i=0; i+5<=usable && pgno+1+i/5<=mxPage; i+=5){
+      const char *zType = "???";
+      unsigned int iFrom = decodeInt32(&a[i+1]);
+      switch( a[i] ){
+        case 1:  zType = "b-tree root page";        break;
+        case 2:  zType = "freelist page";           break;
+        case 3:  zType = "first page of overflow";  break;
+        case 4:  zType = "later page of overflow";  break;
+        case 5:  zType = "b-tree non-root page";    break;
+      }
+      printf("%5d: %s, parent=%u\n", pgno+1+i/5, zType, iFrom);
+    }
+    free(a);
+  }
+}
+
 /*
 ** Print a usage comment
 */
@@ -682,6 +752,7 @@ static void usage(const char *argv0){
    "args:\n"
    "    dbheader        Show database header\n"
    "    pgidx           Index of how each page is used\n"
+    "    ptrmap          Show all PTRMAP page content\n"
    "    NNN..MMM        Show hex of pages NNN through MMM\n"
    "    NNN..end        Show hex of pages NNN through end of file\n"
    "    NNNb            Decode btree page NNN\n"
@@ -731,6 +802,14 @@ int main(int argc, char **argv){
        page_usage_report(argv[1]);
        continue;
      }
+      if( strcmp(argv[i], "ptrmap")==0 ){
+        ptrmap_coverage_report(argv[1]);
+        continue;
+      }
+      if( strcmp(argv[i], "help")==0 ){
+        usage(argv[0]);
+        continue;
+      }
      if( !isdigit(argv[i][0]) ){
        fprintf(stderr, "%s: unknown option: [%s]\n", argv[0], argv[i]);
        continue;
Author	SHA1	Message	Date
mistachkin	8d0b81d764	Prevent Tcl file encoding issues in the tests for the unicode() and char() functions. FossilOrigin-Name: d2e7dfca5a92074a7984032deb6a4e3681389c72	2013-02-26 05:44:33 +00:00
mistachkin	c9545442f9	Remove extra use of the sqlite3_value_int64() function. FossilOrigin-Name: 6d7973524a7d3bf3158fdac58975945da7f51740	2013-02-26 05:42:30 +00:00
drh	fbc1ddf079	Add new SQL functions unicode() and char(). FossilOrigin-Name: be2493905281e12c7f4c146ab17c8872e52da350	2013-02-25 14:39:47 +00:00
drh	c6a67173bd	Strengthen the final test case in index5.test. Also provide additional diagnostic information out the output. FossilOrigin-Name: 47b6418242bb2cd718d1a73b0cb73a43ee74e503	2013-02-25 13:55:59 +00:00
dan	132d93838d	Fix an error in test script index5.test. FossilOrigin-Name: d87e5acf2802d2887e20f79a8bd4990b2cd47b91	2013-02-25 13:44:56 +00:00
dan	f2874b0e1d	Merge the incr-vacuum-opt branch with the trunk. FossilOrigin-Name: 26e235b7a4cd4d0dc9725774d70174c4d369cb98	2013-02-25 13:31:30 +00:00
dan	295fc442b5	Add test file incrvacuum3.test. No code changes. FossilOrigin-Name: bf57534188e044fb341315bfc05b7927e66a04e0	2013-02-25 12:06:55 +00:00
dan	896f99e942	Catch a dropped error code in backup.c. FossilOrigin-Name: ac8ca3ecee4d81bf522b330033e5d85638063670	2013-02-25 07:12:40 +00:00
dan	9138471141	Fix a case where database corruption may cause an assert() to fail. FossilOrigin-Name: ba33bb059ed4f4547da2880dbc8bd827c06fae34	2013-02-24 11:50:43 +00:00
dan	e0b605917e	Fix a case in the incremental vacuum code where database corruption was going unreported. FossilOrigin-Name: 4cd30c72629a7f44b18026a70103aa0bb8a3b959	2013-02-23 19:43:47 +00:00
dan	22d2730047	Fix a cut-and-paste bug causing the library to fail to report database corruption in a few cases. FossilOrigin-Name: f921df59560d536f5b80eef8c995cbe3ff591bae	2013-02-23 19:11:47 +00:00
dan	b88e24fd59	Enhancements to test scripts. No code changes. FossilOrigin-Name: ccab94c10d54e585de918bbf82dec188287d93b2	2013-02-23 18:58:11 +00:00
dan	87ade19eb9	Fix off-by-one bug in [c3939d2491] uncovered by th3. FossilOrigin-Name: 66f9faa9a969d004486ee38d492fb3eda6cdebab	2013-02-23 17:49:16 +00:00
dan	bc1a3c6ce2	If a rollback mode transaction reduces the size of the database file, avoid actually truncating the file until after the transaction has been committed (but before the db has been unlocked). This means pages that are removed from the database by truncating the file need not be journalled. FossilOrigin-Name: b73847f17b7ae1298dfc52de6c4c4bc809bc77a3	2013-02-23 16:40:46 +00:00
dan	f38b65a85d	Fix a problem with the previous commit. FossilOrigin-Name: 720a3ceafc35b81936ed2eb1f07a7187d104f0a0	2013-02-22 20:57:47 +00:00
dan	51f0b6d568	Avoid moving pages more than once in an incremental vacuum operation. FossilOrigin-Name: c3939d249119b47bd57baa11a5ed7cc6014fc795	2013-02-22 20:16:34 +00:00
drh	d495d8c9a3	Two new SQL functions: unicode() and char(). FossilOrigin-Name: 209b21085b9767f10f6ffb7c7cac756fcb74ded5	2013-02-22 19:34:25 +00:00
drh	d5d0f6432c	On Minix, disable the ".timer" command in the shell in order to avoid calling getrusage(). FossilOrigin-Name: 9bd9bd9cab8c804c1a51d472199459176044a633	2013-02-20 00:54:21 +00:00
drh	344a97b4b0	Enhance the showdb tool with options to show PTRMAP usage and content. FossilOrigin-Name: 06bd91305ed6752315c5224be5f89e87cafa6687	2013-02-19 22:26:51 +00:00
drh	103a70fa62	Fix the showdb utility so that it displays the correct secondary usage of a page when reporting on an error of a page being used more than once. FossilOrigin-Name: 4507f0b3d409cd14bb2b9c29c92c76aa3ccb22b6	2013-02-19 20:25:16 +00:00
drh	00e637f0f0	Enhance the pgidx of the showdb utility so that it provides better information even if the sqlite_master table is corrupt. FossilOrigin-Name: d14263a719101d9c70054f2fc37e7788f73aab28	2013-02-19 18:45:11 +00:00
drh	5c5760aa81	Do not rollback the schema if a parsing error occurs while parsing the schema and writable_schema is set. FossilOrigin-Name: 680d3ab56b578bd4b0170559f9e35615ff43cf2c	2013-02-19 18:34:45 +00:00
drh	4bde37027c	Truncate over-length source lines in sqliteInt.h to 80 characters or less. FossilOrigin-Name: d71abab08518f0be1eb73c7068566ab813652318	2013-02-19 18:34:12 +00:00
drh	5cee8a7af5	Add the edit_cost_table= command to the spellfix1 virtual table, permitting the edit distance cost table to be changed at runtime. FossilOrigin-Name: 89d6368250f9ba2f49e930bbe5524f3da615ce70	2013-02-19 11:51:27 +00:00
drh	c8ac0d1649	Fix an unsafe VM register deallocation. FossilOrigin-Name: cfba2c8dadbf1fa6f1602a327933570820a3b134	2013-02-16 02:41:01 +00:00
mistachkin	4600255a88	Check for a NULL handle prior to manually calling winClose. FossilOrigin-Name: 843e1c543aabab8cd62f28742d5818887d36bcb7	2013-02-15 04:21:01 +00:00
drh	c95e01da22	Verify that the schema has not changed before running PRAGMA table_info, index_list, index_info, and foreign_key_list. FossilOrigin-Name: 82952d08f3e3aa80a7f51e80dbc89742cb4a09f0	2013-02-14 16:16:05 +00:00
mistachkin	9ce59a947f	On WinCE, fill in the winFile structure after winceCreateLock has succeeded. FossilOrigin-Name: ec13544ce336e8a02b34d574ced5912b06c82c0e	2013-02-13 22:54:03 +00:00
drh	4207a3fb9b	Add recent API additions to the extension mechanism. FossilOrigin-Name: 7e10a62d0eb1cb2bdafb6752b78a9d368e9f21f5	2013-02-13 14:04:28 +00:00
drh	a4b05e2874	Allow an index to be used for sorting even if prior terms of the index are constrained by IN operators. FossilOrigin-Name: 2cef8b68f0e1216cf68bb7dd45a5a9a330748070	2013-02-13 01:00:35 +00:00
drh	25129243b7	Improve memory allocation error handling on WinCE. FossilOrigin-Name: cdbca259da80df901837034d2616da434cb82216	2013-02-12 22:20:01 +00:00
drh	785097da6b	A bug report coming from from Firefox ([https://bugzilla.mozilla.org/show_bug.cgi?id=840377]) describes an infinite recursion. We are unable to reproduce the problem. Nevertheless, this check-in should prevent the problem from ever occurring again. FossilOrigin-Name: b7222a2bd035e7a32dc9416b25a488d9d017aad1	2013-02-12 22:09:48 +00:00
drh	5edf8435d3	Fix a potential NULL-pointer dereference following an OOM error in the query planner logic for virtual tables with OR-connected terms. FossilOrigin-Name: 71b6c26053fdf2a5a84116e005bad1f2ca873a66	2013-02-08 23:18:18 +00:00
drh	3667033bfc	Make sure the virtual tables that take advantage of IN operators sort the RHS of the IN operator in the correct order according to the ORDER BY clause. FossilOrigin-Name: b016b7546d6fbfba06019398b9ac239b0cbe9086	2013-02-08 20:39:02 +00:00
drh	2d96b934c8	Loop through the elements on the RHS of an IN operator in reverse order when the ORDER BY clauses specifies DESC. FossilOrigin-Name: f78395c8896666bb1359b83fbcd58d5e3dbc39d3	2013-02-08 18:48:23 +00:00
drh	c3e552ff5c	Allow the "a=?1 OR a=?2" to "a IN (?1,?2)" transformation to work on virtual tables again. This was formerly restricted because virtual tables could not optimize IN terms. (See check-in [fad88e71cf195e].) But IN terms are now used by virtual tables (as of check-in [3d65c70343]) so the restriction can now be removed. FossilOrigin-Name: a917c1f09254b54e03e31b119cc49c551269d82e	2013-02-08 16:04:19 +00:00
drh	1b8fc65b92	Allow an index to be used for sorting even if prior terms of the index are constrained by IN operators. FossilOrigin-Name: 98bf668ab1a8683b46ee8c94cb60f8215aabc517	2013-02-07 21:15:14 +00:00