From 5f1731f67000d8e0c7cd9ce14ffd48e37bf42866 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Apr 2015 15:24:53 +0000 Subject: [PATCH 01/10] Disable e_walauto.test on OpenBSD, as it requires a coherent cache. FossilOrigin-Name: 90701227085b8b8eb10a8eebe8d55f38b4778574 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/e_walauto.test | 8 ++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 83880531e8..e116e655ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sand\ssuppression\sof\sendless\sloops\sin\sclearDatabasePage(). -D 2015-04-01T13:21:33.901 +C Disable\se_walauto.test\son\sOpenBSD,\sas\sit\srequires\sa\scoherent\scache. +D 2015-04-02T15:24:53.782 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -485,7 +485,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 -F test/e_walauto.test ca70cf75c07a6cb1874ced101dd426da76625649 +F test/e_walauto.test 6544af03423abc61b53cfb976839385ddc2a0a70 F test/e_walckpt.test 65e29b6631e51f210f83e4ff11571e647ba93608 F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea @@ -1248,7 +1248,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a828e73dc1ae50189bdf73f60caeb7308738ad7a -R 9ae26e7b05df4833a239ec9186729882 -U drh -Z b113a7338e765a6278a4e0ffd57f1cf8 +P 30011ad2f55cfcacaf23a58ebcc17b17a7b9355e +R 0b149500fabb905b0771216875924384 +U dan +Z 259e8553f23f2ead0f4917d2e8d05aa6 diff --git a/manifest.uuid b/manifest.uuid index 925ee38b4d..67f1e1b12f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30011ad2f55cfcacaf23a58ebcc17b17a7b9355e \ No newline at end of file +90701227085b8b8eb10a8eebe8d55f38b4778574 \ No newline at end of file diff --git a/test/e_walauto.test b/test/e_walauto.test index b624b2469c..a1f4eb7079 100644 --- a/test/e_walauto.test +++ b/test/e_walauto.test @@ -15,6 +15,14 @@ source $testdir/tester.tcl source $testdir/wal_common.tcl set testprefix e_walauto +# Do not run this test on OpenBSD, as it depends on read() and mmap both +# accessing the same coherent view of the "test.db-shm" file. This doesn't +# work on OpenBSD. +# +if {$tcl_platform(os) == "OpenBSD"} { + finish_test + return +} proc read_nbackfill {} { seek $::shmfd 96 From eaf26402370336186d8119e3130eaf050595ff44 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Apr 2015 18:33:40 +0000 Subject: [PATCH 02/10] Add source code to the "showlocks" utility program in the tool/ subdirectory. FossilOrigin-Name: 6868cc66d2be67b7f03776c982962ffa4b30de11 --- manifest | 13 +++++----- manifest.uuid | 2 +- tool/showlocks.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 tool/showlocks.c diff --git a/manifest b/manifest index e116e655ac..73436f25df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\se_walauto.test\son\sOpenBSD,\sas\sit\srequires\sa\scoherent\scache. -D 2015-04-02T15:24:53.782 +C Add\ssource\scode\sto\sthe\s"showlocks"\sutility\sprogram\sin\sthe\stool/\ssubdirectory. +D 2015-04-03T18:33:40.031 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1227,6 +1227,7 @@ F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c 63cdef19e7fbca0c164b096ef8aef3bb9e9dd222 F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 +F tool/showlocks.c 9920bcc64f58378ff1118caead34147201f48c68 F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -1248,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30011ad2f55cfcacaf23a58ebcc17b17a7b9355e -R 0b149500fabb905b0771216875924384 -U dan -Z 259e8553f23f2ead0f4917d2e8d05aa6 +P 90701227085b8b8eb10a8eebe8d55f38b4778574 +R f70ddacff9e1db6211fe7e92067ecf89 +U drh +Z 4cde991c3d8a79a805d8a208c5613c13 diff --git a/manifest.uuid b/manifest.uuid index 67f1e1b12f..1722db95be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90701227085b8b8eb10a8eebe8d55f38b4778574 \ No newline at end of file +6868cc66d2be67b7f03776c982962ffa4b30de11 \ No newline at end of file diff --git a/tool/showlocks.c b/tool/showlocks.c new file mode 100644 index 0000000000..752c535cc3 --- /dev/null +++ b/tool/showlocks.c @@ -0,0 +1,64 @@ +/* +** This file implements a simple command-line utility that shows all of the +** Posix Advisory Locks on a file. +** +** Usage: +** +** showlocks FILENAME +** +** To compile: gcc -o showlocks showlocks.c +*/ +#include +#include +#include +#include +#include + +/* This utility only looks for locks in the first 2 billion bytes */ +#define MX_LCK 2147483647 + +/* +** Print all locks on the inode of "fd" that occur in between +** lwr and upr, inclusive. +*/ +static int showLocksInRange(int fd, off_t lwr, off_t upr){ + int cnt = 0; + struct flock x; + + x.l_type = F_WRLCK; + x.l_whence = SEEK_SET; + x.l_start = lwr; + x.l_len = upr-lwr; + fcntl(fd, F_GETLK, &x); + if( x.l_type==F_UNLCK ) return 0; + printf("start: %-12d len: %-5d pid: %-5d type: %s\n", + (int)x.l_start, (int)x.l_len, + x.l_pid, x.l_type==F_WRLCK ? "WRLCK" : "RDLCK"); + cnt++; + if( x.l_start>lwr ){ + cnt += showLocksInRange(fd, lwr, x.l_start-1); + } + if( x.l_start+x.l_len Date: Fri, 3 Apr 2015 20:33:33 +0000 Subject: [PATCH 03/10] Disable the SQLITE_FCNTL_WAL_BLOCK feature for now. It needs more work and is not yet ready for release. FossilOrigin-Name: 4ae9a3acc4eeeb7998769eb856c97c2233476f72 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- test/walblock.test | 5 +---- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 73436f25df..31dda70760 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssource\scode\sto\sthe\s"showlocks"\sutility\sprogram\sin\sthe\stool/\ssubdirectory. -D 2015-04-03T18:33:40.031 +C Disable\sthe\sSQLITE_FCNTL_WAL_BLOCK\sfeature\sfor\snow.\s\sIt\sneeds\smore\swork\sand\nis\snot\syet\sready\sfor\srelease. +D 2015-04-03T20:33:33.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c a4dadbc2da41599e99093e91e276c38c17a73b89 +F src/os_unix.c 25b80a3d167da44226a2084dc9e89a6cb1f02e2e F src/os_win.c 03d27be3a20048ef52a648d5f0a15f5edda9f2a3 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4120a49ecd37697e28f5ed807f470b9c0b88410c @@ -1144,7 +1144,7 @@ F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 -F test/walblock.test ffc761cd467a93ccd8cd998a23be2f21b95a83b1 +F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d F test/walcksum.test 9afeb96240296c08c72fc524d199c912cfe34daa F test/walcrash.test 451d79e528add5c42764cea74aa2750754171b25 F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 @@ -1249,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 90701227085b8b8eb10a8eebe8d55f38b4778574 -R f70ddacff9e1db6211fe7e92067ecf89 +P 6868cc66d2be67b7f03776c982962ffa4b30de11 +R f821ca69e41f19033ce555eacc91c4ed U drh -Z 4cde991c3d8a79a805d8a208c5613c13 +Z 7ec9ecc18bf834ba571fcbb8e1e49df1 diff --git a/manifest.uuid b/manifest.uuid index 1722db95be..174ac4daaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6868cc66d2be67b7f03776c982962ffa4b30de11 \ No newline at end of file +4ae9a3acc4eeeb7998769eb856c97c2233476f72 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a9c883a935..1d867d7cd8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3783,7 +3783,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ case SQLITE_FCNTL_WAL_BLOCK: { - pFile->ctrlFlags |= UNIXFILE_BLOCK; + /* pFile->ctrlFlags |= UNIXFILE_BLOCK; // Deferred feature */ return SQLITE_OK; } case SQLITE_FCNTL_LOCKSTATE: { diff --git a/test/walblock.test b/test/walblock.test index 0b0b2241e6..23167a8830 100644 --- a/test/walblock.test +++ b/test/walblock.test @@ -15,6 +15,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl +finish_test; return; # Feature currently not implemented. ifcapable !wal {finish_test ; return } if {$::tcl_platform(platform)!="unix"} { finish_test ; return } set testprefix walblock @@ -111,7 +112,3 @@ do_test "1.2.3.(blocking 0.5 seconds)" { finish_test - - - - From b33c50f215b45acd1b6f47008562e64d895d3fba Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Apr 2015 16:43:16 +0000 Subject: [PATCH 04/10] Fix a problem with resolving ORDER BY clauses that feature COLLATE clauses attached to compound SELECT statements. FossilOrigin-Name: 427b50fba7362e5b447e79d39050f25ed2ef10af --- manifest | 23 ++++++++++-------- manifest.uuid | 2 +- src/resolve.c | 25 ++++++++++++++++++++ src/select.c | 2 ++ src/sqliteInt.h | 1 + test/selectA.test | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 31dda70760..ad90ce9662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sSQLITE_FCNTL_WAL_BLOCK\sfeature\sfor\snow.\s\sIt\sneeds\smore\swork\sand\nis\snot\syet\sready\sfor\srelease. -D 2015-04-03T20:33:33.240 +C Fix\sa\sproblem\swith\sresolving\sORDER\sBY\sclauses\sthat\sfeature\sCOLLATE\sclauses\sattached\sto\scompound\sSELECT\sstatements. +D 2015-04-04T16:43:16.254 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,14 +228,14 @@ F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 8da9a2687a396daa19860f4dc90975d319304744 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c f4d79e31ffa5820c2e3d1740baa5e9b190425f2b +F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 72ffb62e2879956302140e9f6e6ae88aee36b0e5 +F src/select.c c28c52e353287434fac8473e56ee4be848d12c9d F src/shell.c 84a1593bd86aaa14f4da8a8f9b16fbc239d262aa F src/sqlite.h.in 278602140d49575e8708e643161f4263e428a02a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d02d2b85cb02a38bc442cf9302ec8209baf6771d +F src/sqliteInt.h 107b02ed6c64162b653acc2368e982de529e14f6 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c e7a09215315a978057fb42c640f890160dbcc45e @@ -845,7 +845,7 @@ F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test 64b88a80271c1710966e50e633380696b60a12a4 +F test/selectA.test e452bdb975f488ea46d091382a9185b5853ed2c7 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -1249,7 +1249,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6868cc66d2be67b7f03776c982962ffa4b30de11 -R f821ca69e41f19033ce555eacc91c4ed -U drh -Z 7ec9ecc18bf834ba571fcbb8e1e49df1 +P 4ae9a3acc4eeeb7998769eb856c97c2233476f72 +R 0a683c8bd67cbd2be5ea29e95266b636 +T *branch * compound-order-by-fix +T *sym-compound-order-by-fix * +T -sym-trunk * +U dan +Z 235b3663512a162fe4fda68eec524b54 diff --git a/manifest.uuid b/manifest.uuid index 174ac4daaf..dcc45e2918 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ae9a3acc4eeeb7998769eb856c97c2233476f72 \ No newline at end of file +427b50fba7362e5b447e79d39050f25ed2ef10af \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 47df7243f8..a7b14cd005 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1186,6 +1186,20 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sqlite3ResolveExprNames(&sNC, p->pOffset) ){ return WRC_Abort; } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && isCompound==0 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } /* Recursively resolve names in all subqueries */ @@ -1268,6 +1282,17 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg; + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } + /* Process the ORDER BY clause for singleton SELECT statements. ** The ORDER BY clause for compounds SELECT statements is handled ** below, after all of the result-sets for all of the elements of diff --git a/src/select.c b/src/select.c index 8fd0f15918..90aaa842a6 100644 --- a/src/select.c +++ b/src/select.c @@ -3884,6 +3884,8 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ p->pPrior = 0; p->pNext = 0; p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3841c91f14..1c0ad44e3a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2389,6 +2389,7 @@ struct Select { #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ #define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ +#define SF_Converted 0x2000 /* By convertCompoundSelectToSubquery() */ /* diff --git a/test/selectA.test b/test/selectA.test index 6e593e8e22..0338338902 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -1375,4 +1375,64 @@ do_execsql_test 4.2.2 { } {/2 . 3 . 4 . 5 . 6 . 7 ./} +proc strip_rnd {explain} { + regexp -all {sqlite_sq_[0123456789ABCDEF]*} $explain sqlite_sq +} + +proc do_same_test {tn q1 args} { + set r2 [strip_rnd [db eval "EXPLAIN $q1"]] + set i 1 + foreach q $args { + set tst [subst -nocommands {strip_rnd [db eval "EXPLAIN $q"]}] + uplevel do_test $tn.$i [list $tst] [list $r2] + incr i + } +} + +do_execsql_test 5.0 { + CREATE TABLE t8(a, b); + CREATE TABLE t9(c, d); +} {} + +do_same_test 5.1 { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY a; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY t8.a; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY 1; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY c; +} { + SELECT a, b FROM t8 INTERSECT SELECT c, d FROM t9 ORDER BY t9.c; +} + +do_same_test 5.2 { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY a COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY t8.a COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY 1 COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY c COLLATE NOCASE +} { + SELECT a, b FROM t8 UNION SELECT c, d FROM t9 ORDER BY t9.c COLLATE NOCASE +} + +do_same_test 5.3 { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY b, c COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY 2, 1 COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY d, a COLLATE NOCASE +} { + SELECT a, b FROM t8 EXCEPT SELECT * FROM t9 ORDER BY t9.d, c COLLATE NOCASE +} { + SELECT * FROM t8 EXCEPT SELECT c, d FROM t9 ORDER BY d, t8.a COLLATE NOCASE +} + +do_catchsql_test 5.4 { + SELECT * FROM t8 UNION SELECT * FROM t9 ORDER BY a+b COLLATE NOCASE +} {1 {1st ORDER BY term does not match any column in the result set}} + + finish_test From 4b0000c9a89800c2c7603b2fd722431776685d52 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Apr 2015 09:05:29 +0000 Subject: [PATCH 05/10] Fix a problem with fts3 prefix terms within phrase queries on "order=DESC" tables with a mix of negative and positive rowids. FossilOrigin-Name: 0cdf502885ea7e5805d7ba3719f055f5d48fc78d --- ext/fts3/fts3.c | 57 ++++++++++++++++++++++++++++++++++++-------- manifest | 17 +++++++------ manifest.uuid | 2 +- test/fts3prefix.test | 18 ++++++++++++++ 4 files changed, 76 insertions(+), 18 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 46b499e96b..7a15379b83 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2502,26 +2502,33 @@ static int fts3DoclistOrMerge( ** ** The right-hand input doclist is overwritten by this function. */ -static void fts3DoclistPhraseMerge( +static int fts3DoclistPhraseMerge( int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ - char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; + char *aRight = *paRight; char *pEnd1 = &aLeft[nLeft]; char *pEnd2 = &aRight[*pnRight]; char *p1 = aLeft; char *p2 = aRight; char *p; int bFirstOut = 0; - char *aOut = aRight; + char *aOut; assert( nDist>0 ); - + if( bDescDoclist ){ + aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); @@ -2550,6 +2557,12 @@ static void fts3DoclistPhraseMerge( } *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; } /* @@ -2674,8 +2687,22 @@ static int fts3TermSelectMerge( ){ if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -3931,14 +3958,17 @@ static void fts3EvalAllocateReaders( ** This function assumes that pList points to a buffer allocated using ** sqlite3_malloc(). This function takes responsibility for eventually ** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. */ -static void fts3EvalPhraseMergeToken( +static int fts3EvalPhraseMergeToken( Fts3Table *pTab, /* FTS Table pointer */ Fts3Phrase *p, /* Phrase to merge pList/nList into */ int iToken, /* Token pList/nList corresponds to */ char *pList, /* Pointer to doclist */ int nList /* Number of bytes in pList */ ){ + int rc = SQLITE_OK; assert( iToken!=p->iDoclistToken ); if( pList==0 ){ @@ -3977,13 +4007,16 @@ static void fts3EvalPhraseMergeToken( nDiff = p->iDoclistToken - iToken; } - fts3DoclistPhraseMerge(pTab->bDescIdx, nDiff, pLeft, nLeft, pRight,&nRight); + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); sqlite3_free(pLeft); p->doclist.aAll = pRight; p->doclist.nAll = nRight; } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; } /* @@ -4009,7 +4042,7 @@ static int fts3EvalPhraseLoad( char *pThis = 0; rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ - fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } assert( pToken->pSegcsr==0 ); @@ -4811,9 +4844,13 @@ static int fts3EvalSelectDeferred( char *pList = 0; rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } if( rc==SQLITE_OK ){ int nCount; - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); nCount = fts3DoclistCountDocids( pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll ); diff --git a/manifest b/manifest index 762233f676..12c20e5b8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sresolving\sORDER\sBY\sclauses\sthat\sfeature\sCOLLATE\sclauses\sattached\sto\scompound\sSELECT\sstatements. -D 2015-04-04T16:49:04.697 +C Fix\sa\sproblem\swith\sfts3\sprefix\sterms\swithin\sphrase\squeries\son\s"order=DESC"\stables\swith\sa\smix\sof\snegative\sand\spositive\srowids. +D 2015-04-06T09:05:29.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c d3f6f0e95d366f3c2028d916c36a0844bf805840 +F ext/fts3/fts3.c 23bd9d37a777342f5c22a648e9b4b005dde9e58f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 394858c12a17740f7a1f6bd372c4606d4425a8d1 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -594,7 +594,7 @@ F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test 58544fa4d254000fa4e7f494b0a832f7ba61d45e F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 -F test/fts3prefix.test 9f68e3598a139c23ec47d09299420e0fc4c72a83 +F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce F test/fts3query.test c838b18f2b859e15fd31c64be3d79ef1556803ca F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 @@ -1249,7 +1249,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ae9a3acc4eeeb7998769eb856c97c2233476f72 427b50fba7362e5b447e79d39050f25ed2ef10af -R 0a683c8bd67cbd2be5ea29e95266b636 +P edc1de2a588fd50c0049bb2be76d3f6783443165 +R a8252a4468bea13d638722c58d3ce9cc +T *branch * fts3-prefix-query-fix +T *sym-fts3-prefix-query-fix * +T -sym-trunk * U dan -Z c9ac74415bdb7893d918eacf5a95862c +Z 7c1c374a4c4efe674ae583b0f9d983d2 diff --git a/manifest.uuid b/manifest.uuid index ff36ec293f..2a42d14f33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edc1de2a588fd50c0049bb2be76d3f6783443165 \ No newline at end of file +0cdf502885ea7e5805d7ba3719f055f5d48fc78d \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test index 8ffabe8d65..e8d2b2b5fb 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -274,4 +274,22 @@ do_execsql_test 6.5.2 { SELECT md5sum(quote(root)) FROM t1_segdir; } [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}] + +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t6 USING fts4(x,order=DESC); + INSERT INTO t6(docid, x) VALUES(-1,'a b'); + INSERT INTO t6(docid, x) VALUES(1, 'b'); +} +do_execsql_test 7.1 { + SELECT docid FROM t6 WHERE t6 MATCH '"a* b"'; +} {-1} +do_execsql_test 7.2 { + SELECT docid FROM t6 WHERE t6 MATCH 'a*'; +} {-1} +do_execsql_test 7.3 { + SELECT docid FROM t6 WHERE t6 MATCH 'a* b'; +} {-1} + + + finish_test From 74b42275eca70ba323730c5a1088f1baf9937383 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Apr 2015 12:41:17 +0000 Subject: [PATCH 06/10] Guard against excessive width and precision in floating-point conversions in the printf routines. FossilOrigin-Name: c494171f77dc2e5e04cb6d865e688448f04e5920 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 7 ++++--- test/printf.test | 3 +++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 839dc44cc6..31581470a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts3\sprefix\sterms\swithin\sphrase\squeries\son\s"order=DESC"\stables\swith\sa\smix\sof\snegative\sand\spositive\srowids. -D 2015-04-06T11:04:51.176 +C Guard\sagainst\sexcessive\swidth\sand\sprecision\sin\sfloating-point\sconversions\nin\sthe\sprintf\sroutines. +D 2015-04-07T12:41:17.461 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/pcache1.c 69d137620a305f814398bd29a0c998038c0695e9 F src/pragma.c ac4f3f856b4234e85f55b0f069698a4766011100 F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 -F src/printf.c 8da9a2687a396daa19860f4dc90975d319304744 +F src/printf.c 32f69fcba9ddfb71f9466b867f7ee7b7ffee30de F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -797,7 +797,7 @@ F test/permutations.test f9cc1dd987986c9d4949211c7a4ed55ec9aecba1 F test/pragma.test ad99d05e411c7687302124be56f3b362204be041 F test/pragma2.test f624a496a95ee878e81e59961eade66d5c00c028 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c -F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 +F test/printf.test 2f11179e8b7210f3ea262d96b7f7b4cb78b34500 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -1249,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P edc1de2a588fd50c0049bb2be76d3f6783443165 0cdf502885ea7e5805d7ba3719f055f5d48fc78d -R 3d0aeff53f9d343a974360d23614c066 -U dan -Z ac6175e8f164395f5f3f99cbe84625aa +P 3ad829e50faca538db3abb2afb898b5521550c5c +R 1f357953eb69048c3e73348b4613f5e5 +U drh +Z 21bd74616576d30557b17c20f6eb18f9 diff --git a/manifest.uuid b/manifest.uuid index 60fdaf0098..45e0581d32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ad829e50faca538db3abb2afb898b5521550c5c \ No newline at end of file +c494171f77dc2e5e04cb6d865e688448f04e5920 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 81efa057fc..0ff9887eb7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -450,7 +450,7 @@ void sqlite3VXPrintf( else prefix = 0; } if( xtype==etGENERIC && precision>0 ) precision--; - for(idx=precision, rounder=0.5; idx>0; idx--, rounder*=0.1){} + for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} if( xtype==etFLOAT ) realvalue += rounder; /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; @@ -505,8 +505,9 @@ void sqlite3VXPrintf( }else{ e2 = exp; } - if( MAX(e2,0)+precision+width > etBUFSIZE - 15 ){ - bufpt = zExtra = sqlite3Malloc( MAX(e2,0)+precision+width+15 ); + if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ + bufpt = zExtra + = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); if( bufpt==0 ){ setStrAccumError(pAccum, STRACCUM_NOMEM); return; diff --git a/test/printf.test b/test/printf.test index 73222720ab..5dfc6d06cf 100644 --- a/test/printf.test +++ b/test/printf.test @@ -526,6 +526,9 @@ do_test printf-2.1.2.8 { do_test printf-2.1.2.9 { sqlite3_mprintf_double {abc: %d %d (%1.1g) :xyz} 1 1 1.0e-20 } {abc: 1 1 (1e-20) :xyz} +do_test printf-2.1.2.10 { + sqlite3_mprintf_double {abc: %*.*f} 2000000000 1000000000 1.0e-20 +} {abc: } do_test printf-2.1.3.1 { sqlite3_mprintf_double {abc: (%*.*f) :xyz} 1 1 1.0 } {abc: (1.0) :xyz} From a30d22a7a6b1eff78d2869ab4e56782522d5a758 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Apr 2015 13:28:41 +0000 Subject: [PATCH 07/10] Further changes to guard against integer overflow in the width and precision of printf() arguments. FossilOrigin-Name: 5ce4e7d7651e5c72a59f03f7aeb366291e62ab57 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 9 +++++++-- test/printf.test | 21 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 31581470a9..3aa52a9c50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Guard\sagainst\sexcessive\swidth\sand\sprecision\sin\sfloating-point\sconversions\nin\sthe\sprintf\sroutines. -D 2015-04-07T12:41:17.461 +C Further\schanges\sto\sguard\sagainst\sinteger\soverflow\sin\sthe\swidth\sand\sprecision\nof\sprintf()\sarguments. +D 2015-04-07T13:28:41.878 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/pcache1.c 69d137620a305f814398bd29a0c998038c0695e9 F src/pragma.c ac4f3f856b4234e85f55b0f069698a4766011100 F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 -F src/printf.c 32f69fcba9ddfb71f9466b867f7ee7b7ffee30de +F src/printf.c 0f3476d9c8befc12708a3d614c22859b0cb79f19 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -797,7 +797,7 @@ F test/permutations.test f9cc1dd987986c9d4949211c7a4ed55ec9aecba1 F test/pragma.test ad99d05e411c7687302124be56f3b362204be041 F test/pragma2.test f624a496a95ee878e81e59961eade66d5c00c028 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c -F test/printf.test 2f11179e8b7210f3ea262d96b7f7b4cb78b34500 +F test/printf.test 5ab2b4666ca544645c4af2d78198f93b1e030d6e F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -1249,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ad829e50faca538db3abb2afb898b5521550c5c -R 1f357953eb69048c3e73348b4613f5e5 +P c494171f77dc2e5e04cb6d865e688448f04e5920 +R 7b4db908cf9a9f885bafc7fac7b1a77f U drh -Z 21bd74616576d30557b17c20f6eb18f9 +Z 16061aae0366142fcfaf02b796330263 diff --git a/manifest.uuid b/manifest.uuid index 45e0581d32..dddcbe141c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c494171f77dc2e5e04cb6d865e688448f04e5920 \ No newline at end of file +5ce4e7d7651e5c72a59f03f7aeb366291e62ab57 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 0ff9887eb7..13eb491cda 100644 --- a/src/printf.c +++ b/src/printf.c @@ -390,6 +390,7 @@ void sqlite3VXPrintf( if( precision0 ) precision--; + testcase( precision>0xfff ); for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} if( xtype==etFLOAT ) realvalue += rounder; /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ @@ -739,7 +741,7 @@ void sqlite3VXPrintf( */ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zNew; - assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ testcase(p->accError==STRACCUM_TOOBIG); testcase(p->accError==STRACCUM_NOMEM); @@ -788,7 +790,10 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ ** Append N copies of character c to the given string buffer. */ void sqlite3AppendChar(StrAccum *p, int N, char c){ - if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + testcase( p->nChar + (i64)N > 0x7fffffff ); + if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ + return; + } while( (N--)>0 ) p->zText[p->nChar++] = c; } diff --git a/test/printf.test b/test/printf.test index 5dfc6d06cf..dadc23194e 100644 --- a/test/printf.test +++ b/test/printf.test @@ -472,6 +472,18 @@ do_test printf-1.16.7 { sqlite3_mprintf_int {abc: (%#6d) (%#6x) (%#6o) :xyz}\ 0xff676981 0xff676981 0xff676981 } {abc: (-9999999) (0xff676981) (037731664601) :xyz} +do_test printf-1.17.1 { + sqlite3_mprintf_int {abd: %2147483647d %2147483647x %2147483647o} 1 1 1 +} {} +do_test printf-1.17.2 { + sqlite3_mprintf_int {abd: %*d %x} 2147483647 1 1 +} {} +do_test printf-1.17.3 { + sqlite3_mprintf_int {abd: %*d %x} -2147483648 1 1 +} {} +do_test printf-1.17.4 { + sqlite3_mprintf_int {abd: %.2147483648d %x %x} 1 1 1 +} {abd: 1 1 1} do_test printf-2.1.1.1 { sqlite3_mprintf_double {abc: (%*.*f) :xyz} 1 1 0.001 } {abc: (0.0) :xyz} @@ -3469,6 +3481,15 @@ do_test printf-3.5 { do_test printf-3.6 { sqlite3_mprintf_str {%d %d A String: (%-30s)} 1 2 {This is the string} } [format {%d %d A String: (%-30s)} 1 2 {This is the string}] +do_test printf-3.7 { + sqlite3_mprintf_str {%d A String: (%*s)} 1 2147483647 {This is the string} +} [] +do_test printf-3.8 { + sqlite3_mprintf_str {%d A String: (%*s)} 1 -2147483648 {This is the string} +} [] +do_test printf-3.9 { + sqlite3_mprintf_str {%d A String: (%.*s)} 1 -2147483648 {This is the string} +} {1 A String: (This is the string)} do_test snprintf-3.11 { sqlite3_snprintf_str 2 {x%d %d %s} 10 10 {This is the string} } {x} From 8c069147cecc868648b2ef701b77d74be073095f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Apr 2015 14:38:57 +0000 Subject: [PATCH 08/10] Another change to avoid a problem caused by integer overflow in the printf() code. FossilOrigin-Name: 95625ef3adc3c408d67e70f877f390445fbb8292 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 10 ++++++++-- test/printf.test | 9 ++++++--- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3aa52a9c50..476d45c6a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\schanges\sto\sguard\sagainst\sinteger\soverflow\sin\sthe\swidth\sand\sprecision\nof\sprintf()\sarguments. -D 2015-04-07T13:28:41.878 +C Another\schange\sto\savoid\sa\sproblem\scaused\sby\sinteger\soverflow\sin\sthe\sprintf()\scode. +D 2015-04-07T14:38:57.638 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/pcache1.c 69d137620a305f814398bd29a0c998038c0695e9 F src/pragma.c ac4f3f856b4234e85f55b0f069698a4766011100 F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 -F src/printf.c 0f3476d9c8befc12708a3d614c22859b0cb79f19 +F src/printf.c 62f93b9276807b053e9c9625b48f4843ef82e978 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -797,7 +797,7 @@ F test/permutations.test f9cc1dd987986c9d4949211c7a4ed55ec9aecba1 F test/pragma.test ad99d05e411c7687302124be56f3b362204be041 F test/pragma2.test f624a496a95ee878e81e59961eade66d5c00c028 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c -F test/printf.test 5ab2b4666ca544645c4af2d78198f93b1e030d6e +F test/printf.test b38d5d81a8cc20a040667c9acad68ddf5b18ccdd F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -1249,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c494171f77dc2e5e04cb6d865e688448f04e5920 -R 7b4db908cf9a9f885bafc7fac7b1a77f -U drh -Z 16061aae0366142fcfaf02b796330263 +P 5ce4e7d7651e5c72a59f03f7aeb366291e62ab57 +R d822c3498c99ae588b224a89f4618062 +U dan +Z d5717f1cffb0747ad695b3c7e862b042 diff --git a/manifest.uuid b/manifest.uuid index dddcbe141c..50a22e5cce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ce4e7d7651e5c72a59f03f7aeb366291e62ab57 \ No newline at end of file +95625ef3adc3c408d67e70f877f390445fbb8292 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 13eb491cda..3eab6d47d6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -270,6 +270,8 @@ void sqlite3VXPrintf( c = *++fmt; } } + if( width<0 ) width = 0; /* force to non-negative after int overflow */ + /* Get the precision */ if( c=='.' ){ precision = 0; @@ -280,7 +282,6 @@ void sqlite3VXPrintf( }else{ precision = va_arg(ap,int); } - if( precision<0 ) precision = -precision; c = *++fmt; }else{ while( c>='0' && c<='9' ){ @@ -288,6 +289,12 @@ void sqlite3VXPrintf( c = *++fmt; } } + + /* If a negative precision has been specified, use its absolute value + ** instead. This is (probably) not standard printf() behaviour, but + ** it is what sqlite3_mprintf() and friends have always done. If the + ** precision specified is -2147483648, use 0. */ + if( precision<0 ) precision = (-precision) & 0x7fffffff; }else{ precision = -1; } @@ -390,7 +397,6 @@ void sqlite3VXPrintf( if( precision Date: Tue, 7 Apr 2015 15:39:29 +0000 Subject: [PATCH 09/10] Avoid signed integer overflow when converting oversized in-line integer widths and precisions in printf(). FossilOrigin-Name: 8e4ac2ce24415926247961b00a62425ae85d6ffb --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 22 ++++++++++++---------- test/printf.test | 8 ++++---- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 476d45c6a9..aa53c43007 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\schange\sto\savoid\sa\sproblem\scaused\sby\sinteger\soverflow\sin\sthe\sprintf()\scode. -D 2015-04-07T14:38:57.638 +C Avoid\ssigned\sinteger\soverflow\swhen\sconverting\soversized\sin-line\sinteger\nwidths\sand\sprecisions\sin\sprintf(). +D 2015-04-07T15:39:29.937 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/pcache1.c 69d137620a305f814398bd29a0c998038c0695e9 F src/pragma.c ac4f3f856b4234e85f55b0f069698a4766011100 F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 -F src/printf.c 62f93b9276807b053e9c9625b48f4843ef82e978 +F src/printf.c 8ae1fa9d30c1200a9268a390ba9e9cea9197b27a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 41aa91af56d960e9414ce1d7c17cfb68e0d1c6cb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -797,7 +797,7 @@ F test/permutations.test f9cc1dd987986c9d4949211c7a4ed55ec9aecba1 F test/pragma.test ad99d05e411c7687302124be56f3b362204be041 F test/pragma2.test f624a496a95ee878e81e59961eade66d5c00c028 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c -F test/printf.test b38d5d81a8cc20a040667c9acad68ddf5b18ccdd +F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -1249,7 +1249,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ce4e7d7651e5c72a59f03f7aeb366291e62ab57 -R d822c3498c99ae588b224a89f4618062 -U dan -Z d5717f1cffb0747ad695b3c7e862b042 +P 95625ef3adc3c408d67e70f877f390445fbb8292 +R 5875861747bb686954783d9ce4259b86 +U drh +Z e5281cee406ead55e17262e5bdbb2163 diff --git a/manifest.uuid b/manifest.uuid index 50a22e5cce..786af92027 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95625ef3adc3c408d67e70f877f390445fbb8292 \ No newline at end of file +8e4ac2ce24415926247961b00a62425ae85d6ffb \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 3eab6d47d6..9714fa1565 100644 --- a/src/printf.c +++ b/src/printf.c @@ -261,16 +261,18 @@ void sqlite3VXPrintf( } if( width<0 ){ flag_leftjustify = 1; - width = -width; + width = width >= -2147483647 ? -width : 0; } c = *++fmt; }else{ + unsigned wx = 0; while( c>='0' && c<='9' ){ - width = width*10 + c - '0'; + wx = wx*10 + c - '0'; c = *++fmt; } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; } - if( width<0 ) width = 0; /* force to non-negative after int overflow */ /* Get the precision */ if( c=='.' ){ @@ -283,18 +285,18 @@ void sqlite3VXPrintf( precision = va_arg(ap,int); } c = *++fmt; + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } }else{ + unsigned px = 0; while( c>='0' && c<='9' ){ - precision = precision*10 + c - '0'; + px = px*10 + c - '0'; c = *++fmt; } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; } - - /* If a negative precision has been specified, use its absolute value - ** instead. This is (probably) not standard printf() behaviour, but - ** it is what sqlite3_mprintf() and friends have always done. If the - ** precision specified is -2147483648, use 0. */ - if( precision<0 ) precision = (-precision) & 0x7fffffff; }else{ precision = -1; } diff --git a/test/printf.test b/test/printf.test index 3bf13f4eb9..6103d8acf8 100644 --- a/test/printf.test +++ b/test/printf.test @@ -483,7 +483,7 @@ do_test printf-1.17.3 { } {abd: 1 1} do_test printf-1.17.4 { sqlite3_mprintf_int {abd: %.2147483648d %x %x} 1 1 1 -} {abd: 1 1 1} +} {/.*/} do_test printf-2.1.1.1 { sqlite3_mprintf_double {abc: (%*.*f) :xyz} 1 1 0.001 } {abc: (0.0) :xyz} @@ -3489,7 +3489,7 @@ do_test printf-3.8 { } {1 A String: (This is the string)} do_test printf-3.9 { sqlite3_mprintf_str {%d A String: (%.*s)} 1 -2147483648 {This is the string} -} {1 A String: ()} +} {1 A String: (This is the string)} do_test snprintf-3.11 { sqlite3_snprintf_str 2 {x%d %d %s} 10 10 {This is the string} } {x} @@ -3710,8 +3710,8 @@ do_test printf-13.6 { sqlite3_mprintf_hexdouble %.20f fff8000000000000 } {NaN} do_test printf-13.7 { - sqlite3_mprintf_hexdouble %3000000000.10000f 4693b8b5b5056e17 -} "100000000000000000000000000000000.[string repeat 0 10000]" + sqlite3_mprintf_hexdouble %2147483648.10000f 4693b8b5b5056e17 +} {/100000000000000000000000000000000.00/} do_test printf-14.1 { sqlite3_mprintf_str {abc-%y-123} 0 0 {not used} From 5efbd06ab31aa0853d79b5361142b44c211a33b6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Apr 2015 12:16:33 +0000 Subject: [PATCH 10/10] Version 3.8.9 FossilOrigin-Name: 8a8ffc862e96f57aa698f93de10dee28e69f6e09 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index aa53c43007..e3dc7c37f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssigned\sinteger\soverflow\swhen\sconverting\soversized\sin-line\sinteger\nwidths\sand\sprecisions\sin\sprintf(). -D 2015-04-07T15:39:29.937 +C Version\s3.8.9 +D 2015-04-08T12:16:33.323 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 00d12636df7a5b08af09116bcd6c7bfd49b8b3b4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1249,7 +1249,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 95625ef3adc3c408d67e70f877f390445fbb8292 +P 8e4ac2ce24415926247961b00a62425ae85d6ffb R 5875861747bb686954783d9ce4259b86 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.9 * U drh -Z e5281cee406ead55e17262e5bdbb2163 +Z 05045ab8d5dbbaefca88cc23b8dca09c diff --git a/manifest.uuid b/manifest.uuid index 786af92027..a38b33773e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e4ac2ce24415926247961b00a62425ae85d6ffb \ No newline at end of file +8a8ffc862e96f57aa698f93de10dee28e69f6e09 \ No newline at end of file