From 8c983ddb670097185d0bd26b11b4e65186192820 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Sep 2022 19:28:18 +0000 Subject: [PATCH 1/4] An improved technique for ensuring that the Rekey() operation does not overwrite an existing page number. This approach does not change the semantics of the underlying pluggable pcache and it is easier to prove correct. This replaces the changes at [aadd38f99a3e5abc] and [81cff9f1955874aa]. FossilOrigin-Name: 7dbddde79e3ca1b81e00288616fc27434698ce6dcb47714728fce8602ae91f45 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/pcache.c | 12 ++++++------ src/pcache1.c | 19 ++----------------- 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index e6d926c22b..13721a29d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\stool/omittest-msvc.tcl\sscript\sthat\swill\stry\sto\sbuild\susing\sMSVC\nusing\svarious\scompile-time\soptions,\sto\sensure\sthat\sthe\scompile-time\soptions\nall\sbuild\swithout\serrors. -D 2022-09-05T22:54:36.015 +C An\simproved\stechnique\sfor\sensuring\sthat\sthe\sRekey()\soperation\sdoes\snot\noverwrite\san\sexisting\spage\snumber.\s\sThis\sapproach\sdoes\snot\schange\sthe\nsemantics\sof\sthe\sunderlying\spluggable\spcache\sand\sit\sis\seasier\sto\sprove\ncorrect.\s\sThis\sreplaces\sthe\schanges\sat\n[aadd38f99a3e5abc]\sand\s[81cff9f1955874aa]. +D 2022-09-07T19:28:18.399 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -578,9 +578,9 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 -F src/pcache.c 22a6ebe498d1d26c85fd1e3bcb246d97b882c060027c1e1688fbea905f5ac3cf +F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 -F src/pcache1.c 849a26ea9dc1e6a176b75dc576672a598170b0b46aeef87a981dd25e0af0ccf9 +F src/pcache1.c 5996285f0a1873b1c7925568d1145865105581d588c1671df0c79b43d507174e F src/pragma.c 9bf7d8a2a9ad3bc36df3ec0d61817a44c38a1da527d59c26c203047f906e334a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 971d5819a4bda88038c2283d71fc0a2974ffc3dd480f9bd941341017abacfd1b @@ -2000,9 +2000,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f74a5ea8c986dc33d3afcda169c38abbe55728c56716cf9991a5e2ef7fc4917a 82b89f8a074858a81d841dfc26436e8d39ce4907b8f989eba4d848db16758303 -R 6ca49c20c002174082011fae5352b9a2 -T +closed 82b89f8a074858a81d841dfc26436e8d39ce4907b8f989eba4d848db16758303 +P 6b00ecb59fd303f7985902c35a46db9e729201d4beaedea46596b728d9e4b1c8 +R 8bc604264e2fc000e1d53debee8fb293 U drh -Z 072f191f34018bff25742e9863a6762e +Z 8b6d452122c12811257b08bbb3024121 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1a48ad4407..15ae82f961 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b00ecb59fd303f7985902c35a46db9e729201d4beaedea46596b728d9e4b1c8 \ No newline at end of file +7dbddde79e3ca1b81e00288616fc27434698ce6dcb47714728fce8602ae91f45 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 8c57f5b1eb..0407e06b2f 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -655,14 +655,14 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); - sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); if( pOther ){ - PgHdr *pPg = (PgHdr*)pOther->pExtra; - pPg->pgno = p->pgno; - if( pPg->pPage==0 ){ - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pOther, 0); - } + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); } + sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); diff --git a/src/pcache1.c b/src/pcache1.c index a47087fa11..551f1b896b 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -1125,7 +1125,7 @@ static void pcache1Rekey( assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ hOld = iOld%pCache->nHash; pp = &pCache->apHash[hOld]; @@ -1134,23 +1134,8 @@ static void pcache1Rekey( } *pp = pPage->pNext; + assert( pcache1FetchNoMutex(p, iOld, 0)==0 ); /* iOld not in cache */ hNew = iNew%pCache->nHash; - pp = &pCache->apHash[hNew]; - while( *pp ){ - if( (*pp)->iKey==iNew ){ - /* If there is already another pcache entry at iNew, change it to iOld, - ** thus swapping the positions of iNew and iOld */ - PgHdr1 *pOld = *pp; - *pp = pOld->pNext; - pOld->pNext = pCache->apHash[hOld]; - pCache->apHash[hOld] = pOld; - pOld->iKey = iOld; - break; - }else{ - pp = &(*pp)->pNext; - } - } - pPage->iKey = iNew; pPage->pNext = pCache->apHash[hNew]; pCache->apHash[hNew] = pPage; From b53e0992b294312c439355435492dfb1b37cd7fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 7 Sep 2022 20:11:22 +0000 Subject: [PATCH 2/4] Enhance an assert() to impose for tighter constraints on the operation of pcache. FossilOrigin-Name: 42105eb43a916a7cd839fa6c582eaffaef17f50b23c3e05142400c379feb73c7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache1.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 13721a29d5..4b001c1fe8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C An\simproved\stechnique\sfor\sensuring\sthat\sthe\sRekey()\soperation\sdoes\snot\noverwrite\san\sexisting\spage\snumber.\s\sThis\sapproach\sdoes\snot\schange\sthe\nsemantics\sof\sthe\sunderlying\spluggable\spcache\sand\sit\sis\seasier\sto\sprove\ncorrect.\s\sThis\sreplaces\sthe\schanges\sat\n[aadd38f99a3e5abc]\sand\s[81cff9f1955874aa]. -D 2022-09-07T19:28:18.399 +C Enhance\san\sassert()\sto\simpose\sfor\stighter\sconstraints\son\sthe\soperation\sof\spcache. +D 2022-09-07T20:11:22.675 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -580,7 +580,7 @@ F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 8e67d820030d2655b9942ffe61c1e7e6b96cea2f2f72183533299393907d0564 F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 -F src/pcache1.c 5996285f0a1873b1c7925568d1145865105581d588c1671df0c79b43d507174e +F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 9bf7d8a2a9ad3bc36df3ec0d61817a44c38a1da527d59c26c203047f906e334a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 971d5819a4bda88038c2283d71fc0a2974ffc3dd480f9bd941341017abacfd1b @@ -2000,8 +2000,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b00ecb59fd303f7985902c35a46db9e729201d4beaedea46596b728d9e4b1c8 -R 8bc604264e2fc000e1d53debee8fb293 +P 7dbddde79e3ca1b81e00288616fc27434698ce6dcb47714728fce8602ae91f45 +R 3d3ecc6a79d82686d9e0b48f58e1f898 U drh -Z 8b6d452122c12811257b08bbb3024121 +Z 5fb0b8be05b999eccf9a7905ecda7c89 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 15ae82f961..377fe80f5e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7dbddde79e3ca1b81e00288616fc27434698ce6dcb47714728fce8602ae91f45 \ No newline at end of file +42105eb43a916a7cd839fa6c582eaffaef17f50b23c3e05142400c379feb73c7 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 551f1b896b..adbe953959 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -1134,7 +1134,7 @@ static void pcache1Rekey( } *pp = pPage->pNext; - assert( pcache1FetchNoMutex(p, iOld, 0)==0 ); /* iOld not in cache */ + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ hNew = iNew%pCache->nHash; pPage->iKey = iNew; pPage->pNext = pCache->apHash[hNew]; From bdb6368d725fea2eaec9cc651dfadbe9954ae138 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 9 Sep 2022 17:50:29 +0000 Subject: [PATCH 3/4] Fix harmless compiler warning seen with MSVC. FossilOrigin-Name: 73c4c68d3b4c16caf8281dabddd7365d24691e5ee2da89b1783eca66bfbd9280 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4b001c1fe8..f4a1ccce49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\san\sassert()\sto\simpose\sfor\stighter\sconstraints\son\sthe\soperation\sof\spcache. -D 2022-09-07T20:11:22.675 +C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. +D 2022-09-09T17:50:29.375 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -573,7 +573,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c 102f7e5c5b59c18ea3dbc929dc3be8acb3afc0e0b6ad572e032335c9c27f44f1 -F src/os_win.c e9454cb141908e8eef2102180bad353a36480612d5b736e4c2bd5777d9b25a34 +F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 @@ -2000,8 +2000,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dbddde79e3ca1b81e00288616fc27434698ce6dcb47714728fce8602ae91f45 -R 3d3ecc6a79d82686d9e0b48f58e1f898 -U drh -Z 5fb0b8be05b999eccf9a7905ecda7c89 +P 42105eb43a916a7cd839fa6c582eaffaef17f50b23c3e05142400c379feb73c7 +R fd5aff581be86e15a91eeaa3f20f7912 +U mistachkin +Z bd0ebecde4dd76aef42010e0aed5496b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 377fe80f5e..4b00c103ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42105eb43a916a7cd839fa6c582eaffaef17f50b23c3e05142400c379feb73c7 \ No newline at end of file +73c4c68d3b4c16caf8281dabddd7365d24691e5ee2da89b1783eca66bfbd9280 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2827b0b49b..d71fb39220 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5763,7 +5763,8 @@ static int winFullPathname( char *zFull /* Output buffer */ ){ int rc; - sqlite3_mutex *pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); + MUTEX_LOGIC( sqlite3_mutex *pMutex; ) + MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); ) sqlite3_mutex_enter(pMutex); rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); sqlite3_mutex_leave(pMutex); From 3e4e697d1c0c76a473a2307937d5bf92543ac19a Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 12 Sep 2022 16:07:24 +0000 Subject: [PATCH 4/4] New run-time options to the speed-check.sh script. FossilOrigin-Name: ff46bc9d2a844dde9dafa157114f4d9ee157205ca5c08ed9ee32f018de310e62 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/speed-check.sh | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f4a1ccce49..6c8b53652d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. -D 2022-09-09T17:50:29.375 +C New\srun-time\soptions\sto\sthe\sspeed-check.sh\sscript. +D 2022-09-12T16:07:24.415 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1958,7 +1958,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c 0253c187ae16fdae9cde89e63e1dfcd3bb35e5416d066415f99e2f8cac6ab03d F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 1b5be34c6223cb1af06da2a10fb77863eb869b1962d055820b0a11cf2336ab45 -F tool/speed-check.sh ff74a68bb95a0341275f4d3c9a7d8a3800bd278aceecf1913295a1f0175bc3e6 +F tool/speed-check.sh 13f8e07dbfe25f3aecda33fb6068894665af61ca1360a7b654be0ad0c3f3ae0b F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -2000,8 +2000,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 42105eb43a916a7cd839fa6c582eaffaef17f50b23c3e05142400c379feb73c7 -R fd5aff581be86e15a91eeaa3f20f7912 -U mistachkin -Z bd0ebecde4dd76aef42010e0aed5496b +P 73c4c68d3b4c16caf8281dabddd7365d24691e5ee2da89b1783eca66bfbd9280 +R 8caad88aec151feccca0c9cd18faaecf +U drh +Z 5694145593725214ff362dc6756dc32c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b00c103ba..ee1e7a8414 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73c4c68d3b4c16caf8281dabddd7365d24691e5ee2da89b1783eca66bfbd9280 \ No newline at end of file +ff46bc9d2a844dde9dafa157114f4d9ee157205ca5c08ed9ee32f018de310e62 \ No newline at end of file diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 6b0fbeb43a..879ffa9562 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -96,6 +96,9 @@ while test "$1" != ""; do --cachesize) shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1" ;; + --stmtcache) + shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtcache $1" + ;; --checkpoint) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --checkpoint" ;; @@ -143,6 +146,9 @@ while test "$1" != ""; do SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree" CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE" ;; + --persist) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --persist" + ;; --orm) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" ;;