From 594f5e24c4c3ef1389c24e0e5d1fd8f4bb5d68bb Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Jul 2023 19:39:53 +0000 Subject: [PATCH 1/5] The OP_Column opcode caches large column values coming from overflow pages. FossilOrigin-Name: ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888 --- manifest | 20 +++++------ manifest.uuid | 2 +- src/btree.c | 2 -- src/btree.h | 2 -- src/vdbe.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++---- src/vdbeInt.h | 18 +++++++++- src/vdbeaux.c | 17 +++++++++- 7 files changed, 131 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 37e9c1ea94..3ed2cf51e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\senhancements\sinto\sthe\sjson-opt\sbranch. -D 2023-07-27T18:19:46.300 +C The\sOP_Column\sopcode\scaches\slarge\scolumn\svalues\scoming\sfrom\soverflow\spages. +D 2023-07-27T19:39:53.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -577,8 +577,8 @@ F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c 2281facb0531d53fb42c03d1f32bc1b5903564d782ec5ff4ffc63171d960e2aa -F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc +F src/btree.c 7a37bdf09f338561880860681cb03499a60c3bb0869e539c58bc1d2cdd705ff2 +F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3b4eff7155c0cea6971dc51f62e3529934a15a6640ec607dd42a767e379cb3a9 F src/build.c a8ae3b32d9aa9bbd2c0e97d7c0dd80def9fbca408425de1608f57ee6f47f45f4 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 @@ -708,11 +708,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 +F src/vdbe.c 2465f86f43892f173be761f05a4e6ba381119d6e8f5df7d172cc9ab123c9037a F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 -F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 +F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 -F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 +F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1bf85d4e388714a88f8940dcdec353c3e0267456697eff6963d34637912aecc9 fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c -R 26c73044c10199d992b00944e358a20b +P 5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144 +R 84bfc9450f26b6c2dd62e6fd9e08f614 U drh -Z d6bca8891c39ac6c04dd8ce7fe687700 +Z 46d23d17a391909f5bac31a5a474dfd0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e0906cc4dc..1d4873e515 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144 \ No newline at end of file +ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8817efc71b..c23f86e1d2 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4826,7 +4826,6 @@ void sqlite3BtreeCursorUnpin(BtCursor *pCur){ pCur->curFlags &= ~BTCF_Pinned; } -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the ** payload to which the cursor is pointing. @@ -4838,7 +4837,6 @@ i64 sqlite3BtreeOffset(BtCursor *pCur){ return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + (i64)(pCur->info.pPayload - pCur->pPage->aData); } -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ /* ** Return the number of bytes of payload for the entry that pCur is diff --git a/src/btree.h b/src/btree.h index b9078f9010..b45ace7e1d 100644 --- a/src/btree.h +++ b/src/btree.h @@ -321,9 +321,7 @@ int sqlite3BtreePrevious(BtCursor*, int flags); i64 sqlite3BtreeIntegerKey(BtCursor*); void sqlite3BtreeCursorPin(BtCursor*); void sqlite3BtreeCursorUnpin(BtCursor*); -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC i64 sqlite3BtreeOffset(BtCursor*); -#endif int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); u32 sqlite3BtreePayloadSize(BtCursor*); diff --git a/src/vdbe.c b/src/vdbe.c index 075a632117..f479440dab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -695,6 +695,83 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ return h; } + +/* +** For OP_Column, factor out the case where content is loaded from +** overflow pages, so that the code to implement this case is separate +** the common case where all content fits on the page. Factoring out +** the code reduces register pressure and helps the common case +** to run faster. +*/ +static SQLITE_NOINLINE int vdbeColumnFromOverflow( + VdbeCursor *pC, /* The BTree cursor from which we are reading */ + int iCol, /* The column to read */ + int t, /* The serial-type code for the column value */ + i64 iOffset, /* Offset to the start of the content value */ + Mem *pDest /* Store the value into this register. */ +){ + int rc; + sqlite3 *db = pDest->db; + int encoding = pDest->enc; + int len = sqlite3VdbeSerialTypeLen(t); + assert( pC->eCurType==CURTYPE_BTREE ); + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; + if( len > 4000 ){ + /* Cache large column values that are on overflow pages using + ** an RCStr (reference counted string) so that if they are reloaded, + ** that do not have to be copied a second time. The overhead of + ** creating and managing the cache is such that this is only + ** profitable for larger TEXT and BLOB values. + */ + VdbeTxtBlbCache *pCache; + char *pBuf; + if( pC->colCache==0 ){ + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); + if( pC->pCache==0 ) return SQLITE_NOMEM; + pC->colCache = 1; + } + pCache = pC->pCache; + if( pCache->pCValue==0 + || pCache->iCol!=iCol + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) + ){ + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); + if( pBuf==0 ) return SQLITE_NOMEM; + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); + if( rc ) return rc; + pBuf[len] = 0; + pBuf[len+1] = 0; + pBuf[len+2] = 0; + pCache->iCol = iCol; + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); + }else{ + pBuf = pCache->pCValue; + } + assert( t>=12 ); + sqlite3RCStrRef(pBuf); + if( t&1 ){ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + (void(*)(void*))sqlite3RCStrUnref); + pDest->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, + (void(*)(void*))sqlite3RCStrUnref); + } + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); + if( rc ) return rc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->z[len] = 0; + pDest->flags |= MEM_Term; + } + } + pDest->flags &= ~MEM_Ephem; + return rc; +} + + /* ** Return the symbolic name for the data type of a pMem */ @@ -2902,6 +2979,10 @@ op_column_restart: ** dynamically allocated. */ pC->aRow = 0; pC->szRow = 0; + if( pC->colCache && pC->pCache && pC->pCache->pCValue ){ + sqlite3RCStrUnref(pC->pCache->pCValue); + pC->pCache->pCValue = 0; + } /* Make sure a corrupt database has not given us an oversize header. ** Do this now to avoid an oversize memory allocation. @@ -3061,6 +3142,7 @@ op_column_restart: }else{ u8 p5; pDest->enc = encoding; + assert( pDest->db==db ); /* This branch happens only when content is on overflow pages */ if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 && (p5==OPFLAG_TYPEOFARG @@ -3084,14 +3166,12 @@ op_column_restart: */ sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); }else{ - if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ - pDest->flags |= MEM_Term; + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], pDest); + if( rc ){ + if( rc==SQLITE_NOMEM ) goto no_mem; + if( rc==SQLITE_TOOBIG ) goto too_big; + goto abort_due_to_error; } - pDest->flags &= ~MEM_Ephem; } } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 3a5b961a25..f2ca656752 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -56,6 +56,9 @@ typedef struct VdbeSorter VdbeSorter; /* Elements of the linked list at Vdbe.pAuxData */ typedef struct AuxData AuxData; +/* A cache of large TEXT or BLOB values in a VdbeCursor */ +typedef struct VdbeTxtBlbCache VdbeTxtBlbCache; + /* Types of VDBE cursors */ #define CURTYPE_BTREE 0 #define CURTYPE_SORTER 1 @@ -87,6 +90,7 @@ struct VdbeCursor { Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ union { /* pBtx for isEphermeral. pAltMap otherwise */ Btree *pBtx; /* Separate file holding temporary table */ @@ -127,6 +131,7 @@ struct VdbeCursor { #ifdef SQLITE_ENABLE_COLUMN_USED_MASK u64 maskUsed; /* Mask of columns used by this cursor */ #endif + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for @@ -139,12 +144,23 @@ struct VdbeCursor { #define IsNullCursor(P) \ ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) - /* ** A value for VdbeCursor.cacheStatus that means the cache is always invalid. */ #define CACHE_STALE 0 +/* +** Large TEXT or BLOB values can be slow to load, so we want to avoid +** loading them more than once. For that reason, large TEXT and BLOB values +** can be stored in a cache defined by this object, and attached to the +** VdbeCursor using the pCache field. +*/ +struct VdbeTxtBlbCache { + char *pCValue; /* A RCStr buffer to hold the value */ + i64 iOffset; /* File offset of the row being cached */ + int iCol; /* Column for which the cache is valid */ +}; + /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a0eff155dd..52d3c9e063 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1498,7 +1498,6 @@ void sqlite3VdbeReleaseRegisters( } #endif /* SQLITE_DEBUG */ - /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -2723,7 +2722,23 @@ void sqlite3VdbeMakeReady( void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); } +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ + VdbeTxtBlbCache *pCache = pCx->pCache; + assert( pCx->colCache ); + pCx->colCache = 0; + pCx->pCache = 0; + if( pCache->pCValue ){ + sqlite3RCStrUnref(pCache->pCValue); + pCache->pCValue = 0; + } + sqlite3DbFree(p->db, pCache); + sqlite3VdbeFreeCursorNN(p, pCx); +} void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ + if( pCx->colCache ){ + freeCursorWithCache(p, pCx); + return; + } switch( pCx->eCurType ){ case CURTYPE_SORTER: { sqlite3VdbeSorterClose(p->db, pCx); From 6bc4baf313d6ebb137679a8f9bbb73e77f52fc18 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Jul 2023 20:28:29 +0000 Subject: [PATCH 2/5] If the input JSON to a json function that uses cache comes from an RCStr value, then use that RCStr value in the parse rather than making a copy. FossilOrigin-Name: 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/json.c | 26 ++++++++++++++++++++------ src/sqliteInt.h | 1 + src/vdbemem.c | 18 ++++++++++++++++++ 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3ed2cf51e8..8cc0fe016d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sOP_Column\sopcode\scaches\slarge\scolumn\svalues\scoming\sfrom\soverflow\spages. -D 2023-07-27T19:39:53.698 +C If\sthe\sinput\sJSON\sto\sa\sjson\sfunction\sthat\suses\scache\scomes\sfrom\san\sRCStr\nvalue,\sthen\suse\sthat\sRCStr\svalue\sin\sthe\sparse\srather\sthan\smaking\sa\scopy. +D 2023-07-27T20:28:29.957 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c992d2a87a6888bf16a5c7270ef523eb838cf49e6643420c3cf917124f41f97f +F src/json.c 481dffa457276d850836953bbd2a61a00a98f8cf09c64c29c6f05ac02a84affd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -643,7 +643,7 @@ F src/shell.c.in 62b58d7be1e44ec3eb729411278b079c39949e683853445323fd3f250b674a2 F src/sqlite.h.in f999ef3642f381d69679b2516b430dbcb6c5a2a951b7f5e43dc4751b474a5774 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 30d7b0d586a4d03a384dcb60088c81b6fc6f74ce85cc3a0b3242eedc3cc24dbd +F src/sqliteInt.h 6140eb4058d995f3004ee2732fa098b884f26eb497a7fbb3a51c781ec824d14b F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -714,7 +714,7 @@ F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 33da4f30ddba2670bc1e617c3262b66aef2a8039043d4ff93e5c97974991089d +F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5739a16ad270a5aadcbb46b28c34fa6ba975422788dcbccb1a8e0d1e6ed75144 -R 84bfc9450f26b6c2dd62e6fd9e08f614 +P ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888 +R bc4f27407c20176b7c3faf6538598e39 U drh -Z 46d23d17a391909f5bac31a5a474dfd0 +Z 559a7376c279a69632a349e06eee6ec2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1d4873e515..9223cf7c8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888 \ No newline at end of file +509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 493a1b99c4..24a68d37cd 100644 --- a/src/json.c +++ b/src/json.c @@ -174,6 +174,7 @@ struct JsonParse { u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ @@ -619,6 +620,11 @@ static void jsonParseReset(JsonParse *pParse){ sqlite3_free(pParse->aUp); pParse->aUp = 0; } + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->bJsonIsRCStr = 0; + } if( pParse->zAlt ){ sqlite3RCStrUnref(pParse->zAlt); pParse->zAlt = 0; @@ -1884,6 +1890,8 @@ static JsonParse *jsonParseCached( int iMinKey = 0; u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; + int bJsonRCStr; + int rc; if( zJson==0 ) return 0; for(iKey=0; iKeynJson==nJson && (p->hasMod==0 || bUnedited==0) - && memcmp(p->zJson,zJson,nJson)==0 + && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) ){ p->nErr = 0; p->useMod = 0; @@ -1930,16 +1938,22 @@ static JsonParse *jsonParseCached( /* The input JSON was not found anywhere in the cache. We will need ** to parse it ourselves and generate a new JsonParse object. */ - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref); + p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); return 0; } memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson); - p->zJson[nJson] = 0; - if( jsonParse(p, pErrCtx, p->zJson) ){ + if( bJsonRCStr ){ + p->zJson = sqlite3RCStrRef(zJson); + }else{ + p->zJson = (char*)&p[1]; + memcpy(p->zJson, zJson, nJson+1); + } + rc = jsonParse(p, pErrCtx, p->zJson); + p->bJsonIsRCStr = bJsonRCStr; + if( rc ){ if( pErrCtx==0 ){ p->nErr = 1; assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a19a16d50d..78ae7ad69e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -5200,6 +5200,7 @@ void sqlite3FileSuffix3(const char*, char*); u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); +int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); int sqlite3ValueBytes(sqlite3_value*, u8); void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); diff --git a/src/vdbemem.c b/src/vdbemem.c index c73e59c362..87dfbbebd8 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1368,6 +1368,24 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && (pVal->flags & (MEM_Str|MEM_Blob))!=0 + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ From caf7e26c4f38d1cb894109e84c27727eaf72478f Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Jul 2023 23:51:36 +0000 Subject: [PATCH 3/5] Switch to using jsonParseCached() for json_patch(). FossilOrigin-Name: 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/json.c | 75 +++++++++++++++++++++++---------------------------- 3 files changed, 41 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 8cc0fe016d..b519533d15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sinput\sJSON\sto\sa\sjson\sfunction\sthat\suses\scache\scomes\sfrom\san\sRCStr\nvalue,\sthen\suse\sthat\sRCStr\svalue\sin\sthe\sparse\srather\sthan\smaking\sa\scopy. -D 2023-07-27T20:28:29.957 +C Switch\sto\susing\sjsonParseCached()\sfor\sjson_patch(). +D 2023-07-27T23:51:36.392 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 481dffa457276d850836953bbd2a61a00a98f8cf09c64c29c6f05ac02a84affd +F src/json.c 60b6b0815ddaf546396e3c7328065a114b581721dc1fa20697a52610fb2259aa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab1edcc7fedcf27922d5db4bc1bc673b1495ca9c66eb6debdda7b7776c068888 -R bc4f27407c20176b7c3faf6538598e39 +P 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821 +R a667d9301618252314cdf4d506cbebf3 U drh -Z 559a7376c279a69632a349e06eee6ec2 +Z 152f86b69395ef2af8d236118a3bf25d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9223cf7c8e..56d469bf1c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821 \ No newline at end of file +2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 24a68d37cd..cc39264c05 100644 --- a/src/json.c +++ b/src/json.c @@ -1751,26 +1751,15 @@ json_parse_restart: ** are any errors. If an error occurs, free all memory held by pParse, ** but not pParse itself. ** -** pParse is uninitialized when this routine is called. -** -** pParse->nJPRef set to 1. The caller becomes the owner of the -** the JsonParse object. -** -** pParse->bOwnsJson is set to bTakeJson. If bTakeJson is 1, the newly -** initialized JsonParse object will become the owner of the zJson input -** string. If bTakeJson is 0, then the caller is responsible for -** preserving zJson for the lifetime of the JsonParse object. +** pParse must be initialized to an empty parse object prior to calling +** this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - char *zJson /* Input JSON text to be parsed */ + sqlite3_context *pCtx /* Report errors here */ ){ int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; - pParse->nJPRef = 1; + const char *zJson = pParse->zJson; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ @@ -1891,7 +1880,6 @@ static JsonParse *jsonParseCached( u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; int bJsonRCStr; - int rc; if( zJson==0 ) return 0; for(iKey=0; iKeyzJson = sqlite3RCStrRef(zJson); + p->bJsonIsRCStr = 1; }else{ p->zJson = (char*)&p[1]; memcpy(p->zJson, zJson, nJson+1); } - rc = jsonParse(p, pErrCtx, p->zJson); - p->bJsonIsRCStr = bJsonRCStr; - if( rc ){ + p->nJPRef = 1; + if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ @@ -2677,29 +2665,27 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ + JsonParse *pX; /* The JSON that is being patched */ + JsonParse *pY; /* The patch */ JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - x.useMod = 1; - y.useMod = 1; - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult && x.oom==0 ){ - jsonDebugPrintParse(&x); + pX = jsonParseCached(ctx, argv[0], ctx, 1); + if( pX==0 ) return; + pY = jsonParseCached(ctx, argv[1], ctx, 1); + if( pY==0 ) return; + pX->useMod = 1; + pX->hasMod = 1; + pY->useMod = 1; + pResult = jsonMergePatch(pX, 0, pY->aNode); + assert( pResult!=0 || pX->oom ); + if( pResult && pX->oom==0 ){ + jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); - jsonReturnJson(&x, pResult, ctx, 0); + jsonReturnJson(pX, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } - jsonParseReset(&x); - jsonParseReset(&y); } @@ -3326,7 +3312,6 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); p->iRowid = 0; @@ -3646,11 +3631,19 @@ static int jsonEachFilter( if( idxNum==0 ) return SQLITE_OK; z = (const char*)sqlite3_value_text(argv[0]); if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + if( sqlite3ValueIsOfClass(argv[0], (void(*)(void*))sqlite3RCStrUnref) ){ + p->sParse.zJson = sqlite3RCStrRef((char*)z); + }else{ + n = sqlite3_value_bytes(argv[0]); + p->sParse.zJson = sqlite3RCStrNew( n+1 ); + if( p->sParse.zJson==0 ) return SQLITE_NOMEM; + memcpy(p->sParse.zJson, z, (size_t)n+1); + } + p->sParse.bJsonIsRCStr = 1; + p->zJson = p->sParse.zJson; + if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); From 93269a9e411ffe393e1e331f160377c8feae69f0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Jul 2023 00:54:38 +0000 Subject: [PATCH 4/5] Remove some unnecessary and incorrect code that was mistakenly added in the previous check-in. FossilOrigin-Name: d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 ---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b519533d15..6797755a7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Switch\sto\susing\sjsonParseCached()\sfor\sjson_patch(). -D 2023-07-27T23:51:36.392 +C Remove\ssome\sunnecessary\sand\sincorrect\scode\sthat\swas\smistakenly\sadded\sin\sthe\nprevious\scheck-in. +D 2023-07-28T00:54:38.146 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -708,7 +708,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 2465f86f43892f173be761f05a4e6ba381119d6e8f5df7d172cc9ab123c9037a +F src/vdbe.c cd0396758da6a95e6c4e3bec7c3d6e767c3b39930c295c2425f83bb086cdc6ba F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 509ae9c1470dd79d320e84371e1e6662fb85fa0571df5ed8c4d946d10cdfe821 -R a667d9301618252314cdf4d506cbebf3 +P 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3 +R 865864ade75427cea7a0ed386e21a96e U drh -Z 152f86b69395ef2af8d236118a3bf25d +Z 7b84fe17156a3402b4d2c891447b8313 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 56d469bf1c..cd8eaaa0af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3 \ No newline at end of file +d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index f479440dab..b8ee05e415 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2979,10 +2979,6 @@ op_column_restart: ** dynamically allocated. */ pC->aRow = 0; pC->szRow = 0; - if( pC->colCache && pC->pCache && pC->pCache->pCValue ){ - sqlite3RCStrUnref(pC->pCache->pCValue); - pC->pCache->pCValue = 0; - } /* Make sure a corrupt database has not given us an oversize header. ** Do this now to avoid an oversize memory allocation. From b1926192fa1a8c0cdb3faf9dbe111a57b4ab9e5a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Jul 2023 11:30:07 +0000 Subject: [PATCH 5/5] Mark an unreachable branch as ALWAYS(). FossilOrigin-Name: 8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6797755a7c..f0021c4b83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sunnecessary\sand\sincorrect\scode\sthat\swas\smistakenly\sadded\sin\sthe\nprevious\scheck-in. -D 2023-07-28T00:54:38.146 +C Mark\san\sunreachable\sbranch\sas\sALWAYS(). +D 2023-07-28T11:30:07.088 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -714,7 +714,7 @@ F src/vdbeInt.h c30ef736774d876f923b42758b9210fe456104b39906e3901d21279443a17d47 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c 23f17d418d5b97138b7dbbd4c84d80c66dc7990653c705436c8162103b3cc9ba F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c aee9ac636666616494d9a395d29efc3fe9e1404a9f043db81c82560b43b78f35 +F src/vdbemem.c adb5877dc54330865e45e0fba50f6455c1d9d8b83434c59f0e3d2d2a0b9f0e58 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2ed9c59e6a51037e63485d0a92dae25443116beddfca1ac73d2f5d5be38d4ad3 -R 865864ade75427cea7a0ed386e21a96e +P d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45 +R 7d9c43b78a4f1c2315778caf36cecb44 U drh -Z 7b84fe17156a3402b4d2c891447b8313 +Z c4502aa3c9cdb29d4cc267ca47a361d2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd8eaaa0af..326a691f5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3f458dcc379dd7648262f52579ec55ba931852065ed278367a0629228d0ed45 \ No newline at end of file +8fd06d1151c521515d9f7181575a04e451b522335135a27248052b067d4005ae \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 87dfbbebd8..07fb8feee8 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1376,7 +1376,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ */ int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ if( ALWAYS(pVal!=0) - && (pVal->flags & (MEM_Str|MEM_Blob))!=0 + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) && (pVal->flags & MEM_Dyn)!=0 && pVal->xDel==xFree ){