From 1854837b5ad3d13058e5b027f0fa003bfbf19342 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 10:20:56 +0000 Subject: [PATCH 1/4] Work toward getting json_extract() to operate directly on the BLOB, omitting the translation into a JsonNode array. FossilOrigin-Name: c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 --- manifest | 17 +- manifest.uuid | 2 +- src/json.c | 333 ++++++++++++++++++++++++++++++++++++++++ test/json/json-q1-b.txt | 2 +- 4 files changed, 345 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b1cbae3a58..0ab3584199 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\stests\sfor\sJSONB\sadded. -D 2023-09-27T16:55:13.845 +C Work\stoward\sgetting\sjson_extract()\sto\soperate\sdirectly\son\sthe\sBLOB,\somitting\nthe\stranslation\sinto\sa\sJsonNode\sarray. +D 2023-09-28T10:20:56.267 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6fb31345d252dc6992c707ec7e826ec9f2283b026092e5105f9905664a2210c5 +F src/json.c 1f174d19f143e4968ae391de3d504563f320c6dbc2ba437231b40f19631a6dc4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1300,7 +1300,7 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f -F test/json/json-q1-b.txt d1394d4ade1c9617539b19b48e0dd2df4f6ea918860978722e7a97d60618ca83 +F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test dc9d5a2a5b1fd1b54dbd71c538b17933cc98d84b4c1f821ead754933663dca55 @@ -2123,8 +2123,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700bdbd7383f66a0da675c197204da4e7b6ed757155145ee98d572de32a5d0ae -R cd229cd0127bc955a2b70dc896603e55 +P 7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d +R b9d3bc6ee4123d23cc875353cfd5e9aa +T *branch * jsonb-direct-extract +T *sym-jsonb-direct-extract * +T -sym-jsonb * U drh -Z 76a0e61dc557712a0090ea7a119c4cd2 +Z 3a4258d315e3af636c1707e0dba68e38 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ccac07358c..066de478dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d \ No newline at end of file +c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0b5068a9f0..a6b91e76eb 100644 --- a/src/json.c +++ b/src/json.c @@ -3493,6 +3493,335 @@ static void jsonRenderNodeAsBlob( } } +/* +** Error returns from jsonLookupBlobStep() +*/ +#define JSON_BLOB_ERROR 0xffffffff +#define JSON_BLOB_NOTFOUND 0xfffffffe +#define JSON_BLOB_PATHERROR 0xfffffffd + +/* +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. +** +** Return JSON_BLOB_NOTFOUND if no such element exists. +*/ +static u32 jsonLookupBlobStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ + const char *zPath, /* The path to search */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, k, nKey, sz, n, iEnd; + const char *zKey; + u8 x; + + if( pParse->oom ) return 0; + if( zPath[0]==0 ) return iRoot; + if( zPath[0]=='.' ){ + x = pParse->aBlob[iRoot]; + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + testcase( nKey==0 ); + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + } + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; + iEnd = j+sz; + while( jaBlob[j] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + k = j+n; + if( k+sz>=iEnd ) return JSON_BLOB_ERROR; + if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( j+n+sz>iEnd ) return JSON_BLOB_ERROR; + return jsonLookupBlobStep(pParse, j, &zPath[i], pzErr); + } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + }else if( zPath[0]=='[' ){ + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; + } + if( i<2 || zPath[i]!=']' ){ +#if 0 + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + j += jsonNodeSize(&pBase[j]); + } + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pBase->eU==2 ); + iBase = pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( sqlite3Isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + }else{ + *pzErr = zPath; + return 0; + } +#endif + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot+n; + iEnd = j+sz; + while( jiEnd ) return JSON_BLOB_ERROR; + }else{ + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + return JSON_BLOB_NOTFOUND; +} + +/* +** +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx /* Return value for this function */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x = (char)pParse->aBlob[i+n]; + if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + else if( x=='+' && ALWAYS(sz>0) ){ n++; sz--; } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3_malloc( nOut+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(iIn=iOut=0; iIn>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); + iIn += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + iIn++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + iIn += 2; + continue; + }else{ + continue; + } + } /* end if( c=='\\' ) */ + zOut[iOut++] = c; + } /* end for() */ + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + sqlite3_result_blob(pCtx, &pParse->aBlob[i+n], sz, SQLITE_TRANSIENT); + break; + } + } +} + +/* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. +*/ +static void jsonExtractFromBlob( + sqlite3_context *ctx, + sqlite3_value *pJson, + sqlite3_value *pPath, + int flags +){ + const char *zPath = (const char*)sqlite3_value_text(pPath); + const char *zErr = 0; + u32 i; + JsonParse px; + if( zPath==0 ) return; + if( zPath[0]=='$' ) zPath++; + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(pJson); + px.aBlob = (u8*)sqlite3_value_blob(pJson); + if( px.aBlob==0 ) return; + i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + if( i>$label FROM data1 WHERE x->>$label IS NOT NULL; CREATE TEMP TABLE t2(x JSON TEXT); From 2dc60ec57fa75fa659c5c5c66d702eeaaa3347fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 15:56:35 +0000 Subject: [PATCH 2/4] Improvements to json_extract() to better support JSONB. Still not 100% working. FossilOrigin-Name: 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 49 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 0ab3584199..d05d5a4eac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\sgetting\sjson_extract()\sto\soperate\sdirectly\son\sthe\sBLOB,\somitting\nthe\stranslation\sinto\sa\sJsonNode\sarray. -D 2023-09-28T10:20:56.267 +C Improvements\sto\sjson_extract()\sto\sbetter\ssupport\sJSONB.\sStill\snot\s100%\sworking. +D 2023-09-28T15:56:35.608 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 1f174d19f143e4968ae391de3d504563f320c6dbc2ba437231b40f19631a6dc4 +F src/json.c c9d70a47aabc97b51cd21a33be7602db4760a980920b44b497dbb331a91c12d8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,11 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d -R b9d3bc6ee4123d23cc875353cfd5e9aa -T *branch * jsonb-direct-extract -T *sym-jsonb-direct-extract * -T -sym-jsonb * +P c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 +R df67f115b3402f37b4410f24659e50ec U drh -Z 3a4258d315e3af636c1707e0dba68e38 +Z a8b3beeec30b8fa8de8cb3c4ca40d323 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 066de478dc..a7573f00b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 \ No newline at end of file +8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a6b91e76eb..33c9404105 100644 --- a/src/json.c +++ b/src/json.c @@ -3634,7 +3634,34 @@ static u32 jsonLookupBlobStep( } /* -** +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. +*/ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob +){ + JsonParse x; + JsonString s; + + if( aBlob==0 ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonInit(&s, ctx); + jsonRenderBlob(&x, 0, &s); + jsonResult(&s); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. */ static void jsonReturnFromBlob( JsonParse *pParse, /* Complete JSON parse tree */ @@ -3785,7 +3812,12 @@ static void jsonReturnFromBlob( } case JSONB_ARRAY: case JSONB_OBJECT: { - sqlite3_result_blob(pCtx, &pParse->aBlob[i+n], sz, SQLITE_TRANSIENT); + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } break; } } @@ -3992,23 +4024,13 @@ static void jsonbTest2( int argc, sqlite3_value **argv ){ - JsonParse *pParse; const u8 *aBlob; int nBlob; - JsonParse x; - JsonString s; UNUSED_PARAMETER(argc); aBlob = (const u8*)sqlite3_value_blob(argv[0]); - if( aBlob==0 ) return; nBlob = sqlite3_value_bytes(argv[0]); - pParse = &x; - memset(&x, 0, sizeof(x)); - x.aBlob = (u8*)aBlob; - x.nBlob = nBlob; - jsonInit(&s, ctx); - jsonRenderBlob(pParse, 0, &s); - jsonResult(&s); + jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); } /* @@ -5401,6 +5423,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1, 0, jsonErrorFunc), JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1, JSON_BLOB, jsonExtractFunc), JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), JFUNCTION(json_insert, -1, 0, jsonSetFunc), From 59862e6d224f7bdbd368cc2827ea06a3a2c82d29 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 17:07:43 +0000 Subject: [PATCH 3/4] Miscellaneous bugs fixed. FossilOrigin-Name: 5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 22 ++++++++++++++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d05d5a4eac..c1019d925c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sjson_extract()\sto\sbetter\ssupport\sJSONB.\sStill\snot\s100%\sworking. -D 2023-09-28T15:56:35.608 +C Miscellaneous\sbugs\sfixed. +D 2023-09-28T17:07:43.443 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c9d70a47aabc97b51cd21a33be7602db4760a980920b44b497dbb331a91c12d8 +F src/json.c 946e0bbc5070db333a370e2f7b806ad07d2e9fb93d057cf52a9e5fb58fd43768 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 -R df67f115b3402f37b4410f24659e50ec +P 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 +R e6233e4f5d16d81e42c25a8ff039c998 U drh -Z a8b3beeec30b8fa8de8cb3c4ca40d323 +Z beeeb6a9ecf43431f000f927a20c9a15 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a7573f00b7..5bf7e31c95 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 \ No newline at end of file +5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 33c9404105..322fbc5602 100644 --- a/src/json.c +++ b/src/json.c @@ -518,6 +518,14 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + char *zBuf = sqlite3_malloc64( N+1 ); + if( zBuf==0 ){ + p->bErr = 1; + return; + } + memcpy(zBuf, zIn, N); + zBuf[N] = 0; + zIn = zBuf; if( zIn[0]=='+' ){ zIn++; N--; @@ -539,6 +547,7 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ } assert( N>0 ); jsonAppendRawNZ(p, zIn, N); + sqlite3_free(zBuf); } /* @@ -958,12 +967,16 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; + char *zz; + sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( pNode->eU==1 ); - z = pNode->u.zJContent; + zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); + z = zz; if( z[0]=='-' ){ z++; bNeg = 1; } else if( z[0]=='+' ){ z++; } rc = sqlite3DecOrHexToI64(z, &i); + sqlite3DbFree(db, zz); if( rc<=1 ){ sqlite3_result_int64(pCtx, bNeg ? -i : i); }else if( rc==3 && bNeg ){ @@ -1862,7 +1875,7 @@ static int jsonParse( i = jsonParseValue(pParse, 0); } if( pParse->oom ) i = -1; - if( i>0 ){ + if( !pParse->isBinary && i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; if( zJson[i] ){ @@ -3359,6 +3372,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ break; } } + pParse->aBlob[i] = 0; return i+x+sz; } @@ -3450,7 +3464,7 @@ static void jsonRenderNodeAsBlob( case JSON_ARRAY: { u32 j = 1; u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_ARRAY, 0x7fffffff); + jsonBlobAppendNodeType(pOut, JSONB_ARRAY, pParse->nJson*2); iStart = pOut->nBlob; for(;;){ while( j<=pNode->n ){ @@ -3471,7 +3485,7 @@ static void jsonRenderNodeAsBlob( case JSON_OBJECT: { u32 j = 1; u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_OBJECT, 0x7fffffff); + jsonBlobAppendNodeType(pOut, JSONB_OBJECT, pParse->nJson*2); iStart = pOut->nBlob; for(;;){ while( j<=pNode->n ){ From 6b1db922286ac88505854e93ce017e11e4b18cb5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 17:23:46 +0000 Subject: [PATCH 4/4] Extract directly from BLOB is now complete and appears to work. FossilOrigin-Name: 3de58ec99444b16dfcda1e226420e2343450b77abd3faf33a88b6d18339ef17c --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 61 ++++++++++++++++++++++++--------------------------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index c1019d925c..f6be12b744 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Miscellaneous\sbugs\sfixed. -D 2023-09-28T17:07:43.443 +C Extract\sdirectly\sfrom\sBLOB\sis\snow\scomplete\sand\sappears\sto\swork. +D 2023-09-28T17:23:46.244 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 946e0bbc5070db333a370e2f7b806ad07d2e9fb93d057cf52a9e5fb58fd43768 +F src/json.c 7e68346bfc4f030e454c68751ab7082af52b9925ff28bff88eef03258e486d7c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 -R e6233e4f5d16d81e42c25a8ff039c998 +P 5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b +R 0d79abb184ac0c9db2e32b0ee612448c U drh -Z beeeb6a9ecf43431f000f927a20c9a15 +Z dfb6861bdfa5add7990cbc313b482bb0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5bf7e31c95..6de749eb76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b \ No newline at end of file +3de58ec99444b16dfcda1e226420e2343450b77abd3faf33a88b6d18339ef17c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 322fbc5602..e43240c71c 100644 --- a/src/json.c +++ b/src/json.c @@ -3507,6 +3507,21 @@ static void jsonRenderNodeAsBlob( } } +/* +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. +*/ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; iiEnd ) return JSON_BLOB_ERROR; }else if( zPath[0]=='[' ){ + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ @@ -3586,49 +3604,28 @@ static u32 jsonLookupBlobStep( i++; } if( i<2 || zPath[i]!=']' ){ -#if 0 if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pBase->eU==2 ); - iBase = pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; + k = jsonbArrayCount(pParse, iRoot); + i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ unsigned int x = 0; - j = 3; + i = 3; do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( sqlite3Isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; + x = x*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( x>k ) return 0; + k -= x; } - if( zPath[j]!=']' ){ + if( zPath[i]!=']' ){ *pzErr = zPath; - return 0; + return JSON_BLOB_PATHERROR; } }else{ *pzErr = zPath; - return 0; + return JSON_BLOB_PATHERROR; } -#endif - *pzErr = zPath; - return JSON_BLOB_PATHERROR; } - x = pParse->aBlob[iRoot] & 0x0f; - if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; - n = jsonbPayloadSize(pParse, iRoot, &sz); j = iRoot+n; iEnd = j+sz; while( j