From 6831cad03bc8c477eb10842c0a9bdaa6f372211c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Sep 2023 17:51:39 +0000 Subject: [PATCH 001/191] Initial development code for an experimental binary BLOB encoding for JSON. FossilOrigin-Name: 8131b3c272f47db2618886046a9713285ce120cb87d721484ee7444273290681 --- manifest | 17 +- manifest.uuid | 2 +- src/json.c | 670 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 681 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fc2752435a..78b4dedcd2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sfor\sjava\sFts5ExtensionApi\sAPI. -D 2023-09-18T20:42:06.838 +C Initial\sdevelopment\scode\sfor\san\sexperimental\sbinary\sBLOB\sencoding\sfor\sJSON. +D 2023-09-21T17:51:39.740 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 51141f1c09ccb177057e5813e6302a5e32e5ba88cc4a756318a35081010fc6df +F src/json.c aa191fb04ff8b3857c2e05a51b347a23992807a2c12cda5247286ba4768075e7 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c923893f3604b278277de1bb919ef713bf7a4296b7ff71451cfe19bc2ff03190 -R dce10d8ae94249d41d42f7dff2440548 -U dan -Z 7d6c76f5fce3d6a1618c1597ed085ea5 +P f9d62b853ce8bfbfdc9f137e984e7a1b51d70e88c38b136b4fad1e8ae6ee8913 +R 707eaf997af5bd7aea243a294bfb17a0 +T *branch * jsonb +T *sym-jsonb * +T -sym-trunk * +U drh +Z b6e8348ea42422278215287176b59cff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 81b26d4090..37c7ebe312 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9d62b853ce8bfbfdc9f137e984e7a1b51d70e88c38b136b4fad1e8ae6ee8913 \ No newline at end of file +8131b3c272f47db2618886046a9713285ce120cb87d721484ee7444273290681 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 253fce9f49..c7c6becbd4 100644 --- a/src/json.c +++ b/src/json.c @@ -122,6 +122,19 @@ struct JsonCleanup { #define JSON_ARRAY 7 #define JSON_OBJECT 8 +/* JSON BLOB node types +*/ +#define JSONB_NULL 0 +#define JSONB_TRUE 1 +#define JSONB_FALSE 2 +#define JSONB_NUMBER 3 /* VarInt size + text */ +#define JSONB_NUMBER5 4 /* VarInt size + text (JSON5 formatted) */ +#define JSONB_TEXT 5 /* VarInt size + text (Raw and unescaped) */ +#define JSONB_TEXTJ 6 /* VarInt size + text (JSON formatted) */ +#define JSONB_TEXT5 7 /* VarInt size + text (JSON5 formatted) */ +#define JSONB_ARRAY 8 /* u32 size + content */ +#define JSONB_OBJECT 9 /* u32 size + content */ + /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -210,6 +223,10 @@ struct JsonParse { u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ u32 iHold; /* Age of this entry in the cache for LRU replacement */ + /* Binary format */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ + u8 *aBlob; /* BLOB representation of zJson */ }; /* @@ -1804,6 +1821,582 @@ static int jsonParse( return 0; } +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u32 t; + if( N<=pParse->nBlobAlloc ) return 0; + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; + } + if( taBlob, t ); + if( aNew==0 ){ pParse->oom = 1; return 1; } + pParse->aBlob = aNew; + pParse->nBlobAlloc = t; + return 0; +} + +/* Expand pParse->aBlob and append N bytes. +** +** Return the number of errors. +*/ +static SQLITE_NOINLINE int jsonBlobExpandAndAppend( + JsonParse *pParse, + const u8 *aData, + u32 N +){ + if( jsonBlobExpand(pParse, pParse->nBlob+N) ) return 1; + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append a single character. Return 1 if an error occurs. +*/ +static int jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, &c, 1); + } + pParse->aBlob[pParse->nBlob++] = c; + return 0; +} + +/* Append bytes. Return 1 if an error occurs. +*/ +static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ + if( pParse->nBlob+N > pParse->nBlobAlloc ){ + return jsonBlobExpandAndAppend(pParse, aData, N); + } + memcpy(&pParse->aBlob[pParse->nBlob], aData, N); + pParse->nBlob += N; + return 0; +} + +/* Append a u32. Return 1 if an error occurs. +*/ +static int jsonBlobAppendU32(JsonParse *pParse, u32 x){ + u8 a[4]; + a[3] = x & 0xff; + x >>= 8; + a[2] = x & 0xff; + x >>= 8; + a[1] = x & 0xff; + x >>= 8; + a[0] = x & 0xff; + return jsonBlobAppendNBytes(pParse, a, 4); +} + +/* Write a u32 and offset i. +*/ +static int jsonBlobWriteU32(JsonParse *pParse, u32 x, u32 i){ + u8 *a = &pParse->aBlob[i]; + a[3] = x & 0xff; + x >>= 8; + a[2] = x & 0xff; + x >>= 8; + a[1] = x & 0xff; + x >>= 8; + a[0] = x & 0xff; + return 0; +} + +/* Append a VarInt. Return 1 if an error occurs. +*/ +static int jsonBlobAppendVarint(JsonParse *pParse, u32 x){ + u8 a[5]; + if( x<=0x7f ) return jsonBlobAppendOneByte(pParse, x & 0x7f); + a[0] = 0x80 | (x & 0x7f); + x >>= 7; + a[1] = x & 0x7f; + x >>= 7; + if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 2); + a[1] |= 0x80; + a[2] = x & 0x7f; + x >>= 7; + if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 3); + a[2] |= 0x80; + a[3] = x & 0x7f; + x >>= 7; + if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 4); + a[3] |= 0x80; + a[4] = x & 0x7f; + return jsonBlobAppendNBytes(pParse, a, 5); +} + +/* +** Parse a single JSON text value which begins at pParse->zJson[i] into +** its equivalent BLOB representation in pParse->aBlob[]. The parse is +** appended to pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. +** +** Return the index of the first character past the end of the value parsed, +** or one of the following special result codes: +** +** 0 End of input +** -1 Syntax error +** -2 '}' seen +** -3 ']' seen +** -4 ',' seen +** -5 ':' seen +*/ +static int jsonParseValueB(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + u8 t; + const char *z = pParse->zJson; +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { + /* Parse object */ + jsonBlobAppendOneByte(pParse, JSONB_OBJECT); + iThis = pParse->nBlob; + jsonBlobAppendU32(pParse, 0); + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + for(j=i+1;;j++){ + u32 iBlob = pParse->nBlob; + x = jsonParseValueB(pParse, j); + if( x<=0 ){ + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nBlob!=(u32)iThis+1 ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + ){ + k++; + } + assert( iBlob==pParse->nBlob ); + jsonBlobAppendOneByte(pParse, JSONB_TEXT5); + jsonBlobAppendVarint(pParse, k-j); + jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } + } + if( pParse->oom ) return -1; + t = pParse->aBlob[iBlob]; + if( tJSONB_TEXT5 ){ + pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==':' ){ + j++; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonParseValueB(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: + x = jsonParseValueB(pParse, j); + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonParseValueB(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobWriteU32(pParse, pParse->nBlob - iThis, iThis); + pParse->iDepth--; + return j+1; + } + case '[': { + /* Parse array */ + jsonBlobAppendOneByte(pParse, JSONB_ARRAY); + iThis = pParse->nBlob; + jsonBlobAppendU32(pParse, 0); + if( pParse->oom ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + for(j=i+1;;j++){ + x = jsonParseValueB(pParse, j); + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nBlob!=iThis+4 ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonParseValueB(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobWriteU32(pParse, pParse->nBlob - iThis, iThis); + pParse->iDepth--; + return j+1; + } + case '\'': { + u8 opcode; + char cDelim; + pParse->hasNonstd = 1; + opcode = JNODE_JSON5; + goto parse_string; + case '"': + /* Parse string */ + opcode = JSONB_TEXT; + parse_string: + cDelim = z[i]; + for(j=i+1; 1; j++){ + if( jsonIsOk[(unsigned char)z[j]] ) continue; + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else{ + pParse->iErr = j; + return -1; + } + }else if( c<=0x1f ){ + /* Control characters are not allowed in strings */ + pParse->iErr = j; + return -1; + } + } + jsonBlobAppendOneByte(pParse, opcode); + jsonBlobAppendVarint(pParse, j+1-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j+1-i); + return j+1; + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_TRUE); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonBlobAppendOneByte(pParse, JSONB_FALSE); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenDP, seenE, jnFlags; + pParse->hasNonstd = 1; + jnFlags = 0; + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + seenE = 0; + seenDP = JSON_REAL; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + /* Parse number */ + jnFlags = 0; + parse_number: + seenDP = JSON_INT; + seenE = 0; + assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + + if( c<='0' ){ + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( seenDP==JSON_INT ); + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + jsonBlobAppendOneByte(pParse, JSONB_NUMBER5); + if( z[i]=='-' ){ + jsonBlobAppendVarint(pParse, 8); + jsonBlobAppendNBytes(pParse, (const u8*)"-9.0e999", 8); + }else{ + jsonBlobAppendVarint(pParse, 7); + jsonBlobAppendNBytes(pParse, (const u8*)"9.0e999", 7); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } + } + parse_number_2: + for(j=i+1;; j++){ + c = z[j]; + if( sqlite3Isdigit(c) ) continue; + if( c=='.' ){ + if( seenDP==JSON_REAL ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; + seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } + continue; + } + break; + } + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + if( jnFlags & JNODE_JSON5 ){ + jsonBlobAppendOneByte(pParse, JSONB_NUMBER5); + }else{ + jsonBlobAppendOneByte(pParse, JSONB_NUMBER); + } + jsonBlobAppendVarint(pParse, j-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); + return j; + } + case '}': { + pParse->iErr = i; + return -2; /* End of {...} */ + } + case ']': { + pParse->iErr = i; + return -3; /* End of [...] */ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { + return 0; /* End of file */ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; khasNonstd = 1; + return i + nn; + } + pParse->iErr = i; + return -1; /* Syntax error */ + } + } /* End switch(z[i]) */ +} + /* Mark node i of pParse as being a child of iParent. Call recursively ** to fill in all the descendants of node i. @@ -1832,6 +2425,50 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ } } +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. +** +** pParse must be initialized to an empty parse object prior to calling +** this routine. +*/ +static int jsonParseB( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx /* Report errors here */ +){ + int i; + const char *zJson = pParse->zJson; + jsonBlobAppendOneByte(pParse, 0x4a); + jsonBlobAppendOneByte(pParse, 0x01); + i = jsonParseValueB(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( fast_isspace(zJson[i]) ) i++; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + /* ** Compute the parentage of all nodes in a completed parse. */ @@ -2408,6 +3045,38 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ +/* SQL Function: jsonb_test(TEXT_JSON) +** +** Parse TEXT JSON into the BLOB format and return the resulting BLOB. +** Development testing only. +*/ +static void jsonbTest1( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *pParse; + int nJson; + const char *zJson; + JsonParse x; + UNUSED_PARAMETER(argc); + + zJson = (const char*)sqlite3_value_text(argv[0]); + if( zJson==0 ) return; + nJson = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.zJson = (char*)zJson; + x.nJson = nJson; + if( jsonParseB(pParse, ctx)==0 && pParse->aBlob!=0 ){ + sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } + jsonParseReset(pParse); +} + /* ** Implementation of the json_QUOTE(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting @@ -3815,6 +4484,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1, 0, jsonTypeFunc), JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), + JFUNCTION(jsonb_test1, 1, 0, jsonbTest1), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), From 769a8dea818701ca91221dd691b907aac3033afd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 21 Sep 2023 18:16:35 +0000 Subject: [PATCH 002/191] Reorganize the code to put the new JSONB routines together, for easier editing. FossilOrigin-Name: dc23e783d4147d363856abe109586fc79a5b535b492beee0cf7a0234c0210667 --- manifest | 15 +- manifest.uuid | 2 +- src/json.c | 992 +++++++++++++++++++++++++------------------------- 3 files changed, 505 insertions(+), 504 deletions(-) diff --git a/manifest b/manifest index 78b4dedcd2..00ee965eea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\sdevelopment\scode\sfor\san\sexperimental\sbinary\sBLOB\sencoding\sfor\sJSON. -D 2023-09-21T17:51:39.740 +C Reorganize\sthe\scode\sto\sput\sthe\snew\sJSONB\sroutines\stogether,\sfor\seasier\sediting. +D 2023-09-21T18:16:35.240 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c aa191fb04ff8b3857c2e05a51b347a23992807a2c12cda5247286ba4768075e7 +F src/json.c 909c1ed65cdff9745534ae31783b394c0412307af67392202c38eb4a5a051682 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,11 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f9d62b853ce8bfbfdc9f137e984e7a1b51d70e88c38b136b4fad1e8ae6ee8913 -R 707eaf997af5bd7aea243a294bfb17a0 -T *branch * jsonb -T *sym-jsonb * -T -sym-trunk * +P 8131b3c272f47db2618886046a9713285ce120cb87d721484ee7444273290681 +R f3c29be4023c5b4b5aa9f93c4ce8cc1a U drh -Z b6e8348ea42422278215287176b59cff +Z 6a7efc86b292f902336443f2d5840fea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 37c7ebe312..00f1a8286b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8131b3c272f47db2618886046a9713285ce120cb87d721484ee7444273290681 \ No newline at end of file +dc23e783d4147d363856abe109586fc79a5b535b492beee0cf7a0234c0210667 \ No newline at end of file diff --git a/src/json.c b/src/json.c index c7c6becbd4..9ea085753b 100644 --- a/src/json.c +++ b/src/json.c @@ -1779,6 +1779,33 @@ json_parse_restart: } /* End switch(z[i]) */ } +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there ** are any errors. If an error occurs, free all memory held by pParse, @@ -1821,6 +1848,477 @@ static int jsonParse( return 0; } +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* First cache entry */ +#define JSON_CACHE_SZ 4 /* Max number of cache entries */ + +/* +** Obtain a complete parse of the JSON found in the pJson argument +** +** Use the sqlite3_get_auxdata() cache to find a preexisting parse +** if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse. +** Also register the new parse so that it will be available for +** future sqlite3_get_auxdata() calls. +** +** If an error occurs and pErrCtx!=0 then report the error on pErrCtx +** and return NULL. +** +** The returned pointer (if it is not NULL) is owned by the cache in +** most cases, not the caller. The caller does NOT need to invoke +** jsonParseFree(), in most cases. +** +** Except, if an error occurs and pErrCtx==0 then return the JsonParse +** object with JsonParse.nErr non-zero and the caller will own the JsonParse +** object. In that case, it will be the responsibility of the caller to +** invoke jsonParseFree(). To summarize: +** +** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the +** cache. Call does not need to +** free it. +** +** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller +** and so the caller must free it. +*/ +static JsonParse *jsonParseCached( + sqlite3_context *pCtx, /* Context to use for cache search */ + sqlite3_value *pJson, /* Function param containing JSON text */ + sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ + int bUnedited /* No prior edits allowed */ +){ + char *zJson = (char*)sqlite3_value_text(pJson); + int nJson = sqlite3_value_bytes(pJson); + JsonParse *p; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + int bJsonRCStr; + + if( zJson==0 ) return 0; + for(iKey=0; iKeynJson==nJson + && (p->hasMod==0 || bUnedited==0) + && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) + ){ + p->nErr = 0; + p->useMod = 0; + pMatch = p; + }else + if( pMatch==0 + && p->zAlt!=0 + && bUnedited==0 + && p->nAlt==nJson + && memcmp(p->zAlt, zJson, nJson)==0 + ){ + p->nErr = 0; + p->useMod = 1; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + if( pMatch ){ + /* The input JSON text was found in the cache. Use the preexisting + ** parse of this JSON */ + pMatch->nErr = 0; + pMatch->iHold = iMaxHold+1; + assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ + return pMatch; + } + + /* The input JSON was not found anywhere in the cache. We will need + ** to parse it ourselves and generate a new JsonParse object. + */ + bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref); + p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, sizeof(*p)); + if( bJsonRCStr ){ + p->zJson = sqlite3RCStrRef(zJson); + p->bJsonIsRCStr = 1; + }else{ + p->zJson = (char*)&p[1]; + memcpy(p->zJson, zJson, nJson+1); + } + p->nJPRef = 1; + if( jsonParse(p, pErrCtx) ){ + if( pErrCtx==0 ){ + p->nErr = 1; + assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ + return p; + } + jsonParseFree(p); + return 0; + } + p->nJson = nJson; + p->iHold = iMaxHold+1; + /* Transfer ownership of the new JsonParse to the cache */ + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, + (void(*)(void*))jsonParseFree); + return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); +} + +/* +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. +*/ +static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; + }else{ + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + } +} +static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ + if( p1->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p2, p1->u.zJContent, p1->n); + }else if( p2->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p1, p2->u.zJContent, p2->n); + }else{ + return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; + } +} + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. +*/ +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot; + if( pParse->oom ) return 0; + pRoot = &pParse->aNode[iRoot]; + if( pRoot->jnFlags & (JNODE_REPLACE|JNODE_REMOVE) && pParse->useMod ){ + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ + u32 idx = (u32)(pRoot - pParse->aNode); + i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pRoot = &pParse->aNode[i+1]; + iRoot = i+1; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } + if( pRoot->jnFlags & JNODE_REMOVE ){ + return 0; + } + } + if( zPath[0]==0 ) return pRoot; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + testcase( nKey==0 ); + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pRoot->eU==2 ); + iRoot = pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + assert( pParse->useMod ); + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + assert( pRoot->eU==0 ); + pRoot->u.iAppend = iStart; + pRoot->jnFlags |= JNODE_APPEND; + VVA( pRoot->eU = 2 ); + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' ){ + i = 0; + j = 1; + while( sqlite3Isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( j<2 || zPath[j]!=']' ){ + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + j += jsonNodeSize(&pBase[j]); + } + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pBase->eU==2 ); + iBase = pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( sqlite3Isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + }else{ + *pzErr = zPath; + return 0; + } + } + if( pRoot->eType!=JSON_ARRAY ) return 0; + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) + ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pRoot->eU==2 ); + iRoot = pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + assert( pParse->useMod ); + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + assert( pRoot->eU==0 ); + pRoot->u.iAppend = iStart; + pRoot->jnFlags |= JNODE_APPEND; + VVA( pRoot->eU = 2 ); + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} + +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} + +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; + +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + + /* ** Expand pParse->aBlob so that it holds at least N bytes. ** @@ -2398,33 +2896,6 @@ json_parse_restart: } -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - /* ** Parse a complete JSON string. Return 0 on success or non-zero if there ** are any errors. If an error occurs, free all memory held by pParse, @@ -2469,473 +2940,6 @@ static int jsonParseB( return 0; } -/* -** Compute the parentage of all nodes in a completed parse. -*/ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; - } - jsonParseFillInParentage(pParse, 0, 0); - return SQLITE_OK; -} - -/* -** Magic number used for the JSON parse cache in sqlite3_get_auxdata() -*/ -#define JSON_CACHE_ID (-429938) /* First cache entry */ -#define JSON_CACHE_SZ 4 /* Max number of cache entries */ - -/* -** Obtain a complete parse of the JSON found in the pJson argument -** -** Use the sqlite3_get_auxdata() cache to find a preexisting parse -** if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse. -** Also register the new parse so that it will be available for -** future sqlite3_get_auxdata() calls. -** -** If an error occurs and pErrCtx!=0 then report the error on pErrCtx -** and return NULL. -** -** The returned pointer (if it is not NULL) is owned by the cache in -** most cases, not the caller. The caller does NOT need to invoke -** jsonParseFree(), in most cases. -** -** Except, if an error occurs and pErrCtx==0 then return the JsonParse -** object with JsonParse.nErr non-zero and the caller will own the JsonParse -** object. In that case, it will be the responsibility of the caller to -** invoke jsonParseFree(). To summarize: -** -** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the -** cache. Call does not need to -** free it. -** -** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller -** and so the caller must free it. -*/ -static JsonParse *jsonParseCached( - sqlite3_context *pCtx, /* Context to use for cache search */ - sqlite3_value *pJson, /* Function param containing JSON text */ - sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ - int bUnedited /* No prior edits allowed */ -){ - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); - JsonParse *p; - JsonParse *pMatch = 0; - int iKey; - int iMinKey = 0; - u32 iMinHold = 0xffffffff; - u32 iMaxHold = 0; - int bJsonRCStr; - - if( zJson==0 ) return 0; - for(iKey=0; iKeynJson==nJson - && (p->hasMod==0 || bUnedited==0) - && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) - ){ - p->nErr = 0; - p->useMod = 0; - pMatch = p; - }else - if( pMatch==0 - && p->zAlt!=0 - && bUnedited==0 - && p->nAlt==nJson - && memcmp(p->zAlt, zJson, nJson)==0 - ){ - p->nErr = 0; - p->useMod = 1; - pMatch = p; - }else if( p->iHoldiHold; - iMinKey = iKey; - } - if( p->iHold>iMaxHold ){ - iMaxHold = p->iHold; - } - } - if( pMatch ){ - /* The input JSON text was found in the cache. Use the preexisting - ** parse of this JSON */ - pMatch->nErr = 0; - pMatch->iHold = iMaxHold+1; - assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ - return pMatch; - } - - /* The input JSON was not found anywhere in the cache. We will need - ** to parse it ourselves and generate a new JsonParse object. - */ - bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref); - p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); - if( p==0 ){ - sqlite3_result_error_nomem(pCtx); - return 0; - } - memset(p, 0, sizeof(*p)); - if( bJsonRCStr ){ - p->zJson = sqlite3RCStrRef(zJson); - p->bJsonIsRCStr = 1; - }else{ - p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+1); - } - p->nJPRef = 1; - if( jsonParse(p, pErrCtx) ){ - if( pErrCtx==0 ){ - p->nErr = 1; - assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ - return p; - } - jsonParseFree(p); - return 0; - } - p->nJson = nJson; - p->iHold = iMaxHold+1; - /* Transfer ownership of the new JsonParse to the cache */ - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, - (void(*)(void*))jsonParseFree); - return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); -} - -/* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. -*/ -static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } -} -static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ - if( p1->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p2, p1->u.zJContent, p1->n); - }else if( p2->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p1, p2->u.zJContent, p2->n); - }else{ - return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; - } -} - -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - -/* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. -** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. -*/ -static JsonNode *jsonLookupStep( - JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ -){ - u32 i, j, nKey; - const char *zKey; - JsonNode *pRoot; - if( pParse->oom ) return 0; - pRoot = &pParse->aNode[iRoot]; - if( pRoot->jnFlags & (JNODE_REPLACE|JNODE_REMOVE) && pParse->useMod ){ - while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ - u32 idx = (u32)(pRoot - pParse->aNode); - i = pParse->iSubst; - while( 1 /*exit-by-break*/ ){ - assert( inNode ); - assert( pParse->aNode[i].eType==JSON_SUBST ); - assert( pParse->aNode[i].eU==4 ); - assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ - pRoot = &pParse->aNode[i+1]; - iRoot = i+1; - break; - } - i = pParse->aNode[i].u.iPrev; - } - } - if( pRoot->jnFlags & JNODE_REMOVE ){ - return 0; - } - } - if( zPath[0]==0 ) return pRoot; - if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; - zPath++; - if( zPath[0]=='"' ){ - zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} - nKey = i-1; - if( zPath[i] ){ - i++; - }else{ - *pzErr = zPath; - return 0; - } - testcase( nKey==0 ); - }else{ - zKey = zPath; - for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} - nKey = i; - if( nKey==0 ){ - *pzErr = zPath; - return 0; - } - } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); - } - j++; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pRoot->eU==2 ); - iRoot = pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - assert( pParse->useMod ); - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; - } - }else if( zPath[0]=='[' ){ - i = 0; - j = 1; - while( sqlite3Isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; - } - if( j<2 || zPath[j]!=']' ){ - if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pBase->eU==2 ); - iBase = pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; - if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int x = 0; - j = 3; - do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( sqlite3Isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; - } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; - } - }else{ - *pzErr = zPath; - return 0; - } - } - if( pRoot->eType!=JSON_ARRAY ) return 0; - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n - && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) - ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pRoot->eU==2 ); - iRoot = pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - assert( pParse->useMod ); - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart; - pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); - } - return pNode; - } - }else{ - *pzErr = zPath; - } - return 0; -} - -/* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. -*/ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ -){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; - } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; - } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); -} - -/* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). -*/ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); -} - -/* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. -** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. -** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. -*/ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ -){ - const char *zErr = 0; - JsonNode *pNode = 0; - char *zMsg; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; - -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); - if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); - }else{ - sqlite3_result_error_nomem(pCtx); - } - return 0; -} - - -/* -** Report the wrong number of arguments for json_insert(), json_replace() -** or json_set(). -*/ -static void jsonWrongNumArgs( - sqlite3_context *pCtx, - const char *zFuncName -){ - char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", - zFuncName); - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); -} - -/* -** Mark all NULL entries in the Object passed in as JNODE_REMOVE. -*/ -static void jsonRemoveAllNulls(JsonNode *pNode){ - int i, n; - assert( pNode->eType==JSON_OBJECT ); - n = pNode->n; - for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ - switch( pNode[i].eType ){ - case JSON_NULL: - pNode[i].jnFlags |= JNODE_REMOVE; - break; - case JSON_OBJECT: - jsonRemoveAllNulls(&pNode[i]); - break; - } - } -} - - /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ From 8c55945220bcd60ff8069f9879631edbd29169be Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Sep 2023 11:20:35 +0000 Subject: [PATCH 003/191] Improvements to the JSON binary BLOB format design. FossilOrigin-Name: 2c89ae5d02f6a40ef869e2a162e2c72871df60572b27959fd1d7171f495ce881 --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 489 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 398 insertions(+), 105 deletions(-) diff --git a/manifest b/manifest index 00ee965eea..3c715192be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorganize\sthe\scode\sto\sput\sthe\snew\sJSONB\sroutines\stogether,\sfor\seasier\sediting. -D 2023-09-21T18:16:35.240 +C Improvements\sto\sthe\sJSON\sbinary\sBLOB\sformat\sdesign. +D 2023-09-22T11:20:35.748 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 909c1ed65cdff9745534ae31783b394c0412307af67392202c38eb4a5a051682 +F src/json.c 4a25369c2f65c930f07f0b5b36e3a4ba86522f432035ac9204c9940c9d2dfb43 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8131b3c272f47db2618886046a9713285ce120cb87d721484ee7444273290681 -R f3c29be4023c5b4b5aa9f93c4ce8cc1a +P dc23e783d4147d363856abe109586fc79a5b535b492beee0cf7a0234c0210667 +R 9fd8daf58d6e7ea7726891b7477fe4a5 U drh -Z 6a7efc86b292f902336443f2d5840fea +Z 3129fb1588c8eb084faa14e3a492d4c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 00f1a8286b..b60349f423 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc23e783d4147d363856abe109586fc79a5b535b492beee0cf7a0234c0210667 \ No newline at end of file +2c89ae5d02f6a40ef869e2a162e2c72871df60572b27959fd1d7171f495ce881 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9ea085753b..7be2cebeb6 100644 --- a/src/json.c +++ b/src/json.c @@ -127,13 +127,15 @@ struct JsonCleanup { #define JSONB_NULL 0 #define JSONB_TRUE 1 #define JSONB_FALSE 2 -#define JSONB_NUMBER 3 /* VarInt size + text */ -#define JSONB_NUMBER5 4 /* VarInt size + text (JSON5 formatted) */ -#define JSONB_TEXT 5 /* VarInt size + text (Raw and unescaped) */ -#define JSONB_TEXTJ 6 /* VarInt size + text (JSON formatted) */ -#define JSONB_TEXT5 7 /* VarInt size + text (JSON5 formatted) */ -#define JSONB_ARRAY 8 /* u32 size + content */ -#define JSONB_OBJECT 9 /* u32 size + content */ +#define JSONB_INT 3 +#define JSONB_INT5 4 +#define JSONB_FLOAT 5 +#define JSONB_FLOAT5 6 +#define JSONB_TEXT 7 +#define JSONB_TEXTJ 8 +#define JSONB_TEXT5 9 +#define JSONB_ARRAY 10 +#define JSONB_OBJECT 11 /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -2377,55 +2379,64 @@ static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ return 0; } -/* Append a u32. Return 1 if an error occurs. +/* Append an node type byte together with the payload size. */ -static int jsonBlobAppendU32(JsonParse *pParse, u32 x){ - u8 a[4]; - a[3] = x & 0xff; - x >>= 8; - a[2] = x & 0xff; - x >>= 8; - a[1] = x & 0xff; - x >>= 8; - a[0] = x & 0xff; - return jsonBlobAppendNBytes(pParse, a, 4); -} - -/* Write a u32 and offset i. -*/ -static int jsonBlobWriteU32(JsonParse *pParse, u32 x, u32 i){ - u8 *a = &pParse->aBlob[i]; - a[3] = x & 0xff; - x >>= 8; - a[2] = x & 0xff; - x >>= 8; - a[1] = x & 0xff; - x >>= 8; - a[0] = x & 0xff; - return 0; -} - -/* Append a VarInt. Return 1 if an error occurs. -*/ -static int jsonBlobAppendVarint(JsonParse *pParse, u32 x){ +static void jsonBlobAppendNodeType( + JsonParse *pParse, + u8 eType, + u32 szPayload +){ u8 a[5]; - if( x<=0x7f ) return jsonBlobAppendOneByte(pParse, x & 0x7f); - a[0] = 0x80 | (x & 0x7f); - x >>= 7; - a[1] = x & 0x7f; - x >>= 7; - if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 2); - a[1] |= 0x80; - a[2] = x & 0x7f; - x >>= 7; - if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 3); - a[2] |= 0x80; - a[3] = x & 0x7f; - x >>= 7; - if( x==0 ) return jsonBlobAppendNBytes(pParse, a, 4); - a[3] |= 0x80; - a[4] = x & 0x7f; - return jsonBlobAppendNBytes(pParse, a, 5); + if( szPayload<=11 ){ + jsonBlobAppendOneByte(pParse, eType | (szPayload<<4)); + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 2); + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 3); + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + jsonBlobAppendNBytes(pParse, a, 5); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static void jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a = &pParse->aBlob[i]; + u8 szType = a[0]>>4; + if( szType<=11 ){ + assert( szPayload<=11 ); + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( szType==0xc ){ + assert( szPayload<=0xff ); + assert( i+1nBlob ); + a[1] = szPayload & 0xff; + }else if( szType==0xd ){ + assert( szPayload<=0xffff ); + assert( i+2nBlob ); + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + assert( szType==0xe ); + assert( i+4nBlob ); + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } } /* @@ -2447,7 +2458,7 @@ static int jsonBlobAppendVarint(JsonParse *pParse, u32 x){ static int jsonParseValueB(JsonParse *pParse, u32 i){ char c; u32 j; - int iThis; + u32 iThis, iStart; int x; u8 t; const char *z = pParse->zJson; @@ -2455,13 +2466,13 @@ json_parse_restart: switch( (u8)z[i] ){ case '{': { /* Parse object */ - jsonBlobAppendOneByte(pParse, JSONB_OBJECT); iThis = pParse->nBlob; - jsonBlobAppendU32(pParse, 0); + jsonBlobAppendNodeType(pParse, JSONB_OBJECT, (pParse->nJson-i)*2); if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; return -1; } + iStart = pParse->nBlob; for(j=i+1;;j++){ u32 iBlob = pParse->nBlob; x = jsonParseValueB(pParse, j); @@ -2482,8 +2493,7 @@ json_parse_restart: k++; } assert( iBlob==pParse->nBlob ); - jsonBlobAppendOneByte(pParse, JSONB_TEXT5); - jsonBlobAppendVarint(pParse, k-j); + jsonBlobAppendNodeType(pParse, JSONB_TEXT5, k-j); jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); pParse->hasNonstd = 1; x = k; @@ -2493,7 +2503,7 @@ json_parse_restart: } } if( pParse->oom ) return -1; - t = pParse->aBlob[iBlob]; + t = pParse->aBlob[iBlob] & 0x0f; if( tJSONB_TEXT5 ){ pParse->iErr = j; return -1; @@ -2549,15 +2559,17 @@ json_parse_restart: pParse->iErr = j; return -1; } - jsonBlobWriteU32(pParse, pParse->nBlob - iThis, iThis); + if( pParse->nErr==0 ){ + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + } pParse->iDepth--; return j+1; } case '[': { /* Parse array */ - jsonBlobAppendOneByte(pParse, JSONB_ARRAY); iThis = pParse->nBlob; - jsonBlobAppendU32(pParse, 0); + jsonBlobAppendNodeType(pParse, JSONB_ARRAY, pParse->nJson - i); + iStart = pParse->nBlob; if( pParse->oom ) return -1; if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; @@ -2601,7 +2613,9 @@ json_parse_restart: pParse->iErr = j; return -1; } - jsonBlobWriteU32(pParse, pParse->nBlob - iThis, iThis); + if( pParse->iErr==0 ){ + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + } pParse->iDepth--; return j+1; } @@ -2609,7 +2623,7 @@ json_parse_restart: u8 opcode; char cDelim; pParse->hasNonstd = 1; - opcode = JNODE_JSON5; + opcode = JSONB_TEXT; goto parse_string; case '"': /* Parse string */ @@ -2647,8 +2661,7 @@ json_parse_restart: return -1; } } - jsonBlobAppendOneByte(pParse, opcode); - jsonBlobAppendVarint(pParse, j+1-i); + jsonBlobAppendNodeType(pParse, opcode, j+1-i); jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j+1-i); return j+1; } @@ -2669,16 +2682,15 @@ json_parse_restart: return -1; } case '+': { - u8 seenDP, seenE, jnFlags; + u8 seenE; pParse->hasNonstd = 1; - jnFlags = 0; + t = 0x01; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ goto parse_number; case '.': if( sqlite3Isdigit(z[i+1]) ){ pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ seenE = 0; - seenDP = JSON_REAL; goto parse_number_2; } pParse->iErr = i; @@ -2695,9 +2707,8 @@ json_parse_restart: case '8': case '9': /* Parse number */ - jnFlags = 0; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ parse_number: - seenDP = JSON_INT; seenE = 0; assert( '-' < '0' ); assert( '+' < '0' ); @@ -2707,9 +2718,9 @@ json_parse_restart: if( c<='0' ){ if( c=='0' ){ if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ - assert( seenDP==JSON_INT ); + assert( t==0x00 ); pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t = 0x01; for(j=i+3; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; }else if( sqlite3Isdigit(z[i+1]) ){ @@ -2725,19 +2736,18 @@ json_parse_restart: && sqlite3StrNICmp(&z[i+1], "inf",3)==0 ){ pParse->hasNonstd = 1; - jsonBlobAppendOneByte(pParse, JSONB_NUMBER5); if( z[i]=='-' ){ - jsonBlobAppendVarint(pParse, 8); - jsonBlobAppendNBytes(pParse, (const u8*)"-9.0e999", 8); + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 6); + jsonBlobAppendNBytes(pParse, (const u8*)"-9e999", 6); }else{ - jsonBlobAppendVarint(pParse, 7); - jsonBlobAppendNBytes(pParse, (const u8*)"9.0e999", 7); + jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 5); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); } return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); } if( z[i+1]=='.' ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x03; goto parse_number_2; } pParse->iErr = i; @@ -2749,7 +2759,7 @@ json_parse_restart: return -1; }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; for(j=i+4; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; } @@ -2761,18 +2771,18 @@ json_parse_restart: c = z[j]; if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( seenDP==JSON_REAL ){ + if( (t & 0x02)!=0 ){ pParse->iErr = j; return -1; } - seenDP = JSON_REAL; + t |= 0x02; continue; } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; }else{ pParse->iErr = j; return -1; @@ -2782,7 +2792,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - seenDP = JSON_REAL; + t |= 0x02; seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ @@ -2800,19 +2810,17 @@ json_parse_restart: if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; + t |= 0x01; }else{ pParse->iErr = j; return -1; } } parse_number_finish: - if( jnFlags & JNODE_JSON5 ){ - jsonBlobAppendOneByte(pParse, JSONB_NUMBER5); - }else{ - jsonBlobAppendOneByte(pParse, JSONB_NUMBER); - } - jsonBlobAppendVarint(pParse, j-i); + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + jsonBlobAppendNodeType(pParse, JSONB_INT+t, j-i); jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); return j; } @@ -2880,9 +2888,9 @@ json_parse_restart: } if( sqlite3Isalnum(z[i+nn]) ) continue; if( aNanInfName[k].eType==JSON_REAL ){ - jsonBlobAppendOneByte(pParse, JSONB_NUMBER); - jsonBlobAppendOneByte(pParse, 7); - jsonBlobAppendNBytes(pParse, (const u8*)"9.0e999", 7); + jsonBlobAppendOneByte(pParse, JSONB_FLOAT); + jsonBlobAppendOneByte(pParse, 5); + jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); }else{ jsonBlobAppendOneByte(pParse, JSONB_NULL); } @@ -2910,8 +2918,6 @@ static int jsonParseB( ){ int i; const char *zJson = pParse->zJson; - jsonBlobAppendOneByte(pParse, 0x4a); - jsonBlobAppendOneByte(pParse, 0x01); i = jsonParseValueB(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ @@ -2940,6 +2946,293 @@ static int jsonParseB( return 0; } +#if 0 +/* +** Convert the binary BLOB representation of JSON beginning at +** aBlob[0] (and extending for no more than nBlob bytes) into +** a pure JSON string. The string is appended to pOut. +*/ +static void jsonRenderBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + u8 *aBlob, /* The binary encoded JSON to be rendered */ + u32 nBlob, /* Maximum size of aBlob[...] */ + JsonString *pOut /* Write JSON here */ +){ + u32 i; + if( nBlob<1 ){ + pOut->bErr = 1; + return; + } + switch( aBlob[0] ){ + case JSONB_NULL: + jsonAppendRawNZ(pOut, "null", 4); + break; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + break; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + break; + } + case JSONB_NUMBER: + case JSONB_TEXTJ: { + u32 sz, n; + n = jsonbDecodeVarint(&aBlob[1], nBlob-1, &sz); + if( 1+n+sz>nBlob ){ + pOut->bErr = 1; + return; + } + jsonAppendRaw(pOut, &aBlob[1+n], sz); + break; + + case JSON_STRING: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->jnFlags & JNODE_LABEL ){ + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); + }else{ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); + }else{ + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + } + break; + } + case JSON_REAL: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); + }else{ + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + } + break; + } + case JSON_INT: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); + }else{ + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + } + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(pParse, &pNode[j], pOut); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(pParse, &pNode[j], pOut); + jsonAppendChar(pOut, ':'); + jsonRenderNode(pParse, &pNode[j+1], pOut); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, '}'); + break; + } + } +} +#endif /* 0 */ + + +#if 0 +/* +** Convert the JSON BLOB node an aBlob (size no greater than nBlob) +** into the return value from the function call. +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx /* Return value for this function */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + int rc; + int bNeg = 0; + const char *z; + + assert( pNode->eU==1 ); + z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; bNeg = 1; } + else if( z[0]=='+' ){ z++; } + rc = sqlite3DecOrHexToI64(z, &i); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -i : i); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; + } + break; + } + case JSON_REAL: { + double r; + const char *z; + assert( pNode->eU==1 ); + to_double: + z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + assert( pNode->eU==1 ); + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + assert( pNode->eU==1 ); + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z; + char *zOut; + u32 j; + u32 nOut = n; + assert( pNode->eU==1 ); + z = pNode->u.zJContent; + zOut = sqlite3_malloc( nOut+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); + i += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + i++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + i += 2; + continue; + }else{ + continue; + } + } /* end if( c=='\\' ) */ + zOut[j++] = c; + } /* end for() */ + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pParse, pNode, pCtx, 0); + break; + } + } +} +#endif /* 0 */ + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ From 90189be7ced5e35aa1d321f3ff6dccca3d1ac241 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Sep 2023 12:16:56 +0000 Subject: [PATCH 004/191] Begin adding code to render binary JSON back into text. Very incomplete. This is an incremental check-in. FossilOrigin-Name: b817dd865ed60fc4da0b662a9edec0fceb8921b02ce98133bdd565988939fd0f --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 364 ++++++++++++++------------------------------------ 3 files changed, 110 insertions(+), 268 deletions(-) diff --git a/manifest b/manifest index 3c715192be..6f81a43706 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sJSON\sbinary\sBLOB\sformat\sdesign. -D 2023-09-22T11:20:35.748 +C Begin\sadding\scode\sto\srender\sbinary\sJSON\sback\sinto\stext.\s\sVery\sincomplete.\nThis\sis\san\sincremental\scheck-in. +D 2023-09-22T12:16:56.105 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4a25369c2f65c930f07f0b5b36e3a4ba86522f432035ac9204c9940c9d2dfb43 +F src/json.c 351f2b99846c6b1afdbab553c26f7cf87c303f0caa005171017a30b5d9a1aece F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc23e783d4147d363856abe109586fc79a5b535b492beee0cf7a0234c0210667 -R 9fd8daf58d6e7ea7726891b7477fe4a5 +P 2c89ae5d02f6a40ef869e2a162e2c72871df60572b27959fd1d7171f495ce881 +R b78d29e507a66c69faecba07ef4a8c9a U drh -Z 3129fb1588c8eb084faa14e3a492d4c9 +Z 8e9aefce50e8026f8d315b79ff0edc06 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b60349f423..757c0d1778 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c89ae5d02f6a40ef869e2a162e2c72871df60572b27959fd1d7171f495ce881 \ No newline at end of file +b817dd865ed60fc4da0b662a9edec0fceb8921b02ce98133bdd565988939fd0f \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7be2cebeb6..3f17d8b601 100644 --- a/src/json.c +++ b/src/json.c @@ -2946,292 +2946,104 @@ static int jsonParseB( return 0; } -#if 0 +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the number of additional +** bytes following the typecode that are used to hold size information. +** Enhance, if the return value is N, then the start of the payload +** is at i+N+1. +*/ +static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + if( i>pParse->nBlob ){ + pParse->iErr = 1; + *pSz = 0; + return 0; + } + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 0; + }else if( x==12 ){ + if( i+1>pParse->nBlob ){ + pParse->iErr = 1; + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 1; + }else if( x==13 ){ + if( i+2>pParse->nBlob ){ + pParse->iErr = 1; + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 2; + }else{ + if( i+4>pParse->nBlob ){ + pParse->iErr = 1; + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 4; + } + if( i+sz+n>pParse->nBlob ){ + sz = pParse->nBlob - (i+n); + pParse->iErr = 1; + } + *pSz = sz; + return n; +} + + /* ** Convert the binary BLOB representation of JSON beginning at ** aBlob[0] (and extending for no more than nBlob bytes) into ** a pure JSON string. The string is appended to pOut. */ -static void jsonRenderBlob( +static u32 jsonRenderBlob( JsonParse *pParse, /* the complete parse of the JSON */ - u8 *aBlob, /* The binary encoded JSON to be rendered */ - u32 nBlob, /* Maximum size of aBlob[...] */ + u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ - u32 i; - if( nBlob<1 ){ + if( i>=pParse->nBlob ){ pOut->bErr = 1; - return; + return 0; } - switch( aBlob[0] ){ - case JSONB_NULL: + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { jsonAppendRawNZ(pOut, "null", 4); - break; + return i+1; } case JSONB_TRUE: { jsonAppendRawNZ(pOut, "true", 4); - break; + return i+1; } case JSONB_FALSE: { jsonAppendRawNZ(pOut, "false", 5); - break; + return i+1; } - case JSONB_NUMBER: + case JSONB_INT: + case JSONB_FLOAT: case JSONB_TEXTJ: { u32 sz, n; - n = jsonbDecodeVarint(&aBlob[1], nBlob-1, &sz); - if( 1+n+sz>nBlob ){ - pOut->bErr = 1; - return; - } - jsonAppendRaw(pOut, &aBlob[1+n], sz); - break; + n = jsonbPayloadSize(pParse, i, &sz); + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n+1], sz); + return i+n+sz; + } - case JSON_STRING: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->jnFlags & JNODE_LABEL ){ - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - jsonAppendChar(pOut, '"'); - }else{ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - } - }else if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); - }else{ - assert( pNode->n>0 ); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_REAL: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); - }else{ - assert( pNode->n>0 ); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_INT: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); - }else{ - assert( pNode->n>0 ); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); - jsonAppendChar(pOut, ':'); - jsonRenderNode(pParse, &pNode[j+1], pOut); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; - } - } -} -#endif /* 0 */ - - -#if 0 -/* -** Convert the JSON BLOB node an aBlob (size no greater than nBlob) -** into the return value from the function call. -*/ -static void jsonReturnFromBlob( - JsonParse *pParse, /* Complete JSON parse tree */ - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx /* Return value for this function */ -){ - switch( pNode->eType ){ default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - int rc; - int bNeg = 0; - const char *z; - - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; bNeg = 1; } - else if( z[0]=='+' ){ z++; } - rc = sqlite3DecOrHexToI64(z, &i); - if( rc<=1 ){ - sqlite3_result_int64(pCtx, bNeg ? -i : i); - }else if( rc==3 && bNeg ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - }else{ - goto to_double; - } - break; - } - case JSON_REAL: { - double r; - const char *z; - assert( pNode->eU==1 ); - to_double: - z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z; - char *zOut; - u32 j; - u32 nOut = n; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - zOut = sqlite3_malloc( nOut+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } - for(i=1, j=0; i>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - u32 vlo; - if( (v&0xfc00)==0xd800 - && i>18); - zOut[j++] = 0x80 | ((v>>12)&0x3f); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = 0xe0 | (v>>12); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - } - continue; - }else if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - }else if( c=='v' ){ - c = '\v'; - }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ - /* pass through unchanged */ - }else if( c=='0' ){ - c = 0; - }else if( c=='x' ){ - c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); - i += 2; - }else if( c=='\r' && z[i+1]=='\n' ){ - i++; - continue; - }else if( 0xe2==(u8)c ){ - assert( 0x80==(u8)z[i+1] ); - assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); - i += 2; - continue; - }else{ - continue; - } - } /* end if( c=='\\' ) */ - zOut[j++] = c; - } /* end for() */ - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); - } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx, 0); break; } } + return 0; } -#endif /* 0 */ + /**************************************************************************** ** SQL functions used for testing and debugging @@ -3342,7 +3154,7 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ -/* SQL Function: jsonb_test(TEXT_JSON) +/* SQL Function: jsonb_test1(TEXT_JSON) ** ** Parse TEXT JSON into the BLOB format and return the resulting BLOB. ** Development testing only. @@ -3374,6 +3186,35 @@ static void jsonbTest1( jsonParseReset(pParse); } +/* SQL Function: jsonb_test2(BLOB_JSON) +** +** Render BLOB_JSON back into text. +** Development testing only. +*/ +static void jsonbTest2( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *pParse; + const u8 *aBlob; + int nBlob; + JsonParse x; + JsonString s; + UNUSED_PARAMETER(argc); + + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + if( aBlob==0 ) return; + nBlob = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonInit(&s, ctx); + jsonRenderBlob(pParse, 0, &s); + jsonResult(&s); +} + /* ** Implementation of the json_QUOTE(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting @@ -4782,6 +4623,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), JFUNCTION(jsonb_test1, 1, 0, jsonbTest1), + JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), From c1c1d4d4a6e01a35729e4833b902901ce2cc62fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Sep 2023 14:33:39 +0000 Subject: [PATCH 005/191] Add the ability to render a binary BLOB back into valid canonical JSON. FossilOrigin-Name: 0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 205 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 177 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 6f81a43706..85abeb56b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sadding\scode\sto\srender\sbinary\sJSON\sback\sinto\stext.\s\sVery\sincomplete.\nThis\sis\san\sincremental\scheck-in. -D 2023-09-22T12:16:56.105 +C Add\sthe\sability\sto\srender\sa\sbinary\sBLOB\sback\sinto\svalid\scanonical\sJSON. +D 2023-09-22T14:33:39.536 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 351f2b99846c6b1afdbab553c26f7cf87c303f0caa005171017a30b5d9a1aece +F src/json.c 574ed4c90fc5a18d40665557c56987d6ad1cd78644fdd698b755002538252de0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2c89ae5d02f6a40ef869e2a162e2c72871df60572b27959fd1d7171f495ce881 -R b78d29e507a66c69faecba07ef4a8c9a +P b817dd865ed60fc4da0b662a9edec0fceb8921b02ce98133bdd565988939fd0f +R f9832ae6b1e64d76b924944a3c617b61 U drh -Z 8e9aefce50e8026f8d315b79ff0edc06 +Z 86dcd161b824c66e770c11b16ee3a065 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 757c0d1778..3622ea106f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b817dd865ed60fc4da0b662a9edec0fceb8921b02ce98133bdd565988939fd0f \ No newline at end of file +0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3f17d8b601..6bfa0f6ce8 100644 --- a/src/json.c +++ b/src/json.c @@ -124,18 +124,19 @@ struct JsonCleanup { /* JSON BLOB node types */ -#define JSONB_NULL 0 -#define JSONB_TRUE 1 -#define JSONB_FALSE 2 -#define JSONB_INT 3 -#define JSONB_INT5 4 -#define JSONB_FLOAT 5 -#define JSONB_FLOAT5 6 -#define JSONB_TEXT 7 -#define JSONB_TEXTJ 8 -#define JSONB_TEXT5 9 -#define JSONB_ARRAY 10 -#define JSONB_OBJECT 11 +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ /* The "subtype" set for JSON values */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ @@ -2493,7 +2494,7 @@ json_parse_restart: k++; } assert( iBlob==pParse->nBlob ); - jsonBlobAppendNodeType(pParse, JSONB_TEXT5, k-j); + jsonBlobAppendNodeType(pParse, JSONB_TEXTJ, k-j); jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); pParse->hasNonstd = 1; x = k; @@ -2504,7 +2505,7 @@ json_parse_restart: } if( pParse->oom ) return -1; t = pParse->aBlob[iBlob] & 0x0f; - if( tJSONB_TEXT5 ){ + if( tJSONB_TEXTRAW ){ pParse->iErr = j; return -1; } @@ -2661,8 +2662,8 @@ json_parse_restart: return -1; } } - jsonBlobAppendNodeType(pParse, opcode, j+1-i); - jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j+1-i); + jsonBlobAppendNodeType(pParse, opcode, j-1-i); + jsonBlobAppendNBytes(pParse, (const u8*)&z[i+1], j-1-i); return j+1; } case 't': { @@ -2684,7 +2685,7 @@ json_parse_restart: case '+': { u8 seenE; pParse->hasNonstd = 1; - t = 0x01; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ goto parse_number; case '.': if( sqlite3Isdigit(z[i+1]) ){ @@ -2820,6 +2821,7 @@ json_parse_restart: assert( JSONB_INT+0x01==JSONB_INT5 ); assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; jsonBlobAppendNodeType(pParse, JSONB_INT+t, j-i); jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); return j; @@ -2948,10 +2950,8 @@ static int jsonParseB( /* The byte at index i is a node type-code. This routine ** determines the payload size for that node and writes that -** payload size in to *pSz. It returns the number of additional -** bytes following the typecode that are used to hold size information. -** Enhance, if the return value is N, then the start of the payload -** is at i+N+1. +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. */ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ u8 x; @@ -2960,37 +2960,37 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ if( i>pParse->nBlob ){ pParse->iErr = 1; *pSz = 0; - return 0; + return 1; } x = pParse->aBlob[i]>>4; if( x<=11 ){ sz = x; - n = 0; + n = 1; }else if( x==12 ){ if( i+1>pParse->nBlob ){ pParse->iErr = 1; *pSz = 0; - return 0; + return 1; } sz = pParse->aBlob[i+1]; - n = 1; + n = 2; }else if( x==13 ){ if( i+2>pParse->nBlob ){ pParse->iErr = 1; *pSz = 0; - return 0; + return 1; } sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; - n = 2; + n = 3; }else{ if( i+4>pParse->nBlob ){ pParse->iErr = 1; *pSz = 0; - return 0; + return 1; } sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; - n = 4; + n = 5; } if( i+sz+n>pParse->nBlob ){ sz = pParse->nBlob - (i+n); @@ -3011,6 +3011,8 @@ static u32 jsonRenderBlob( u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ + u32 sz, n, j, iEnd; + if( i>=pParse->nBlob ){ pOut->bErr = 1; return 0; @@ -3029,19 +3031,152 @@ static u32 jsonRenderBlob( return i+1; } case JSONB_INT: - case JSONB_FLOAT: - case JSONB_TEXTJ: { - u32 sz, n; + case JSONB_FLOAT: { n = jsonbPayloadSize(pParse, i, &sz); - jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n+1], sz); - return i+n+sz; + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + int k; + sqlite3_uint64 u = 0; + n = jsonbPayloadSize(pParse, i, &sz); + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( zIn[0]=='-' ){ + zIn++; + sz--; + jsonAppendChar(pOut, '-'); + } + for(k=2; kaBlob[i+n]; + if( zIn[0]=='-' ){ + zIn++; + sz--; + jsonAppendChar(pOut, '-'); + } + if( zIn[0]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(k=0; k0 ){ + jsonAppendRawNZ(pOut, zIn, sz); + } + break; + } + case JSONB_TEXT: + case JSONB_TEXTJ: { + n = jsonbPayloadSize(pParse, i, &sz); + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + n = jsonbPayloadSize(pParse, i, &sz); + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz>0 ){ + for(k=0; k0 ){ + jsonAppendRawNZ(pOut, zIn, k); + zIn += k; + sz -= k; + if( sz==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u0009", 6); + break; + case 'x': + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( zIn[2]=='\n' ){ + zIn++; + sz--; + } + break; + case '\n': + break; + case 0xe2: + assert( sz>=4 ); + assert( 0x80==(u8)zIn[2] ); + assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + zIn += 2; + sz -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + zIn += 2; + sz -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + n = jsonbPayloadSize(pParse, i, &sz); + j = i+n; + iEnd = j+sz; + while( 1 /* exit by break */ ){ + j = jsonRenderBlob(pParse, j, pOut); + if( j>=iEnd ) break; + jsonAppendChar(pOut, ','); + } + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + n = jsonbPayloadSize(pParse, i, &sz); + j = i+n; + iEnd = j+sz; + while( 1 /* edit by break */ ){ + j = jsonRenderBlob(pParse, j, pOut); + if( j>=iEnd ) break; + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + jsonAppendChar(pOut, '}'); + break; } default: { + n = jsonbPayloadSize(pParse, i, &sz); break; } } - return 0; + return i+n+sz; } From e367e453e140bb39a9df756713d28b75cff7d2e4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 22 Sep 2023 16:20:48 +0000 Subject: [PATCH 006/191] Fix minor parse-to-BLOB bugs. FossilOrigin-Name: 8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 85abeb56b9..1e08af59f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\srender\sa\sbinary\sBLOB\sback\sinto\svalid\scanonical\sJSON. -D 2023-09-22T14:33:39.536 +C Fix\sminor\sparse-to-BLOB\sbugs. +D 2023-09-22T16:20:48.168 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 574ed4c90fc5a18d40665557c56987d6ad1cd78644fdd698b755002538252de0 +F src/json.c 522ebfebd77224594e2fcd9ea18e201ac8b997d5f1cf2c71d8eba00e874b54b2 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b817dd865ed60fc4da0b662a9edec0fceb8921b02ce98133bdd565988939fd0f -R f9832ae6b1e64d76b924944a3c617b61 +P 0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 +R e81e3fd90e402a6e846b93a0d3beab83 U drh -Z 86dcd161b824c66e770c11b16ee3a065 +Z 285487434c996c07680e15a392e180f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3622ea106f..94c65ef3d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 \ No newline at end of file +8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6bfa0f6ce8..ac8976d03c 100644 --- a/src/json.c +++ b/src/json.c @@ -2480,7 +2480,7 @@ json_parse_restart: if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; - if( pParse->nBlob!=(u32)iThis+1 ) pParse->hasNonstd = 1; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; break; } j += json5Whitespace(&z[j]); @@ -2560,7 +2560,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - if( pParse->nErr==0 ){ + if( pParse->oom==0 ){ jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); } pParse->iDepth--; @@ -2581,7 +2581,7 @@ json_parse_restart: if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; - if( pParse->nBlob!=iThis+4 ) pParse->hasNonstd = 1; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; break; } if( x!=(-1) ) pParse->iErr = j; @@ -2614,7 +2614,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - if( pParse->iErr==0 ){ + if( pParse->oom==0 ){ jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); } pParse->iDepth--; @@ -3148,11 +3148,11 @@ static u32 jsonRenderBlob( n = jsonbPayloadSize(pParse, i, &sz); j = i+n; iEnd = j+sz; - while( 1 /* exit by break */ ){ + while( j=iEnd ) break; jsonAppendChar(pOut, ','); } + if( sz>0 ) pOut->nUsed--; jsonAppendChar(pOut, ']'); break; } @@ -3162,11 +3162,11 @@ static u32 jsonRenderBlob( n = jsonbPayloadSize(pParse, i, &sz); j = i+n; iEnd = j+sz; - while( 1 /* edit by break */ ){ + while( j=iEnd ) break; jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } + if( sz>0 ) pOut->nUsed--; jsonAppendChar(pOut, '}'); break; } From 5933581cf035d1479752ebdd3bce84ae609c215b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Sep 2023 13:23:29 +0000 Subject: [PATCH 007/191] As a temporary measure, try to translate the BLOB JSON format into the legacy node format for processing. FossilOrigin-Name: 14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 225 ++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 198 insertions(+), 41 deletions(-) diff --git a/manifest b/manifest index 1e08af59f7..3fd91c08fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sparse-to-BLOB\sbugs. -D 2023-09-22T16:20:48.168 +C As\sa\stemporary\smeasure,\stry\sto\stranslate\sthe\sBLOB\sJSON\sformat\sinto\sthe\nlegacy\snode\sformat\sfor\sprocessing. +D 2023-09-25T13:23:29.913 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 522ebfebd77224594e2fcd9ea18e201ac8b997d5f1cf2c71d8eba00e874b54b2 +F src/json.c 23efc117c4641118c2154938decc80fb3b29b1a420bcd0eaefe24ef808d1ada4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b70cb77a4c8e3f17932f1ecca3942e0b0b03de637fb9656a130fe045f7ef826 -R e81e3fd90e402a6e846b93a0d3beab83 +P 8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 +R 31821ddd534254fa82bb41b5d469060a U drh -Z 285487434c996c07680e15a392e180f6 +Z 25f12fc9cea10e56c4709ee1d21954ff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 94c65ef3d9..1d3c76b2dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 \ No newline at end of file +14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f \ No newline at end of file diff --git a/src/json.c b/src/json.c index ac8976d03c..afa152624e 100644 --- a/src/json.c +++ b/src/json.c @@ -220,6 +220,7 @@ struct JsonParse { u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ + u8 isBinary; /* True if zJson is the binary encoding */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ int nAlt; /* Length of alternative JSON string zAlt, in bytes */ @@ -1809,6 +1810,9 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ } } +/* Forward reference */ +static int jsonParseValueFromBinary(JsonParse *pParse, u32 i); + /* ** Parse a complete JSON string. Return 0 on success or non-zero if there ** are any errors. If an error occurs, free all memory held by pParse, @@ -1823,7 +1827,13 @@ static int jsonParse( ){ int i; const char *zJson = pParse->zJson; - i = jsonParseValue(pParse, 0); + if( pParse->isBinary ){ + pParse->aBlob = (u8*)pParse->zJson; + pParse->nBlob = pParse->nJson; + i = jsonParseValueFromBinary(pParse, 0); + }else{ + i = jsonParseValue(pParse, 0); + } if( pParse->oom ) i = -1; if( i>0 ){ assert( pParse->iDepth==0 ); @@ -1872,6 +1882,10 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_ID (-429938) /* First cache entry */ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ +/* Forward reference */ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); + + /* ** Obtain a complete parse of the JSON found in the pJson argument ** @@ -1906,8 +1920,8 @@ static JsonParse *jsonParseCached( sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ int bUnedited /* No prior edits allowed */ ){ - char *zJson = (char*)sqlite3_value_text(pJson); - int nJson = sqlite3_value_bytes(pJson); + char *zJson; + int nJson; JsonParse *p; JsonParse *pMatch = 0; int iKey; @@ -1915,6 +1929,16 @@ static JsonParse *jsonParseCached( u32 iMinHold = 0xffffffff; u32 iMaxHold = 0; int bJsonRCStr; + int isBinary; + + if( jsonFuncArgMightBeBinary(pJson) ){ + zJson = (char*)sqlite3_value_blob(pJson); + isBinary = 1; + }else{ + zJson = (char*)sqlite3_value_text(pJson); + isBinary = 0; + } + nJson = sqlite3_value_bytes(pJson); if( zJson==0 ) return 0; for(iKey=0; iKeybJsonIsRCStr = 1; }else{ p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+1); + memcpy(p->zJson, zJson, nJson+(isBinary==0)); } p->nJPRef = 1; + p->isBinary = isBinary; + p->nJson = nJson; if( jsonParse(p, pErrCtx) ){ if( pErrCtx==0 ){ p->nErr = 1; @@ -1985,7 +2011,6 @@ static JsonParse *jsonParseCached( jsonParseFree(p); return 0; } - p->nJson = nJson; p->iHold = iMaxHold+1; /* Transfer ownership of the new JsonParse to the cache */ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, @@ -2951,16 +2976,15 @@ static int jsonParseB( /* The byte at index i is a node type-code. This routine ** determines the payload size for that node and writes that ** payload size in to *pSz. It returns the offset from i to the -** beginning of the payload. +** beginning of the payload. Return 0 on error. */ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; if( i>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } x = pParse->aBlob[i]>>4; if( x<=11 ){ @@ -2968,25 +2992,22 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ n = 1; }else if( x==12 ){ if( i+1>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = pParse->aBlob[i+1]; n = 2; }else if( x==13 ){ if( i+2>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; n = 3; }else{ if( i+4>pParse->nBlob ){ - pParse->iErr = 1; *pSz = 0; - return 1; + return 0; } sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; @@ -2994,7 +3015,6 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ } if( i+sz+n>pParse->nBlob ){ sz = pParse->nBlob - (i+n); - pParse->iErr = 1; } *pSz = sz; return n; @@ -3179,6 +3199,124 @@ static u32 jsonRenderBlob( return i+n+sz; } +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + aBlob = sqlite3_value_blob(pJson); + if( (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + return sz+n==(u32)nBlob; +} + +/* Parse a single element of binary JSON into the legacy Node structure. +** Return the index of the first byte past the end of the binary JSON element. +** +** This routine is a temporary translator while the legacy Node encoding +** is still in use. It will be removed after all processing translates +** to the new BLOB encoding. +*/ +static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ + u8 t; /* Node type */ + u32 sz; /* Node size */ + u32 x; /* Index of payload start */ + + const char *zPayload; + x = jsonbPayloadSize(pParse, i, &sz); + if( x==0 ) return -1; + t = pParse->zJson[i] & 0x0f; + zPayload = &pParse->zJson[i+x]; + switch( t ){ + case JSONB_NULL: { + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + break; + } + case JSONB_TRUE: { + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + break; + } + case JSONB_FALSE: { + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + break; + } + case JSONB_INT: { + jsonParseAddNode(pParse, JSON_INT, sz, zPayload); + break; + } + case JSONB_INT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_FLOAT: { + jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); + break; + } + case JSONB_FLOAT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); + break; + } + case JSONB_TEXT: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXTJ: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_TEXT5: { + pParse->hasNonstd = 1; + jsonParseAddNode(pParse, JSON_STRING | ((JNODE_ESCAPE|JNODE_JSON5)<<8), + sz, zPayload); + break; + } + case JSONB_TEXTRAW: { + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + break; + } + case JSONB_ARRAY: { + int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + u32 j = i+x; + while( jaNode[iThis].n = pParse->nNode - (u32)iThis - 1; + break; + } + case JSONB_OBJECT: { + int iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + u32 j = i+x, k = 0; + while( jaNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; + } + j = (u32)r; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + break; + } + } + return i+x+sz; +} /**************************************************************************** ** SQL functions used for testing and debugging @@ -3289,12 +3427,23 @@ static void jsonTest1Func( ** Scalar SQL function implementations ****************************************************************************/ -/* SQL Function: jsonb_test1(TEXT_JSON) +/* SQL Function: jsonb(JSON) ** -** Parse TEXT JSON into the BLOB format and return the resulting BLOB. -** Development testing only. +** Convert the input argument into JSONB (the SQLite binary encoding of +** JSON). +** +** If the input is TEXT, or NUMERIC, try to parse it as JSON. If the fails, +** raise an error. Otherwise, return the resulting BLOB value. +** +** If the input is a BLOB, check to see if the input is a plausible +** JSONB. If it is, return it unchanged. Raise an error if it is not. +** Note that there could be internal inconsistencies in the BLOB - this +** routine does not do a full byte-for-byte validity check of the +** JSON blob. +** +** If the input is NULL, return NULL. */ -static void jsonbTest1( +static void jsonbFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv @@ -3305,20 +3454,28 @@ static void jsonbTest1( JsonParse x; UNUSED_PARAMETER(argc); - zJson = (const char*)sqlite3_value_text(argv[0]); - if( zJson==0 ) return; - nJson = sqlite3_value_bytes(argv[0]); - pParse = &x; - memset(&x, 0, sizeof(x)); - x.zJson = (char*)zJson; - x.nJson = nJson; - if( jsonParseB(pParse, ctx)==0 && pParse->aBlob!=0 ){ - sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->nBlobAlloc = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + /* No-op */ + }else if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_value(ctx, argv[0]); + }else{ + zJson = (const char*)sqlite3_value_text(argv[0]); + if( zJson==0 ) return; + nJson = sqlite3_value_bytes(argv[0]); + pParse = &x; + memset(&x, 0, sizeof(x)); + x.zJson = (char*)zJson; + x.nJson = nJson; + if( jsonParseB(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } + jsonParseReset(pParse); } - jsonParseReset(pParse); } /* SQL Function: jsonb_test2(BLOB_JSON) @@ -3351,7 +3508,7 @@ static void jsonbTest2( } /* -** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting ** double-quotes around strings and returning the unquoted string "null" ** when given a NULL input. @@ -4757,7 +4914,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1, 0, jsonTypeFunc), JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), - JFUNCTION(jsonb_test1, 1, 0, jsonbTest1), + JFUNCTION(jsonb, 1, 0, jsonbFunc), JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), From 09f23d2a368ca28d2d1cbaef0073b9cc71efd835 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 25 Sep 2023 17:14:17 +0000 Subject: [PATCH 008/191] When parsing JSON text into the BLOB format, only use node type JSONB_TEXTJ for an unquoted object label if the object label contains escape sequences. FossilOrigin-Name: a82ebbac3c542ec7f86d1e8414d7fd166db48450115ee3b26d12b5bb445f5896 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 27 ++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 3fd91c08fe..9cc27fb47c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C As\sa\stemporary\smeasure,\stry\sto\stranslate\sthe\sBLOB\sJSON\sformat\sinto\sthe\nlegacy\snode\sformat\sfor\sprocessing. -D 2023-09-25T13:23:29.913 +C When\sparsing\sJSON\stext\sinto\sthe\sBLOB\sformat,\sonly\suse\snode\stype\sJSONB_TEXTJ\nfor\san\sunquoted\sobject\slabel\sif\sthe\sobject\slabel\scontains\sescape\ssequences. +D 2023-09-25T17:14:17.033 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 23efc117c4641118c2154938decc80fb3b29b1a420bcd0eaefe24ef808d1ada4 +F src/json.c ee823133ed6f56ffb42c99fa78814667be70cd1b8106714cff459ee35f5a063e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2121,8 +2121,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b53b2e6600c324ff7864840d98a3f03896b9792fcb60b70cc1f6227b3bd4ca1 -R 31821ddd534254fa82bb41b5d469060a +P 14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f +R 0f073ae43da95216a48accf876602e9c U drh -Z 25f12fc9cea10e56c4709ee1d21954ff +Z 7639209733392bafef0d3cec4e521036 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1d3c76b2dd..93079e8a7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14f2e95a9e531ef0d3fa7f1249f23c073a50c31b2109eefc2f258cada635ac2f \ No newline at end of file +a82ebbac3c542ec7f86d1e8414d7fd166db48450115ee3b26d12b5bb445f5896 \ No newline at end of file diff --git a/src/json.c b/src/json.c index afa152624e..cfd37754f8 100644 --- a/src/json.c +++ b/src/json.c @@ -2465,6 +2465,21 @@ static void jsonBlobChangePayloadSize( } } +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !sqlite3Isxdigit(z[1]) ) return 0; + if( !sqlite3Isxdigit(z[2]) ) return 0; + if( !sqlite3Isxdigit(z[3]) ) return 0; + if( !sqlite3Isxdigit(z[4]) ) return 0; + *pOp = JSONB_TEXTJ; + return 1; +} + /* ** Parse a single JSON text value which begins at pParse->zJson[i] into ** its equivalent BLOB representation in pParse->aBlob[]. The parse is @@ -2503,23 +2518,25 @@ json_parse_restart: u32 iBlob = pParse->nBlob; x = jsonParseValueB(pParse, j); if( x<=0 ){ + int op; if( x==(-2) ){ j = pParse->iErr; if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; break; } j += json5Whitespace(&z[j]); - if( sqlite3JsonId1(z[j]) - || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) ){ int k = j+1; while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) - || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) ){ k++; } assert( iBlob==pParse->nBlob ); - jsonBlobAppendNodeType(pParse, JSONB_TEXTJ, k-j); + jsonBlobAppendNodeType(pParse, op, k-j); jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); pParse->hasNonstd = 1; x = k; @@ -3276,7 +3293,7 @@ static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ break; } case JSONB_TEXTJ: { - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); + jsonParseAddNode(pParse, JSON_STRING | (JNODE_ESCAPE<<8), sz, zPayload); break; } case JSONB_TEXT5: { From 42156fd90c65caa8f9f86a76624408f4dd8a3ef5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 26 Sep 2023 19:30:46 +0000 Subject: [PATCH 009/191] Add in many jsonb_xxxx() interfaces. Still uses the internal JsonNode representation for transformations and search, but it does at least conform to the desired API design. Largely untested. FossilOrigin-Name: e6045b4e1bf3a8e33926fc12b3c039f5e1002eaecbe277ffa82b0ec271a29d17 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 225 ++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 197 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 06d1240d09..80c8adf66b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\senhancements\sinto\sjsonb\sbranch. -D 2023-09-26T15:13:04.116 +C Add\sin\smany\sjsonb_xxxx()\sinterfaces.\s\sStill\suses\sthe\sinternal\sJsonNode\nrepresentation\sfor\stransformations\sand\ssearch,\sbut\sit\sdoes\sat\sleast\sconform\nto\sthe\sdesired\sAPI\sdesign.\s\sLargely\suntested. +D 2023-09-26T19:30:46.857 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c ee823133ed6f56ffb42c99fa78814667be70cd1b8106714cff459ee35f5a063e +F src/json.c 6fb31345d252dc6992c707ec7e826ec9f2283b026092e5105f9905664a2210c5 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2122,8 +2122,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a82ebbac3c542ec7f86d1e8414d7fd166db48450115ee3b26d12b5bb445f5896 7ad38254c37153efa72291d09800693ca60894359548eda877d59defa8c70d49 -R 2f21564c602d3b30115c3421d174134f +P ac242c4d47ec36aab1c2fa5e65e7b595e686f49473b75bd63708d05c59ce3f0f +R 3743d1571a08e262e44d6bfa0fe7cd5d U drh -Z bdae2aab159970d0c164367006c6960b +Z 4bd7fe1646cada0525f1a26ea76a3776 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4de851b60c..a8e9449ee7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac242c4d47ec36aab1c2fa5e65e7b595e686f49473b75bd63708d05c59ce3f0f \ No newline at end of file +e6045b4e1bf3a8e33926fc12b3c039f5e1002eaecbe277ffa82b0ec271a29d17 \ No newline at end of file diff --git a/src/json.c b/src/json.c index cfd37754f8..0b5068a9f0 100644 --- a/src/json.c +++ b/src/json.c @@ -159,6 +159,16 @@ static const char * const jsonType[] = { #define JNODE_LABEL 0x20 /* Is a label of an object */ #define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ +/* +** Bit values for the flags passed into jsonExtractFunc() or +** jsonSetFunc() via the user-data value. +*/ +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ + /* A single node of parsed JSON. An array of these nodes describes ** a parse of JSON + edits. @@ -847,6 +857,13 @@ static void jsonRenderNode( } } +/* Forward reference */ +static void jsonRenderNodeAsBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + JsonNode *pNode, /* The node to render */ + JsonParse *pOut /* Write the BLOB rendering of JSON here */ +); + /* ** Return a JsonNode and all its descendants as a JSON string. */ @@ -856,12 +873,22 @@ static void jsonReturnJson( sqlite3_context *pCtx, /* Return value for this function */ int bGenerateAlt /* Also store the rendered text in zAlt */ ){ + int flags; JsonString s; if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); return; } - if( pParse->nErr==0 ){ + if( pParse->nErr ){ + return; + } + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + JsonParse x; + memset(&x, 0, sizeof(x)); + jsonRenderNodeAsBlob(pParse, pNode, &x); + sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); + }else{ jsonInit(&s, pCtx); jsonRenderNode(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ @@ -1811,7 +1838,7 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ } /* Forward reference */ -static int jsonParseValueFromBinary(JsonParse *pParse, u32 i); +static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); /* ** Parse a complete JSON string. Return 0 on success or non-zero if there @@ -1830,7 +1857,7 @@ static int jsonParse( if( pParse->isBinary ){ pParse->aBlob = (u8*)pParse->zJson; pParse->nBlob = pParse->nJson; - i = jsonParseValueFromBinary(pParse, 0); + i = jsonParseValueFromBlob(pParse, 0); }else{ i = jsonParseValue(pParse, 0); } @@ -3247,7 +3274,7 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ ** is still in use. It will be removed after all processing translates ** to the new BLOB encoding. */ -static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ +static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ u8 t; /* Node type */ u32 sz; /* Node size */ u32 x; /* Index of payload start */ @@ -3310,7 +3337,7 @@ static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); u32 j = i+x; while( jaNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; @@ -3335,6 +3362,137 @@ static int jsonParseValueFromBinary(JsonParse *pParse, u32 i){ return i+x+sz; } +/* +** Convert pNode that belongs to pParse into the BLOB representation. +** The BLOB representation is added to the pOut->aBlob[] array. +*/ +static void jsonRenderNodeAsBlob( + JsonParse *pParse, /* the complete parse of the JSON */ + JsonNode *pNode, /* The node to render */ + JsonParse *pOut /* Write the BLOB rendering of JSON here */ +){ + assert( pNode!=0 ); + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ + u32 idx = (u32)(pNode - pParse->aNode); + u32 i = pParse->iSubst; + while( 1 /*exit-by-break*/ ){ + assert( inNode ); + assert( pParse->aNode[i].eType==JSON_SUBST ); + assert( pParse->aNode[i].eU==4 ); + assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ + pNode = &pParse->aNode[i+1]; + break; + } + i = pParse->aNode[i].u.iPrev; + } + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonBlobAppendNodeType(pOut, JSONB_NULL, 0); + break; + } + case JSON_TRUE: { + jsonBlobAppendNodeType(pOut, JSONB_TRUE, 0); + break; + } + case JSON_FALSE: { + jsonBlobAppendNodeType(pOut, JSONB_FALSE, 0); + break; + } + case JSON_STRING: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_RAW ){ + if( memchr(pNode->u.zJContent, '"', pNode->n)==0 + && memchr(pNode->u.zJContent, '\\', pNode->n)==0 + ){ + op = JSONB_TEXT; + }else{ + op = JSONB_TEXTRAW; + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_TEXT5; + }else{ + op = JSONB_TEXTJ; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_REAL: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_FLOAT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_FLOAT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_INT: { + int op; + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + op = JSONB_INT5; + }else{ + assert( pNode->n>0 ); + op = JSONB_INT; + } + jsonBlobAppendNodeType(pOut, op, pNode->n); + jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_ARRAY, 0x7fffffff); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + case JSON_OBJECT: { + u32 j = 1; + u32 iStart, iThis = pOut->nBlob; + jsonBlobAppendNodeType(pOut, JSONB_OBJECT, 0x7fffffff); + iStart = pOut->nBlob; + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); + jsonRenderNodeAsBlob(pParse, &pNode[j+1], pOut); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pNode->eU==2 ); + pNode = &pParse->aNode[pNode->u.iAppend]; + j = 1; + } + jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); + break; + } + } +} + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ @@ -3614,15 +3772,6 @@ static void jsonArrayLengthFunc( sqlite3_result_int64(ctx, n); } -/* -** Bit values for the flags passed into jsonExtractFunc() or -** jsonSetFunc() via the user-data value. -*/ -#define JSON_JSON 0x01 /* Result is always JSON */ -#define JSON_SQL 0x02 /* Result is always SQL */ -#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ -#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ - /* ** json_extract(JSON, PATH, ...) ** "->"(JSON,PATH) @@ -4913,26 +5062,32 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), - JFUNCTION(jsonb, 1, 0, jsonbFunc), - JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), + JFUNCTION(json, 1, 0, jsonRemoveFunc), + JFUNCTION(json_array, -1, 0, jsonArrayFunc), + JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1, 0, jsonErrorFunc), + JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1, 0, jsonSetFunc), + JFUNCTION(json_object, -1, 0, jsonObjectFunc), + JFUNCTION(json_patch, 2, 0, jsonPatchFunc), + JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), + JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), + JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), + JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1, 0, jsonTypeFunc), + JFUNCTION(json_type, 2, 0, jsonTypeFunc), + JFUNCTION(json_valid, 1, 0, jsonValidFunc), + JFUNCTION(jsonb, 1, JSON_BLOB, jsonbFunc), + JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), + JFUNCTION(jsonb_insert, -1, JSON_BLOB, jsonSetFunc), + JFUNCTION(jsonb_patch, 2, JSON_BLOB, jsonPatchFunc), + JFUNCTION(jsonb_quote, 1, JSON_BLOB, jsonQuoteFunc), + JFUNCTION(jsonb_remove, -1, JSON_BLOB, jsonRemoveFunc), + JFUNCTION(jsonb_replace, -1, JSON_BLOB, jsonReplaceFunc), + JFUNCTION(jsonb_set, -1, JSON_ISSET|JSON_BLOB, jsonSetFunc), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), From 733da8d31a58930aae98a139a7aa58ec4b08a141 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 27 Sep 2023 16:55:13 +0000 Subject: [PATCH 010/191] Performance tests for JSONB added. FossilOrigin-Name: 7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d --- manifest | 13 +++++++------ manifest.uuid | 2 +- test/json/json-q1-b.txt | 25 +++++++++++++++++++++++++ test/json/json-speed-check.sh | 18 ++++++++++++++---- 4 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 test/json/json-q1-b.txt diff --git a/manifest b/manifest index fd4fd00030..b1cbae3a58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\spartial-index\sconstant\svalue\sfixes\sfrom\strunk\sinto\sthe\sjsonb\sbranch. -D 2023-09-26T19:46:38.407 +C Performance\stests\sfor\sJSONB\sadded. +D 2023-09-27T16:55:13.845 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1300,8 +1300,9 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f +F test/json/json-q1-b.txt d1394d4ade1c9617539b19b48e0dd2df4f6ea918860978722e7a97d60618ca83 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 -F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x +F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test dc9d5a2a5b1fd1b54dbd71c538b17933cc98d84b4c1f821ead754933663dca55 F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe @@ -2122,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6045b4e1bf3a8e33926fc12b3c039f5e1002eaecbe277ffa82b0ec271a29d17 f459d0806cf044fd07743e4c91d0a5a6ddf45b3b41004bde4278f190d99a4cf5 -R 1d55abb00dbf638a471cf92ac6ee2784 +P 700bdbd7383f66a0da675c197204da4e7b6ed757155145ee98d572de32a5d0ae +R cd229cd0127bc955a2b70dc896603e55 U drh -Z 2f2ccec484af750dd462546f4baf8eb1 +Z 76a0e61dc557712a0090ea7a119c4cd2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0d71c6abf..ccac07358c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -700bdbd7383f66a0da675c197204da4e7b6ed757155145ee98d572de32a5d0ae \ No newline at end of file +7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d \ No newline at end of file diff --git a/test/json/json-q1-b.txt b/test/json/json-q1-b.txt new file mode 100644 index 0000000000..c86a2db44f --- /dev/null +++ b/test/json/json-q1-b.txt @@ -0,0 +1,25 @@ +.mode qbox +.timer on +.param set $label 'q87' +SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL; + +CREATE TEMP TABLE t2(x JSON TEXT); +WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<25000), + array1(y) AS ( + SELECT json_group_array( + json_object('x',x,'y',random(),'z',hex(randomblob(50))) + ) + FROM c + ), + c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5) +INSERT INTO t2(x) + SELECT jsonb(json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6)) + FROM array1, c2; +CREATE INDEX t2x1 ON t2(x->>'a'); +CREATE INDEX t2x2 ON t2(x->>'b'); +CREATE INDEX t2x3 ON t2(x->>'e'); +CREATE INDEX t2x4 ON t2(x->>'f'); +UPDATE t2 SET x=jsonb_replace(x,'$.f',(x->>'f')+1); +UPDATE t2 SET x=jsonb_set(x,'$.e',(x->>'f')-1); +UPDATE t2 SET x=jsonb_remove(x,'$.d'); diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh index f7e7f1be58..c57bfecd7a 100755 --- a/test/json/json-speed-check.sh +++ b/test/json/json-speed-check.sh @@ -40,6 +40,7 @@ doCachegrind=1 doVdbeProfile=0 doWal=1 doDiff=1 +doJsonB=0 while test "$1" != ""; do case $1 in --nodiff) @@ -54,6 +55,9 @@ while test "$1" != ""; do --gcc7) CC=gcc-7 ;; + --jsonb) + doJsonB=1 + ;; -*) CC_OPTS="$CC_OPTS $1" ;; @@ -69,12 +73,18 @@ rm -f cachegrind.out.* jsonshell $CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread ls -l jsonshell | tee -a summary-$NAME.txt home=`echo $0 | sed -e 's,/[^/]*$,,'` -echo ./jsonshell json100mb.db "<$home/json-q1.txt" -valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \ - 2>&1 | tee -a summary-$NAME.txt +if test $doJsonB -eq 1; then + echo ./jsonshell json100mb_b.db "<$home/json-q1-b.txt" + valgrind --tool=cachegrind ./jsonshell json100mb_b.db <$home/json-q1-b.txt \ + 2>&1 | tee -a summary-$NAME.txt +else + echo ./jsonshell json100mb.db "<$home/json-q1.txt" + valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \ + 2>&1 | tee -a summary-$NAME.txt +fi cg_anno.tcl cachegrind.out.* >jout-$NAME.txt echo '*****************************************************' >>jout-$NAME.txt sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt -if test "$NAME" != "$BASELINE"; then +if test "$NAME" != "$BASELINE" -a $doDiff -ne 0; then fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt fi From 1854837b5ad3d13058e5b027f0fa003bfbf19342 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 10:20:56 +0000 Subject: [PATCH 011/191] Work toward getting json_extract() to operate directly on the BLOB, omitting the translation into a JsonNode array. FossilOrigin-Name: c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 --- manifest | 17 +- manifest.uuid | 2 +- src/json.c | 333 ++++++++++++++++++++++++++++++++++++++++ test/json/json-q1-b.txt | 2 +- 4 files changed, 345 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b1cbae3a58..0ab3584199 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\stests\sfor\sJSONB\sadded. -D 2023-09-27T16:55:13.845 +C Work\stoward\sgetting\sjson_extract()\sto\soperate\sdirectly\son\sthe\sBLOB,\somitting\nthe\stranslation\sinto\sa\sJsonNode\sarray. +D 2023-09-28T10:20:56.267 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6fb31345d252dc6992c707ec7e826ec9f2283b026092e5105f9905664a2210c5 +F src/json.c 1f174d19f143e4968ae391de3d504563f320c6dbc2ba437231b40f19631a6dc4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1300,7 +1300,7 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f -F test/json/json-q1-b.txt d1394d4ade1c9617539b19b48e0dd2df4f6ea918860978722e7a97d60618ca83 +F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test dc9d5a2a5b1fd1b54dbd71c538b17933cc98d84b4c1f821ead754933663dca55 @@ -2123,8 +2123,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 700bdbd7383f66a0da675c197204da4e7b6ed757155145ee98d572de32a5d0ae -R cd229cd0127bc955a2b70dc896603e55 +P 7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d +R b9d3bc6ee4123d23cc875353cfd5e9aa +T *branch * jsonb-direct-extract +T *sym-jsonb-direct-extract * +T -sym-jsonb * U drh -Z 76a0e61dc557712a0090ea7a119c4cd2 +Z 3a4258d315e3af636c1707e0dba68e38 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ccac07358c..066de478dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d \ No newline at end of file +c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0b5068a9f0..a6b91e76eb 100644 --- a/src/json.c +++ b/src/json.c @@ -3493,6 +3493,335 @@ static void jsonRenderNodeAsBlob( } } +/* +** Error returns from jsonLookupBlobStep() +*/ +#define JSON_BLOB_ERROR 0xffffffff +#define JSON_BLOB_NOTFOUND 0xfffffffe +#define JSON_BLOB_PATHERROR 0xfffffffd + +/* +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. +** +** Return JSON_BLOB_NOTFOUND if no such element exists. +*/ +static u32 jsonLookupBlobStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ + const char *zPath, /* The path to search */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, k, nKey, sz, n, iEnd; + const char *zKey; + u8 x; + + if( pParse->oom ) return 0; + if( zPath[0]==0 ) return iRoot; + if( zPath[0]=='.' ){ + x = pParse->aBlob[iRoot]; + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + testcase( nKey==0 ); + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + } + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; + iEnd = j+sz; + while( jaBlob[j] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + k = j+n; + if( k+sz>=iEnd ) return JSON_BLOB_ERROR; + if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( j+n+sz>iEnd ) return JSON_BLOB_ERROR; + return jsonLookupBlobStep(pParse, j, &zPath[i], pzErr); + } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + j += n+sz; + } + if( j>iEnd ) return JSON_BLOB_ERROR; + }else if( zPath[0]=='[' ){ + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; + } + if( i<2 || zPath[i]!=']' ){ +#if 0 + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + j += jsonNodeSize(&pBase[j]); + } + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + if( pParse->useMod==0 ) break; + assert( pBase->eU==2 ); + iBase = pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( sqlite3Isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + }else{ + *pzErr = zPath; + return 0; + } +#endif + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot+n; + iEnd = j+sz; + while( jiEnd ) return JSON_BLOB_ERROR; + }else{ + *pzErr = zPath; + return JSON_BLOB_PATHERROR; + } + return JSON_BLOB_NOTFOUND; +} + +/* +** +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx /* Return value for this function */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x = (char)pParse->aBlob[i+n]; + if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } + else if( x=='+' && ALWAYS(sz>0) ){ n++; sz--; } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) return; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3_malloc( nOut+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(iIn=iOut=0; iIn>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else{ + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + } + continue; + }else if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ + /* pass through unchanged */ + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); + iIn += 2; + }else if( c=='\r' && z[i+1]=='\n' ){ + iIn++; + continue; + }else if( 0xe2==(u8)c ){ + assert( 0x80==(u8)z[i+1] ); + assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + iIn += 2; + continue; + }else{ + continue; + } + } /* end if( c=='\\' ) */ + zOut[iOut++] = c; + } /* end for() */ + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + sqlite3_result_blob(pCtx, &pParse->aBlob[i+n], sz, SQLITE_TRANSIENT); + break; + } + } +} + +/* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. +*/ +static void jsonExtractFromBlob( + sqlite3_context *ctx, + sqlite3_value *pJson, + sqlite3_value *pPath, + int flags +){ + const char *zPath = (const char*)sqlite3_value_text(pPath); + const char *zErr = 0; + u32 i; + JsonParse px; + if( zPath==0 ) return; + if( zPath[0]=='$' ) zPath++; + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(pJson); + px.aBlob = (u8*)sqlite3_value_blob(pJson); + if( px.aBlob==0 ) return; + i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + if( i>$label FROM data1 WHERE x->>$label IS NOT NULL; CREATE TEMP TABLE t2(x JSON TEXT); From 2dc60ec57fa75fa659c5c5c66d702eeaaa3347fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 15:56:35 +0000 Subject: [PATCH 012/191] Improvements to json_extract() to better support JSONB. Still not 100% working. FossilOrigin-Name: 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 49 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 0ab3584199..d05d5a4eac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\sgetting\sjson_extract()\sto\soperate\sdirectly\son\sthe\sBLOB,\somitting\nthe\stranslation\sinto\sa\sJsonNode\sarray. -D 2023-09-28T10:20:56.267 +C Improvements\sto\sjson_extract()\sto\sbetter\ssupport\sJSONB.\sStill\snot\s100%\sworking. +D 2023-09-28T15:56:35.608 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 1f174d19f143e4968ae391de3d504563f320c6dbc2ba437231b40f19631a6dc4 +F src/json.c c9d70a47aabc97b51cd21a33be7602db4760a980920b44b497dbb331a91c12d8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,11 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c1be8e361db87458ac9d8fcee080c2b558936539c852bb80f0f7941d61bf15d -R b9d3bc6ee4123d23cc875353cfd5e9aa -T *branch * jsonb-direct-extract -T *sym-jsonb-direct-extract * -T -sym-jsonb * +P c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 +R df67f115b3402f37b4410f24659e50ec U drh -Z 3a4258d315e3af636c1707e0dba68e38 +Z a8b3beeec30b8fa8de8cb3c4ca40d323 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 066de478dc..a7573f00b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 \ No newline at end of file +8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a6b91e76eb..33c9404105 100644 --- a/src/json.c +++ b/src/json.c @@ -3634,7 +3634,34 @@ static u32 jsonLookupBlobStep( } /* -** +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. +*/ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob +){ + JsonParse x; + JsonString s; + + if( aBlob==0 ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonInit(&s, ctx); + jsonRenderBlob(&x, 0, &s); + jsonResult(&s); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. */ static void jsonReturnFromBlob( JsonParse *pParse, /* Complete JSON parse tree */ @@ -3785,7 +3812,12 @@ static void jsonReturnFromBlob( } case JSONB_ARRAY: case JSONB_OBJECT: { - sqlite3_result_blob(pCtx, &pParse->aBlob[i+n], sz, SQLITE_TRANSIENT); + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } break; } } @@ -3992,23 +4024,13 @@ static void jsonbTest2( int argc, sqlite3_value **argv ){ - JsonParse *pParse; const u8 *aBlob; int nBlob; - JsonParse x; - JsonString s; UNUSED_PARAMETER(argc); aBlob = (const u8*)sqlite3_value_blob(argv[0]); - if( aBlob==0 ) return; nBlob = sqlite3_value_bytes(argv[0]); - pParse = &x; - memset(&x, 0, sizeof(x)); - x.aBlob = (u8*)aBlob; - x.nBlob = nBlob; - jsonInit(&s, ctx); - jsonRenderBlob(pParse, 0, &s); - jsonResult(&s); + jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); } /* @@ -5401,6 +5423,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1, 0, jsonErrorFunc), JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1, JSON_BLOB, jsonExtractFunc), JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), JFUNCTION(json_insert, -1, 0, jsonSetFunc), From 59862e6d224f7bdbd368cc2827ea06a3a2c82d29 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 17:07:43 +0000 Subject: [PATCH 013/191] Miscellaneous bugs fixed. FossilOrigin-Name: 5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 22 ++++++++++++++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d05d5a4eac..c1019d925c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sjson_extract()\sto\sbetter\ssupport\sJSONB.\sStill\snot\s100%\sworking. -D 2023-09-28T15:56:35.608 +C Miscellaneous\sbugs\sfixed. +D 2023-09-28T17:07:43.443 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c9d70a47aabc97b51cd21a33be7602db4760a980920b44b497dbb331a91c12d8 +F src/json.c 946e0bbc5070db333a370e2f7b806ad07d2e9fb93d057cf52a9e5fb58fd43768 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1feba70f55a8e5f4696d48e4706855415d173ac8ac3c2656787c242a883b4f5 -R df67f115b3402f37b4410f24659e50ec +P 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 +R e6233e4f5d16d81e42c25a8ff039c998 U drh -Z a8b3beeec30b8fa8de8cb3c4ca40d323 +Z beeeb6a9ecf43431f000f927a20c9a15 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a7573f00b7..5bf7e31c95 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 \ No newline at end of file +5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 33c9404105..322fbc5602 100644 --- a/src/json.c +++ b/src/json.c @@ -518,6 +518,14 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + char *zBuf = sqlite3_malloc64( N+1 ); + if( zBuf==0 ){ + p->bErr = 1; + return; + } + memcpy(zBuf, zIn, N); + zBuf[N] = 0; + zIn = zBuf; if( zIn[0]=='+' ){ zIn++; N--; @@ -539,6 +547,7 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ } assert( N>0 ); jsonAppendRawNZ(p, zIn, N); + sqlite3_free(zBuf); } /* @@ -958,12 +967,16 @@ static void jsonReturn( int rc; int bNeg = 0; const char *z; + char *zz; + sqlite3 *db = sqlite3_context_db_handle(pCtx); assert( pNode->eU==1 ); - z = pNode->u.zJContent; + zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); + z = zz; if( z[0]=='-' ){ z++; bNeg = 1; } else if( z[0]=='+' ){ z++; } rc = sqlite3DecOrHexToI64(z, &i); + sqlite3DbFree(db, zz); if( rc<=1 ){ sqlite3_result_int64(pCtx, bNeg ? -i : i); }else if( rc==3 && bNeg ){ @@ -1862,7 +1875,7 @@ static int jsonParse( i = jsonParseValue(pParse, 0); } if( pParse->oom ) i = -1; - if( i>0 ){ + if( !pParse->isBinary && i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; if( zJson[i] ){ @@ -3359,6 +3372,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ break; } } + pParse->aBlob[i] = 0; return i+x+sz; } @@ -3450,7 +3464,7 @@ static void jsonRenderNodeAsBlob( case JSON_ARRAY: { u32 j = 1; u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_ARRAY, 0x7fffffff); + jsonBlobAppendNodeType(pOut, JSONB_ARRAY, pParse->nJson*2); iStart = pOut->nBlob; for(;;){ while( j<=pNode->n ){ @@ -3471,7 +3485,7 @@ static void jsonRenderNodeAsBlob( case JSON_OBJECT: { u32 j = 1; u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_OBJECT, 0x7fffffff); + jsonBlobAppendNodeType(pOut, JSONB_OBJECT, pParse->nJson*2); iStart = pOut->nBlob; for(;;){ while( j<=pNode->n ){ From 6b1db922286ac88505854e93ce017e11e4b18cb5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 17:23:46 +0000 Subject: [PATCH 014/191] Extract directly from BLOB is now complete and appears to work. FossilOrigin-Name: 3de58ec99444b16dfcda1e226420e2343450b77abd3faf33a88b6d18339ef17c --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 61 ++++++++++++++++++++++++--------------------------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index c1019d925c..f6be12b744 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Miscellaneous\sbugs\sfixed. -D 2023-09-28T17:07:43.443 +C Extract\sdirectly\sfrom\sBLOB\sis\snow\scomplete\sand\sappears\sto\swork. +D 2023-09-28T17:23:46.244 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 946e0bbc5070db333a370e2f7b806ad07d2e9fb93d057cf52a9e5fb58fd43768 +F src/json.c 7e68346bfc4f030e454c68751ab7082af52b9925ff28bff88eef03258e486d7c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c82576176539c4d132b14d46adbf31366c4bcaa59a61dd639dc9cc308fe8825 -R e6233e4f5d16d81e42c25a8ff039c998 +P 5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b +R 0d79abb184ac0c9db2e32b0ee612448c U drh -Z beeeb6a9ecf43431f000f927a20c9a15 +Z dfb6861bdfa5add7990cbc313b482bb0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5bf7e31c95..6de749eb76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c0815fa2e422d81198a43a2c04a022e319fcbcadfd4be4437f2e663892ca26b \ No newline at end of file +3de58ec99444b16dfcda1e226420e2343450b77abd3faf33a88b6d18339ef17c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 322fbc5602..e43240c71c 100644 --- a/src/json.c +++ b/src/json.c @@ -3507,6 +3507,21 @@ static void jsonRenderNodeAsBlob( } } +/* +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. +*/ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; iiEnd ) return JSON_BLOB_ERROR; }else if( zPath[0]=='[' ){ + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ @@ -3586,49 +3604,28 @@ static u32 jsonLookupBlobStep( i++; } if( i<2 || zPath[i]!=']' ){ -#if 0 if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pBase->eU==2 ); - iBase = pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; + k = jsonbArrayCount(pParse, iRoot); + i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ unsigned int x = 0; - j = 3; + i = 3; do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( sqlite3Isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; + x = x*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( x>k ) return 0; + k -= x; } - if( zPath[j]!=']' ){ + if( zPath[i]!=']' ){ *pzErr = zPath; - return 0; + return JSON_BLOB_PATHERROR; } }else{ *pzErr = zPath; - return 0; + return JSON_BLOB_PATHERROR; } -#endif - *pzErr = zPath; - return JSON_BLOB_PATHERROR; } - x = pParse->aBlob[iRoot] & 0x0f; - if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; - n = jsonbPayloadSize(pParse, iRoot, &sz); j = iRoot+n; iEnd = j+sz; while( j Date: Thu, 28 Sep 2023 18:23:52 +0000 Subject: [PATCH 015/191] Allow the sqlite3_user_data() function to be invoked with a NULL argument or with an sqlite3_context pointer from a virtual table. It returns NULL in both cases. FossilOrigin-Name: 2f49687371ada65fef374336c28b352c48ab98dc31282ac82397035efe04ba11 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9246cdfdde..6ac2aacfe0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthe\sjsonb\sbranch\sfor\sthe\scompiler\swarning\sfixes\nfrom\stwo\sdays\sago. -D 2023-09-28T17:41:45.378 +C Allow\sthe\ssqlite3_user_data()\sfunction\sto\sbe\sinvoked\swith\sa\sNULL\sargument\nor\swith\san\ssqlite3_context\spointer\sfrom\sa\svirtual\stable.\s\sIt\sreturns\sNULL\nin\sboth\scases. +D 2023-09-28T18:23:52.254 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -783,7 +783,7 @@ F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c cd112eb00d20fc5cc44f631d0e713838602637328b0f127c2f3c2aa8cea3cc91 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c -F src/vdbeapi.c 4184402246172220418c0ef49ff4cf1a19ced9a4ac6c843c2f0773fb5c543f37 +F src/vdbeapi.c 7fdd1eec6b967a5455ccc45a43b84fd32746d6749a423104333f9ba5cec56c9f F src/vdbeaux.c 5b415e09b5b9d5be6c0f4fcbf18ea9d7d16f6a29ced2f14a3b2041020f63e9c1 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c 317b9f48708139db6239ade40c7980b4bc8233168383690d588dad6d8437f722 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5853065b7ee326c04ddfcde64c178f487af04cd3adc3cc99f559907484ec169d b488b9fb71652eca90d2bf73d32f3d748badf517859dc833c58e021b0e017194 -R 9fc8b28e091a22aa2df6890c9cd30c84 +P cee113cc315b04fd75ccc172cf4529bf15b2050bf274433496c31a282e281ab8 +R 447caeefd53e356f21199c52281cba5f U drh -Z 296260cf4c2437a82708d346d0a00f10 +Z 98132ac7eca8ce40c36975540dff67b4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8230cfa03a..f38a3e887b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cee113cc315b04fd75ccc172cf4529bf15b2050bf274433496c31a282e281ab8 \ No newline at end of file +2f49687371ada65fef374336c28b352c48ab98dc31282ac82397035efe04ba11 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 1213dbe6d1..6b32506f9f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -845,8 +845,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ ** pointer to it. */ void *sqlite3_user_data(sqlite3_context *p){ - assert( p && p->pFunc ); - return p->pFunc->pUserData; + return (p && p->pFunc) ? p->pFunc->pUserData : 0; } /* From 8cdab142863a98b84baf047ce443ee37ed141ab1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 28 Sep 2023 19:11:36 +0000 Subject: [PATCH 016/191] Fix some minor memory issues so that all legacy tests now pass. FossilOrigin-Name: 1744bfc669346ff221f28d45fd978863e876a2d2f0b82bcf0e5ee6f0326900cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 10 +++++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6ac2aacfe0..5be61e1831 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\ssqlite3_user_data()\sfunction\sto\sbe\sinvoked\swith\sa\sNULL\sargument\nor\swith\san\ssqlite3_context\spointer\sfrom\sa\svirtual\stable.\s\sIt\sreturns\sNULL\nin\sboth\scases. -D 2023-09-28T18:23:52.254 +C Fix\ssome\sminor\smemory\sissues\sso\sthat\sall\slegacy\stests\snow\spass. +D 2023-09-28T19:11:36.692 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7e68346bfc4f030e454c68751ab7082af52b9925ff28bff88eef03258e486d7c +F src/json.c 6ecde7714aed5238881b936edc3f9bf85c2ece02f2b1a41e10f5d5f3e8f89513 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cee113cc315b04fd75ccc172cf4529bf15b2050bf274433496c31a282e281ab8 -R 447caeefd53e356f21199c52281cba5f +P 2f49687371ada65fef374336c28b352c48ab98dc31282ac82397035efe04ba11 +R 3d149364fb767414041d170ea764743b U drh -Z 98132ac7eca8ce40c36975540dff67b4 +Z 6ffc9f7e51690a482c053773cfbc0f4b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f38a3e887b..fe5f150f03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f49687371ada65fef374336c28b352c48ab98dc31282ac82397035efe04ba11 \ No newline at end of file +1744bfc669346ff221f28d45fd978863e876a2d2f0b82bcf0e5ee6f0326900cc \ No newline at end of file diff --git a/src/json.c b/src/json.c index e43240c71c..3d44489e6f 100644 --- a/src/json.c +++ b/src/json.c @@ -543,10 +543,10 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ assert( rc==2 ); jsonAppendRawNZ(p, "9.0e999", 7); } - return; + }else{ + assert( N>0 ); + jsonAppendRawNZ(p, zIn, N); } - assert( N>0 ); - jsonAppendRawNZ(p, zIn, N); sqlite3_free(zBuf); } @@ -972,6 +972,10 @@ static void jsonReturn( assert( pNode->eU==1 ); zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); + if( zz==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } z = zz; if( z[0]=='-' ){ z++; bNeg = 1; } else if( z[0]=='+' ){ z++; } From ecce6022d5c198b541ae74a0174f9949f7df5376 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 29 Sep 2023 11:17:43 +0000 Subject: [PATCH 017/191] Describe the JSONB encoding in a header comment to the json.c source file. FossilOrigin-Name: 1c0cba3461d6111b3aeb77726880221f1240355f0b57e060febbdeb12fb688c0 --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 5be61e1831..2b9bd7fd66 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sminor\smemory\sissues\sso\sthat\sall\slegacy\stests\snow\spass. -D 2023-09-28T19:11:36.692 +C Describe\sthe\sJSONB\sencoding\sin\sa\sheader\scomment\sto\sthe\sjson.c\ssource\sfile. +D 2023-09-29T11:17:43.227 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6ecde7714aed5238881b936edc3f9bf85c2ece02f2b1a41e10f5d5f3e8f89513 +F src/json.c 6d2643118360760d589829d5a10d612a0270729ce11b303553502eae70c8f899 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2f49687371ada65fef374336c28b352c48ab98dc31282ac82397035efe04ba11 -R 3d149364fb767414041d170ea764743b +P 1744bfc669346ff221f28d45fd978863e876a2d2f0b82bcf0e5ee6f0326900cc +R 88ae9ef60dd50edd212e3ca11b3078d3 U drh -Z 6ffc9f7e51690a482c053773cfbc0f4b +Z 174eaa9aa981071dd243f3ac21aacaae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fe5f150f03..a42d3e4da1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1744bfc669346ff221f28d45fd978863e876a2d2f0b82bcf0e5ee6f0326900cc \ No newline at end of file +1c0cba3461d6111b3aeb77726880221f1240355f0b57e060febbdeb12fb688c0 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3d44489e6f..393e451663 100644 --- a/src/json.c +++ b/src/json.c @@ -15,11 +15,105 @@ ** This file began as an extension in ext/misc/json1.c in 2015. That ** extension proved so useful that it has now been moved into the core. ** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.44.0 (pending), these routines also accept +** BLOB values that have JSON encoded using a binary representation we +** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk +** format SQLite JSONB is completely different and incompatible with +** PostgreSQL JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. Roughtly speaking, though, a JSONB blob +** and the equivalent RFC-8259 text string take up the same amount of space +** on disk. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text literal with RFC-8259 escape codes +** 9: TEXT5 -- Text literal with JSON5 and RFC-8259 escapes +** 10: TEXTRAW -- Text literal with unescaped ', ", or \ characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. */ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" From ae5f55e2271d0fd6788a76dc00f61947dcca68d7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 29 Sep 2023 12:45:14 +0000 Subject: [PATCH 018/191] Improvements to comments and procedure names for clarity in the JSON implementation. FossilOrigin-Name: 9b620d813ef483f1277c1683c5e926a882f07f3b90804dea0c91b325ff8e45a4 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 285 +++++++++++++++++++++++++++----------------------- 3 files changed, 161 insertions(+), 138 deletions(-) diff --git a/manifest b/manifest index 2b9bd7fd66..7fcd2bd9c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Describe\sthe\sJSONB\sencoding\sin\sa\sheader\scomment\sto\sthe\sjson.c\ssource\sfile. -D 2023-09-29T11:17:43.227 +C Improvements\sto\scomments\sand\sprocedure\snames\sfor\sclarity\sin\sthe\sJSON\nimplementation. +D 2023-09-29T12:45:14.794 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6d2643118360760d589829d5a10d612a0270729ce11b303553502eae70c8f899 +F src/json.c 92d7c2ea8db842cd6a7a7b664bf43d5ae75e097981f001a5ab3860bd222132ca F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1744bfc669346ff221f28d45fd978863e876a2d2f0b82bcf0e5ee6f0326900cc -R 88ae9ef60dd50edd212e3ca11b3078d3 +P 1c0cba3461d6111b3aeb77726880221f1240355f0b57e060febbdeb12fb688c0 +R d73ce77d34a070de20c2504f2815e166 U drh -Z 174eaa9aa981071dd243f3ac21aacaae +Z 100adbc30a124e4f2d5cc09bea6ade0f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a42d3e4da1..7f67a4ccc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c0cba3461d6111b3aeb77726880221f1240355f0b57e060febbdeb12fb688c0 \ No newline at end of file +9b620d813ef483f1277c1683c5e926a882f07f3b90804dea0c91b325ff8e45a4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 393e451663..97a6aebf4e 100644 --- a/src/json.c +++ b/src/json.c @@ -33,9 +33,8 @@ ** for delimiters using persnickety syntax rules. JSONB seems to be about ** 3x faster than text JSON as a result. JSONB is also tends to be slightly ** smaller than text JSON, by 5% or 10%, but there are corner cases where -** JSONB can be slightly larger. Roughtly speaking, though, a JSONB blob -** and the equivalent RFC-8259 text string take up the same amount of space -** on disk. +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. ** ** ** THE JSONB ENCODING: @@ -55,9 +54,9 @@ ** 5: FLOAT -- RFC-8259 floating point literal ** 6: FLOAT5 -- JSON5 floating point literal ** 7: TEXT -- Text literal acceptable to both SQL and JSON -** 8: TEXTJ -- Text literal with RFC-8259 escape codes -** 9: TEXT5 -- Text literal with JSON5 and RFC-8259 escapes -** 10: TEXTRAW -- Text literal with unescaped ', ", or \ characters +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters ** 11: ARRAY ** 12: OBJECT ** @@ -118,10 +117,26 @@ #ifndef SQLITE_OMIT_JSON #include "sqliteInt.h" +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, @@ -168,11 +183,12 @@ static const char jsonIsOk[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - +/* Put code used only for testing inside the JSON_VVA() macro. +*/ #if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define VVA(X) +# define JSON_VVA(X) #else -# define VVA(X) X +# define JSON_VVA(X) X #endif /* Objects */ @@ -204,7 +220,7 @@ struct JsonCleanup { void *pArg; /* Argument to xOp() */ }; -/* JSON type values +/* JSON type values for JsonNode.eType */ #define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ #define JSON_NULL 1 @@ -216,33 +232,18 @@ struct JsonCleanup { #define JSON_ARRAY 7 #define JSON_OBJECT 8 -/* JSON BLOB node types -*/ -#define JSONB_NULL 0 /* "null" */ -#define JSONB_TRUE 1 /* "true" */ -#define JSONB_FALSE 2 /* "false" */ -#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ -#define JSONB_INT5 4 /* integer in 0x000 notation */ -#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ -#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ -#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ -#define JSONB_TEXTJ 8 /* Text with JSON escapes */ -#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ -#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ -#define JSONB_ARRAY 11 /* An array */ -#define JSONB_OBJECT 12 /* An object */ - -/* The "subtype" set for JSON values */ -#define JSON_SUBTYPE 74 /* Ascii for "J" */ - -/* -** Names of the various JSON types: +/* Human-readalbe names for the JsonNode types: */ static const char * const jsonType[] = { "subst", "null", "true", "false", "integer", "real", "text", "array", "object" }; +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* Bit values for the JsonNode.jnFlag field */ #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ @@ -350,9 +351,10 @@ struct JsonParse { ** Utility routines for dealing with JsonString objects **************************************************************************/ -/* Set the JsonString object to an empty string +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. */ -static void jsonZero(JsonString *p){ +static void jsonStringZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; @@ -361,39 +363,39 @@ static void jsonZero(JsonString *p){ /* Initialize the JsonString object */ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; p->bErr = 0; - jsonZero(p); + jsonStringZero(p); } /* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(JsonString *p){ +static void jsonStringReset(JsonString *p){ if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); - jsonZero(p); + jsonStringZero(p); } /* Report an out-of-memory (OOM) condition */ -static void jsonOom(JsonString *p){ +static void jsonStringOom(JsonString *p){ p->bErr = 1; sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); + jsonStringReset(p); } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(JsonString *p, u32 N){ +static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ if( p->bErr ) return 1; zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ - jsonOom(p); + jsonStringOom(p); return SQLITE_NOMEM; } memcpy(zNew, p->zBuf, (size_t)p->nUsed); @@ -403,7 +405,7 @@ static int jsonGrow(JsonString *p, u32 N){ p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); if( p->zBuf==0 ){ p->bErr = 1; - jsonZero(p); + jsonStringZero(p); return SQLITE_NOMEM; } } @@ -413,20 +415,20 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static SQLITE_NOINLINE void jsonAppendExpand( +static SQLITE_NOINLINE void jsonStringExpandAndAppend( JsonString *p, const char *zIn, u32 N ){ assert( N>0 ); - if( jsonGrow(p,N) ) return; + if( jsonStringGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -435,7 +437,7 @@ static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ assert( N>0 ); if( N+p->nUsed >= p->nAlloc ){ - jsonAppendExpand(p,zIn,N); + jsonStringExpandAndAppend(p,zIn,N); }else{ memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -447,7 +449,7 @@ static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ */ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; va_start(ap, zFormat); sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); va_end(ap); @@ -457,7 +459,7 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ - if( jsonGrow(p,1) ) return; + if( jsonStringGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } static void jsonAppendChar(JsonString *p, char c){ @@ -480,7 +482,7 @@ static int jsonForceRCStr(JsonString *p){ if( p->bStatic==0 ) return 1; p->nAlloc = 0; p->nUsed++; - jsonGrow(p, p->nUsed); + jsonStringGrow(p, p->nUsed); p->nUsed--; return p->bStatic==0; } @@ -504,7 +506,8 @@ static void jsonAppendSeparator(JsonString *p){ */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 i; - if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; + if( zIn==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; for(i=0; izBuf[p->nUsed++] = c; }else if( c=='"' || c=='\\' ){ json_simple_escape: - if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + if( (p->nUsed+N+3-i > p->nAlloc) && jsonStringGrow(p,N+3-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ @@ -533,7 +536,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ c = aSpecial[c]; goto json_simple_escape; } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + if( (p->nUsed+N+7+i > p->nAlloc) && jsonStringGrow(p,N+7-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = 'u'; p->zBuf[p->nUsed++] = '0'; @@ -680,10 +683,10 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ /* -** Append a function parameter value to the JSON string under -** construction. +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. */ -static void jsonAppendValue( +static void jsonAppendSqlValue( JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ @@ -716,7 +719,7 @@ static void jsonAppendValue( if( p->bErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->bErr = 2; - jsonReset(p); + jsonStringReset(p); } break; } @@ -724,11 +727,12 @@ static void jsonAppendValue( } -/* Make the JSON in p the result of the SQL function. +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. ** -** The JSON string is reset. +** The JsonString is reset. */ -static void jsonResult(JsonString *p){ +static void jsonReturnString(JsonString *p){ if( p->bErr==0 ){ if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, @@ -743,7 +747,7 @@ static void jsonResult(JsonString *p){ if( p->bErr==1 ){ sqlite3_result_error_nomem(p->pCtx); } - jsonReset(p); + jsonStringReset(p); } /************************************************************************** @@ -842,7 +846,7 @@ static int jsonParseAddCleanup( ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static void jsonRenderNode( +static void jsonRenderNodeAsText( JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ JsonString *pOut /* Write JSON here */ @@ -922,7 +926,7 @@ static void jsonRenderNode( while( j<=pNode->n ){ if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonRenderNodeAsText(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } @@ -942,9 +946,9 @@ static void jsonRenderNode( while( j<=pNode->n ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNode(pParse, &pNode[j], pOut); + jsonRenderNodeAsText(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNode(pParse, &pNode[j+1], pOut); + jsonRenderNodeAsText(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -968,9 +972,14 @@ static void jsonRenderNodeAsBlob( ); /* -** Return a JsonNode and all its descendants as a JSON string. +** Make the return value of an SQL function be the JSON encoded by pNode. +** +** By default, the node is rendered as RFC-8259 JSON text (canonical +** JSON text without any JSON-5 enhancements). However if the +** JSON_BLOB flag is set in the user-data for the function, then the +** node is rendered into the JSONB format and returned as a BLOB. */ -static void jsonReturnJson( +static void jsonReturnNodeAsJson( JsonParse *pParse, /* The complete JSON */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx, /* Return value for this function */ @@ -992,13 +1001,13 @@ static void jsonReturnJson( jsonRenderNodeAsBlob(pParse, pNode, &x); sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); }else{ - jsonInit(&s, pCtx); - jsonRenderNode(pParse, pNode, &s); + jsonStringInit(&s, pCtx); + jsonRenderNodeAsText(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ pParse->zAlt = sqlite3RCStrRef(s.zBuf); pParse->nAlt = s.nUsed; } - jsonResult(&s); + jsonReturnString(&s); sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } } @@ -1035,9 +1044,16 @@ static u32 jsonHexToInt4(const char *z){ } /* -** Make the JsonNode the return value of the function. +** Make the return value from an SQL function be the SQL value of +** JsonNode pNode. +** +** If pNode is an atom (not an array or object) then the value returned +** is a pure SQL value - an SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT, or +** SQLITE_NULL. However, if pNode is a JSON array or object, then the +** value returned is either RFC-8259 JSON text or a BLOB in the JSONB +** format, depending on the JSON_BLOB flag of the function user-data. */ -static void jsonReturn( +static void jsonReturnNodeAsSql( JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx /* Return value for this function */ @@ -1194,7 +1210,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - jsonReturnJson(pParse, pNode, pCtx, 0); + jsonReturnNodeAsJson(pParse, pNode, pCtx, 0); break; } } @@ -1265,7 +1281,7 @@ static int jsonParseAddNode( assert( p!=0 ); p->eType = (u8)(eType & 0xff); p->jnFlags = (u8)(eType >> 8); - VVA( p->eU = zContent ? 1 : 0 ); + JSON_VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; return pParse->nNode++; @@ -1490,7 +1506,7 @@ static const struct NanInfName { ** -4 ',' seen ** -5 ':' seen */ -static int jsonParseValue(JsonParse *pParse, u32 i){ +static int jsonParseValueFromText(JsonParse *pParse, u32 i){ char c; u32 j; int iThis; @@ -1509,7 +1525,7 @@ json_parse_restart: } for(j=i+1;;j++){ u32 nNode = pParse->nNode; - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; @@ -1552,7 +1568,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1560,7 +1576,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1579,7 +1595,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1606,7 +1622,7 @@ json_parse_restart: } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1630,7 +1646,7 @@ json_parse_restart: break; } } - x = jsonParseValue(pParse, j); + x = jsonParseValueFromText(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1807,7 +1823,10 @@ json_parse_restart: } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') + && ALWAYS(j-2>=i) + && sqlite3Isdigit(z[j-2]) + ){ pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; }else{ @@ -1952,12 +1971,14 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); /* -** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory held by pParse, -** but not pParse itself. +** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB +** blob) into the JsonNode representation. ** -** pParse must be initialized to an empty parse object prior to calling -** this routine. +** Return 0 on success or non-zero if there are any errors. +** If an error occurs, free all memory held by pParse, but not pParse itself. +** +** pParse must be initialized with pParse->zJson set to the input text or +** blob prior to calling this routine. */ static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ @@ -1970,7 +1991,7 @@ static int jsonParse( pParse->nBlob = pParse->nJson; i = jsonParseValueFromBlob(pParse, 0); }else{ - i = jsonParseValue(pParse, 0); + i = jsonParseValueFromText(pParse, 0); } if( pParse->oom ) i = -1; if( !pParse->isBinary && i>0 ){ @@ -2279,7 +2300,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); pParse->aNode[iLabel].jnFlags |= JNODE_RAW; } return pNode; @@ -2298,7 +2319,9 @@ static JsonNode *jsonLookupStep( if( pRoot->eType!=JSON_ARRAY ) return 0; for(;;){ while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ + i++; + } j += jsonNodeSize(&pBase[j]); } if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; @@ -2360,7 +2383,7 @@ static JsonNode *jsonLookupStep( assert( pRoot->eU==0 ); pRoot->u.iAppend = iStart; pRoot->jnFlags |= JNODE_APPEND; - VVA( pRoot->eU = 2 ); + JSON_VVA( pRoot->eU = 2 ); } return pNode; } @@ -3758,9 +3781,9 @@ static void jsonReturnTextJsonFromBlob( memset(&x, 0, sizeof(x)); x.aBlob = (u8*)aBlob; x.nBlob = nBlob; - jsonInit(&s, ctx); + jsonStringInit(&s, ctx); jsonRenderBlob(&x, 0, &s); - jsonResult(&s); + jsonReturnString(&s); } @@ -4050,7 +4073,7 @@ static void jsonParseFunc( printf("iSubst = %u\n", p->iSubst); printf("iHold = %u\n", p->iHold); jsonDebugPrintNodeEntries(p->aNode, p->nNode); - jsonReturnJson(p, p->aNode, ctx, 1); + jsonReturnNodeAsJson(p, p->aNode, ctx, 1); } /* @@ -4156,9 +4179,9 @@ static void jsonQuoteFunc( JsonString jx; UNUSED_PARAMETER(argc); - jsonInit(&jx, ctx); - jsonAppendValue(&jx, argv[0]); - jsonResult(&jx); + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -4175,14 +4198,14 @@ static void jsonArrayFunc( int i; JsonString jx; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=0; i $.LABEL // PG compatible ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience */ - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); if( sqlite3Isdigit(zPath[0]) ){ jsonAppendRawNZ(&jx, "$[", 2); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); @@ -4295,27 +4318,27 @@ static void jsonExtractFunc( jsonAppendChar(&jx, 0); } pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); - jsonReset(&jx); + jsonStringReset(&jx); }else{ pNode = jsonLookup(p, zPath, 0, ctx); } if( pNode ){ if( flags & JSON_JSON ){ - jsonReturnJson(p, pNode, ctx, 0); + jsonReturnNodeAsJson(p, pNode, ctx, 0); }else{ - jsonReturn(p, pNode, ctx); + jsonReturnNodeAsSql(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); + if( p->nErr==0 && pNode ) jsonReturnNodeAsSql(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each ** element of the array being the value selected by one of the PATHs */ int i; - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '['); for(i=1; inErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNode(p, pNode, &jx); + jsonRenderNodeAsText(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } } if( i==argc ){ jsonAppendChar(&jx, ']'); - jsonResult(&jx); + jsonReturnString(&jx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } - jsonReset(&jx); + jsonStringReset(&jx); } } @@ -4399,7 +4422,7 @@ static JsonNode *jsonMergePatch( pParse->aNode[iStart].n = 1+nApnd; pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; pParse->aNode[iRoot].u.iAppend = iStart; - VVA( pParse->aNode[iRoot].eU = 2 ); + JSON_VVA( pParse->aNode[iRoot].eU = 2 ); iRoot = iStart; pTarget = &pParse->aNode[iTarget]; } @@ -4435,7 +4458,7 @@ static void jsonPatchFunc( if( pResult && pX->oom==0 ){ jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); - jsonReturnJson(pX, pResult, ctx, 0); + jsonReturnNodeAsJson(pX, pResult, ctx, 0); }else{ sqlite3_result_error_nomem(ctx); } @@ -4462,12 +4485,12 @@ static void jsonObjectFunc( "of arguments", -1); return; } - jsonInit(&jx, ctx); + jsonStringInit(&jx, ctx); jsonAppendChar(&jx, '{'); for(i=0; iaNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); } remove_done: jsonDebugPrintParse(p); @@ -4640,7 +4663,7 @@ static void jsonReplaceFunc( jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); replace_err: jsonDebugPrintParse(pParse); } @@ -4692,7 +4715,7 @@ static void jsonSetFunc( } } jsonDebugPrintParse(pParse); - jsonReturnJson(pParse, pParse->aNode, ctx, 1); + jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1); jsonSetDone: /* no cleanup required */; @@ -4830,13 +4853,13 @@ static void jsonArrayStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '['); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - jsonAppendValue(pStr, argv[0]); + jsonAppendSqlValue(pStr, argv[0]); } } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ @@ -4936,7 +4959,7 @@ static void jsonObjectStep( pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); + jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); }else if( pStr->nUsed>1 ){ jsonAppendChar(pStr, ','); @@ -4946,7 +4969,7 @@ static void jsonObjectStep( n = (u32)sqlite3_value_bytes(argv[0]); jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); + jsonAppendSqlValue(pStr, argv[1]); } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ @@ -5110,7 +5133,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ if( pUp->eType==JSON_ARRAY ){ assert( pUp->eU==0 || pUp->eU==3 ); testcase( pUp->eU==3 ); - VVA( pUp->eU = 3 ); + JSON_VVA( pUp->eU = 3 ); if( iUp==p->i-1 ){ pUp->u.iKey = 0; }else{ @@ -5208,7 +5231,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -5224,7 +5247,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -5235,7 +5258,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(&p->sParse, pThis, ctx); + jsonReturnNodeAsSql(&p->sParse, pThis, ctx); break; } case JEACH_ID: { @@ -5251,7 +5274,7 @@ static int jsonEachColumn( } case JEACH_FULLKEY: { JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); if( p->bRecursive ){ jsonEachComputePath(p, &x, p->i); }else{ @@ -5266,15 +5289,15 @@ static int jsonEachColumn( jsonAppendObjectPathElement(&x, pThis); } } - jsonResult(&x); + jsonReturnString(&x); break; } case JEACH_PATH: { if( p->bRecursive ){ JsonString x; - jsonInit(&x, ctx); + jsonStringInit(&x, ctx); jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); + jsonReturnString(&x); break; } /* For json_each() path and root are the same so fall through @@ -5444,7 +5467,7 @@ static int jsonEachFilter( p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ assert( pNode->eU==0 ); - VVA( pNode->eU = 3 ); + JSON_VVA( pNode->eU = 3 ); pNode->u.iKey = 0; p->iEnd = p->i + pNode->n + 1; if( p->bRecursive ){ From a99e2fb576d793299a18a47f62330d3093a7a36e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 29 Sep 2023 16:37:22 +0000 Subject: [PATCH 019/191] Additional refactoring and cleanup. FossilOrigin-Name: 45dd1760875b1ad899a10189c6f5a0a9a0677903794fb5a06ffacd39952a7882 --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 113 +++++++++++++++++++++++++------------------------- 3 files changed, 63 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 7fcd2bd9c7..8a5c90e20d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments\sand\sprocedure\snames\sfor\sclarity\sin\sthe\sJSON\nimplementation. -D 2023-09-29T12:45:14.794 +C Additional\srefactoring\sand\scleanup. +D 2023-09-29T16:37:22.006 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 92d7c2ea8db842cd6a7a7b664bf43d5ae75e097981f001a5ab3860bd222132ca +F src/json.c d7de85731741831d299cfd76520f6af7e72b3b175f3cbfe22d0454e8abbb6090 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c0cba3461d6111b3aeb77726880221f1240355f0b57e060febbdeb12fb688c0 -R d73ce77d34a070de20c2504f2815e166 +P 9b620d813ef483f1277c1683c5e926a882f07f3b90804dea0c91b325ff8e45a4 +R bcca84e0a588bc606d4e22707cc86ed4 U drh -Z 100adbc30a124e4f2d5cc09bea6ade0f +Z 4c6f8cb9c19d993b3edfba11c172dd33 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f67a4ccc1..9118fea8e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b620d813ef483f1277c1683c5e926a882f07f3b90804dea0c91b325ff8e45a4 \ No newline at end of file +45dd1760875b1ad899a10189c6f5a0a9a0677903794fb5a06ffacd39952a7882 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 97a6aebf4e..b381e7b28e 100644 --- a/src/json.c +++ b/src/json.c @@ -2657,7 +2657,7 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** -4 ',' seen ** -5 ':' seen */ -static int jsonParseValueB(JsonParse *pParse, u32 i){ +static int jsonTranslateTextValueToBlob(JsonParse *pParse, u32 i){ char c; u32 j; u32 iThis, iStart; @@ -2677,7 +2677,7 @@ json_parse_restart: iStart = pParse->nBlob; for(j=i+1;;j++){ u32 iBlob = pParse->nBlob; - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x<=0 ){ int op; if( x==(-2) ){ @@ -2723,7 +2723,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -2731,7 +2731,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -2750,7 +2750,7 @@ json_parse_restart: break; } } - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -2780,7 +2780,7 @@ json_parse_restart: return -1; } for(j=i+1;;j++){ - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -2804,7 +2804,7 @@ json_parse_restart: break; } } - x = jsonParseValueB(pParse, j); + x = jsonTranslateTextValueToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -3117,13 +3117,13 @@ json_parse_restart: ** pParse must be initialized to an empty parse object prior to calling ** this routine. */ -static int jsonParseB( +static int jsonConvertTextToBlob( JsonParse *pParse, /* Initialize and fill this JsonParse object */ sqlite3_context *pCtx /* Report errors here */ ){ int i; const char *zJson = pParse->zJson; - i = jsonParseValueB(pParse, 0); + i = jsonTranslateTextValueToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ assert( pParse->iDepth==0 ); @@ -4089,6 +4089,25 @@ static void jsonTest1Func( UNUSED_PARAMETER(argc); sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } + +/* SQL Function: jsonb_test2(BLOB_JSON) +** +** Render BLOB_JSON back into text. +** Development testing only. +*/ +static void jsonbTest2( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + const u8 *aBlob; + int nBlob; + UNUSED_PARAMETER(argc); + + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + nBlob = sqlite3_value_bytes(argv[0]); + jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); +} #endif /* SQLITE_DEBUG */ /**************************************************************************** @@ -4134,7 +4153,7 @@ static void jsonbFunc( memset(&x, 0, sizeof(x)); x.zJson = (char*)zJson; x.nJson = nJson; - if( jsonParseB(pParse, ctx) ){ + if( jsonConvertTextToBlob(pParse, ctx) ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); @@ -4146,25 +4165,6 @@ static void jsonbFunc( } } -/* SQL Function: jsonb_test2(BLOB_JSON) -** -** Render BLOB_JSON back into text. -** Development testing only. -*/ -static void jsonbTest2( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - const u8 *aBlob; - int nBlob; - UNUSED_PARAMETER(argc); - - aBlob = (const u8*)sqlite3_value_blob(argv[0]); - nBlob = sqlite3_value_bytes(argv[0]); - jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); -} - /* ** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting @@ -5549,36 +5549,35 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error_position,1, 0, jsonErrorFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(jsonb_extract, -1, JSON_BLOB, jsonExtractFunc), - JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), - JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), - JFUNCTION(jsonb, 1, JSON_BLOB, jsonbFunc), - JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), - JFUNCTION(jsonb_insert, -1, JSON_BLOB, jsonSetFunc), - JFUNCTION(jsonb_patch, 2, JSON_BLOB, jsonPatchFunc), - JFUNCTION(jsonb_quote, 1, JSON_BLOB, jsonQuoteFunc), - JFUNCTION(jsonb_remove, -1, JSON_BLOB, jsonRemoveFunc), - JFUNCTION(jsonb_replace, -1, JSON_BLOB, jsonReplaceFunc), + JFUNCTION(json, 1, 0, jsonRemoveFunc), + JFUNCTION(jsonb, 1, JSON_BLOB, jsonbFunc), + JFUNCTION(json_array, -1, 0, jsonArrayFunc), + JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1, 0, jsonErrorFunc), + JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1, JSON_BLOB, jsonExtractFunc), + JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1, 0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1, JSON_BLOB, jsonSetFunc), + JFUNCTION(json_object, -1, 0, jsonObjectFunc), + JFUNCTION(json_patch, 2, 0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2, JSON_BLOB, jsonPatchFunc), + JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), + JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1, JSON_BLOB, jsonRemoveFunc), + JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1, JSON_BLOB, jsonReplaceFunc), + JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), JFUNCTION(jsonb_set, -1, JSON_ISSET|JSON_BLOB, jsonSetFunc), + JFUNCTION(json_type, 1, 0, jsonTypeFunc), + JFUNCTION(json_type, 2, 0, jsonTypeFunc), + JFUNCTION(json_valid, 1, 0, jsonValidFunc), #if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(json_parse, 1, 0, jsonParseFunc), + JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(jsonb_test2, 1, 0, jsonbTest2), #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, From ee50aad86544f188fa87644376fa3a7879abb043 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 29 Sep 2023 19:47:25 +0000 Subject: [PATCH 020/191] The u.zJContent field of JsonNode for a string or label should NOT include the quotation mark delimiters. Ever. This is an inefficiency that really ought to be fixed on trunk, but that can wait until this branch lands. FossilOrigin-Name: 96f545f6f839dab4829861361ee3d7a56840217c5f954f334616e77d23c5fe29 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 50 +++++++++++++++++++++++--------------------------- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 8a5c90e20d..97725044f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Additional\srefactoring\sand\scleanup. -D 2023-09-29T16:37:22.006 +C The\su.zJContent\sfield\sof\sJsonNode\sfor\sa\sstring\sor\slabel\sshould\sNOT\sinclude\nthe\squotation\smark\sdelimiters.\s\sEver.\s\sThis\sis\san\sinefficiency\sthat\sreally\nought\sto\sbe\sfixed\son\strunk,\sbut\sthat\scan\swait\suntil\sthis\sbranch\slands. +D 2023-09-29T19:47:25.994 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d7de85731741831d299cfd76520f6af7e72b3b175f3cbfe22d0454e8abbb6090 +F src/json.c 2112b76a0d0ed938df844dcac57170ef825c6fb92f0bb1e2c35360d77fa8abbd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b620d813ef483f1277c1683c5e926a882f07f3b90804dea0c91b325ff8e45a4 -R bcca84e0a588bc606d4e22707cc86ed4 +P 45dd1760875b1ad899a10189c6f5a0a9a0677903794fb5a06ffacd39952a7882 +R 0ad6f1d70a03e9858beb1d83ab248cfc U drh -Z 4c6f8cb9c19d993b3edfba11c172dd33 +Z ecabd724f6e30a2efa3505d39eb1f7e4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9118fea8e8..fd367b5df4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45dd1760875b1ad899a10189c6f5a0a9a0677903794fb5a06ffacd39952a7882 \ No newline at end of file +96f545f6f839dab4829861361ee3d7a56840217c5f954f334616e77d23c5fe29 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b381e7b28e..9b988c1b66 100644 --- a/src/json.c +++ b/src/json.c @@ -435,7 +435,7 @@ static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ } } static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ - assert( N>0 ); + if( N==0 ) return; if( N+p->nUsed >= p->nAlloc ){ jsonStringExpandAndAppend(p,zIn,N); }else{ @@ -557,8 +557,6 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ u32 i; jsonAppendChar(p, '"'); - zIn++; - N -= 2; while( N>0 ){ for(i=0; i0 ){ @@ -894,8 +892,9 @@ static void jsonRenderNodeAsText( }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ - assert( pNode->n>0 ); + jsonAppendChar(pOut, '"'); jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); } break; } @@ -1118,7 +1117,7 @@ static void jsonReturnNodeAsSql( }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ @@ -1135,7 +1134,7 @@ static void jsonReturnNodeAsSql( sqlite3_result_error_nomem(pCtx); break; } - for(i=1, j=0; ieU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; } static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ if( p1->jnFlags & JNODE_RAW ){ @@ -5169,26 +5163,28 @@ static void jsonAppendObjectPathElement( JsonString *pStr, JsonNode *pNode ){ - int jj, nn; + int nn; const char *z; + int bNeedQuote = 0; assert( pNode->eType==JSON_STRING ); assert( pNode->jnFlags & JNODE_LABEL ); assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - if( (pNode->jnFlags & JNODE_RAW)==0 ){ - assert( nn>=2 ); - assert( z[0]=='"' || z[0]=='\'' ); - assert( z[nn-1]=='"' || z[0]=='\'' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jjjnFlags & JNODE_RAW ){ + /* no-op */ + }else if( nn==0 || !sqlite3Isalpha(z[0]) ){ + bNeedQuote = 1; + }else{ + int jj; + for(jj=1; jj Date: Fri, 29 Sep 2023 22:37:18 +0000 Subject: [PATCH 021/191] Update json_each() and json_tree() so that they work with JSONB inputs. FossilOrigin-Name: bb5e50ff56dff95d954aacdd4c5461790f953cef8d7b89da000d8d587fcdf9b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 13 +++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 97725044f7..96d7117b35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\su.zJContent\sfield\sof\sJsonNode\sfor\sa\sstring\sor\slabel\sshould\sNOT\sinclude\nthe\squotation\smark\sdelimiters.\s\sEver.\s\sThis\sis\san\sinefficiency\sthat\sreally\nought\sto\sbe\sfixed\son\strunk,\sbut\sthat\scan\swait\suntil\sthis\sbranch\slands. -D 2023-09-29T19:47:25.994 +C Update\sjson_each()\sand\sjson_tree()\sso\sthat\sthey\swork\swith\sJSONB\sinputs. +D 2023-09-29T22:37:18.176 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2112b76a0d0ed938df844dcac57170ef825c6fb92f0bb1e2c35360d77fa8abbd +F src/json.c 50fb4f1c25eb749e0186fe68f095325d40ebe59aa3a53063f00f382cfd2c36fd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 45dd1760875b1ad899a10189c6f5a0a9a0677903794fb5a06ffacd39952a7882 -R 0ad6f1d70a03e9858beb1d83ab248cfc +P 96f545f6f839dab4829861361ee3d7a56840217c5f954f334616e77d23c5fe29 +R 9027bab5f11df4bc07814add94e083b5 U drh -Z ecabd724f6e30a2efa3505d39eb1f7e4 +Z d0258d856fbcf0571a1fd4c95a27f4d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fd367b5df4..a8c16bbd39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96f545f6f839dab4829861361ee3d7a56840217c5f954f334616e77d23c5fe29 \ No newline at end of file +bb5e50ff56dff95d954aacdd4c5461790f953cef8d7b89da000d8d587fcdf9b8 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9b988c1b66..a790157a50 100644 --- a/src/json.c +++ b/src/json.c @@ -5402,12 +5402,19 @@ static int jsonEachFilter( const char *z; const char *zRoot = 0; sqlite3_int64 n; + int isBinary; UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + z = (const char*)sqlite3_value_blob(argv[0]); + isBinary = 1; + }else{ + z = (const char*)sqlite3_value_text(argv[0]); + isBinary = 0; + } if( z==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; @@ -5417,9 +5424,11 @@ static int jsonEachFilter( n = sqlite3_value_bytes(argv[0]); p->sParse.zJson = sqlite3RCStrNew( n+1 ); if( p->sParse.zJson==0 ) return SQLITE_NOMEM; - memcpy(p->sParse.zJson, z, (size_t)n+1); + memcpy(p->sParse.zJson, z, (size_t)n+(isBinary==0)); + p->sParse.nJson = n; } p->sParse.bJsonIsRCStr = 1; + p->sParse.isBinary = isBinary; p->zJson = p->sParse.zJson; if( jsonParse(&p->sParse, 0) ){ int rc = SQLITE_NOMEM; From 7e86d3fc6961dc64a45cfefd6ceeea375167ffc4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 30 Sep 2023 14:34:39 +0000 Subject: [PATCH 022/191] Finish adding jsonb_ versions for all JSON routines that return JSON text. FossilOrigin-Name: 6daa7b69695e1b68dba317abbcad4d0205c91963d4a9eb2d595a3ec10fa0fdf4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 45 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 96d7117b35..668f82b009 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sjson_each()\sand\sjson_tree()\sso\sthat\sthey\swork\swith\sJSONB\sinputs. -D 2023-09-29T22:37:18.176 +C Finish\sadding\sjsonb_\sversions\sfor\sall\sJSON\sroutines\sthat\sreturn\sJSON\stext. +D 2023-09-30T14:34:39.518 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 50fb4f1c25eb749e0186fe68f095325d40ebe59aa3a53063f00f382cfd2c36fd +F src/json.c d4de26b1c8a18949b80a89d457c4714a00cc65c8b32b965a5f921556daeb48f8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96f545f6f839dab4829861361ee3d7a56840217c5f954f334616e77d23c5fe29 -R 9027bab5f11df4bc07814add94e083b5 +P bb5e50ff56dff95d954aacdd4c5461790f953cef8d7b89da000d8d587fcdf9b8 +R 29e50b8d59965f3c05293a631078e5bd U drh -Z d0258d856fbcf0571a1fd4c95a27f4d8 +Z bae80fe0062bce2a09237dc9749d4f41 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a8c16bbd39..dadf8fc937 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb5e50ff56dff95d954aacdd4c5461790f953cef8d7b89da000d8d587fcdf9b8 \ No newline at end of file +6daa7b69695e1b68dba317abbcad4d0205c91963d4a9eb2d595a3ec10fa0fdf4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a790157a50..0089584e62 100644 --- a/src/json.c +++ b/src/json.c @@ -724,6 +724,8 @@ static void jsonAppendSqlValue( } } +/* Forward reference */ +static void jsonReturnStringAsBlob(JsonString*); /* Make the text in p (which is probably a generated JSON text string) ** the result of the SQL function. @@ -732,7 +734,10 @@ static void jsonAppendSqlValue( */ static void jsonReturnString(JsonString *p){ if( p->bErr==0 ){ - if( p->bStatic ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ @@ -3145,6 +3150,25 @@ static int jsonConvertTextToBlob( return 0; } +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. +*/ +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + (void)jsonTranslateTextValueToBlob(&px, 0); + if( px.oom ){ + sqlite3_free(px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, sqlite3_free); + } +} + /* The byte at index i is a node type-code. This routine ** determines the payload size for that node and writes that ** payload size in to *pSz. It returns the offset from i to the @@ -4860,11 +4884,16 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->bErr ){ if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -4970,10 +4999,16 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->bErr ){ if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, pStr->bStatic ? SQLITE_TRANSIENT : @@ -5557,6 +5592,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json, 1, 0, jsonRemoveFunc), JFUNCTION(jsonb, 1, JSON_BLOB, jsonbFunc), JFUNCTION(json_array, -1, 0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1, JSON_BLOB, jsonArrayFunc), JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1, 0, jsonErrorFunc), @@ -5567,6 +5603,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_insert, -1, 0, jsonSetFunc), JFUNCTION(jsonb_insert, -1, JSON_BLOB, jsonSetFunc), JFUNCTION(json_object, -1, 0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1, JSON_BLOB, jsonObjectFunc), JFUNCTION(json_patch, 2, 0, jsonPatchFunc), JFUNCTION(jsonb_patch, 2, JSON_BLOB, jsonPatchFunc), JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), @@ -5587,7 +5624,13 @@ void sqlite3RegisterJsonFunctions(void){ WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), WAGGREGATE(json_group_object, 2, 0, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC) }; From d8f26350a0831acec6fe51f637f73aa48a9b091e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 30 Sep 2023 16:50:17 +0000 Subject: [PATCH 023/191] Restore the trunk version of sqlite3_user_data(). Fix the xColumn for virtual tables so that the sqlite3_context contains a valid but NULL user data pointer. FossilOrigin-Name: 15ffd932fecc82a5791b2024f55d2ec80887e8eb7345de68d6f5cac4912cfbe8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 3 +++ src/vdbeapi.c | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 668f82b009..c46034ef3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Finish\sadding\sjsonb_\sversions\sfor\sall\sJSON\sroutines\sthat\sreturn\sJSON\stext. -D 2023-09-30T14:34:39.518 +C Restore\sthe\strunk\sversion\sof\ssqlite3_user_data().\s\sFix\sthe\sxColumn\sfor\svirtual\ntables\sso\sthat\sthe\ssqlite3_context\scontains\sa\svalid\sbut\sNULL\suser\sdata\spointer. +D 2023-09-30T16:50:17.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -780,10 +780,10 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c e87f66258c37f87724f46e849572c3ece4c74ef5614ba41eb221e98f0dbc95de F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c cd112eb00d20fc5cc44f631d0e713838602637328b0f127c2f3c2aa8cea3cc91 +F src/vdbe.c 03e66cfa55542c3ae089a1926d06459f8d67cdbad9d52ddc8a7559b7505e90f7 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c -F src/vdbeapi.c 7fdd1eec6b967a5455ccc45a43b84fd32746d6749a423104333f9ba5cec56c9f +F src/vdbeapi.c b82f465cfaa77cf88c414bc53cb8515523e6840956283cfc7e8d284f72640735 F src/vdbeaux.c 5b415e09b5b9d5be6c0f4fcbf18ea9d7d16f6a29ced2f14a3b2041020f63e9c1 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c 317b9f48708139db6239ade40c7980b4bc8233168383690d588dad6d8437f722 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb5e50ff56dff95d954aacdd4c5461790f953cef8d7b89da000d8d587fcdf9b8 -R 29e50b8d59965f3c05293a631078e5bd +P 6daa7b69695e1b68dba317abbcad4d0205c91963d4a9eb2d595a3ec10fa0fdf4 +R 156c1068608d73b6c04a73abb658869e U drh -Z bae80fe0062bce2a09237dc9749d4f41 +Z 48c7e5fa14a791868247c634dc1c52ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dadf8fc937..121b0eb585 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6daa7b69695e1b68dba317abbcad4d0205c91963d4a9eb2d595a3ec10fa0fdf4 \ No newline at end of file +15ffd932fecc82a5791b2024f55d2ec80887e8eb7345de68d6f5cac4912cfbe8 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6463136507..f2d94237a8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8290,6 +8290,7 @@ case OP_VColumn: { /* ncycle */ const sqlite3_module *pModule; Mem *pDest; sqlite3_context sContext; + FuncDef nullFunc; VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur!=0 ); @@ -8307,6 +8308,8 @@ case OP_VColumn: { /* ncycle */ memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; sContext.enc = encoding; + nullFunc.pUserData = 0; + sContext.pFunc = &nullFunc; assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 ); if( pOp->p5 & OPFLAG_NOCHNG ){ sqlite3VdbeMemSetNull(pDest); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 6b32506f9f..91843f5636 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -845,7 +845,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ ** pointer to it. */ void *sqlite3_user_data(sqlite3_context *p){ - return (p && p->pFunc) ? p->pFunc->pUserData : 0; + return p->pFunc->pUserData; } /* From a35ae44150c9afa5b74dd364f015767371dbe97d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 30 Sep 2023 18:13:35 +0000 Subject: [PATCH 024/191] Changes so that fts5 can handle tokens with embedded '\0' bytes. FossilOrigin-Name: c027c092c4af53bd6ae3cc6e2b4439167d9eeb0f9de549b6a2c2a72a67ee886c --- ext/fts5/fts5Int.h | 1 + ext/fts5/fts5_hash.c | 49 +++++---- ext/fts5/fts5_index.c | 23 ++-- ext/fts5/fts5_tcl.c | 173 +++++++++++++++++++++++++++++- ext/fts5/test/fts5origintext.test | 116 ++++++++++++++++++++ manifest | 24 +++-- manifest.uuid | 2 +- 7 files changed, 345 insertions(+), 43 deletions(-) create mode 100644 ext/fts5/test/fts5origintext.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 8bbafbaaf4..1687168d5f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -645,6 +645,7 @@ void sqlite3Fts5HashScanNext(Fts5Hash*); int sqlite3Fts5HashScanEof(Fts5Hash*); void sqlite3Fts5HashScanEntry(Fts5Hash *, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ); diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index 7e50c36608..f6224f1275 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -36,10 +36,15 @@ struct Fts5Hash { /* ** Each entry in the hash table is represented by an object of the -** following type. Each object, its key (a nul-terminated string) and -** its current data are stored in a single memory allocation. The -** key immediately follows the object in memory. The position list -** data immediately follows the key data in memory. +** following type. Each object, its key, and its current data are stored +** in a single memory allocation. The key immediately follows the object +** in memory. The position list data immediately follows the key data +** in memory. +** +** The key is Fts5HashEntry.nKey bytes in size. It consists of a single +** byte identifying the index (either the main term index or a prefix-index), +** followed by the term data. For example: "0token". There is no +** nul-terminator - in this case nKey=6. ** ** The data that follows the key is in a similar, but not identical format ** to the doclist data stored in the database. It is: @@ -174,8 +179,7 @@ static int fts5HashResize(Fts5Hash *pHash){ unsigned int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), - (int)strlen(fts5EntryKey(p))); + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -259,7 +263,7 @@ int sqlite3Fts5HashWrite( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ char *zKey = fts5EntryKey(p); if( zKey[0]==bByte - && p->nKey==nToken + && p->nKey==nToken+1 && memcmp(&zKey[1], pToken, nToken)==0 ){ break; @@ -289,9 +293,9 @@ int sqlite3Fts5HashWrite( zKey[0] = bByte; memcpy(&zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); - p->nKey = nToken; + p->nKey = nToken+1; zKey[nToken+1] = '\0'; - p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); + p->nData = nToken+1 + sizeof(Fts5HashEntry); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -408,12 +412,17 @@ static Fts5HashEntry *fts5HashEntryMerge( *ppOut = p1; p1 = 0; }else{ - int i = 0; char *zKey1 = fts5EntryKey(p1); char *zKey2 = fts5EntryKey(p2); - while( zKey1[i]==zKey2[i] ) i++; + int nMin = MIN(p1->nKey, p2->nKey); - if( ((u8)zKey1[i])>((u8)zKey2[i]) ){ + int cmp = memcmp(zKey1, zKey2, nMin); + if( cmp==0 ){ + cmp = p1->nKey - p2->nKey; + } + assert( cmp!=0 ); + + if( cmp>0 ){ /* p2 is smaller */ *ppOut = p2; ppOut = &p2->pScanNext; @@ -457,7 +466,7 @@ static int fts5HashEntrySort( Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ if( pTerm==0 - || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; @@ -496,12 +505,11 @@ int sqlite3Fts5HashQuery( for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ zKey = fts5EntryKey(p); - assert( p->nKey+1==(int)strlen(zKey) ); - if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break; + if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break; } if( p ){ - int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1; + int nHashPre = sizeof(Fts5HashEntry) + nTerm; int nList = p->nData - nHashPre; u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); if( pRet ){ @@ -562,19 +570,22 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){ void sqlite3Fts5HashScanEntry( Fts5Hash *pHash, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ char *zKey = fts5EntryKey(p); - int nTerm = (int)strlen(zKey); + int nTerm = p->nKey; fts5HashAddPoslistSize(pHash, p, 0); *pzTerm = zKey; - *ppDoclist = (const u8*)&zKey[nTerm+1]; - *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); + *pnTerm = nTerm; + *ppDoclist = (const u8*)&zKey[nTerm]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm); }else{ *pzTerm = 0; + *pnTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index f527709237..9db5925b94 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2177,15 +2177,16 @@ static void fts5SegIterNext_None( }else{ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; - sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -2251,11 +2252,12 @@ static void fts5SegIterNext( }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList = 0; assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); @@ -2265,8 +2267,7 @@ static void fts5SegIterNext( pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); *pbNewTerm = 1; } @@ -2711,8 +2712,7 @@ static void fts5SegIterHashInit( const u8 *pList = 0; p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); - sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); - n = (z ? (int)strlen((const char*)z) : 0); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList); if( pList ){ pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf ){ @@ -5313,10 +5313,10 @@ static void fts5FlushSecureDelete( Fts5Index *p, Fts5Structure *pStruct, const char *zTerm, + int nTerm, i64 iRowid ){ const int f = FTS5INDEX_QUERY_SKIPHASH; - int nTerm = (int)strlen(zTerm); Fts5Iter *pIter = 0; /* Used to find term instance */ fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); @@ -5390,8 +5390,7 @@ static void fts5FlushOneHash(Fts5Index *p){ int nDoclist; /* Size of doclist in bytes */ /* Get the term and doclist for this entry. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = (int)strlen(zTerm); + sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist); if( bSecureDelete==0 ){ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); if( p->rc!=SQLITE_OK ) break; @@ -5421,7 +5420,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( bSecureDelete ){ if( eDetail==FTS5_DETAIL_NONE ){ if( iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; continue; diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 80c600dbb1..fb4bea8e9e 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -1117,6 +1117,176 @@ static int SQLITE_TCLAPI f5tRegisterTok( return TCL_OK; } +typedef struct OriginTextCtx OriginTextCtx; +struct OriginTextCtx { + sqlite3 *db; + fts5_api *pApi; +}; + +typedef struct OriginTextTokenizer OriginTextTokenizer; +struct OriginTextTokenizer { + Fts5Tokenizer *pTok; /* Underlying tokenizer object */ + fts5_tokenizer tokapi; /* API implementation for pTok */ +}; + +/* +** Delete the OriginTextCtx object indicated by the only argument. +*/ +static void f5tOrigintextTokenizerDelete(void *pCtx){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + ckfree(p); +} + +static int f5tOrigintextCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + OriginTextCtx *p = (OriginTextCtx*)pCtx; + OriginTextTokenizer *pTok = 0; + void *pTokCtx = 0; + int rc = SQLITE_OK; + + pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + if( pTok==0 ){ + rc = SQLITE_NOMEM; + }else if( nArg<1 ){ + rc = SQLITE_ERROR; + }else{ + /* Locate the underlying tokenizer */ + rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi); + } + + /* Create the new tokenizer instance */ + if( rc==SQLITE_OK ){ + rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pTok); + pTok = 0; + } + *ppOut = (Fts5Tokenizer*)pTok; + return rc; +} + +static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + if( p->pTok ){ + p->tokapi.xDelete(p->pTok); + } + sqlite3_free(p); +} + +typedef struct OriginTextCb OriginTextCb; +struct OriginTextCb { + void *pCtx; + const char *pText; + int nText; + int (*xToken)(void *, int, const char *, int, int, int); + + char *aBuf; /* Buffer to use */ + int nBuf; /* Allocated size of aBuf[] */ +}; + +static int xOriginToken( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input */ +){ + OriginTextCb *p = (OriginTextCb*)pCtx; + int ret = 0; + + if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){ + /* Token exactly matches document text. Pass it through as is. */ + ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); + }else{ + int nReq = nToken + 1 + (iEnd-iStart); + if( nReq>p->nBuf ){ + sqlite3_free(p->aBuf); + p->aBuf = sqlite3_malloc(nReq*2); + if( p->aBuf==0 ) return SQLITE_NOMEM; + p->nBuf = nReq*2; + } + + memcpy(p->aBuf, pToken, nToken); + p->aBuf[nToken] = '\0'; + memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart); + ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd); + } + + return ret; +} + + +static int f5tOrigintextTokenize( + Fts5Tokenizer *pTokenizer, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)(void *, int, const char *, int, int, int) +){ + OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; + OriginTextCb cb; + int ret; + + memset(&cb, 0, sizeof(cb)); + cb.pCtx = pCtx; + cb.pText = pText; + cb.nText = nText; + cb.xToken = xToken; + + ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken); + sqlite3_free(cb.aBuf); + return ret; +} + +/* +** sqlite3_fts5_register_origintext DB +** +** Description... +*/ +static int SQLITE_TCLAPI f5tRegisterOriginText( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + fts5_api *pApi = 0; + int rc; + fts5_tokenizer tok = {0, 0, 0}; + OriginTextCtx *pCtx = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; + + pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx)); + pCtx->db = db; + pCtx->pApi = pApi; + + tok.xCreate = f5tOrigintextCreate; + tok.xDelete = f5tOrigintextDelete; + tok.xTokenize = f5tOrigintextTokenize; + rc = pApi->xCreateTokenizer( + pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete + ); + + Tcl_ResetResult(interp); + if( rc!=SQLITE_OK ){ + Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0); + return TCL_ERROR; + } + return TCL_OK; +} + /* ** Entry point. */ @@ -1133,7 +1303,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){ { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, { "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, - { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 } + { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }, + { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 } }; int i; F5tTokenizerContext *pContext; diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test new file mode 100644 index 0000000000..791b850c76 --- /dev/null +++ b/ext/fts5/test/fts5origintext.test @@ -0,0 +1,116 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); +} + +do_execsql_test 1.1 { + INSERT INTO ft VALUES('Hello world'); +} + +do_execsql_test 1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b + +do_execsql_test 1.3 { + select b(term) from vocab; +} { + hello.Hello + world +} + +#------------------------------------------------------------------------- +reset_db + +# Return a random integer between 0 and n-1. +# +proc random {n} { + expr {abs(int(rand()*$n))} +} + +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} + +proc term {} { + set first_letter { + a b c d e f g h i j k l m n o p q r s t u v w x y z + A B C D E F G H I J K L M N O P Q R S T U V W X Y Z + } + + set term [select_one $first_letter] + append term [random 100] +} + +proc document {} { + set nTerm [expr [random 5] + 5] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +sqlite3_fts5_register_origintext db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + INSERT INTO ft(ft, rank) VALUES('pgsz', 128); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); +} + +do_test 2.1 { + for {set ii 0} {$ii < 500} {incr ii} { + execsql { INSERT INTO ft VALUES( document() ) } + } +} {} + +do_execsql_test 2.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.3 { + INSERT INTO ft(ft, rank) VALUES('merge', 16); +} + +do_execsql_test 2.4 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.5 { + INSERT INTO ft(ft) VALUES('optimize'); +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b +#execsql_pp { SELECT b(term) FROM vocab } + +finish_test + diff --git a/manifest b/manifest index 2eae6bfcf8..bf21569e58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sJNI\sbinding\sto\scompile\swithout\sSQLITE_ENABLE_PREUPDATE_HOOK.\sAdd\sbuild\soption\sto\sdisable\sall\soptional\sENABLE\sflags. -D 2023-09-30T17:08:29.126 +C Changes\sso\sthat\sfts5\scan\shandle\stokens\swith\sembedded\s'\\0'\sbytes. +D 2023-09-30T18:13:35.306 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,16 +88,16 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520 -F ext/fts5/fts5Int.h 78a63cc0795186cde5384816a9403a68c65774b35d952e05b81a1b4b158e07c8 +F ext/fts5/fts5Int.h 66a38b285e2b860baa29745d8eff27f5b0809268e7820498494d9acfaccf8a5c F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6 -F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d -F ext/fts5/fts5_index.c a86bcd5637625ce1037649d55974ab8da1fa8d1375cb334aae47ef376642e93b +F ext/fts5/fts5_hash.c 76765856397eff56f526b0640b23a1677d737d35e07bc00e4b4b2e0fc5fda60d +F ext/fts5/fts5_index.c 16d775ecbccf7d3698a03bcae3c3fbee0749df748b93b29d0e82a37e02eaaa94 F ext/fts5/fts5_main.c 799ec88d2309055f6406bddb0bd6ed80148c5da5eb14594c3c5309a6e944d489 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5 -F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae +F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c a2bed8edb25f6432e8cdb62aad5916935c19dba8dac2b8324950cfff397e25ff F ext/fts5/fts5_tokenize.c 5e251efb0f1af99a25ed50010ba6b1ad1250aca5921af1988fdcabe5ebc3cb43 @@ -187,6 +187,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca +F ext/fts5/test/fts5origintext.test 9a6edc85ccc4afb10e71d54d98d8170f850272e55b120520f367afbb12526674 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2122,8 +2123,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5e387275f69ab2d3159b4b67b8cbfc6270410b61e5ac1f988616e8d051f6572e -R 02618be495064fd0c511f49fba8a92b2 -U stephan -Z fb900a6927398da79962f35041fce8dc +P c04022b7407f77eaf0175e831ebcd6bbdc0af1cef0d42c5c11102aa8484f24ca +R ee3b13ddf778c77c1640cd7c7844c1f5 +T *branch * fts5-token-data +T *sym-fts5-token-data * +T -sym-trunk * +U dan +Z 7d5bd217a552215de3d888e155abaef5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92d537682b..9db9eb3c64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c04022b7407f77eaf0175e831ebcd6bbdc0af1cef0d42c5c11102aa8484f24ca \ No newline at end of file +c027c092c4af53bd6ae3cc6e2b4439167d9eeb0f9de549b6a2c2a72a67ee886c \ No newline at end of file From 9b176f863b41dc34dd7e5abcbb593500751848e6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 1 Oct 2023 18:59:34 +0000 Subject: [PATCH 025/191] Fix problems following OOM in JSONB parsing. FossilOrigin-Name: 0d8cd6b5fb592f88f593ceaad9cdfa95dcfdf14169c3a56a5becbd66d62fd02b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 12 +++++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c46034ef3b..892d400ef4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restore\sthe\strunk\sversion\sof\ssqlite3_user_data().\s\sFix\sthe\sxColumn\sfor\svirtual\ntables\sso\sthat\sthe\ssqlite3_context\scontains\sa\svalid\sbut\sNULL\suser\sdata\spointer. -D 2023-09-30T16:50:17.069 +C Fix\sproblems\sfollowing\sOOM\sin\sJSONB\sparsing. +D 2023-10-01T18:59:34.631 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d4de26b1c8a18949b80a89d457c4714a00cc65c8b32b965a5f921556daeb48f8 +F src/json.c 253ca59ae7f4be5fa7e0d630f9073f6d2f8b6dd89b47d80192f3b037271fe45e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6daa7b69695e1b68dba317abbcad4d0205c91963d4a9eb2d595a3ec10fa0fdf4 -R 156c1068608d73b6c04a73abb658869e +P 15ffd932fecc82a5791b2024f55d2ec80887e8eb7345de68d6f5cac4912cfbe8 +R 32ce5e0c8d0e91f62fdd054af5b27453 U drh -Z 48c7e5fa14a791868247c634dc1c52ad +Z 6f251c6f66875dddb0d77e3789c48938 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 121b0eb585..ffd32b4c8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15ffd932fecc82a5791b2024f55d2ec80887e8eb7345de68d6f5cac4912cfbe8 \ No newline at end of file +0d8cd6b5fb592f88f593ceaad9cdfa95dcfdf14169c3a56a5becbd66d62fd02b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0089584e62..9f41ca4343 100644 --- a/src/json.c +++ b/src/json.c @@ -1612,7 +1612,9 @@ json_parse_restart: pParse->iErr = j; return -1; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } pParse->iDepth--; return j+1; } @@ -3493,7 +3495,9 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ if( r<=0 ) return -1; j = (u32)r; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } break; } case JSONB_OBJECT: { @@ -3507,7 +3511,9 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ } j = (u32)r; } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + if( !pParse->oom ){ + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + } break; } } From 5624b0b4cd997f749511fdb62cca63602f0cb079 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 12:40:04 +0000 Subject: [PATCH 026/191] Gather forward references into a single place for the JSON code. Allow JSONB arguments to json_array() and json_object() and similar. FossilOrigin-Name: c352201b8c299c330d9abbff6dbcbcbcf00ada53183d3cd91020ec772e066357 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 41 ++++++++++++++++++----------------------- test/json101.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 892d400ef4..ac267d347c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sfollowing\sOOM\sin\sJSONB\sparsing. -D 2023-10-01T18:59:34.631 +C Gather\sforward\sreferences\sinto\sa\ssingle\splace\sfor\sthe\sJSON\scode.\s\sAllow\nJSONB\sarguments\sto\sjson_array()\sand\sjson_object()\sand\ssimilar. +D 2023-10-02T12:40:04.670 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 253ca59ae7f4be5fa7e0d630f9073f6d2f8b6dd89b47d80192f3b037271fe45e +F src/json.c e815a5e2f5bac5747e8290216eeb7de209aadc537a3f6a37b2880473bb2a8ed3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1303,7 +1303,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test dc9d5a2a5b1fd1b54dbd71c538b17933cc98d84b4c1f821ead754933663dca55 +F test/json101.test 728d90ecd6881a507e4081ed25ce0ac33ec501bbe897568ac47a71d9034561cc F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15ffd932fecc82a5791b2024f55d2ec80887e8eb7345de68d6f5cac4912cfbe8 -R 32ce5e0c8d0e91f62fdd054af5b27453 +P 0d8cd6b5fb592f88f593ceaad9cdfa95dcfdf14169c3a56a5becbd66d62fd02b +R cb29cf34fe33404e23405b0f8bd1299f U drh -Z 6f251c6f66875dddb0d77e3789c48938 +Z e34235bdd2a6e0a4ec57afd79666bb6d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ffd32b4c8c..cd8719aa96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d8cd6b5fb592f88f593ceaad9cdfa95dcfdf14169c3a56a5becbd66d62fd02b \ No newline at end of file +c352201b8c299c330d9abbff6dbcbcbcf00ada53183d3cd91020ec772e066357 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9f41ca4343..8070c7bdf7 100644 --- a/src/json.c +++ b/src/json.c @@ -347,6 +347,17 @@ struct JsonParse { */ #define JSON_MAX_DEPTH 1000 +/************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static void jsonRenderNodeAsBlob(JsonParse*,JsonNode*,JsonParse*); +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); +static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); +static u32 jsonRenderBlob(JsonParse*,u32,JsonString*); + /************************************************************************** ** Utility routines for dealing with JsonString objects **************************************************************************/ @@ -714,7 +725,13 @@ static void jsonAppendSqlValue( break; } default: { - if( p->bErr==0 ){ + if( jsonFuncArgMightBeBinary(pValue) ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(pValue); + px.nBlob = sqlite3_value_bytes(pValue); + jsonRenderBlob(&px, 0, p); + }else if( p->bErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->bErr = 2; jsonStringReset(p); @@ -724,8 +741,6 @@ static void jsonAppendSqlValue( } } -/* Forward reference */ -static void jsonReturnStringAsBlob(JsonString*); /* Make the text in p (which is probably a generated JSON text string) ** the result of the SQL function. @@ -968,13 +983,6 @@ static void jsonRenderNodeAsText( } } -/* Forward reference */ -static void jsonRenderNodeAsBlob( - JsonParse *pParse, /* the complete parse of the JSON */ - JsonNode *pNode, /* The node to render */ - JsonParse *pOut /* Write the BLOB rendering of JSON here */ -); - /* ** Make the return value of an SQL function be the JSON encoded by pNode. ** @@ -1220,9 +1228,6 @@ static void jsonReturnNodeAsSql( } } -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - /* ** A macro to hint to the compiler that a function should not be ** inlined. @@ -1973,9 +1978,6 @@ static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ } } -/* Forward reference */ -static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); - /* ** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB ** blob) into the JsonNode representation. @@ -2047,10 +2049,6 @@ static int jsonParseFindParents(JsonParse *pParse){ #define JSON_CACHE_ID (-429938) /* First cache entry */ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ -/* Forward reference */ -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); - - /* ** Obtain a complete parse of the JSON found in the pJson argument ** @@ -2202,9 +2200,6 @@ static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ } } -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such diff --git a/test/json101.test b/test/json101.test index 4da1d132cc..5a7d770dde 100644 --- a/test/json101.test +++ b/test/json101.test @@ -36,6 +36,9 @@ do_execsql_test json101-1.2 { do_catchsql_test json101-1.3 { SELECT json_array(1,printf('%.1000c','x'),x'abcd',3); } {1 {JSON cannot hold BLOB values}} +do_catchsql_test json101-1.3b { + SELECT jsonb_array(1,printf('%.1000c','x'),x'abcd',3); +} {1 {JSON cannot hold BLOB values}} do_execsql_test json101-1.4 { SELECT json_array(-9223372036854775808,9223372036854775807,0,1,-1, 0.0, 1.0, -1.0, -1e99, +2e100, @@ -47,13 +50,43 @@ do_execsql_test json101-1.4 { 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', 99); } {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]} +do_execsql_test json101-1.4b { + SELECT json(jsonb_array(-9223372036854775808,9223372036854775807,0,1,-1, + 0.0, 1.0, -1.0, -1e99, +2e100, + 'one','two','three', + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, NULL, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', + 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', + 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', + 99)); +} {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]} do_execsql_test json101-2.1 { SELECT json_object('a',1,'b',2.5,'c',null,'d','String Test'); } {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}} +do_execsql_test json101-2.1b { + SELECT json(jsonb_object('a',1,'b',2.5,'c',null,'d','String Test')); +} {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}} do_catchsql_test json101-2.2 { SELECT json_object('a',printf('%.1000c','x'),2,2.5); } {1 {json_object() labels must be TEXT}} +do_catchsql_test json101-2.2b { + SELECT jsonb_object('a',printf('%.1000c','x'),2,2.5); +} {1 {json_object() labels must be TEXT}} +do_execsql_test json101-2.2.2 { + SELECT json_object('a',json_array('xyx',77,4.5),'x',2.5); +} {{{"a":["xyx",77,4.5],"x":2.5}}} +do_execsql_test json101-2.2.2b { + SELECT json(jsonb_object('a',json_array('xyx',77,4.5),'x',2.5)); +} {{{"a":["xyx",77,4.5],"x":2.5}}} +do_execsql_test json101-2.2.3 { + SELECT json_object('a',jsonb_array('xyx',77,4.5),'x',2.5); +} {{{"a":["xyx",77,4.5],"x":2.5}}} +do_execsql_test json101-2.2.3b { + SELECT json(jsonb_object('a',jsonb_array('xyx',77,4.5),'x',2.5)); +} {{{"a":["xyx",77,4.5],"x":2.5}}} +exit do_catchsql_test json101-2.3 { SELECT json_object('a',1,'b'); } {1 {json_object() requires an even number of arguments}} From 8eeafb754459486a530ec3264cde5587793522d7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 13:20:43 +0000 Subject: [PATCH 027/191] Improvements to error handling for BLOB inputs on JSON. FossilOrigin-Name: 14f20ecbfab44934e86f1ac7a3f745b989aa8190c6df119ff5aa8100fa248d93 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 19 +++++++------------ test/json101.test | 6 ++++-- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index ac267d347c..b0b1b78488 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Gather\sforward\sreferences\sinto\sa\ssingle\splace\sfor\sthe\sJSON\scode.\s\sAllow\nJSONB\sarguments\sto\sjson_array()\sand\sjson_object()\sand\ssimilar. -D 2023-10-02T12:40:04.670 +C Improvements\sto\serror\shandling\sfor\sBLOB\sinputs\son\sJSON. +D 2023-10-02T13:20:43.907 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e815a5e2f5bac5747e8290216eeb7de209aadc537a3f6a37b2880473bb2a8ed3 +F src/json.c 057ce1f7bcf30abb8e0418e8b84892e6cda66e0fcac45465574ea3f2e37d6f85 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1303,7 +1303,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 728d90ecd6881a507e4081ed25ce0ac33ec501bbe897568ac47a71d9034561cc +F test/json101.test da33cc6d114e82228fc56b6b2700b2fc2c352636aeca59cb157fc6237df688af F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d8cd6b5fb592f88f593ceaad9cdfa95dcfdf14169c3a56a5becbd66d62fd02b -R cb29cf34fe33404e23405b0f8bd1299f +P c352201b8c299c330d9abbff6dbcbcbcf00ada53183d3cd91020ec772e066357 +R f0da6e230789d1a4973c52d698202614 U drh -Z e34235bdd2a6e0a4ec57afd79666bb6d +Z 0e4cae102419e8824f19fd85f33d7b9c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd8719aa96..348760d2b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c352201b8c299c330d9abbff6dbcbcbcf00ada53183d3cd91020ec772e066357 \ No newline at end of file +14f20ecbfab44934e86f1ac7a3f745b989aa8190c6df119ff5aa8100fa248d93 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8070c7bdf7..1c7693df7a 100644 --- a/src/json.c +++ b/src/json.c @@ -3207,7 +3207,8 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ n = 5; } if( i+sz+n>pParse->nBlob ){ - sz = pParse->nBlob - (i+n); + sz = 0; + n = 0; } *pSz = sz; return n; @@ -3226,9 +3227,10 @@ static u32 jsonRenderBlob( ){ u32 sz, n, j, iEnd; - if( i>=pParse->nBlob ){ + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ pOut->bErr = 1; - return 0; + return pParse->nBlob+1; } switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { @@ -3245,14 +3247,12 @@ static u32 jsonRenderBlob( } case JSONB_INT: case JSONB_FLOAT: { - n = jsonbPayloadSize(pParse, i, &sz); jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); break; } case JSONB_INT5: { /* Integer literal in hexadecimal notation */ int k; sqlite3_uint64 u = 0; - n = jsonbPayloadSize(pParse, i, &sz); const char *zIn = (const char*)&pParse->aBlob[i+n]; if( zIn[0]=='-' ){ zIn++; @@ -3267,7 +3267,6 @@ static u32 jsonRenderBlob( } case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ int k; - n = jsonbPayloadSize(pParse, i, &sz); const char *zIn = (const char*)&pParse->aBlob[i+n]; if( zIn[0]=='-' ){ zIn++; @@ -3294,7 +3293,6 @@ static u32 jsonRenderBlob( } case JSONB_TEXT: case JSONB_TEXTJ: { - n = jsonbPayloadSize(pParse, i, &sz); jsonAppendChar(pOut, '"'); jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); jsonAppendChar(pOut, '"'); @@ -3303,7 +3301,6 @@ static u32 jsonRenderBlob( case JSONB_TEXT5: { const char *zIn; u32 k; - n = jsonbPayloadSize(pParse, i, &sz); zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz>0 ){ @@ -3358,7 +3355,6 @@ static u32 jsonRenderBlob( } case JSONB_ARRAY: { jsonAppendChar(pOut, '['); - n = jsonbPayloadSize(pParse, i, &sz); j = i+n; iEnd = j+sz; while( j0 && iaBlob[i] & 0x0f ){ case JSONB_NULL: { sqlite3_result_null(pCtx); diff --git a/test/json101.test b/test/json101.test index 5a7d770dde..5bf6635601 100644 --- a/test/json101.test +++ b/test/json101.test @@ -86,13 +86,15 @@ do_execsql_test json101-2.2.3 { do_execsql_test json101-2.2.3b { SELECT json(jsonb_object('a',jsonb_array('xyx',77,4.5),'x',2.5)); } {{{"a":["xyx",77,4.5],"x":2.5}}} -exit do_catchsql_test json101-2.3 { SELECT json_object('a',1,'b'); } {1 {json_object() requires an even number of arguments}} do_catchsql_test json101-2.4 { SELECT json_object('a',printf('%.1000c','x'),'b',x'abcd'); } {1 {JSON cannot hold BLOB values}} +do_execsql_test json101-2.5 { + SELECT json_object('a',printf('%.10c','x'),'b',jsonb_array(1,2,3)); +} {{{"a":"xxxxxxxxxx","b":[1,2,3]}}} do_execsql_test json101-3.1 { SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); @@ -431,7 +433,7 @@ do_execsql_test json101-9.4 { SELECT json_quote(null); } {"null"} do_catchsql_test json101-9.5 { - SELECT json_quote(x'30313233'); + SELECT json_quote(x'3031323334'); } {1 {JSON cannot hold BLOB values}} do_catchsql_test json101-9.6 { SELECT json_quote(123,456) From c97407fdbad35baaefcea20738db81a02c853c6d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 13:35:06 +0000 Subject: [PATCH 028/191] Allow json_replace() to accept JSONB arguments as the new value. FossilOrigin-Name: 95eb7b37fab29931924311f541d52173ef77a448efc8771b1a1783ccd786d23d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 15 +++++++++++---- test/json101.test | 6 ++++++ 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b0b1b78488..d67b7d412c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\serror\shandling\sfor\sBLOB\sinputs\son\sJSON. -D 2023-10-02T13:20:43.907 +C Allow\sjson_replace()\sto\saccept\sJSONB\sarguments\sas\sthe\snew\svalue. +D 2023-10-02T13:35:06.348 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 057ce1f7bcf30abb8e0418e8b84892e6cda66e0fcac45465574ea3f2e37d6f85 +F src/json.c 643f1553f9a704ea558937b16831d26f9db90dd97e05d77cfef984f84af37564 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1303,7 +1303,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test da33cc6d114e82228fc56b6b2700b2fc2c352636aeca59cb157fc6237df688af +F test/json101.test 54b27bd681ce3fe33629b58fecdba2502d0ce8d445f69a155dbd9861a868ea27 F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c352201b8c299c330d9abbff6dbcbcbcf00ada53183d3cd91020ec772e066357 -R f0da6e230789d1a4973c52d698202614 +P 14f20ecbfab44934e86f1ac7a3f745b989aa8190c6df119ff5aa8100fa248d93 +R cc5fa69b041d057bb643a7cdf2281f1c U drh -Z 0e4cae102419e8824f19fd85f33d7b9c +Z 2b0bdbc85e19cadd323c6d81b0285ad4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 348760d2b2..7c8b3403e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14f20ecbfab44934e86f1ac7a3f745b989aa8190c6df119ff5aa8100fa248d93 \ No newline at end of file +95eb7b37fab29931924311f541d52173ef77a448efc8771b1a1783ccd786d23d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1c7693df7a..c7bf260c9c 100644 --- a/src/json.c +++ b/src/json.c @@ -4620,7 +4620,10 @@ static void jsonReplaceNode( k = jsonParseAddNode(p, JSON_STRING, n, zCopy); assert( k>0 || p->oom ); if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; - }else{ + break; + } + replace_with_json: + { JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); if( pPatch==0 ){ p->oom = 1; @@ -4637,9 +4640,13 @@ static void jsonReplaceNode( break; } default: { - jsonParseAddNode(p, JSON_NULL, 0, 0); - sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); - p->nErr++; + if( jsonFuncArgMightBeBinary(pValue) ){ + goto replace_with_json; + }else{ + jsonParseAddNode(p, JSON_NULL, 0, 0); + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); + p->nErr++; + } break; } } diff --git a/test/json101.test b/test/json101.test index 5bf6635601..a6fec19507 100644 --- a/test/json101.test +++ b/test/json101.test @@ -99,9 +99,15 @@ do_execsql_test json101-2.5 { do_execsql_test json101-3.1 { SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); } {{{"a":"[3,4,5]","b":2}}} +do_execsql_test json101-3.1b { + SELECT json(jsonb_replace('{"a":1,"b":2}','$.a','[3,4,5]')); +} {{{"a":"[3,4,5]","b":2}}} do_execsql_test json101-3.2 { SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]')); } {{{"a":[3,4,5],"b":2}}} +do_execsql_test json101-3.2b { + SELECT json_replace('{"a":1,"b":2}','$.a',jsonb('[3,4,5]')); +} {{{"a":[3,4,5],"b":2}}} do_execsql_test json101-3.3 { SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); } {text} From dee29e8c7acbe30112ed875496e86f77cc3bdcae Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 14:51:28 +0000 Subject: [PATCH 029/191] New test cases for JSONB. FossilOrigin-Name: d624c31e5c49e1ce63b4b72caa42a61c5167866f47d842fbcfe4e826fd079d7c --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- test/json101.test | 56 ++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index d67b7d412c..a167aca684 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sjson_replace()\sto\saccept\sJSONB\sarguments\sas\sthe\snew\svalue. -D 2023-10-02T13:35:06.348 +C New\stest\scases\sfor\sJSONB. +D 2023-10-02T14:51:28.119 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 643f1553f9a704ea558937b16831d26f9db90dd97e05d77cfef984f84af37564 +F src/json.c a75a4bfff4d4f435c4e22d69e0b54c0bccfa978725e4a89c052764f31c7f3bc9 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1303,7 +1303,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 54b27bd681ce3fe33629b58fecdba2502d0ce8d445f69a155dbd9861a868ea27 +F test/json101.test e8ccd09f965c594f38ef486ddf7913f0fcac97be20a785a41c3d7cd4289e82de F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14f20ecbfab44934e86f1ac7a3f745b989aa8190c6df119ff5aa8100fa248d93 -R cc5fa69b041d057bb643a7cdf2281f1c +P 95eb7b37fab29931924311f541d52173ef77a448efc8771b1a1783ccd786d23d +R 2841b59f4f7afcfcc5b77dd83ccca9c0 U drh -Z 2b0bdbc85e19cadd323c6d81b0285ad4 +Z d9c4ddddb83fc3b47a718cf56777e023 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7c8b3403e2..3e4d4770c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95eb7b37fab29931924311f541d52173ef77a448efc8771b1a1783ccd786d23d \ No newline at end of file +d624c31e5c49e1ce63b4b72caa42a61c5167866f47d842fbcfe4e826fd079d7c \ No newline at end of file diff --git a/src/json.c b/src/json.c index c7bf260c9c..46c65befd7 100644 --- a/src/json.c +++ b/src/json.c @@ -3890,7 +3890,7 @@ static void jsonReturnFromBlob( c = z[++iIn]; if( c=='u' ){ u32 v = jsonHexToInt4(z+iIn+1); - i += 4; + iIn += 4; if( v==0 ) break; if( v<=0x7f ){ zOut[iOut++] = (char)v; diff --git a/test/json101.test b/test/json101.test index a6fec19507..d5ec36d8b5 100644 --- a/test/json101.test +++ b/test/json101.test @@ -111,13 +111,22 @@ do_execsql_test json101-3.2b { do_execsql_test json101-3.3 { SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); } {text} +do_execsql_test json101-3.3b { + SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); +} {text} do_execsql_test json101-3.4 { SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b'); } {object} +do_execsql_test json101-3.4b { + SELECT json_type(jsonb_set('{"a":1,"b":2}','$.b',jsonb('{"x":3,"y":4}')),'$.b'); +} {object} ifcapable vtab { -do_execsql_test json101-3.5 { - SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456)); -} {{$} {} | {$.x} 456 |} + do_execsql_test json101-3.5 { + SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456)); + } {{$} {} | {$.x} 456 |} + do_execsql_test json101-3.5b { + SELECT fullkey, atom, '|' FROM json_tree(jsonb_set('{}','$.x',123,'$.x',456)); + } {{$} {} | {$.x} 456 |} } # Per rfc7159, any JSON value is allowed at the top level, and whitespace @@ -169,6 +178,13 @@ do_execsql_test json101-4.10 { WHERE json_extract(x,'$')<>x AND json_type(x) IN ('object','array'); } {4} +do_execsql_test json101-4.10b { + CREATE TABLE j1b AS SELECT jsonb(x) AS "x" FROM j1; + SELECT count(*) FROM j1b WHERE json_type(x) IN ('object','array'); + SELECT json(x) FROM j1b + WHERE json_extract(x,'$')<>json(x) + AND json_type(x) IN ('object','array'); +} {4} do_execsql_test json101-5.1 { CREATE TABLE j2(id INTEGER PRIMARY KEY, json, src); @@ -296,11 +312,17 @@ do_execsql_test json101-5.1 { } ]','https://adobe.github.io/Spry/samples/data_region/JSONDataSetSample.html'); SELECT count(*) FROM j2; -} {3} + CREATE TABLE j2b(id INTEGER PRIMARY KEY, json, src); + INSERT INTO J2b(id,json,src) SELECT id, jsonb(json), src FROM j2; + SELECT count(*) FROM j2b; +} {3 3} do_execsql_test json101-5.2 { SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id; } {1 1 object | 2 1 object | 3 1 array |} +do_execsql_test json101-5.2b { + SELECT id, json_valid(json), json_type(json), '|' FROM j2b ORDER BY id; +} {1 1 object | 2 1 object | 3 1 array |} ifcapable !vtab { finish_test @@ -315,6 +337,12 @@ do_execsql_test json101-5.3 { WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']' ELSE '.'||key END); } {} +do_execsql_test json101-5.3b { + SELECT j2b.rowid, jx.rowid, fullkey, path, key + FROM j2b, json_tree(j2b.json) AS jx + WHERE fullkey!=(path || CASE WHEN typeof(key)=='integer' THEN '['||key||']' + ELSE '.'||key END); +} {} do_execsql_test json101-5.4 { SELECT j2.rowid, jx.rowid, fullkey, path, key FROM j2, json_each(j2.json) AS jx @@ -409,6 +437,13 @@ do_execsql_test json101-8.1 { UPDATE t8 SET b=json_array(a); SELECT b FROM t8; } {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}} +do_execsql_test json101-8.1b { + DROP TABLE IF EXISTS t8; + CREATE TABLE t8(a,b); + INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz'); + UPDATE t8 SET b=jsonb_array(a); + SELECT json(b) FROM t8; +} {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}} do_execsql_test json101-8.2 { SELECT a=json_extract(b,'$[0]') FROM t8; } {1} @@ -807,14 +842,23 @@ do_execsql_test json101-12.100 { } }'); } {} + do_execsql_test json101-12.110 { SELECT json_remove(x, '$.settings.layer2."dis.legomenon".forceDisplay') FROM t12; } {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}} +do_execsql_test json101-12.110b { + SELECT json_remove(jsonb(x), '$.settings.layer2."dis.legomenon".forceDisplay') + FROM t12; +} {{{"settings":{"layer2":{"hapax.legomenon":{"forceDisplay":true,"transliterate":true,"add.footnote":true,"summary.report":true},"dis.legomenon":{"transliterate":false,"add.footnote":false,"summary.report":true},"tris.legomenon":{"forceDisplay":true,"transliterate":false,"add.footnote":false,"summary.report":false}}}}}} do_execsql_test json101-12.120 { SELECT json_extract(x, '$.settings.layer2."tris.legomenon"."summary.report"') FROM t12; } {0} +do_execsql_test json101-12.120b { + SELECT json_extract(jsonb(x), '$.settings.layer2."tris.legomenon"."summary.report"') + FROM t12; +} {0} # 2018-01-26 # ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41 @@ -1054,8 +1098,4 @@ do_execsql_test json101-21.27 { SELECT json_group_object(x,y) FROM c; } {{{"a":1,"b":2.0,"c":null,:"three","e":"four"}}} - - - - finish_test From e5c384e9a52816ca989953154617fa66ee4a75a2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 20:16:06 +0000 Subject: [PATCH 030/191] Fix jsonb_insert() so that it does not behave like jsonb_set(). New test cases added. FossilOrigin-Name: 54197149b811d30b6c4487eedf5692b164ed0f90cfcc541aa3157094f5f17f6a --- manifest | 14 ++--- manifest.uuid | 2 +- src/json.c | 3 +- test/json102.test | 133 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 141 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a167aca684..02e0ac823a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sJSONB. -D 2023-10-02T14:51:28.119 +C Fix\sjsonb_insert()\sso\sthat\sit\sdoes\snot\sbehave\slike\sjsonb_set().\nNew\stest\scases\sadded. +D 2023-10-02T20:16:06.301 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a75a4bfff4d4f435c4e22d69e0b54c0bccfa978725e4a89c052764f31c7f3bc9 +F src/json.c a2e70610cc14d0f97feb17bf0000c0e034f1cb5ac1b318bcf3ccf7a014f01a80 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1304,7 +1304,7 @@ F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b7244 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test e8ccd09f965c594f38ef486ddf7913f0fcac97be20a785a41c3d7cd4289e82de -F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b +F test/json102.test d89476e2fb515c8c2660b1dcdd9772ac8f17bb538d12e7b7f44dfa46fc899aa8 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95eb7b37fab29931924311f541d52173ef77a448efc8771b1a1783ccd786d23d -R 2841b59f4f7afcfcc5b77dd83ccca9c0 +P d624c31e5c49e1ce63b4b72caa42a61c5167866f47d842fbcfe4e826fd079d7c +R f84ac9b23e77db6a3aeb1eb8d5ee202d U drh -Z d9c4ddddb83fc3b47a718cf56777e023 +Z 7bfe4ac791f85f5c6a70ea3c48417630 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e4d4770c2..bb4d24251b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d624c31e5c49e1ce63b4b72caa42a61c5167866f47d842fbcfe4e826fd079d7c \ No newline at end of file +54197149b811d30b6c4487eedf5692b164ed0f90cfcc541aa3157094f5f17f6a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 46c65befd7..1e40668f9f 100644 --- a/src/json.c +++ b/src/json.c @@ -4712,7 +4712,8 @@ static void jsonSetFunc( const char *zPath; u32 i; int bApnd; - int bIsSet = sqlite3_user_data(ctx)!=0; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; if( (argc&1)==0 ) { diff --git a/test/json102.test b/test/json102.test index ce901efeb1..8ada49d7ae 100644 --- a/test/json102.test +++ b/test/json102.test @@ -21,75 +21,204 @@ source $testdir/tester.tcl do_execsql_test json102-100 { SELECT json_object('ex','[52,3.14159]'); } {{{"ex":"[52,3.14159]"}}} +do_execsql_test json102-100b { + SELECT json(jsonb_object('ex','[52,3.14159]')); +} {{{"ex":"[52,3.14159]"}}} do_execsql_test json102-110 { SELECT json_object('ex',json('[52,3.14159]')); } {{{"ex":[52,3.14159]}}} +do_execsql_test json102-110-2 { + SELECT json(jsonb_object('ex',json('[52,3.14159]'))); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-110-3 { + SELECT json_object('ex',jsonb('[52,3.14159]')); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-110-3 { + SELECT json(jsonb_object('ex',jsonb('[52,3.14159]'))); +} {{{"ex":[52,3.14159]}}} do_execsql_test json102-120 { SELECT json_object('ex',json_array(52,3.14159)); } {{{"ex":[52,3.14159]}}} +do_execsql_test json102-120-2 { + SELECT json(jsonb_object('ex',json_array(52,3.14159))); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-120-3 { + SELECT json_object('ex',jsonb_array(52,3.14159)); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-120-4 { + SELECT json(jsonb_object('ex',jsonb_array(52,3.14159))); +} {{{"ex":[52,3.14159]}}} do_execsql_test json102-130 { SELECT json(' { "this" : "is", "a": [ "test" ] } '); } {{{"this":"is","a":["test"]}}} +do_execsql_test json102-130b { + SELECT json(jsonb(' { "this" : "is", "a": [ "test" ] } ')); +} {{{"this":"is","a":["test"]}}} do_execsql_test json102-140 { SELECT json_array(1,2,'3',4); } {{[1,2,"3",4]}} +do_execsql_test json102-140b { + SELECT json(jsonb_array(1,2,'3',4)); +} {{[1,2,"3",4]}} do_execsql_test json102-150 { SELECT json_array('[1,2]'); } {{["[1,2]"]}} +do_execsql_test json102-150b { + SELECT json(jsonb_array('[1,2]')); +} {{["[1,2]"]}} do_execsql_test json102-160 { SELECT json_array(json_array(1,2)); } {{[[1,2]]}} +do_execsql_test json102-160-2 { + SELECT json_array(jsonb_array(1,2)); +} {{[[1,2]]}} +do_execsql_test json102-160-3 { + SELECT json(jsonb_array(json_array(1,2))); +} {{[[1,2]]}} +do_execsql_test json102-160-4 { + SELECT json(jsonb_array(jsonb_array(1,2))); +} {{[[1,2]]}} do_execsql_test json102-170 { SELECT json_array(1,null,'3','[4,5]','{"six":7.7}'); } {{[1,null,"3","[4,5]","{\"six\":7.7}"]}} +do_execsql_test json102-170b { + SELECT json(jsonb_array(1,null,'3','[4,5]','{"six":7.7}')); +} {{[1,null,"3","[4,5]","{\"six\":7.7}"]}} do_execsql_test json102-180 { SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')); } {{[1,null,"3",[4,5],{"six":7.7}]}} +do_execsql_test json102-180-2 { + SELECT json_array(1,null,'3',jsonb('[4,5]'),json('{"six":7.7}')); +} {{[1,null,"3",[4,5],{"six":7.7}]}} +do_execsql_test json102-180-3 { + SELECT json(jsonb_array(1,null,'3',json('[4,5]'),json('{"six":7.7}'))); +} {{[1,null,"3",[4,5],{"six":7.7}]}} +do_execsql_test json102-180-4 { + SELECT json(jsonb_array(1,null,'3',jsonb('[4,5]'),jsonb('{"six":7.7}'))); +} {{[1,null,"3",[4,5],{"six":7.7}]}} do_execsql_test json102-190 { SELECT json_array_length('[1,2,3,4]'); } {{4}} +do_execsql_test json102-190b { + SELECT json_array_length(jsonb('[1,2,3,4]')); +} {{4}} do_execsql_test json102-191 { SELECT json_array_length( json_remove('[1,2,3,4]','$[2]') ); } {{3}} +do_execsql_test json102-191b { + SELECT json_array_length( jsonb_remove('[1,2,3,4]','$[2]') ); +} {{3}} do_execsql_test json102-200 { SELECT json_array_length('[1,2,3,4]', '$'); } {{4}} +do_execsql_test json102-200b { + SELECT json_array_length(jsonb('[1,2,3,4]'), '$'); +} {{4}} do_execsql_test json102-210 { SELECT json_array_length('[1,2,3,4]', '$[2]'); } {{0}} +do_execsql_test json102-210b { + SELECT json_array_length(jsonb('[1,2,3,4]'), '$[2]'); +} {{0}} do_execsql_test json102-220 { SELECT json_array_length('{"one":[1,2,3]}'); } {{0}} -do_execsql_test json102-230 { - SELECT json_array_length('{"one":[1,2,3]}', '$.one'); +do_execsql_test json102-220 { + SELECT json_array_length('{"one":[1,2,3]}'); +} {{0}} +do_execsql_test json102-230b { + SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.one'); } {{3}} do_execsql_test json102-240 { SELECT json_array_length('{"one":[1,2,3]}', '$.two'); } {{}} +do_execsql_test json102-240b { + SELECT json_array_length(jsonb('{"one":[1,2,3]}'), '$.two'); +} {{}} do_execsql_test json102-250 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); } {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-250-2 { + SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$'); +} {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-250-3 { + SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$')); +} {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-250-4 { + SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$')); +} {{{"a":2,"c":[4,5,{"f":7}]}}} do_execsql_test json102-260 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); } {{[4,5,{"f":7}]}} +do_execsql_test json102-260-2 { + SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c'); +} {{[4,5,{"f":7}]}} +do_execsql_test json102-260-3 { + SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c')); +} {{[4,5,{"f":7}]}} +do_execsql_test json102-260-4 { + SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c')); +} {{[4,5,{"f":7}]}} do_execsql_test json102-270 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); } {{{"f":7}}} +do_execsql_test json102-270-2 { + SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]'); +} {{{"f":7}}} +do_execsql_test json102-270-3 { + SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.c[2]')); +} {{{"f":7}}} +do_execsql_test json102-270-4 { + SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]')); +} {{{"f":7}}} do_execsql_test json102-280 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); } {{7}} +do_execsql_test json102-280b { + SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); +} {{7}} do_execsql_test json102-290 { SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); } {{[[4,5],2]}} +do_execsql_test json102-290-2 { + SELECT json_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a'); +} {{[[4,5],2]}} +do_execsql_test json102-290-3 { + SELECT json(jsonb_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a')); +} {{[[4,5],2]}} +do_execsql_test json102-290-4 { + SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5],"f":7}'),'$.c','$.a')); +} {{[[4,5],2]}} do_execsql_test json102-300 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); } {{}} +do_execsql_test json102-300b { + SELECT jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); +} {{}} do_execsql_test json102-310 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a'); } {{[null,2]}} +do_execsql_test json102-310-2 { + SELECT json_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a'); +} {{[null,2]}} +do_execsql_test json102-310-3 { + SELECT json(jsonb_extract(jsonb('{"a":2,"c":[4,5,{"f":7}]}'), '$.x', '$.a')); +} {{[null,2]}} +do_execsql_test json102-310-43 { + SELECT json(jsonb_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a')); +} {{[null,2]}} do_execsql_test json102-320 { SELECT json_insert('{"a":2,"c":4}', '$.a', 99); } {{{"a":2,"c":4}}} +do_execsql_test json102-320-2 { + SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.a', 99); +} {{{"a":2,"c":4}}} +do_execsql_test json102-320-3 { + SELECT json(jsonb_insert('{"a":2,"c":4}', '$.a', 99)); +} {{{"a":2,"c":4}}} +do_execsql_test json102-320-4 { + SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.a', 99)); +} {{{"a":2,"c":4}}} do_execsql_test json102-330 { SELECT json_insert('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} From 313336f90bc3f6e180529260096610a0aff8cc23 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 2 Oct 2023 23:56:46 +0000 Subject: [PATCH 031/191] New test cases for JSONB. FossilOrigin-Name: 6d4aeff5751722c83ebb0e1f21072b72be418c64dcf28ee032d3a548e1b3a951 --- manifest | 12 +-- manifest.uuid | 2 +- test/json102.test | 219 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 225 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 02e0ac823a..c3140c8eee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sjsonb_insert()\sso\sthat\sit\sdoes\snot\sbehave\slike\sjsonb_set().\nNew\stest\scases\sadded. -D 2023-10-02T20:16:06.301 +C New\stest\scases\sfor\sJSONB. +D 2023-10-02T23:56:46.985 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1304,7 +1304,7 @@ F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b7244 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test e8ccd09f965c594f38ef486ddf7913f0fcac97be20a785a41c3d7cd4289e82de -F test/json102.test d89476e2fb515c8c2660b1dcdd9772ac8f17bb538d12e7b7f44dfa46fc899aa8 +F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d624c31e5c49e1ce63b4b72caa42a61c5167866f47d842fbcfe4e826fd079d7c -R f84ac9b23e77db6a3aeb1eb8d5ee202d +P 54197149b811d30b6c4487eedf5692b164ed0f90cfcc541aa3157094f5f17f6a +R f7716cc560ac345b0556f46a68783d4e U drh -Z 7bfe4ac791f85f5c6a70ea3c48417630 +Z aa79deaa2b4777d46e48ea5272670f4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb4d24251b..cc209c225a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54197149b811d30b6c4487eedf5692b164ed0f90cfcc541aa3157094f5f17f6a \ No newline at end of file +6d4aeff5751722c83ebb0e1f21072b72be418c64dcf28ee032d3a548e1b3a951 \ No newline at end of file diff --git a/test/json102.test b/test/json102.test index 8ada49d7ae..15a54b47c4 100644 --- a/test/json102.test +++ b/test/json102.test @@ -222,87 +222,291 @@ do_execsql_test json102-320-4 { do_execsql_test json102-330 { SELECT json_insert('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-330-2 { + SELECT json_insert(jsonb('{"a":2,"c":4}'), '$.e', 99); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-330-3 { + SELECT json(jsonb_insert('{"a":2,"c":4}', '$.e', 99)); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-330-4 { + SELECT json(jsonb_insert(jsonb('{"a":2,"c":4}'), '$.e', 99)); +} {{{"a":2,"c":4,"e":99}}} do_execsql_test json102-340 { SELECT json_replace('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} +do_execsql_test json102-340-2 { + SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.a', 99); +} {{{"a":99,"c":4}}} +do_execsql_test json102-340-3 { + SELECT json(jsonb_replace('{"a":2,"c":4}', '$.a', 99)); +} {{{"a":99,"c":4}}} +do_execsql_test json102-340-4 { + SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.a', 99)); +} {{{"a":99,"c":4}}} do_execsql_test json102-350 { SELECT json_replace('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4}}} +do_execsql_test json102-350-2 { + SELECT json_replace(jsonb('{"a":2,"c":4}'), '$.e', 99); +} {{{"a":2,"c":4}}} +do_execsql_test json102-350-3 { + SELECT json(jsonb_replace('{"a":2,"c":4}', '$.e', 99)); +} {{{"a":2,"c":4}}} +do_execsql_test json102-350-4 { + SELECT json(jsonb_replace(jsonb('{"a":2,"c":4}'), '$.e', 99)); +} {{{"a":2,"c":4}}} do_execsql_test json102-360 { SELECT json_set('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} +do_execsql_test json102-360-2 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.a', 99); +} {{{"a":99,"c":4}}} +do_execsql_test json102-360-3 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.a', 99)); +} {{{"a":99,"c":4}}} +do_execsql_test json102-360-4 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.a', 99)); +} {{{"a":99,"c":4}}} do_execsql_test json102-370 { SELECT json_set('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-370-2 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.e', 99); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-370-3 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.e', 99)); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-370-4 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.e', 99)); +} {{{"a":2,"c":4,"e":99}}} do_execsql_test json102-380 { SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]'); } {{{"a":2,"c":"[97,96]"}}} +do_execsql_test json102-380-2 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]'); +} {{{"a":2,"c":"[97,96]"}}} +do_execsql_test json102-380-3 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', '[97,96]')); +} {{{"a":2,"c":"[97,96]"}}} +do_execsql_test json102-380-4 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', '[97,96]')); +} {{{"a":2,"c":"[97,96]"}}} do_execsql_test json102-390 { SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]')); } {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-2 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]')); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-3 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json('[97,96]'))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-4 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json('[97,96]'))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-5 { + SELECT json_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]')); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-6 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]')); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-7 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb('[97,96]'))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-390-8 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb('[97,96]'))); +} {{{"a":2,"c":[97,96]}}} do_execsql_test json102-400 { SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96)); } {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-2 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96)); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-3 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', json_array(97,96))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-4 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', json_array(97,96))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-5 { + SELECT json_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96)); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-6 { + SELECT json_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96)); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-7 { + SELECT json(jsonb_set('{"a":2,"c":4}', '$.c', jsonb_array(97,96))); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-400-8 { + SELECT json(jsonb_set(jsonb('{"a":2,"c":4}'), '$.c', jsonb_array(97,96))); +} {{{"a":2,"c":[97,96]}}} do_execsql_test json102-410 { SELECT json_object('a',2,'c',4); } {{{"a":2,"c":4}}} +do_execsql_test json102-410b { + SELECT json(jsonb_object('a',2,'c',4)); +} {{{"a":2,"c":4}}} do_execsql_test json102-420 { SELECT json_object('a',2,'c','{e:5}'); } {{{"a":2,"c":"{e:5}"}}} +do_execsql_test json102-420b { + SELECT json(jsonb_object('a',2,'c','{e:5}')); +} {{{"a":2,"c":"{e:5}"}}} do_execsql_test json102-430 { SELECT json_object('a',2,'c',json_object('e',5)); } {{{"a":2,"c":{"e":5}}}} +do_execsql_test json102-430-2 { + SELECT json(jsonb_object('a',2,'c',json_object('e',5))); +} {{{"a":2,"c":{"e":5}}}} +do_execsql_test json102-430-3 { + SELECT json_object('a',2,'c',jsonb_object('e',5)); +} {{{"a":2,"c":{"e":5}}}} +do_execsql_test json102-430-4 { + SELECT json(jsonb_object('a',2,'c',jsonb_object('e',5))); +} {{{"a":2,"c":{"e":5}}}} do_execsql_test json102-440 { SELECT json_remove('[0,1,2,3,4]','$[2]'); } {{[0,1,3,4]}} +do_execsql_test json102-440-2 { + SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]'); +} {{[0,1,3,4]}} +do_execsql_test json102-440-3 { + SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]')); +} {{[0,1,3,4]}} +do_execsql_test json102-440-4 { + SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]')); +} {{[0,1,3,4]}} do_execsql_test json102-450 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} +do_execsql_test json102-450-2 { + SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]'); +} {{[1,3,4]}} +do_execsql_test json102-450-3 { + SELECT json(jsonb_remove('[0,1,2,3,4]','$[2]','$[0]')); +} {{[1,3,4]}} +do_execsql_test json102-450-4 { + SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]','$[0]')); +} {{[1,3,4]}} do_execsql_test json102-460 { SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]'); } {{[1,2,4]}} +do_execsql_test json102-460-2 { + SELECT json_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]'); +} {{[1,2,4]}} +do_execsql_test json102-460-3 { + SELECT json(jsonb_remove('[0,1,2,3,4]','$[0]','$[2]')); +} {{[1,2,4]}} +do_execsql_test json102-460-4 { + SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[0]','$[2]')); +} {{[1,2,4]}} do_execsql_test json102-470 { SELECT json_remove('{"x":25,"y":42}'); } {{{"x":25,"y":42}}} +do_execsql_test json102-470-2 { + SELECT json_remove(jsonb('{"x":25,"y":42}')); +} {{{"x":25,"y":42}}} +do_execsql_test json102-470-3 { + SELECT json(jsonb_remove('{"x":25,"y":42}')); +} {{{"x":25,"y":42}}} +do_execsql_test json102-470-4 { + SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'))); +} {{{"x":25,"y":42}}} do_execsql_test json102-480 { SELECT json_remove('{"x":25,"y":42}','$.z'); } {{{"x":25,"y":42}}} +do_execsql_test json102-480-2 { + SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.z'); +} {{{"x":25,"y":42}}} +do_execsql_test json102-480-3 { + SELECT json(jsonb_remove('{"x":25,"y":42}','$.z')); +} {{{"x":25,"y":42}}} +do_execsql_test json102-480-4 { + SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.z')); +} {{{"x":25,"y":42}}} do_execsql_test json102-490 { SELECT json_remove('{"x":25,"y":42}','$.y'); } {{{"x":25}}} +do_execsql_test json102-490-2 { + SELECT json_remove(jsonb('{"x":25,"y":42}'),'$.y'); +} {{{"x":25}}} +do_execsql_test json102-490-3 { + SELECT json(jsonb_remove('{"x":25,"y":42}','$.y')); +} {{{"x":25}}} +do_execsql_test json102-490-4 { + SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$.y')); +} {{{"x":25}}} do_execsql_test json102-500 { SELECT json_remove('{"x":25,"y":42}','$'); } {{}} +do_execsql_test json102-500-2 { + SELECT json_remove(jsonb('{"x":25,"y":42}'),'$'); +} {{}} +do_execsql_test json102-500-3 { + SELECT json(jsonb_remove('{"x":25,"y":42}','$')); +} {{}} +do_execsql_test json102-500-4 { + SELECT json(jsonb_remove(jsonb('{"x":25,"y":42}'),'$')); +} {{}} do_execsql_test json102-510 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}'); } {{object}} +do_execsql_test json102-510b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778'); +} {{object}} do_execsql_test json102-520 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$'); } {{object}} +do_execsql_test json102-520b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$'); +} {{object}} do_execsql_test json102-530 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a'); } {{array}} +do_execsql_test json102-530b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a'); +} {{array}} do_execsql_test json102-540 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]'); } {{integer}} +do_execsql_test json102-540b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[0]'); +} {{integer}} do_execsql_test json102-550 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]'); } {{real}} +do_execsql_test json102-550b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[1]'); +} {{real}} do_execsql_test json102-560 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]'); } {{true}} +do_execsql_test json102-560b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[2]'); +} {{true}} do_execsql_test json102-570 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]'); } {{false}} +do_execsql_test json102-570b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[3]'); +} {{false}} do_execsql_test json102-580 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]'); } {{null}} +do_execsql_test json102-580b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[4]'); +} {{null}} do_execsql_test json102-590 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]'); } {{text}} +do_execsql_test json102-590b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[5]'); +} {{text}} do_execsql_test json102-600 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]'); } {{}} +do_execsql_test json102-600b { + SELECT json_type(x'cc0f1761cb0b133235332e350102001778','$.a[6]'); +} {{}} do_execsql_test json102-610 { SELECT json_valid(char(123)||'"x":35'||char(125)); } {{1}} @@ -312,17 +516,24 @@ do_execsql_test json102-620 { ifcapable vtab { do_execsql_test json102-1000 { - CREATE TABLE user(name,phone); + CREATE TABLE user(name,phone,phoneb); INSERT INTO user(name,phone) VALUES ('Alice','["919-555-2345","804-555-3621"]'), ('Bob','["201-555-8872"]'), ('Cindy','["704-555-9983"]'), ('Dave','["336-555-8421","704-555-4321","803-911-4421"]'); + UPDATE user SET phoneb=jsonb(phone); SELECT DISTINCT user.name FROM user, json_each(user.phone) WHERE json_each.value LIKE '704-%' ORDER BY 1; } {Cindy Dave} +do_execsql_test json102-1000b { + SELECT DISTINCT user.name + FROM user, json_each(user.phoneb) + WHERE json_each.value LIKE '704-%' + ORDER BY 1; +} {Cindy Dave} do_execsql_test json102-1010 { UPDATE user @@ -382,6 +593,12 @@ do_execsql_test json102-1110 { WHERE json_tree.type NOT IN ('object','array') ORDER BY +big.rowid, +json_tree.id } $correct_answer +do_execsql_test json102-1110b { + SELECT big.rowid, fullkey, value + FROM big, json_tree(jsonb(big.json)) + WHERE json_tree.type NOT IN ('object','array') + ORDER BY +big.rowid, +json_tree.id +} $correct_answer do_execsql_test json102-1120 { SELECT big.rowid, fullkey, atom FROM big, json_tree(big.json) From 8405e1140a6eefd2ab7e2a02a36e527caba07ca9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 10:43:30 +0000 Subject: [PATCH 032/191] Fix an issue with the use of jsonb_group_array() and jsonb_group_object() when used by window functions. FossilOrigin-Name: 808bd349ba587fbcdc4aea3f9c08e0df01ba08dec181c5af5ea157e89d86ff7b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c3140c8eee..ea14e67936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sJSONB. -D 2023-10-02T23:56:46.985 +C Fix\san\sissue\swith\sthe\suse\sof\sjsonb_group_array()\sand\sjsonb_group_object()\nwhen\sused\sby\swindow\sfunctions. +D 2023-10-03T10:43:30.961 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a2e70610cc14d0f97feb17bf0000c0e034f1cb5ac1b318bcf3ccf7a014f01a80 +F src/json.c 87218d0698545f668879e2a0f51dc5e3982de7d8cccfc0d0a27ede23f26f68d7 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54197149b811d30b6c4487eedf5692b164ed0f90cfcc541aa3157094f5f17f6a -R f7716cc560ac345b0556f46a68783d4e +P 6d4aeff5751722c83ebb0e1f21072b72be418c64dcf28ee032d3a548e1b3a951 +R d0d4e794647619bd04667ee02e2f09dc U drh -Z aa79deaa2b4777d46e48ea5272670f4a +Z 873b50f85fdb68283d7ad2c516b8f579 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cc209c225a..c2bdadfa3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d4aeff5751722c83ebb0e1f21072b72be418c64dcf28ee032d3a548e1b3a951 \ No newline at end of file +808bd349ba587fbcdc4aea3f9c08e0df01ba08dec181c5af5ea157e89d86ff7b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1e40668f9f..50da24b897 100644 --- a/src/json.c +++ b/src/json.c @@ -4897,6 +4897,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); + if( !isFinal ) pStr->nUsed--; return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, @@ -5012,6 +5013,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); + if( !isFinal ) pStr->nUsed--; return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, From ee0f6777f9796f82f4198f7805edf5fe1a24944e Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 10:59:01 +0000 Subject: [PATCH 033/191] Fix a problem in the JSONB parser that comes up following an OOM. FossilOrigin-Name: 355acfb18897254f6b6444a21d781b5e10e930b81952850dd2a40d88bbf2f3db --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ea14e67936..ad6777f075 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\sthe\suse\sof\sjsonb_group_array()\sand\sjsonb_group_object()\nwhen\sused\sby\swindow\sfunctions. -D 2023-10-03T10:43:30.961 +C Fix\sa\sproblem\sin\sthe\sJSONB\sparser\sthat\scomes\sup\sfollowing\san\sOOM. +D 2023-10-03T10:59:01.269 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 87218d0698545f668879e2a0f51dc5e3982de7d8cccfc0d0a27ede23f26f68d7 +F src/json.c 7ed190ae57c81674e978a420f4d1ba2293d09fc5dff8c26801e9e2303346981d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d4aeff5751722c83ebb0e1f21072b72be418c64dcf28ee032d3a548e1b3a951 -R d0d4e794647619bd04667ee02e2f09dc +P 808bd349ba587fbcdc4aea3f9c08e0df01ba08dec181c5af5ea157e89d86ff7b +R 807c8cfa37036711ec0caf325260119e U drh -Z 873b50f85fdb68283d7ad2c516b8f579 +Z 88d0db789da116a139a304208f9109e5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c2bdadfa3b..27183fc527 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -808bd349ba587fbcdc4aea3f9c08e0df01ba08dec181c5af5ea157e89d86ff7b \ No newline at end of file +355acfb18897254f6b6444a21d781b5e10e930b81952850dd2a40d88bbf2f3db \ No newline at end of file diff --git a/src/json.c b/src/json.c index 50da24b897..f25bed1bec 100644 --- a/src/json.c +++ b/src/json.c @@ -3495,7 +3495,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ while( joom ){ pParse->aNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; } j = (u32)r; From eb28787b5ff64a634a70ae7d649eb540f068b0e0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Oct 2023 17:07:54 +0000 Subject: [PATCH 034/191] Update fts5_decode() to allow for embedded 0x00 bytes in tokens. FossilOrigin-Name: e051120067fd87f57b498e505e3960cf4d14e8e33bad940618cc0823253254f7 --- ext/fts5/fts5_index.c | 28 ++++++++++++++++++++++------ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9db5925b94..6cf30b5a00 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -7745,6 +7745,24 @@ static void fts5DecodeRowidList( } #endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){ + int ii; + fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1); + if( *pRc==SQLITE_OK ){ + for(ii=0; iin; ii++){ + if( pTerm->p[ii]==0x00 ){ + pBuf->p[pBuf->n++] = '\\'; + pBuf->p[pBuf->n++] = '0'; + }else{ + pBuf->p[pBuf->n++] = pTerm->p[ii]; + } + } + pBuf->p[pBuf->n] = 0x00; + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + #if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). @@ -7852,9 +7870,8 @@ static void fts5DecodeFunction( iOff += fts5GetVarint32(&a[iOff], nAppend); term.n = nKeep; fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff]); - sqlite3Fts5BufferAppendPrintf( - &rc, &s, " term=%.*s", term.n, (const char*)term.p - ); + sqlite3Fts5BufferAppendPrintf(&rc, &s, " term="); + fts5BufferAppendTerm(&rc, &s, &term); iOff += nAppend; /* Figure out where the doclist for this term ends */ @@ -7962,9 +7979,8 @@ static void fts5DecodeFunction( fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); iOff += nByte; - sqlite3Fts5BufferAppendPrintf( - &rc, &s, " term=%.*s", term.n, (const char*)term.p - ); + sqlite3Fts5BufferAppendPrintf(&rc, &s, " term="); + fts5BufferAppendTerm(&rc, &s, &term); iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff); } diff --git a/manifest b/manifest index bf21569e58..431d902181 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sso\sthat\sfts5\scan\shandle\stokens\swith\sembedded\s'\\0'\sbytes. -D 2023-09-30T18:13:35.306 +C Update\sfts5_decode()\sto\sallow\sfor\sembedded\s0x00\sbytes\sin\stokens. +D 2023-10-03T17:07:54.562 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6 F ext/fts5/fts5_hash.c 76765856397eff56f526b0640b23a1677d737d35e07bc00e4b4b2e0fc5fda60d -F ext/fts5/fts5_index.c 16d775ecbccf7d3698a03bcae3c3fbee0749df748b93b29d0e82a37e02eaaa94 +F ext/fts5/fts5_index.c e472083d371f420d52ec80445b9d2a99b16b23548205cb4064ddcd41bd79f63e F ext/fts5/fts5_main.c 799ec88d2309055f6406bddb0bd6ed80148c5da5eb14594c3c5309a6e944d489 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5 F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f @@ -2123,11 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c04022b7407f77eaf0175e831ebcd6bbdc0af1cef0d42c5c11102aa8484f24ca -R ee3b13ddf778c77c1640cd7c7844c1f5 -T *branch * fts5-token-data -T *sym-fts5-token-data * -T -sym-trunk * +P c027c092c4af53bd6ae3cc6e2b4439167d9eeb0f9de549b6a2c2a72a67ee886c +R 9e81ed5ff713a928831c6c73df8f7a54 U dan -Z 7d5bd217a552215de3d888e155abaef5 +Z 0a0daf2566a3400fafbefb55947df637 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9db9eb3c64..53a545a52f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c027c092c4af53bd6ae3cc6e2b4439167d9eeb0f9de549b6a2c2a72a67ee886c \ No newline at end of file +e051120067fd87f57b498e505e3960cf4d14e8e33bad940618cc0823253254f7 \ No newline at end of file From 1846de49a46bbe1487679d5fdc4283acd38ba307 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 3 Oct 2023 19:06:52 +0000 Subject: [PATCH 035/191] Fixes for fts5 expression parser module to allow embedded 0x00 bytes in tokens. FossilOrigin-Name: 342c8d0783f449817d3f565ff6b9f010a6c690beeea32f1861640810490a8b5f --- ext/fts5/fts5_expr.c | 43 +++++++++++++++++-------------- ext/fts5/test/fts5origintext.test | 4 +++ manifest | 14 +++++----- manifest.uuid | 2 +- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index f5101ba065..745a5d9fa6 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -100,7 +100,8 @@ struct Fts5ExprNode { struct Fts5ExprTerm { u8 bPrefix; /* True for a prefix term */ u8 bFirst; /* True if token must be first in column */ - char *zTerm; /* nul-terminated term */ + char *pTerm; /* Term data */ + int nTerm; /* Size of term in bytes */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; @@ -967,7 +968,7 @@ static int fts5ExprNearInitAll( p->pIter = 0; } rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), + pExpr->pIndex, p->pTerm, p->nTerm, (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), pNear->pColset, @@ -1604,7 +1605,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; - sqlite3_free(pTerm->zTerm); + sqlite3_free(pTerm->pTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; @@ -1735,8 +1736,9 @@ static int fts5ParseTokenize( rc = SQLITE_NOMEM; }else{ memset(pSyn, 0, (size_t)nByte); - pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); - memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + pSyn->nTerm = nToken; + memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } @@ -1761,7 +1763,8 @@ static int fts5ParseTokenize( if( rc==SQLITE_OK ){ pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->nTerm = nToken; } } @@ -1913,9 +1916,7 @@ int sqlite3Fts5ExprClonePhrase( int tflags = 0; Fts5ExprTerm *p; for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); + rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm, p->nTerm, 0, 0); tflags = FTS5_TOKEN_COLOCATED; } if( rc==SQLITE_OK ){ @@ -2296,11 +2297,13 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( if( parseGrowPhraseArray(pParse) ){ fts5ExprPhraseFree(pPhrase); }else{ + Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii]; pParse->apPhrase[pParse->nPhrase++] = pPhrase; pPhrase->nTerm = 1; - pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup( - &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1 + pPhrase->aTerm[0].pTerm = sqlite3Fts5Strndup( + &pParse->rc, p->pTerm, p->nTerm ); + pPhrase->aTerm[0].nTerm = p->nTerm; pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) ); @@ -2485,16 +2488,17 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + nByte += pTerm->nTerm * 2 + 3 + 2; } zQuoted = sqlite3_malloc64(nByte); if( zQuoted ){ int i = 0; for(p=pTerm; p; p=p->pSynonym){ - char *zIn = p->zTerm; + char *zIn = p->pTerm; + char *zEnd = &zIn[p->nTerm]; zQuoted[i++] = '"'; - while( *zIn ){ + while( zInnTerm; iTerm++){ - char *zTerm = pPhrase->aTerm[iTerm].zTerm; - zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; + zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", + p->nTerm, p->pTerm + ); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } @@ -2994,9 +3000,8 @@ static int fts5ExprPopulatePoslistsCb( Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = (int)strlen(pTerm->zTerm); - if( (nTerm==nToken || (nTermbPrefix)) - && memcmp(pTerm->zTerm, pToken, nTerm)==0 + if( (pTerm->nTerm==nToken || (pTerm->nTermbPrefix)) + && memcmp(pTerm->pTerm, pToken, pTerm->nTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 791b850c76..155d74c025 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -45,6 +45,10 @@ do_execsql_test 1.3 { world } +do_execsql_test 1.4 { + SELECT rowid FROM ft('Hello'); +} {1} + #------------------------------------------------------------------------- reset_db diff --git a/manifest b/manifest index 431d902181..e0c37bf1e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sfts5_decode()\sto\sallow\sfor\sembedded\s0x00\sbytes\sin\stokens. -D 2023-10-03T17:07:54.562 +C Fixes\sfor\sfts5\sexpression\sparser\smodule\sto\sallow\sembedded\s0x00\sbytes\sin\stokens. +D 2023-10-03T19:06:52.966 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5Int.h 66a38b285e2b860baa29745d8eff27f5b0809268e7820498494d9acfacc F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 -F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6 +F ext/fts5/fts5_expr.c cc215d39714b428523d2f2ef42b713c83095a28a67bc7f6f2dc4ac036a29f460 F ext/fts5/fts5_hash.c 76765856397eff56f526b0640b23a1677d737d35e07bc00e4b4b2e0fc5fda60d F ext/fts5/fts5_index.c e472083d371f420d52ec80445b9d2a99b16b23548205cb4064ddcd41bd79f63e F ext/fts5/fts5_main.c 799ec88d2309055f6406bddb0bd6ed80148c5da5eb14594c3c5309a6e944d489 @@ -187,7 +187,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 9a6edc85ccc4afb10e71d54d98d8170f850272e55b120520f367afbb12526674 +F ext/fts5/test/fts5origintext.test 3e1ac3230f65a0d644e9bf0738bebb09b4db9d9f123e1307d8630e42269b4afb F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c027c092c4af53bd6ae3cc6e2b4439167d9eeb0f9de549b6a2c2a72a67ee886c -R 9e81ed5ff713a928831c6c73df8f7a54 +P e051120067fd87f57b498e505e3960cf4d14e8e33bad940618cc0823253254f7 +R 5cce41f02eae121cb66e72942ef56113 U dan -Z 0a0daf2566a3400fafbefb55947df637 +Z 80b8e2664e768ed2ac03913fcf0180ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 53a545a52f..4798938837 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e051120067fd87f57b498e505e3960cf4d14e8e33bad940618cc0823253254f7 \ No newline at end of file +342c8d0783f449817d3f565ff6b9f010a6c690beeea32f1861640810490a8b5f \ No newline at end of file From 6e737b92521d8bb3e796ca87b13af36a55aa1531 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 19:37:19 +0000 Subject: [PATCH 036/191] Improved handling of OOM while translating the JsonNode representing into the BLOB representation. FossilOrigin-Name: ef5956710bb542d6045c82937d02218a7ed45af94cf3959b0c180268e04d14e1 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/json.c | 14 +++++++++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 68220a4e0d..40a49f1c48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-10-03T11:36:47.346 +C Improved\shandling\sof\sOOM\swhile\stranslating\sthe\sJsonNode\srepresenting\sinto\nthe\sBLOB\srepresentation. +D 2023-10-03T19:37:19.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -259,12 +259,12 @@ F ext/jni/src/org/sqlite/jni/ProgressHandlerCallback.java 7b9ff2218129ece98ba60c F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86 F ext/jni/src/org/sqlite/jni/RollbackHookCallback.java d12352c0e22840de484ffa9b11ed5058bb0daca2e9f218055d3c54c947a273c4 F ext/jni/src/org/sqlite/jni/SQLFunction.java 544a875d33fd160467d82e2397ac33157b29971d715a821a4fad3c899113ee8c -F ext/jni/src/org/sqlite/jni/SQLTester.java da42be06a2d644e0b915b40508934c1f32391e5308ab8767c1e2e65a281a198f w ext/jni/src/org/sqlite/jni/tester/SQLTester.java +F ext/jni/src/org/sqlite/jni/SQLTester.java da42be06a2d644e0b915b40508934c1f32391e5308ab8767c1e2e65a281a198f F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 9860c1cebd8a38041306f2ee7563f2898fcbdf77e4bfa393fba25b4924edcb5d F ext/jni/src/org/sqlite/jni/ScalarFunction.java 6d387bb499fbe3bc13c53315335233dbf6a0c711e8fa7c521683219b041c614c F ext/jni/src/org/sqlite/jni/TableColumnMetadata.java 54511b4297fa28dcb3f49b24035e34ced10e3fd44fd0e458e784f4d6b0096dab F ext/jni/src/org/sqlite/jni/Tester1.java ced62ed417c3326f93d2e90b3bb64ac2db58ac42a7ad7a5965b24545434e3200 -F ext/jni/src/org/sqlite/jni/TesterFts5.java 854c737bb5c9463ee92a8ee230013e924236dd4b74d4688dd62c17f38d5837db w ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java +F ext/jni/src/org/sqlite/jni/TesterFts5.java 854c737bb5c9463ee92a8ee230013e924236dd4b74d4688dd62c17f38d5837db F ext/jni/src/org/sqlite/jni/TraceV2Callback.java beb0b064c1a5f8bfe585a324ed39a4e33edbe379a3fc60f1401661620d3ca7c0 F ext/jni/src/org/sqlite/jni/UpdateHookCallback.java 8376f4a931f2d5612b295c003c9515ba933ee76d8f95610e89c339727376e36c F ext/jni/src/org/sqlite/jni/WindowFunction.java 488980f4dbb6bdd7067d6cb9c43e4075475e51c54d9b74a5834422654b126246 @@ -289,7 +289,7 @@ F ext/jni/src/org/sqlite/jni/sqlite3_blob.java fc631ad52feea6e3d1d62b0d0e769ac10 F ext/jni/src/org/sqlite/jni/sqlite3_context.java 66ca95ce904044263a4aff684abe262d56f73e6b06bca6cf650761d79d7779ad F ext/jni/src/org/sqlite/jni/sqlite3_stmt.java cf7f076d8b0f2a23faebbd64e12e8b3dd1977378ca828245d186f1b98458127d F ext/jni/src/org/sqlite/jni/sqlite3_value.java 3d1d4903e267bc0bc81d57d21f5e85978eff389a1a6ed46726dbe75f85e6914a -F ext/jni/src/org/sqlite/jni/test-script-interpreter.md f9f25126127045d051e918fe59004a1485311c50a13edbf18c79a6ff9160030e w ext/jni/src/org/sqlite/jni/tester/test-script-interpreter.md +F ext/jni/src/org/sqlite/jni/test-script-interpreter.md f9f25126127045d051e918fe59004a1485311c50a13edbf18c79a6ff9160030e F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43443408f3000139478c4dc745 F ext/jni/src/tests/000-001-ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70 F ext/jni/src/tests/900-001-fts.test bf0ce17a8d082773450e91f2388f5bbb2dfa316d0b676c313c637a91198090f0 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7ed190ae57c81674e978a420f4d1ba2293d09fc5dff8c26801e9e2303346981d +F src/json.c 6a6437fce0eb558b05fc30c1d806ee342e99ccb2dcae1f53b038c60572f84383 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 355acfb18897254f6b6444a21d781b5e10e930b81952850dd2a40d88bbf2f3db e4f9d3946fa570bccea2de17c5536901bdabd46cfe25678fdd37ba0e2bfd99b1 -R 98820669d6cd8b3d30c4719644b02708 +P e6406a9865b75dea2f26d3ee4f4c206958400059c7f92ced88edc8507dd3c82f +R a18f64bf40d3960df89ef1dde4bf90eb U drh -Z e2ba343971c07e25e0b89cfd2106272c +Z d1b86d452a43e36a5aedf44b0fbb363f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 034aa299ab..bea2bfc71f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6406a9865b75dea2f26d3ee4f4c206958400059c7f92ced88edc8507dd3c82f \ No newline at end of file +ef5956710bb542d6045c82937d02218a7ed45af94cf3959b0c180268e04d14e1 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f25bed1bec..c4d874309f 100644 --- a/src/json.c +++ b/src/json.c @@ -1011,7 +1011,12 @@ static void jsonReturnNodeAsJson( JsonParse x; memset(&x, 0, sizeof(x)); jsonRenderNodeAsBlob(pParse, pNode, &x); - sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); + if( x.oom ){ + sqlite3_result_error_nomem(pCtx); + sqlite3_free(x.aBlob); + }else{ + sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); + } }else{ jsonStringInit(&s, pCtx); jsonRenderNodeAsText(pParse, pNode, &s); @@ -2598,8 +2603,11 @@ static void jsonBlobChangePayloadSize( u32 i, u32 szPayload ){ - u8 *a = &pParse->aBlob[i]; - u8 szType = a[0]>>4; + u8 *a; + u8 szType; + if( pParse->oom ) return; + a = &pParse->aBlob[i]; + szType = a[0]>>4; if( szType<=11 ){ assert( szPayload<=11 ); a[0] = (a[0] & 0x0f) | (szPayload<<4); From 51cc3041fe03a7ede42c5286d92bbdf7adbfb276 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 20:58:39 +0000 Subject: [PATCH 037/191] Fix problems in the "json" output column of the json_tree() virtual table for the case when the input is JSONB. FossilOrigin-Name: fefa4475c496aab32fe7b9cc203747ce4faf7448da4f29b807e5486c7686238d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 40a49f1c48..fd92509288 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shandling\sof\sOOM\swhile\stranslating\sthe\sJsonNode\srepresenting\sinto\nthe\sBLOB\srepresentation. -D 2023-10-03T19:37:19.514 +C Fix\sproblems\sin\sthe\s"json"\soutput\scolumn\sof\sthe\sjson_tree()\svirtual\stable\nfor\sthe\scase\swhen\sthe\sinput\sis\sJSONB. +D 2023-10-03T20:58:39.820 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6a6437fce0eb558b05fc30c1d806ee342e99ccb2dcae1f53b038c60572f84383 +F src/json.c 6b11a18b09250869464c7b0e025e76c8697366a2a1e88459ee6ce1e189064332 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6406a9865b75dea2f26d3ee4f4c206958400059c7f92ced88edc8507dd3c82f -R a18f64bf40d3960df89ef1dde4bf90eb +P ef5956710bb542d6045c82937d02218a7ed45af94cf3959b0c180268e04d14e1 +R dc84d245e8ee185581c160d484d587e1 U drh -Z d1b86d452a43e36a5aedf44b0fbb363f +Z 9c1148fb20ac73a79d87122df2949595 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bea2bfc71f..e9a8f2071a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef5956710bb542d6045c82937d02218a7ed45af94cf3959b0c180268e04d14e1 \ No newline at end of file +fefa4475c496aab32fe7b9cc203747ce4faf7448da4f29b807e5486c7686238d \ No newline at end of file diff --git a/src/json.c b/src/json.c index c4d874309f..967a3af006 100644 --- a/src/json.c +++ b/src/json.c @@ -3514,7 +3514,6 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ break; } } - pParse->aBlob[i] = 0; return i+x+sz; } @@ -5356,8 +5355,12 @@ static int jsonEachColumn( break; } case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } break; } } From f362731c1cba8f46cba7b978705573d9e0a1eb51 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 21:54:09 +0000 Subject: [PATCH 038/191] Fix a bug in the jsonReturnFromBlob() function that causes a positive result for a negative value for when a JSON integer is too large and needs to be converted into double. FossilOrigin-Name: dca684da0c29ec78460362f972ea7747be42c13c4d1325da9d62c1ea58022e39 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fd92509288..5334835936 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\sthe\s"json"\soutput\scolumn\sof\sthe\sjson_tree()\svirtual\stable\nfor\sthe\scase\swhen\sthe\sinput\sis\sJSONB. -D 2023-10-03T20:58:39.820 +C Fix\sa\sbug\sin\sthe\sjsonReturnFromBlob()\sfunction\sthat\scauses\sa\spositive\sresult\nfor\sa\snegative\svalue\sfor\swhen\sa\sJSON\sinteger\sis\stoo\slarge\sand\sneeds\sto\sbe\nconverted\sinto\sdouble. +D 2023-10-03T21:54:09.421 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6b11a18b09250869464c7b0e025e76c8697366a2a1e88459ee6ce1e189064332 +F src/json.c 62de483f16a54d30fb98b1ff85a344cb945cfa65a4056dc8950f2b101f0413ec F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef5956710bb542d6045c82937d02218a7ed45af94cf3959b0c180268e04d14e1 -R dc84d245e8ee185581c160d484d587e1 +P fefa4475c496aab32fe7b9cc203747ce4faf7448da4f29b807e5486c7686238d +R f94f7ac8eca40bde91714a188c3f89b5 U drh -Z 9c1148fb20ac73a79d87122df2949595 +Z 6b6f54adde231acb6a7cd797b978cce1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e9a8f2071a..2980b28f6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fefa4475c496aab32fe7b9cc203747ce4faf7448da4f29b807e5486c7686238d \ No newline at end of file +dca684da0c29ec78460362f972ea7747be42c13c4d1325da9d62c1ea58022e39 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 967a3af006..e29940c720 100644 --- a/src/json.c +++ b/src/json.c @@ -3856,6 +3856,7 @@ static void jsonReturnFromBlob( }else if( rc==3 && bNeg ){ sqlite3_result_int64(pCtx, SMALLEST_INT64); }else{ + if( bNeg ){ n--; sz++; } goto to_double; } break; From 4500d87e763914432dd65d181e2bbe928a36500b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Oct 2023 22:40:22 +0000 Subject: [PATCH 039/191] Fix a memory leak in JSON group-aggregates when the output is JSONB. FossilOrigin-Name: 08e7db138b636890cb29a76d65c2069c6e2ff44470fa4bca14f4526fe5f195ae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 12 ++++++++++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5334835936..3991c75517 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\sjsonReturnFromBlob()\sfunction\sthat\scauses\sa\spositive\sresult\nfor\sa\snegative\svalue\sfor\swhen\sa\sJSON\sinteger\sis\stoo\slarge\sand\sneeds\sto\sbe\nconverted\sinto\sdouble. -D 2023-10-03T21:54:09.421 +C Fix\sa\smemory\sleak\sin\sJSON\sgroup-aggregates\swhen\sthe\soutput\sis\sJSONB. +D 2023-10-03T22:40:22.299 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 62de483f16a54d30fb98b1ff85a344cb945cfa65a4056dc8950f2b101f0413ec +F src/json.c 52e4fa529ca0a727ece05607fc3641a7ba6734e39ecc1f3e9ba8c1a7caa1f0cc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fefa4475c496aab32fe7b9cc203747ce4faf7448da4f29b807e5486c7686238d -R f94f7ac8eca40bde91714a188c3f89b5 +P dca684da0c29ec78460362f972ea7747be42c13c4d1325da9d62c1ea58022e39 +R 29d5fdb97c89ceaf3767b5ff5eb276bd U drh -Z 6b6f54adde231acb6a7cd797b978cce1 +Z 573dbd4dc43601643579851c7ec5201b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2980b28f6b..bdc5d99e2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dca684da0c29ec78460362f972ea7747be42c13c4d1325da9d62c1ea58022e39 \ No newline at end of file +08e7db138b636890cb29a76d65c2069c6e2ff44470fa4bca14f4526fe5f195ae \ No newline at end of file diff --git a/src/json.c b/src/json.c index e29940c720..955c35377b 100644 --- a/src/json.c +++ b/src/json.c @@ -4905,7 +4905,11 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); - if( !isFinal ) pStr->nUsed--; + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, @@ -5021,7 +5025,11 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ assert( pStr->bStatic ); }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); - if( !isFinal ) pStr->nUsed--; + if( isFinal ){ + sqlite3RCStrUnref(pStr->zBuf); + }else{ + pStr->nUsed--; + } return; }else if( isFinal ){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, From 9061e22a05f7b0037a9121a8f58f26e37ae8c2f0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 15:02:00 +0000 Subject: [PATCH 040/191] Allow the PG-style syntax for the PATH operand on the right-hand side of the ->> and -> operators. FossilOrigin-Name: bae5071b0834aa3b7fd4bd871f863e1b148c6558989c8f6cdd02dc4da4770953 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 30 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6d4d77ef56..0303e4cd7f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-10-05T11:22:16.707 +C Allow\sthe\sPG-style\ssyntax\sfor\sthe\sPATH\soperand\son\sthe\sright-hand\sside\sof\nthe\s->>\sand\s->\soperators. +D 2023-10-05T15:02:00.537 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 52e4fa529ca0a727ece05607fc3641a7ba6734e39ecc1f3e9ba8c1a7caa1f0cc +F src/json.c 7f765a31a4b0c7df2505a43e74f5059eaff8ed851d60dfc7b6eedfdf9f5b4089 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 08e7db138b636890cb29a76d65c2069c6e2ff44470fa4bca14f4526fe5f195ae a2464bbb825b5976ef974a2e6c17ea150f5e6fcd0dd0f144b9f9c1c22a9c9c82 -R 84b74e4fd51ca93386cc67654677da5e +P be5907b648386e05530d1e321e2a3e563e07e99f08373aaf9a3452adc9228dc3 +R a953453e7f2d067ffe8d15205e74a86b U drh -Z 4d31c96be6e146821c899c0f24a5a9d8 +Z 1c88c01c099cd40d1caaee0961805818 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 973ac14362..12eff4a58c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be5907b648386e05530d1e321e2a3e563e07e99f08373aaf9a3452adc9228dc3 \ No newline at end of file +bae5071b0834aa3b7fd4bd871f863e1b148c6558989c8f6cdd02dc4da4770953 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 955c35377b..3307db0bab 100644 --- a/src/json.c +++ b/src/json.c @@ -3990,12 +3990,38 @@ static void jsonExtractFromBlob( u32 i; JsonParse px; if( zPath==0 ) return; - if( zPath[0]=='$' ) zPath++; memset(&px, 0, sizeof(px)); px.nBlob = sqlite3_value_bytes(pJson); px.aBlob = (u8*)sqlite3_value_blob(pJson); if( px.aBlob==0 ) return; - i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + if( zPath[0]=='$' ){ + zPath++; + i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + }else if( (flags & JSON_ABPATH) ){ + /* The -> and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + */ + JsonString jx; + jsonStringInit(&jx, ctx); + if( sqlite3Isdigit(zPath[0]) ){ + jsonAppendRawNZ(&jx, "[", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendRawNZ(&jx, "]", 2); + zPath = jx.zBuf; + }else if( zPath[0]!='[' ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendChar(&jx, 0); + zPath = jx.zBuf; + } + i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + jsonStringReset(&jx); + } if( i Date: Thu, 5 Oct 2023 15:05:33 +0000 Subject: [PATCH 041/191] Fix a memory leak following a syntax error in jsonb(). FossilOrigin-Name: bf4b36eda8e200b67041ebdf60fc31d0325bd84185272f8d390c56a91d6ac1ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0303e4cd7f..51517eaffb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sPG-style\ssyntax\sfor\sthe\sPATH\soperand\son\sthe\sright-hand\sside\sof\nthe\s->>\sand\s->\soperators. -D 2023-10-05T15:02:00.537 +C Fix\sa\smemory\sleak\sfollowing\sa\ssyntax\serror\sin\sjsonb(). +D 2023-10-05T15:05:33.202 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7f765a31a4b0c7df2505a43e74f5059eaff8ed851d60dfc7b6eedfdf9f5b4089 +F src/json.c 85d3615672a130f2fd22ea6005984048953b745deb48886be72f227ba6f21f9c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be5907b648386e05530d1e321e2a3e563e07e99f08373aaf9a3452adc9228dc3 -R a953453e7f2d067ffe8d15205e74a86b +P bae5071b0834aa3b7fd4bd871f863e1b148c6558989c8f6cdd02dc4da4770953 +R 4bb974d3e61cd02ec317b43011061d2f U drh -Z 1c88c01c099cd40d1caaee0961805818 +Z be9524533dfa255e55d6b8c43429a3ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12eff4a58c..a27cb035f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bae5071b0834aa3b7fd4bd871f863e1b148c6558989c8f6cdd02dc4da4770953 \ No newline at end of file +bf4b36eda8e200b67041ebdf60fc31d0325bd84185272f8d390c56a91d6ac1ee \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3307db0bab..24487f430c 100644 --- a/src/json.c +++ b/src/json.c @@ -4203,12 +4203,13 @@ static void jsonbFunc( x.nJson = nJson; if( jsonConvertTextToBlob(pParse, ctx) ){ sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3_free(pParse->aBlob); }else{ sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->nBlobAlloc = 0; } + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; jsonParseReset(pParse); } } From 0585d9f08f0b7786739621b3e41362079b2cb1e9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 16:33:00 +0000 Subject: [PATCH 042/191] Slightly stricter testing for when a BLOB is valid JSONB: If the element is a null, true, or false, its payload size must be zero. FossilOrigin-Name: 487781be8a93d9b8d870efb9f25822bb4bbcf9d39d571d1017e9c29043ea515d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 51517eaffb..dea192ba1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sfollowing\sa\ssyntax\serror\sin\sjsonb(). -D 2023-10-05T15:05:33.202 +C Slightly\sstricter\stesting\sfor\swhen\sa\sBLOB\sis\svalid\sJSONB:\s\sIf\sthe\selement\nis\sa\snull,\strue,\sor\sfalse,\sits\spayload\ssize\smust\sbe\szero. +D 2023-10-05T16:33:00.108 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 85d3615672a130f2fd22ea6005984048953b745deb48886be72f227ba6f21f9c +F src/json.c 5fce177a1fa0e0853024141e4f603cf04324833c35f1ee2484c20296236d1fa8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bae5071b0834aa3b7fd4bd871f863e1b148c6558989c8f6cdd02dc4da4770953 -R 4bb974d3e61cd02ec317b43011061d2f +P bf4b36eda8e200b67041ebdf60fc31d0325bd84185272f8d390c56a91d6ac1ee +R 82f6c0430b8291dd07f3a6ba0e331be4 U drh -Z be9524533dfa255e55d6b8c43429a3ea +Z fe5195d3e7cce0d43ee394c132952294 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a27cb035f1..943883628e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf4b36eda8e200b67041ebdf60fc31d0325bd84185272f8d390c56a91d6ac1ee \ No newline at end of file +487781be8a93d9b8d870efb9f25822bb4bbcf9d39d571d1017e9c29043ea515d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 24487f430c..c405c5943a 100644 --- a/src/json.c +++ b/src/json.c @@ -3415,6 +3415,8 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ s.nBlob = nBlob; n = jsonbPayloadSize(&s, 0, &sz); if( n==0 ) return 0; + if( sz+n!=(u32)nBlob ) return 0; + if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; return sz+n==(u32)nBlob; } From 14fce24a9bacf6e872154a2a679ff95acb10a7a8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 17:52:39 +0000 Subject: [PATCH 043/191] Fix the text-to-JSONB parser so that it handles some JSON5 floating point literals correctly. FossilOrigin-Name: 564edb3b6dd752e09e445e3a25c2f5394ceede2a2cdff251d6433dadc0b3644f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dea192ba1b..d1a8859d10 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Slightly\sstricter\stesting\sfor\swhen\sa\sBLOB\sis\svalid\sJSONB:\s\sIf\sthe\selement\nis\sa\snull,\strue,\sor\sfalse,\sits\spayload\ssize\smust\sbe\szero. -D 2023-10-05T16:33:00.108 +C Fix\sthe\stext-to-JSONB\sparser\sso\sthat\sit\shandles\ssome\sJSON5\sfloating\spoint\nliterals\scorrectly. +D 2023-10-05T17:52:39.456 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 5fce177a1fa0e0853024141e4f603cf04324833c35f1ee2484c20296236d1fa8 +F src/json.c 5b00523c237b93a0c0f644bfa2adc5cf2bf381e3a2a02791a03a1765d6ddac67 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf4b36eda8e200b67041ebdf60fc31d0325bd84185272f8d390c56a91d6ac1ee -R 82f6c0430b8291dd07f3a6ba0e331be4 +P 487781be8a93d9b8d870efb9f25822bb4bbcf9d39d571d1017e9c29043ea515d +R 6dd39d4bbce9b9abcc4c4c2e13a0b712 U drh -Z fe5195d3e7cce0d43ee394c132952294 +Z 37a9884f8448701e772294f223618265 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 943883628e..233800a07f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -487781be8a93d9b8d870efb9f25822bb4bbcf9d39d571d1017e9c29043ea515d \ No newline at end of file +564edb3b6dd752e09e445e3a25c2f5394ceede2a2cdff251d6433dadc0b3644f \ No newline at end of file diff --git a/src/json.c b/src/json.c index c405c5943a..ba701ba68e 100644 --- a/src/json.c +++ b/src/json.c @@ -2955,7 +2955,7 @@ json_parse_restart: } if( z[i+1]=='.' ){ pParse->hasNonstd = 1; - t |= 0x03; + t |= 0x01; goto parse_number_2; } pParse->iErr = i; From 5b6cfb79068f20f4df34322139ef7c0b9d58ea2c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 18:09:12 +0000 Subject: [PATCH 044/191] Change the json_valid(X) routine to return true whenever X is a blob that could plausibly be a valid JSONB. FossilOrigin-Name: 425f0b85a6f8ad0604c4a5faa18efb90436fedb1fe2612a559147c62cff8b7e7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d1a8859d10..3b52cf09de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stext-to-JSONB\sparser\sso\sthat\sit\shandles\ssome\sJSON5\sfloating\spoint\nliterals\scorrectly. -D 2023-10-05T17:52:39.456 +C Change\sthe\sjson_valid(X)\sroutine\sto\sreturn\strue\swhenever\sX\sis\sa\sblob\sthat\ncould\splausibly\sbe\sa\svalid\sJSONB. +D 2023-10-05T18:09:12.028 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 5b00523c237b93a0c0f644bfa2adc5cf2bf381e3a2a02791a03a1765d6ddac67 +F src/json.c 436cc9b8a4cfef2ac5c8547fcf8acff8a91b9fb9b9d070ebf240e8ab871f23a7 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 487781be8a93d9b8d870efb9f25822bb4bbcf9d39d571d1017e9c29043ea515d -R 6dd39d4bbce9b9abcc4c4c2e13a0b712 +P 564edb3b6dd752e09e445e3a25c2f5394ceede2a2cdff251d6433dadc0b3644f +R 99ed34df38b98d618529ef152948a4c8 U drh -Z 37a9884f8448701e772294f223618265 +Z 05060c3a4460d98ce1ee0304660e3eea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 233800a07f..8c16d8482a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -564edb3b6dd752e09e445e3a25c2f5394ceede2a2cdff251d6433dadc0b3644f \ No newline at end of file +425f0b85a6f8ad0604c4a5faa18efb90436fedb1fe2612a559147c62cff8b7e7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index ba701ba68e..3c179ac16a 100644 --- a/src/json.c +++ b/src/json.c @@ -4812,8 +4812,14 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a well-formed canonical JSON string according -** to RFC-7159. Return 0 otherwise. +** Return 1 if the argument is one of: +** +** * A well-formed canonical JSON string according to RFC-8259 +** (without JSON5 enhancements), or +** +** * A BLOB that plausibly could be a JSONB value. +** +** Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -4829,6 +4835,10 @@ static void jsonValidFunc( #endif return; } + if( jsonFuncArgMightBeBinary(argv[0]) ){ + sqlite3_result_int(ctx, 1); + return; + } p = jsonParseCached(ctx, argv[0], 0, 0); if( p==0 || p->oom ){ sqlite3_result_error_nomem(ctx); From 51d507d432c60e241ad3684a50e0d9a74283303b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 18:33:19 +0000 Subject: [PATCH 045/191] Fix the parsing of non-standard "Infinity" and "NaN" values from text into JSONB. FossilOrigin-Name: df1fbbeb83a2b4a496c9de0d86c7c8c677f8fe3bc770da163dcc1d338a17b58b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3b52cf09de..2f457f4e43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sjson_valid(X)\sroutine\sto\sreturn\strue\swhenever\sX\sis\sa\sblob\sthat\ncould\splausibly\sbe\sa\svalid\sJSONB. -D 2023-10-05T18:09:12.028 +C Fix\sthe\sparsing\sof\snon-standard\s"Infinity"\sand\s"NaN"\svalues\sfrom\stext\sinto\nJSONB. +D 2023-10-05T18:33:19.785 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 436cc9b8a4cfef2ac5c8547fcf8acff8a91b9fb9b9d070ebf240e8ab871f23a7 +F src/json.c 91ba143709783aef3aa7d0e04cc963284a70d1b904f0da14494ede24819b356e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 564edb3b6dd752e09e445e3a25c2f5394ceede2a2cdff251d6433dadc0b3644f -R 99ed34df38b98d618529ef152948a4c8 +P 425f0b85a6f8ad0604c4a5faa18efb90436fedb1fe2612a559147c62cff8b7e7 +R 8b87e1b5ecb5e7b583f0e4691e8e2ff5 U drh -Z 05060c3a4460d98ce1ee0304660e3eea +Z 222bd3b1cad134cfad3fe073645453f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8c16d8482a..37628e34a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -425f0b85a6f8ad0604c4a5faa18efb90436fedb1fe2612a559147c62cff8b7e7 \ No newline at end of file +df1fbbeb83a2b4a496c9de0d86c7c8c677f8fe3bc770da163dcc1d338a17b58b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3c179ac16a..8b14aa3aa9 100644 --- a/src/json.c +++ b/src/json.c @@ -3097,8 +3097,7 @@ json_parse_restart: } if( sqlite3Isalnum(z[i+nn]) ) continue; if( aNanInfName[k].eType==JSON_REAL ){ - jsonBlobAppendOneByte(pParse, JSONB_FLOAT); - jsonBlobAppendOneByte(pParse, 5); + jsonBlobAppendOneByte(pParse, JSONB_FLOAT | 0x50); jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); }else{ jsonBlobAppendOneByte(pParse, JSONB_NULL); From dac27071526c6e0b54827713f8b8c2cee413a9c0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 20:17:01 +0000 Subject: [PATCH 046/191] Turn an unreachable branch into an assert(). FossilOrigin-Name: 0f75199160e48fa8c44f986f1af777adf19e40fbd114a6f58e24d5e6dede779d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2f457f4e43..dc737fc064 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sparsing\sof\snon-standard\s"Infinity"\sand\s"NaN"\svalues\sfrom\stext\sinto\nJSONB. -D 2023-10-05T18:33:19.785 +C Turn\san\sunreachable\sbranch\sinto\san\sassert(). +D 2023-10-05T20:17:01.522 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 91ba143709783aef3aa7d0e04cc963284a70d1b904f0da14494ede24819b356e +F src/json.c e8befacf282a191e830e68f64cf09b1d725c7b4fac0a5617456b218832ecd760 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 425f0b85a6f8ad0604c4a5faa18efb90436fedb1fe2612a559147c62cff8b7e7 -R 8b87e1b5ecb5e7b583f0e4691e8e2ff5 +P df1fbbeb83a2b4a496c9de0d86c7c8c677f8fe3bc770da163dcc1d338a17b58b +R 44eaad4f19392397a894a77de5ede2c8 U drh -Z 222bd3b1cad134cfad3fe073645453f8 +Z 855bfdf3eb65ffc730a52733ff32ac54 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 37628e34a1..ed65424673 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df1fbbeb83a2b4a496c9de0d86c7c8c677f8fe3bc770da163dcc1d338a17b58b \ No newline at end of file +0f75199160e48fa8c44f986f1af777adf19e40fbd114a6f58e24d5e6dede779d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8b14aa3aa9..30612aff91 100644 --- a/src/json.c +++ b/src/json.c @@ -2517,7 +2517,7 @@ static void jsonRemoveAllNulls(JsonNode *pNode){ static int jsonBlobExpand(JsonParse *pParse, u32 N){ u8 *aNew; u32 t; - if( N<=pParse->nBlobAlloc ) return 0; + assert( N>pParse->nBlobAlloc ); if( pParse->nBlobAlloc==0 ){ t = 100; }else{ From f7af8f32704b5675b1acab942d3b8e6931c8edd0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 22:52:43 +0000 Subject: [PATCH 047/191] Improvements to comments. No changes to code. FossilOrigin-Name: ac74d7877685006e43684edd6a2d22be8c9857f9f9eb52fc5b3c182d5e2fdb8d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index dc737fc064..3423bef24f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Turn\san\sunreachable\sbranch\sinto\san\sassert(). -D 2023-10-05T20:17:01.522 +C Improvements\sto\scomments.\s\sNo\schanges\sto\scode. +D 2023-10-05T22:52:43.004 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e8befacf282a191e830e68f64cf09b1d725c7b4fac0a5617456b218832ecd760 +F src/json.c a51a9dd785d3d65f7b4ffc3827610571efae82c5f7469ff8bc9b973de4b43245 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df1fbbeb83a2b4a496c9de0d86c7c8c677f8fe3bc770da163dcc1d338a17b58b -R 44eaad4f19392397a894a77de5ede2c8 +P 0f75199160e48fa8c44f986f1af777adf19e40fbd114a6f58e24d5e6dede779d +R 61adb43472fe06712b001bc8a8daf5aa U drh -Z 855bfdf3eb65ffc730a52733ff32ac54 +Z b88584b3539f893217397c49059fc0f7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ed65424673..b2ac92830c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f75199160e48fa8c44f986f1af777adf19e40fbd114a6f58e24d5e6dede779d \ No newline at end of file +ac74d7877685006e43684edd6a2d22be8c9857f9f9eb52fc5b3c182d5e2fdb8d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 30612aff91..57309e3674 100644 --- a/src/json.c +++ b/src/json.c @@ -1515,10 +1515,10 @@ static const struct NanInfName { ** ** 0 End of input ** -1 Syntax error -** -2 '}' seen -** -3 ']' seen -** -4 ',' seen -** -5 ':' seen +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / */ static int jsonParseValueFromText(JsonParse *pParse, u32 i){ char c; @@ -2656,10 +2656,10 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** ** 0 End of input ** -1 Syntax error -** -2 '}' seen -** -3 ']' seen -** -4 ',' seen -** -5 ':' seen +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / */ static int jsonTranslateTextValueToBlob(JsonParse *pParse, u32 i){ char c; From 27401d38d8a50ff2f03ac4ce34453418249e41d8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Oct 2023 23:05:35 +0000 Subject: [PATCH 048/191] Better error detection when doing a lookup on a JSONB. FossilOrigin-Name: 6e8e0eedbf06e175af6ef230f1ce9a5ad84698b1c6dcd2794d878c746e7b6be4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3423bef24f..218f2d0904 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments.\s\sNo\schanges\sto\scode. -D 2023-10-05T22:52:43.004 +C Better\serror\sdetection\swhen\sdoing\sa\slookup\son\sa\sJSONB. +D 2023-10-05T23:05:35.100 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a51a9dd785d3d65f7b4ffc3827610571efae82c5f7469ff8bc9b973de4b43245 +F src/json.c 1b3349c49b363c654b76f5f9b45c4172d13955679fd4a242b3d92c879f95294c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f75199160e48fa8c44f986f1af777adf19e40fbd114a6f58e24d5e6dede779d -R 61adb43472fe06712b001bc8a8daf5aa +P ac74d7877685006e43684edd6a2d22be8c9857f9f9eb52fc5b3c182d5e2fdb8d +R 01e239b11851146ca21382a5317a867b U drh -Z b88584b3539f893217397c49059fc0f7 +Z 51245adc64c2e53d6bd1f03542705249 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b2ac92830c..cbe2a3c96d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac74d7877685006e43684edd6a2d22be8c9857f9f9eb52fc5b3c182d5e2fdb8d \ No newline at end of file +6e8e0eedbf06e175af6ef230f1ce9a5ad84698b1c6dcd2794d878c746e7b6be4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 57309e3674..738354f009 100644 --- a/src/json.c +++ b/src/json.c @@ -3720,6 +3720,7 @@ static u32 jsonLookupBlobStep( x = pParse->aBlob[j] & 0x0f; if( xJSONB_TEXTRAW ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; k = j+n; if( k+sz>=iEnd ) return JSON_BLOB_ERROR; if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ @@ -3732,6 +3733,7 @@ static u32 jsonLookupBlobStep( j = k+sz; if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; j += n+sz; } if( j>iEnd ) return JSON_BLOB_ERROR; @@ -3776,6 +3778,7 @@ static u32 jsonLookupBlobStep( } k--; n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_BLOB_ERROR; j += n+sz; } if( j>iEnd ) return JSON_BLOB_ERROR; From dd7f09e6c00315eff7930a8c13d4e8d659c541a9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 00:06:26 +0000 Subject: [PATCH 049/191] Improvements to coping with malformed JSONB. FossilOrigin-Name: 563cde404cec4c6559ae0c5fc5edbc878fee874b36562ce2fac5049cc8349343 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 218f2d0904..bc4af507a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\serror\sdetection\swhen\sdoing\sa\slookup\son\sa\sJSONB. -D 2023-10-05T23:05:35.100 +C Improvements\sto\scoping\swith\smalformed\sJSONB. +D 2023-10-06T00:06:26.513 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 1b3349c49b363c654b76f5f9b45c4172d13955679fd4a242b3d92c879f95294c +F src/json.c 6758202d3fa30306ae6379c6f995158b0768e1f75e304ff604c1bb498499fcd1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac74d7877685006e43684edd6a2d22be8c9857f9f9eb52fc5b3c182d5e2fdb8d -R 01e239b11851146ca21382a5317a867b +P 6e8e0eedbf06e175af6ef230f1ce9a5ad84698b1c6dcd2794d878c746e7b6be4 +R 0e32d4d167fb477dce553004066d14c6 U drh -Z 51245adc64c2e53d6bd1f03542705249 +Z 0df001ca326ecb2506a99289e47391a0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cbe2a3c96d..62b4fee4ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6e8e0eedbf06e175af6ef230f1ce9a5ad84698b1c6dcd2794d878c746e7b6be4 \ No newline at end of file +563cde404cec4c6559ae0c5fc5edbc878fee874b36562ce2fac5049cc8349343 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 738354f009..7522c9571a 100644 --- a/src/json.c +++ b/src/json.c @@ -3512,6 +3512,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ if( !pParse->oom ){ pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; } + if( k&1 ) return -1; break; } } @@ -3724,10 +3725,10 @@ static u32 jsonLookupBlobStep( k = j+n; if( k+sz>=iEnd ) return JSON_BLOB_ERROR; if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ - j = k+sz; + j = k+sz; if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); - if( j+n+sz>iEnd ) return JSON_BLOB_ERROR; + if( n==0 || j+n+sz>iEnd ) return JSON_BLOB_ERROR; return jsonLookupBlobStep(pParse, j, &zPath[i], pzErr); } j = k+sz; @@ -4031,7 +4032,7 @@ static void jsonExtractFromBlob( }else if( i==JSON_BLOB_NOTFOUND ){ return; /* Return NULL if not found */ }else if( i==JSON_BLOB_ERROR ){ - sqlite3_result_error(ctx, "malformed JSONB", -1); + sqlite3_result_error(ctx, "malformed JSON", -1); }else{ sqlite3_result_error(ctx, zErr, -1); } @@ -4340,7 +4341,7 @@ static void jsonExtractFunc( JsonString jx; if( argc<2 ) return; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB && argc==2 ){ + if( jsonFuncArgMightBeBinary(argv[0]) && argc==2 ){ jsonExtractFromBlob(ctx, argv[0], argv[1], flags); return; } From e1e2f2dcc5b76ef2bb2ce7c0e59f67b8913b74a2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 14:52:49 +0000 Subject: [PATCH 050/191] Improvements to error handling. FossilOrigin-Name: b41dd237fb6c0dd7daedeaaf81dbc4fdb31cf511fd32cd3d4ee69db34e389915 --- manifest | 12 +++---- manifest.uuid | 2 +- src/json.c | 89 +++++++++++++++++++++++++++------------------------ 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index b6d3e12ce0..555f873fb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\scompiler\swarning\sfixes\sfrom\strunk\sinto\sthe\sjsonb\sbranch. -D 2023-10-06T13:05:42.465 +C Improvements\sto\serror\shandling. +D 2023-10-06T14:52:49.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e89b89a6f36b2281524aea01521690e1d3bc69db2754856e4a03ac271f261346 +F src/json.c fe8cd7b38e947a89fbe8b8099a88ee363d02a6747009ba5246a517de1e15159f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 563cde404cec4c6559ae0c5fc5edbc878fee874b36562ce2fac5049cc8349343 9bf4bfd68080367b58594e0d44b110b3ee9766420f648537fd7bc638dacefb72 -R fa6a8cc346d8604b4e2021455b16ba57 +P 6409d307915f8969f12df2d5ffa961645bd4db7ccfbd6f52237a217b6a867252 +R bb54780cc2adfb5e91d01e5a38070ac9 U drh -Z 77ed8049621ac72b8ae81ce5801b8fae +Z 4eb2d45211dd6d4bee3b0c79a0072e1a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dea74400a2..950949b618 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6409d307915f8969f12df2d5ffa961645bd4db7ccfbd6f52237a217b6a867252 \ No newline at end of file +b41dd237fb6c0dd7daedeaaf81dbc4fdb31cf511fd32cd3d4ee69db34e389915 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3419572759..69c9e67b6b 100644 --- a/src/json.c +++ b/src/json.c @@ -207,10 +207,15 @@ struct JsonString { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ + u8 eErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; +/* Allowed values for JsonString.eErr */ +#define JSTRING_OOM 0x01 /* Out of memory */ +#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ +#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ + /* A deferred cleanup task. A list of JsonCleanup objects might be ** run when the JsonParse object is destroyed. */ @@ -376,7 +381,7 @@ static void jsonStringZero(JsonString *p){ */ static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->bErr = 0; + p->eErr = 0; jsonStringZero(p); } @@ -391,7 +396,7 @@ static void jsonStringReset(JsonString *p){ /* Report an out-of-memory (OOM) condition */ static void jsonStringOom(JsonString *p){ - p->bErr = 1; + p->eErr |= JSTRING_OOM; sqlite3_result_error_nomem(p->pCtx); jsonStringReset(p); } @@ -403,7 +408,7 @@ static int jsonStringGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->bErr ) return 1; + if( p->eErr ) return 1; zNew = sqlite3RCStrNew(nTotal); if( zNew==0 ){ jsonStringOom(p); @@ -415,7 +420,7 @@ static int jsonStringGrow(JsonString *p, u32 N){ }else{ p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); if( p->zBuf==0 ){ - p->bErr = 1; + p->eErr |= JSTRING_OOM; jsonStringZero(p); return SQLITE_NOMEM; } @@ -488,7 +493,7 @@ static void jsonAppendChar(JsonString *p, char c){ */ static int jsonForceRCStr(JsonString *p){ jsonAppendChar(p, 0); - if( p->bErr ) return 0; + if( p->eErr ) return 0; p->nUsed--; if( p->bStatic==0 ) return 1; p->nAlloc = 0; @@ -626,7 +631,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ char *zBuf = sqlite3_malloc64( N+1 ); if( zBuf==0 ){ - p->bErr = 1; + p->eErr |= JSTRING_OOM; return; } memcpy(zBuf, zIn, N); @@ -731,9 +736,9 @@ static void jsonAppendSqlValue( px.aBlob = (u8*)sqlite3_value_blob(pValue); px.nBlob = sqlite3_value_bytes(pValue); jsonRenderBlob(&px, 0, p); - }else if( p->bErr==0 ){ + }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; + p->eErr = JSTRING_ERR; jsonStringReset(p); } break; @@ -748,7 +753,7 @@ static void jsonAppendSqlValue( ** The JsonString is reset. */ static void jsonReturnString(JsonString *p){ - if( p->bErr==0 ){ + if( p->eErr==0 ){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(p); @@ -760,10 +765,13 @@ static void jsonReturnString(JsonString *p){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, (void(*)(void*))sqlite3RCStrUnref, SQLITE_UTF8); + }else{ + sqlite3_result_error_nomem(p->pCtx); } - } - if( p->bErr==1 ){ + }else if( p->eErr & JSTRING_OOM ){ sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_MALFORMED ){ + sqlite3_result_error(p->pCtx, "malformed JSON", -1); } jsonStringReset(p); } @@ -2974,6 +2982,7 @@ json_parse_restart: } } } + parse_number_2: for(j=i+1;; j++){ c = z[j]; @@ -3236,7 +3245,7 @@ static u32 jsonRenderBlob( n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ - pOut->bErr = 1; + pOut->eErr |= JSTRING_MALFORMED; return pParse->nBlob+1; } switch( pParse->aBlob[i] & 0x0f ){ @@ -3258,44 +3267,35 @@ static u32 jsonRenderBlob( break; } case JSONB_INT5: { /* Integer literal in hexadecimal notation */ - int k; + int k = 2; sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; - if( zIn[0]=='-' ){ - zIn++; - sz--; - jsonAppendChar(pOut, '-'); + if( zIn[0]=='+' || zIn[0]=='-' ){ + if( zIn[0]=='-' ) jsonAppendChar(pOut, '-'); + k++; } - for(k=2; kaBlob[i+n]; - if( zIn[0]=='-' ){ - zIn++; - sz--; - jsonAppendChar(pOut, '-'); + if( zIn[0]=='+' || zIn[0]=='-' ){ + if( zIn[0]=='-' ) jsonAppendChar(pOut, '-'); + k++; } - if( zIn[0]=='.' ){ + if( zIn[k]=='.' ){ jsonAppendChar(pOut, '0'); } - for(k=0; k0 ){ - jsonAppendRawNZ(pOut, zIn, sz); - } break; } case JSONB_TEXT: @@ -3808,7 +3808,12 @@ static void jsonReturnTextJsonFromBlob( x.nBlob = nBlob; jsonStringInit(&s, ctx); jsonRenderBlob(&x, 0, &s); - jsonReturnString(&s); + if( x.nErr ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + jsonStringReset(&s); + }else{ + jsonReturnString(&s); + } } @@ -4371,7 +4376,7 @@ static void jsonExtractFunc( jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); jsonAppendChar(&jx, 0); } - pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); + pNode = jx.eErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); jsonStringReset(&jx); }else{ pNode = jsonLookup(p, zPath, 0, ctx); @@ -4942,9 +4947,9 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + if( pStr->eErr ){ + jsonReturnString(pStr); + return; }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ @@ -5062,9 +5067,9 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ jsonAppendChar(pStr, '}'); pStr->pCtx = ctx; flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); + if( pStr->eErr ){ + jsonReturnString(pStr); + return; }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); if( isFinal ){ From e690d5ad80ae90573fce913b0c01d05aaac0c769 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 14:59:40 +0000 Subject: [PATCH 051/191] Fix compiler warnings. FossilOrigin-Name: 5227add3c8d509de2e081249163fafdf30ac3173a6d710957f3c3b6f03e7017e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 17 ++++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 555f873fb7..53be387052 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\serror\shandling. -D 2023-10-06T14:52:49.860 +C Fix\scompiler\swarnings. +D 2023-10-06T14:59:40.414 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fe8cd7b38e947a89fbe8b8099a88ee363d02a6747009ba5246a517de1e15159f +F src/json.c e1a4c5ffadbeaab6dfd9dfb687f8c7a0b9e012ad2ec8b74223e75f86295a5a5e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6409d307915f8969f12df2d5ffa961645bd4db7ccfbd6f52237a217b6a867252 -R bb54780cc2adfb5e91d01e5a38070ac9 +P b41dd237fb6c0dd7daedeaaf81dbc4fdb31cf511fd32cd3d4ee69db34e389915 +R d43877b0997e4344224eb4366c47821e U drh -Z 4eb2d45211dd6d4bee3b0c79a0072e1a +Z 0dbbe0cfb7ed03f93b950daed7da83cd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 950949b618..cd9e86ced9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b41dd237fb6c0dd7daedeaaf81dbc4fdb31cf511fd32cd3d4ee69db34e389915 \ No newline at end of file +5227add3c8d509de2e081249163fafdf30ac3173a6d710957f3c3b6f03e7017e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 69c9e67b6b..b2053af9ec 100644 --- a/src/json.c +++ b/src/json.c @@ -3267,7 +3267,7 @@ static u32 jsonRenderBlob( break; } case JSONB_INT5: { /* Integer literal in hexadecimal notation */ - int k = 2; + u32 k = 2; sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; if( zIn[0]=='+' || zIn[0]=='-' ){ @@ -3281,7 +3281,7 @@ static u32 jsonRenderBlob( break; } case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ - int k = 0; + u32 k = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; if( zIn[0]=='+' || zIn[0]=='-' ){ if( zIn[0]=='-' ) jsonAppendChar(pOut, '-'); @@ -3753,14 +3753,14 @@ static u32 jsonLookupBlobStep( k = jsonbArrayCount(pParse, iRoot); i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int x = 0; + unsigned int nn = 0; i = 3; do{ - x = x*10 + zPath[i] - '0'; + nn = nn*10 + zPath[i] - '0'; i++; }while( sqlite3Isdigit(zPath[i]) ); - if( x>k ) return 0; - k -= x; + if( nn>k ) return 0; + k -= nn; } if( zPath[i]!=']' ){ *pzErr = zPath; @@ -3997,7 +3997,7 @@ static void jsonExtractFromBlob( ){ const char *zPath = (const char*)sqlite3_value_text(pPath); const char *zErr = 0; - u32 i; + u32 i = 0; JsonParse px; if( zPath==0 ) return; memset(&px, 0, sizeof(px)); @@ -4031,6 +4031,9 @@ static void jsonExtractFromBlob( } i = jsonLookupBlobStep(&px, 0, zPath, &zErr); jsonStringReset(&jx); + }else{ + sqlite3_result_error(ctx, "bad path", -1); + return; } if( i Date: Fri, 6 Oct 2023 15:35:42 +0000 Subject: [PATCH 052/191] Fixes to rendering JSON5 extensions encoded as JSONB. FossilOrigin-Name: 5a17e4479aad2d8313170e5de83a1c52f30b55d9d4fb776024fa6622e175c63b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 20 +++++++++++--------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 53be387052..ab452be4c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings. -D 2023-10-06T14:59:40.414 +C Fixes\sto\srendering\sJSON5\sextensions\sencoded\sas\sJSONB. +D 2023-10-06T15:35:42.574 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e1a4c5ffadbeaab6dfd9dfb687f8c7a0b9e012ad2ec8b74223e75f86295a5a5e +F src/json.c 07cbae93a65485082e3a2b7900b0c5e3912a5dd37ea0d948af54fec0fe39855c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b41dd237fb6c0dd7daedeaaf81dbc4fdb31cf511fd32cd3d4ee69db34e389915 -R d43877b0997e4344224eb4366c47821e +P 5227add3c8d509de2e081249163fafdf30ac3173a6d710957f3c3b6f03e7017e +R 4cd522c574d0ec5aee66d5dc3b63f485 U drh -Z 0dbbe0cfb7ed03f93b950daed7da83cd +Z 9624fe80ac16daaa792bb7c1f316d34b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd9e86ced9..630195eeab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5227add3c8d509de2e081249163fafdf30ac3173a6d710957f3c3b6f03e7017e \ No newline at end of file +5a17e4479aad2d8313170e5de83a1c52f30b55d9d4fb776024fa6622e175c63b \ No newline at end of file diff --git a/src/json.c b/src/json.c index b2053af9ec..25ef372d81 100644 --- a/src/json.c +++ b/src/json.c @@ -3308,15 +3308,16 @@ static u32 jsonRenderBlob( case JSONB_TEXT5: { const char *zIn; u32 k; + u32 sz2 = sz; zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); - while( sz>0 ){ - for(k=0; k0 ){ + for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); zIn += k; - sz -= k; - if( sz==0 ) break; + sz2 -= k; + if( sz2==0 ) break; } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ @@ -3330,7 +3331,7 @@ static u32 jsonRenderBlob( jsonAppendRawNZ(pOut, "\\u00", 4); jsonAppendRawNZ(pOut, &zIn[2], 2); zIn += 2; - sz -= 2; + sz2 -= 2; break; case '0': jsonAppendRawNZ(pOut, "\\u0000", 6); @@ -3338,24 +3339,24 @@ static u32 jsonRenderBlob( case '\r': if( zIn[2]=='\n' ){ zIn++; - sz--; + sz2--; } break; case '\n': break; case 0xe2: - assert( sz>=4 ); + assert( sz2>=4 ); assert( 0x80==(u8)zIn[2] ); assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); zIn += 2; - sz -= 2; + sz2 -= 2; break; default: jsonAppendRawNZ(pOut, zIn, 2); break; } zIn += 2; - sz -= 2; + sz2 -= 2; } jsonAppendChar(pOut, '"'); break; @@ -3387,6 +3388,7 @@ static u32 jsonRenderBlob( } default: { + pOut->eErr |= JSTRING_MALFORMED; break; } } From 7b4349735f339a4272060b196d7ede864825f3d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 18:21:47 +0000 Subject: [PATCH 053/191] Incremental improvements to the JSONB logic. FossilOrigin-Name: fe326829c27715e249f727ba797c7df6491e874ec205a0a82ee09c78d61c6e1f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 32 +++++++++++++++++++++----------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index ab452be4c4..7a0a01ea06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\srendering\sJSON5\sextensions\sencoded\sas\sJSONB. -D 2023-10-06T15:35:42.574 +C Incremental\simprovements\sto\sthe\sJSONB\slogic. +D 2023-10-06T18:21:47.497 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 07cbae93a65485082e3a2b7900b0c5e3912a5dd37ea0d948af54fec0fe39855c +F src/json.c 7a37b75ae7a31399af464627f923232f88bdee093af5148445f0df42ee85bd77 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5227add3c8d509de2e081249163fafdf30ac3173a6d710957f3c3b6f03e7017e -R 4cd522c574d0ec5aee66d5dc3b63f485 +P 5a17e4479aad2d8313170e5de83a1c52f30b55d9d4fb776024fa6622e175c63b +R 83990e8b89665f4fb2543aa74cc93235 U drh -Z 9624fe80ac16daaa792bb7c1f316d34b +Z e604af5f00b43366ef6886bc95452ede # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 630195eeab..7926def158 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a17e4479aad2d8313170e5de83a1c52f30b55d9d4fb776024fa6622e175c63b \ No newline at end of file +fe326829c27715e249f727ba797c7df6491e874ec205a0a82ee09c78d61c6e1f \ No newline at end of file diff --git a/src/json.c b/src/json.c index 25ef372d81..62dc44b830 100644 --- a/src/json.c +++ b/src/json.c @@ -3150,7 +3150,7 @@ static int jsonConvertTextToBlob( } } if( i<=0 ){ - if( pCtx!=0 ){ + if( ALWAYS(pCtx!=0) ){ if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); }else{ @@ -3191,7 +3191,7 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; - if( i>pParse->nBlob ){ + if( NEVER(i>pParse->nBlob) ){ *pSz = 0; return 0; } @@ -3200,21 +3200,21 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ sz = x; n = 1; }else if( x==12 ){ - if( i+1>pParse->nBlob ){ + if( i+1>=pParse->nBlob ){ *pSz = 0; return 0; } sz = pParse->aBlob[i+1]; n = 2; }else if( x==13 ){ - if( i+2>pParse->nBlob ){ + if( i+2>=pParse->nBlob ){ *pSz = 0; return 0; } sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; n = 3; }else{ - if( i+4>pParse->nBlob ){ + if( i+4>=pParse->nBlob ){ *pSz = 0; return 0; } @@ -3233,8 +3233,15 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ /* ** Convert the binary BLOB representation of JSON beginning at -** aBlob[0] (and extending for no more than nBlob bytes) into +** aBlob[0] and extending for no more than nBlob bytes into ** a pure JSON string. The string is appended to pOut. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ static u32 jsonRenderBlob( JsonParse *pParse, /* the complete parse of the JSON */ @@ -3270,11 +3277,14 @@ static u32 jsonRenderBlob( u32 k = 2; sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; - if( zIn[0]=='+' || zIn[0]=='-' ){ - if( zIn[0]=='-' ) jsonAppendChar(pOut, '-'); + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); k++; } for(; keErr |= JSTRING_MALFORMED; + } u = u*16 + sqlite3HexToInt(zIn[k]); } jsonPrintf(100,pOut,"%llu",u); @@ -3283,8 +3293,8 @@ static u32 jsonRenderBlob( case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ u32 k = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; - if( zIn[0]=='+' || zIn[0]=='-' ){ - if( zIn[0]=='-' ) jsonAppendChar(pOut, '-'); + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); k++; } if( zIn[k]=='.' ){ @@ -3761,7 +3771,7 @@ static u32 jsonLookupBlobStep( nn = nn*10 + zPath[i] - '0'; i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>k ) return 0; + if( nn>k ) return JSON_BLOB_NOTFOUND; k -= nn; } if( zPath[i]!=']' ){ From 6b8aa95c3c3f79cedb4fcaea84687c671642bcf9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 22:16:09 +0000 Subject: [PATCH 054/191] Improved error detection for JSONB inputs. FossilOrigin-Name: 6945e11aa441ace52b93cdbb81ac45b6f809987f3dbc522251e63f8cb948fddd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7a0a01ea06..5fe562b491 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\simprovements\sto\sthe\sJSONB\slogic. -D 2023-10-06T18:21:47.497 +C Improved\serror\sdetection\sfor\sJSONB\sinputs. +D 2023-10-06T22:16:09.559 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7a37b75ae7a31399af464627f923232f88bdee093af5148445f0df42ee85bd77 +F src/json.c b806a942b8abe97c95eef416fe6b467687e3b6452db35bf53746b19bd6800d5f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a17e4479aad2d8313170e5de83a1c52f30b55d9d4fb776024fa6622e175c63b -R 83990e8b89665f4fb2543aa74cc93235 +P fe326829c27715e249f727ba797c7df6491e874ec205a0a82ee09c78d61c6e1f +R 37a35a123339c5edc35b6af8c811707d U drh -Z e604af5f00b43366ef6886bc95452ede +Z 5c69847c848dbc8186c5994996d772c2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7926def158..8c40873330 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe326829c27715e249f727ba797c7df6491e874ec205a0a82ee09c78d61c6e1f \ No newline at end of file +6945e11aa441ace52b93cdbb81ac45b6f809987f3dbc522251e63f8cb948fddd \ No newline at end of file diff --git a/src/json.c b/src/json.c index 62dc44b830..25aa186e9a 100644 --- a/src/json.c +++ b/src/json.c @@ -3284,8 +3284,10 @@ static u32 jsonRenderBlob( for(; keErr |= JSTRING_MALFORMED; + break; + }else{ + u = u*16 + sqlite3HexToInt(zIn[k]); } - u = u*16 + sqlite3HexToInt(zIn[k]); } jsonPrintf(100,pOut,"%llu",u); break; From 11aa2adca34aed5b723e4772b39176aafc1da84a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Oct 2023 23:02:06 +0000 Subject: [PATCH 055/191] Correct handling of "raw" strings in JSON. This requires three test-case changes in TH3 to add double-quotes to the path outputs from json_tree(). The new behavior is correct, I believe. FossilOrigin-Name: ab2bf3e3596dfa7566f4a130adeccbef4056c7321181112a875477193583f30c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 28 +++++++++++++++------------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5fe562b491..7ea0301feb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\serror\sdetection\sfor\sJSONB\sinputs. -D 2023-10-06T22:16:09.559 +C Correct\shandling\sof\s"raw"\sstrings\sin\sJSON.\s\sThis\srequires\sthree\stest-case\nchanges\sin\sTH3\sto\sadd\sdouble-quotes\sto\sthe\spath\soutputs\sfrom\sjson_tree().\nThe\snew\sbehavior\sis\scorrect,\sI\sbelieve. +D 2023-10-06T23:02:06.178 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c b806a942b8abe97c95eef416fe6b467687e3b6452db35bf53746b19bd6800d5f +F src/json.c 4d2527a8f01797bc4a83ca282f4ea82a5e4a1fdbb05260df241a4a76d2e610b9 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe326829c27715e249f727ba797c7df6491e874ec205a0a82ee09c78d61c6e1f -R 37a35a123339c5edc35b6af8c811707d +P 6945e11aa441ace52b93cdbb81ac45b6f809987f3dbc522251e63f8cb948fddd +R 9a8ad0594c72d0104c485b0093cb4da0 U drh -Z 5c69847c848dbc8186c5994996d772c2 +Z 53c0ada212c66efecbd15c380158c623 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8c40873330..c079911246 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6945e11aa441ace52b93cdbb81ac45b6f809987f3dbc522251e63f8cb948fddd \ No newline at end of file +ab2bf3e3596dfa7566f4a130adeccbef4056c7321181112a875477193583f30c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 25aa186e9a..850f2bbf48 100644 --- a/src/json.c +++ b/src/json.c @@ -910,13 +910,7 @@ static void jsonRenderNodeAsText( case JSON_STRING: { assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->jnFlags & JNODE_LABEL ){ - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - jsonAppendChar(pOut, '"'); - }else{ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - } + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ @@ -1564,7 +1558,7 @@ json_parse_restart: ){ k++; } - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); + jsonParseAddNode(pParse, JSON_STRING, k-j, &z[j]); pParse->hasNonstd = 1; x = k; }else{ @@ -3373,6 +3367,10 @@ static u32 jsonRenderBlob( jsonAppendChar(pOut, '"'); break; } + case JSONB_TEXTRAW: { + jsonAppendString(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } case JSONB_ARRAY: { jsonAppendChar(pOut, '['); j = i+n; @@ -3481,10 +3479,14 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); break; } - case JSONB_TEXT: { + case JSONB_TEXTRAW: { jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); break; } + case JSONB_TEXT: { + jsonParseAddNode(pParse, JSON_STRING, sz, zPayload); + break; + } case JSONB_TEXTJ: { jsonParseAddNode(pParse, JSON_STRING | (JNODE_ESCAPE<<8), sz, zPayload); break; @@ -3495,10 +3497,6 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ sz, zPayload); break; } - case JSONB_TEXTRAW: { - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); - break; - } case JSONB_ARRAY: { int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); u32 j = i+x; @@ -3529,6 +3527,10 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ if( k&1 ) return -1; break; } + default: { + pParse->nErr++; + break; + } } return i+x+sz; } From 522e880bc263a2b7afa3234fe473532cc054348c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 09:13:59 +0000 Subject: [PATCH 056/191] Report unknown JSONB element type when parsing JSONB into JsonNode. FossilOrigin-Name: 1f4d3268257bda234c66c3767451b3d39d7b4c7c73a779ad396d9dfc5315905c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7ea0301feb..25c5daf405 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\shandling\sof\s"raw"\sstrings\sin\sJSON.\s\sThis\srequires\sthree\stest-case\nchanges\sin\sTH3\sto\sadd\sdouble-quotes\sto\sthe\spath\soutputs\sfrom\sjson_tree().\nThe\snew\sbehavior\sis\scorrect,\sI\sbelieve. -D 2023-10-06T23:02:06.178 +C Report\sunknown\sJSONB\selement\stype\swhen\sparsing\sJSONB\sinto\sJsonNode. +D 2023-10-07T09:13:59.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4d2527a8f01797bc4a83ca282f4ea82a5e4a1fdbb05260df241a4a76d2e610b9 +F src/json.c e1a1da1393d89d9845dccb57cb1e37fa9b753294946c2779d2c94dddbdee799b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6945e11aa441ace52b93cdbb81ac45b6f809987f3dbc522251e63f8cb948fddd -R 9a8ad0594c72d0104c485b0093cb4da0 +P ab2bf3e3596dfa7566f4a130adeccbef4056c7321181112a875477193583f30c +R 6dab506825da64397c11d8b8419cab92 U drh -Z 53c0ada212c66efecbd15c380158c623 +Z be12a743f89d2812c7cba83b03ca7a82 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c079911246..68c204be9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab2bf3e3596dfa7566f4a130adeccbef4056c7321181112a875477193583f30c \ No newline at end of file +1f4d3268257bda234c66c3767451b3d39d7b4c7c73a779ad396d9dfc5315905c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 850f2bbf48..3c2af6e845 100644 --- a/src/json.c +++ b/src/json.c @@ -3528,8 +3528,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ break; } default: { - pParse->nErr++; - break; + return -1; } } return i+x+sz; From bae197c175d8e58bfccd873e29d3c2d722f6b2ae Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 11:36:16 +0000 Subject: [PATCH 057/191] Improved error messages from search on JSONB. FossilOrigin-Name: 96cfdc31e305406123f6ef1b920423f71902dc4b34381c3efb720274343d37cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 15 ++++++++++----- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 25c5daf405..5c7372384b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Report\sunknown\sJSONB\selement\stype\swhen\sparsing\sJSONB\sinto\sJsonNode. -D 2023-10-07T09:13:59.445 +C Improved\serror\smessages\sfrom\ssearch\son\sJSONB. +D 2023-10-07T11:36:16.795 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e1a1da1393d89d9845dccb57cb1e37fa9b753294946c2779d2c94dddbdee799b +F src/json.c 969f4897a681aa786f77ed7122db07475cf95f473d3a2de2572ffae600ec0ac1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab2bf3e3596dfa7566f4a130adeccbef4056c7321181112a875477193583f30c -R 6dab506825da64397c11d8b8419cab92 +P 1f4d3268257bda234c66c3767451b3d39d7b4c7c73a779ad396d9dfc5315905c +R 473fbbc9196350cff1fe0e5f952691b5 U drh -Z be12a743f89d2812c7cba83b03ca7a82 +Z ebe282f031d63d9584408df2f4948927 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 68c204be9d..fc45fd3b7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f4d3268257bda234c66c3767451b3d39d7b4c7c73a779ad396d9dfc5315905c \ No newline at end of file +96cfdc31e305406123f6ef1b920423f71902dc4b34381c3efb720274343d37cc \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3c2af6e845..fdc7df41b2 100644 --- a/src/json.c +++ b/src/json.c @@ -3716,8 +3716,8 @@ static u32 jsonLookupBlobStep( if( zPath[i] ){ i++; }else{ - *pzErr = zPath; - return 0; + *pzErr = "unterminated \""; + return JSON_BLOB_PATHERROR; } testcase( nKey==0 ); }else{ @@ -3778,11 +3778,11 @@ static u32 jsonLookupBlobStep( k -= nn; } if( zPath[i]!=']' ){ - *pzErr = zPath; + *pzErr = "unterminated ["; return JSON_BLOB_PATHERROR; } }else{ - *pzErr = zPath; + *pzErr = "bad argument to []"; return JSON_BLOB_PATHERROR; } } @@ -3799,7 +3799,7 @@ static u32 jsonLookupBlobStep( } if( j>iEnd ) return JSON_BLOB_ERROR; }else{ - *pzErr = zPath; + *pzErr = "syntax error"; return JSON_BLOB_PATHERROR; } return JSON_BLOB_NOTFOUND; @@ -4056,6 +4056,11 @@ static void jsonExtractFromBlob( return; /* Return NULL if not found */ }else if( i==JSON_BLOB_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); + }else if( i==JSON_BLOB_PATHERROR ){ + char *zMsg = sqlite3_mprintf("in JSON path '%s': %s", + sqlite3_value_text(pPath), zErr); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); }else{ sqlite3_result_error(ctx, zErr, -1); } From 1244b6cbf663132e6f3d8c285eef0162c6fca397 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 17:50:06 +0000 Subject: [PATCH 058/191] Improvements to error messages returned when the ->> operator fails. FossilOrigin-Name: 2f3388f14c843f1c02926e8b929365c06c1f1f4ea6fe6316092c3799c14549d3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 29 ++++++++++------------------- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 1720248ca6..5ccd6fe084 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\smemory\sleak\sfix\sfrom\strunk. -D 2023-10-07T11:37:00.963 +C Improvements\sto\serror\smessages\sreturned\swhen\sthe\s->>\soperator\sfails. +D 2023-10-07T17:50:06.014 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 969f4897a681aa786f77ed7122db07475cf95f473d3a2de2572ffae600ec0ac1 +F src/json.c a5d273004e0b0d4b961efe59b403c5b118218b3db2c2d61b4512ea2815ada8b2 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96cfdc31e305406123f6ef1b920423f71902dc4b34381c3efb720274343d37cc f99ff655d09763c4a22d065041644ece793d84c82c644931e89ccf50c4f4564a -R 9cb62826bab57bac8933243b48ca592e +P 358de1b09f3d5ec0fe459775b0a2a99dfa235817327016b472aaa1ed56d952e6 +R 65b1837fdcd2785b2dbe03dd6bcb09bb U drh -Z cdf0570fe999933e060b2c73f0eb1dc1 +Z f25f60a6b37a3fc4a80bb7878104ebfa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 282aac4978..f4e3476f43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -358de1b09f3d5ec0fe459775b0a2a99dfa235817327016b472aaa1ed56d952e6 \ No newline at end of file +2f3388f14c843f1c02926e8b929365c06c1f1f4ea6fe6316092c3799c14549d3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index fdc7df41b2..fac05a0776 100644 --- a/src/json.c +++ b/src/json.c @@ -3696,8 +3696,7 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ static u32 jsonLookupBlobStep( JsonParse *pParse, /* The JSON to search */ u32 iRoot, /* Begin the search at this element of aBlob[] */ - const char *zPath, /* The path to search */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ + const char *zPath /* The path to search */ ){ u32 i, j, k, nKey, sz, n, iEnd; const char *zKey; @@ -3707,7 +3706,6 @@ static u32 jsonLookupBlobStep( if( zPath[0]==0 ) return iRoot; if( zPath[0]=='.' ){ x = pParse->aBlob[iRoot]; - if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; @@ -3716,7 +3714,6 @@ static u32 jsonLookupBlobStep( if( zPath[i] ){ i++; }else{ - *pzErr = "unterminated \""; return JSON_BLOB_PATHERROR; } testcase( nKey==0 ); @@ -3725,10 +3722,10 @@ static u32 jsonLookupBlobStep( for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; if( nKey==0 ){ - *pzErr = zPath; - return 0; + return JSON_BLOB_PATHERROR; } } + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); j = iRoot + n; iEnd = j+sz; @@ -3744,7 +3741,7 @@ static u32 jsonLookupBlobStep( if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 || j+n+sz>iEnd ) return JSON_BLOB_ERROR; - return jsonLookupBlobStep(pParse, j, &zPath[i], pzErr); + return jsonLookupBlobStep(pParse, j, &zPath[i]); } j = k+sz; if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; @@ -3778,11 +3775,9 @@ static u32 jsonLookupBlobStep( k -= nn; } if( zPath[i]!=']' ){ - *pzErr = "unterminated ["; return JSON_BLOB_PATHERROR; } }else{ - *pzErr = "bad argument to []"; return JSON_BLOB_PATHERROR; } } @@ -3790,7 +3785,7 @@ static u32 jsonLookupBlobStep( iEnd = j+sz; while( jiEnd ) return JSON_BLOB_ERROR; }else{ - *pzErr = "syntax error"; return JSON_BLOB_PATHERROR; } return JSON_BLOB_NOTFOUND; @@ -4011,7 +4005,6 @@ static void jsonExtractFromBlob( int flags ){ const char *zPath = (const char*)sqlite3_value_text(pPath); - const char *zErr = 0; u32 i = 0; JsonParse px; if( zPath==0 ) return; @@ -4021,7 +4014,7 @@ static void jsonExtractFromBlob( if( px.aBlob==0 ) return; if( zPath[0]=='$' ){ zPath++; - i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + i = jsonLookupBlobStep(&px, 0, zPath); }else if( (flags & JSON_ABPATH) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for @@ -4044,7 +4037,7 @@ static void jsonExtractFromBlob( jsonAppendChar(&jx, 0); zPath = jx.zBuf; } - i = jsonLookupBlobStep(&px, 0, zPath, &zErr); + i = jsonLookupBlobStep(&px, 0, zPath); jsonStringReset(&jx); }else{ sqlite3_result_error(ctx, "bad path", -1); @@ -4056,13 +4049,11 @@ static void jsonExtractFromBlob( return; /* Return NULL if not found */ }else if( i==JSON_BLOB_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); - }else if( i==JSON_BLOB_PATHERROR ){ - char *zMsg = sqlite3_mprintf("in JSON path '%s': %s", - sqlite3_value_text(pPath), zErr); + }else{ + char *zMsg = sqlite3_mprintf("bad path syntax: %s", + sqlite3_value_text(pPath)); sqlite3_result_error(ctx, zMsg, -1); sqlite3_free(zMsg); - }else{ - sqlite3_result_error(ctx, zErr, -1); } } From f26833d7837932c7a3ecc6522e439947e34074e0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 19:05:10 +0000 Subject: [PATCH 059/191] Fix the jsonbChangePayloadSize() routine so that it shifts the payload in order to always render the most compact encoding of the payload size. This is necessary as sometimes (as discovered by dbsqlfuzz) the payload size can grow significantly due to json_insert() or json_replace(). FossilOrigin-Name: 8d6d04ca975ec55c419d40d8463c433b0db698c9fb4812ab9f16d4ee5bee460e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 49 ++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 5ccd6fe084..588048d8e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\serror\smessages\sreturned\swhen\sthe\s->>\soperator\sfails. -D 2023-10-07T17:50:06.014 +C Fix\sthe\sjsonbChangePayloadSize()\sroutine\sso\sthat\sit\sshifts\sthe\spayload\sin\sorder\nto\salways\srender\sthe\smost\scompact\sencoding\sof\sthe\spayload\ssize.\s\sThis\sis\nnecessary\sas\ssometimes\s(as\sdiscovered\sby\sdbsqlfuzz)\sthe\spayload\ssize\scan\ngrow\ssignificantly\sdue\sto\sjson_insert()\sor\sjson_replace(). +D 2023-10-07T19:05:10.497 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a5d273004e0b0d4b961efe59b403c5b118218b3db2c2d61b4512ea2815ada8b2 +F src/json.c f526f060002c245769018dc5610bb64c981fb41b239b4d21781c47c6f6e83f1c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 358de1b09f3d5ec0fe459775b0a2a99dfa235817327016b472aaa1ed56d952e6 -R 65b1837fdcd2785b2dbe03dd6bcb09bb +P 2f3388f14c843f1c02926e8b929365c06c1f1f4ea6fe6316092c3799c14549d3 +R 9d37d6a70144b585d8728dd14898ad5e U drh -Z f25f60a6b37a3fc4a80bb7878104ebfa +Z 1ec73fd9ef6d1fd16b94f82c874a9e54 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f4e3476f43..76641696e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f3388f14c843f1c02926e8b929365c06c1f1f4ea6fe6316092c3799c14549d3 \ No newline at end of file +8d6d04ca975ec55c419d40d8463c433b0db698c9fb4812ab9f16d4ee5bee460e \ No newline at end of file diff --git a/src/json.c b/src/json.c index fac05a0776..6f31214f55 100644 --- a/src/json.c +++ b/src/json.c @@ -2607,24 +2607,55 @@ static void jsonBlobChangePayloadSize( ){ u8 *a; u8 szType; + u8 nExtra; + u8 nNeeded; + i8 delta; if( pParse->oom ) return; a = &pParse->aBlob[i]; szType = a[0]>>4; if( szType<=11 ){ - assert( szPayload<=11 ); + nExtra = 0; + }else if( szType==12 ){ + nExtra = 1; + }else if( szType==13 ){ + nExtra = 2; + }else{ + nExtra = 4; + } + if( szPayload<=11 ){ + nNeeded = 0; + }else if( szPayload<=0xff ){ + nNeeded = 1; + }else if( szPayload<=0xffff ){ + nNeeded = 2; + }else{ + nNeeded = 4; + } + delta = nNeeded - nExtra; + if( delta ){ + u32 newSize = pParse->nBlob + delta; + if( delta>0 ){ + if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ + return; /* OOM error. Error state recorded in pParse->oom. */ + } + a = &pParse->aBlob[i]; + memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); + }else{ + memmove(&a[1], &a[1-delta], pParse->nBlob - (i+1-delta)); + } + pParse->nBlob = newSize; + } + if( nNeeded==0 ){ a[0] = (a[0] & 0x0f) | (szPayload<<4); - }else if( szType==0xc ){ - assert( szPayload<=0xff ); - assert( i+1nBlob ); + }else if( nNeeded==1 ){ + a[0] = (a[0] & 0x0f) | 0xc0; a[1] = szPayload & 0xff; - }else if( szType==0xd ){ - assert( szPayload<=0xffff ); - assert( i+2nBlob ); + }else if( nNeeded==2 ){ + a[0] = (a[0] & 0x0f) | 0xd0; a[1] = (szPayload >> 8) & 0xff; a[2] = szPayload & 0xff; }else{ - assert( szType==0xe ); - assert( i+4nBlob ); + a[0] = (a[0] & 0x0f) | 0xe0; a[1] = (szPayload >> 24) & 0xff; a[2] = (szPayload >> 16) & 0xff; a[3] = (szPayload >> 8) & 0xff; From 3efb2c47918e47c391d04b49cbebe24b31c3464b Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 19:40:20 +0000 Subject: [PATCH 060/191] Improved detection of malformed JSONB when parsing it into a JsonNode array. FossilOrigin-Name: ed99a788415e1f8375bd5ec004dd18b1cd0fae4aa94558170882ca487f6dff93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 588048d8e9..3e86195bd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sjsonbChangePayloadSize()\sroutine\sso\sthat\sit\sshifts\sthe\spayload\sin\sorder\nto\salways\srender\sthe\smost\scompact\sencoding\sof\sthe\spayload\ssize.\s\sThis\sis\nnecessary\sas\ssometimes\s(as\sdiscovered\sby\sdbsqlfuzz)\sthe\spayload\ssize\scan\ngrow\ssignificantly\sdue\sto\sjson_insert()\sor\sjson_replace(). -D 2023-10-07T19:05:10.497 +C Improved\sdetection\sof\smalformed\sJSONB\swhen\sparsing\sit\sinto\sa\sJsonNode\sarray. +D 2023-10-07T19:40:20.950 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c f526f060002c245769018dc5610bb64c981fb41b239b4d21781c47c6f6e83f1c +F src/json.c 98ef9894e38f07a5565160b297754937ff3ac2b800241f16a60c6168251d3aa3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2f3388f14c843f1c02926e8b929365c06c1f1f4ea6fe6316092c3799c14549d3 -R 9d37d6a70144b585d8728dd14898ad5e +P 8d6d04ca975ec55c419d40d8463c433b0db698c9fb4812ab9f16d4ee5bee460e +R 6b6b7362eeaf8bd29175dff0f94e9ac1 U drh -Z 1ec73fd9ef6d1fd16b94f82c874a9e54 +Z 3aae7be717232d09cf2cbfe5dd8cc288 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 76641696e4..46067f0f9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d6d04ca975ec55c419d40d8463c433b0db698c9fb4812ab9f16d4ee5bee460e \ No newline at end of file +ed99a788415e1f8375bd5ec004dd18b1cd0fae4aa94558170882ca487f6dff93 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6f31214f55..1cfaaaf58a 100644 --- a/src/json.c +++ b/src/json.c @@ -3481,31 +3481,38 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ zPayload = &pParse->zJson[i+x]; switch( t ){ case JSONB_NULL: { + if( sz>0 ) return -1; jsonParseAddNode(pParse, JSON_NULL, 0, 0); break; } case JSONB_TRUE: { + if( sz>0 ) return -1; jsonParseAddNode(pParse, JSON_TRUE, 0, 0); break; } case JSONB_FALSE: { + if( sz>0 ) return -1; jsonParseAddNode(pParse, JSON_FALSE, 0, 0); break; } case JSONB_INT: { + if( sz==0 ) return -1; jsonParseAddNode(pParse, JSON_INT, sz, zPayload); break; } case JSONB_INT5: { + if( sz==0 ) return -1; pParse->hasNonstd = 1; jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); break; } case JSONB_FLOAT: { + if( sz==0 ) return -1; jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); break; } case JSONB_FLOAT5: { + if( sz==0 ) return -1; pParse->hasNonstd = 1; jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); break; From de8ccf00fbe12a0c4f353224e94412ef0be13090 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 19:46:22 +0000 Subject: [PATCH 061/191] The return from sqlite3_value_blob() in jsonFuncArgMightBeBinary() might be a NULL pointer. Check for that case. FossilOrigin-Name: 7b52b266b066f1385144c1103a3a411306db5f44568366ae1e93cd8cce799bbc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e86195bd3..76e3968de4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sof\smalformed\sJSONB\swhen\sparsing\sit\sinto\sa\sJsonNode\sarray. -D 2023-10-07T19:40:20.950 +C The\sreturn\sfrom\ssqlite3_value_blob()\sin\sjsonFuncArgMightBeBinary()\smight\nbe\sa\sNULL\spointer.\s\sCheck\sfor\sthat\scase. +D 2023-10-07T19:46:22.897 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 98ef9894e38f07a5565160b297754937ff3ac2b800241f16a60c6168251d3aa3 +F src/json.c 6b0c502122ad2e306888e2c32eebd8c5b46c36e6b6f3ae1662c81e4a26d48410 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d6d04ca975ec55c419d40d8463c433b0db698c9fb4812ab9f16d4ee5bee460e -R 6b6b7362eeaf8bd29175dff0f94e9ac1 +P ed99a788415e1f8375bd5ec004dd18b1cd0fae4aa94558170882ca487f6dff93 +R d7e7cc74e75e3ba90c541936a8c94fd0 U drh -Z 3aae7be717232d09cf2cbfe5dd8cc288 +Z 327a5a9c0b0e46cc41c5ed6d2b8be3ed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46067f0f9c..2445478af5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed99a788415e1f8375bd5ec004dd18b1cd0fae4aa94558170882ca487f6dff93 \ No newline at end of file +7b52b266b066f1385144c1103a3a411306db5f44568366ae1e93cd8cce799bbc \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1cfaaaf58a..f2cb796889 100644 --- a/src/json.c +++ b/src/json.c @@ -3451,7 +3451,7 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ nBlob = sqlite3_value_bytes(pJson); if( nBlob<1 ) return 0; aBlob = sqlite3_value_blob(pJson); - if( (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + if( aBlob==0 || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; memset(&s, 0, sizeof(s)); s.aBlob = (u8*)aBlob; s.nBlob = nBlob; From a7e9386e889151ca062688190468e361a7e7e21d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Oct 2023 23:35:07 +0000 Subject: [PATCH 062/191] Remove some unnecessary code. Report errors for invalid JSONB input on an extract. FossilOrigin-Name: cbea16c29eb0507f39b5a1cf744a3bb9bb7c71ac156e84a19d03a37cb1816891 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 20 +++++++------------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 76e3968de4..1df3b331aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sreturn\sfrom\ssqlite3_value_blob()\sin\sjsonFuncArgMightBeBinary()\smight\nbe\sa\sNULL\spointer.\s\sCheck\sfor\sthat\scase. -D 2023-10-07T19:46:22.897 +C Remove\ssome\sunnecessary\scode.\s\sReport\serrors\sfor\sinvalid\sJSONB\sinput\son\nan\sextract. +D 2023-10-07T23:35:07.967 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -670,7 +670,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6b0c502122ad2e306888e2c32eebd8c5b46c36e6b6f3ae1662c81e4a26d48410 +F src/json.c e97d03f1c19e403bfe0f0a1deaf50b3e3d657bb8addd5dfe7f9dcf72bcfa3109 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2124,8 +2124,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed99a788415e1f8375bd5ec004dd18b1cd0fae4aa94558170882ca487f6dff93 -R d7e7cc74e75e3ba90c541936a8c94fd0 +P 7b52b266b066f1385144c1103a3a411306db5f44568366ae1e93cd8cce799bbc +R 878d86e1e87a30f2fb868ea5f585d29f U drh -Z 327a5a9c0b0e46cc41c5ed6d2b8be3ed +Z 509d9208ad89eff1e0f2ca5c213ed021 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2445478af5..db876805c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b52b266b066f1385144c1103a3a411306db5f44568366ae1e93cd8cce799bbc \ No newline at end of file +cbea16c29eb0507f39b5a1cf744a3bb9bb7c71ac156e84a19d03a37cb1816891 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f2cb796889..139ffb4598 100644 --- a/src/json.c +++ b/src/json.c @@ -2800,9 +2800,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - if( pParse->oom==0 ){ - jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); - } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); pParse->iDepth--; return j+1; } @@ -2854,9 +2852,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - if( pParse->oom==0 ){ - jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); - } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); pParse->iDepth--; return j+1; } @@ -3855,12 +3851,7 @@ static void jsonReturnTextJsonFromBlob( x.nBlob = nBlob; jsonStringInit(&s, ctx); jsonRenderBlob(&x, 0, &s); - if( x.nErr ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - jsonStringReset(&s); - }else{ - jsonReturnString(&s); - } + jsonReturnString(&s); } @@ -3903,7 +3894,6 @@ static void jsonReturnFromBlob( int bNeg = 0; char x = (char)pParse->aBlob[i+n]; if( x=='-' && ALWAYS(sz>0) ){ n++; sz--; bNeg = 1; } - else if( x=='+' && ALWAYS(sz>0) ){ n++; sz--; } z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); if( z==0 ) return; rc = sqlite3DecOrHexToI64(z, &iRes); @@ -4031,6 +4021,10 @@ static void jsonReturnFromBlob( } break; } + default: { + sqlite3_result_error(pCtx, "malformed JSON", -1); + break; + } } } From f32aa3434648a03a308d7251d21a3334664e9fcc Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 9 Oct 2023 18:33:01 +0000 Subject: [PATCH 063/191] Systematize the names of some of the translation function in the JSON implementation. FossilOrigin-Name: db44bd1d62084ef69c808f7d07e0a25d5a089dcb8b98b1b5d026777591bbdefc --- manifest | 14 +++--- manifest.uuid | 2 +- src/json.c | 137 ++++++++++++++++++++++++++------------------------ 3 files changed, 78 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index d523129a6f..3959f0c228 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\sfixes\sand\senhancements\sinto\sthe\sjsonb\sbranch,\sand\nespecially\sthe\sJSON\scache\sspill\sUAF\sfix. -D 2023-10-09T12:57:03.290 +C Systematize\sthe\snames\sof\ssome\sof\sthe\stranslation\sfunction\sin\sthe\sJSON\nimplementation. +D 2023-10-09T18:33:01.568 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -245,7 +245,7 @@ F ext/jni/src/org/sqlite/jni/AggregateFunction.java 7312486bc65fecdb91753c0a4515 F ext/jni/src/org/sqlite/jni/AuthorizerCallback.java fde5f758ad170ca45ae00b12194c8ba8d8f3090bd64cc3e002dd9c5e7dff8568 F ext/jni/src/org/sqlite/jni/AutoExtensionCallback.java c0fbfd3779fc92982c7935325a7484dee43eeb80d716989ed31218f453addb94 F ext/jni/src/org/sqlite/jni/BusyHandlerCallback.java 4cb7fc70efd55583fed6033c34a8719da42975ca97ef4781dda0b9f6cc8ec2e8 -F ext/jni/src/org/sqlite/jni/CApi.java c1dde485a3a3f43c46c8d9c527f9ba5bf303fe0409b2c0de253fb7b6e1055f7e w ext/jni/src/org/sqlite/jni/SQLite3Jni.java +F ext/jni/src/org/sqlite/jni/CApi.java c1dde485a3a3f43c46c8d9c527f9ba5bf303fe0409b2c0de253fb7b6e1055f7e F ext/jni/src/org/sqlite/jni/CallbackProxy.java 064a8a00e4c63cc501c30504f93ca996d422c5f010067f969b2d0a10f0868153 F ext/jni/src/org/sqlite/jni/CollationCallback.java 8cf57cb014a645ecc12609eed17308852a597bc5e83d82a4fdb90f7fadc25f9d F ext/jni/src/org/sqlite/jni/CollationNeededCallback.java 0c62245e000d5db52576c728cac20f6a31f31f5cf40ca4cbcd64b22964e82ae5 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 8717fe7a6461f24ba7b92ccd323c8e2417f44f2a959704c5a05a7aac1ca0df12 +F src/json.c bbf52181d04e09852e273f11cf31544a8030ffc911359cb7f77e39f252cce7a6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2128,8 +2128,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cbea16c29eb0507f39b5a1cf744a3bb9bb7c71ac156e84a19d03a37cb1816891 a163fecca90cab9d1b7bf8ebac78d498775eed7b6d81e7920e3401633c3a4b60 -R 1dd322f61ae3b783d06705b01094df38 +P 9422c24f4a8b290dcae61e50ec81be5b314b22c61a2bca1e194e47da1316b6e6 +R 3ec6096d3289810ffc9770092e31c712 U drh -Z 41e410a665a938a0454df231f95bd4b4 +Z 411045c6779d7c6439ce2a3295f7e71a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e2032820b2..533952e38f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9422c24f4a8b290dcae61e50ec81be5b314b22c61a2bca1e194e47da1316b6e6 \ No newline at end of file +db44bd1d62084ef69c808f7d07e0a25d5a089dcb8b98b1b5d026777591bbdefc \ No newline at end of file diff --git a/src/json.c b/src/json.c index d4367324d8..c2b7e39225 100644 --- a/src/json.c +++ b/src/json.c @@ -337,7 +337,7 @@ struct JsonParse { u32 iErr; /* Error location in zJson[] */ u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ u32 iHold; /* Age of this entry in the cache for LRU replacement */ - /* Binary format */ + /* Storage for the binary JSONB format */ u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ u8 *aBlob; /* BLOB representation of zJson */ @@ -356,12 +356,12 @@ struct JsonParse { ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static void jsonRenderNodeAsBlob(JsonParse*,JsonNode*,JsonParse*); +static void jsonXlateNodeToBlob(JsonParse*,JsonNode*,JsonParse*); static int jsonParseAddNode(JsonParse*,u32,u32,const char*); -static int jsonParseValueFromBlob(JsonParse *pParse, u32 i); +static int jsonXlateBlobToNode(JsonParse *pParse, u32 i); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); -static u32 jsonRenderBlob(JsonParse*,u32,JsonString*); +static u32 jsonXlateBlobToText(JsonParse*,u32,JsonString*); /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -735,7 +735,7 @@ static void jsonAppendSqlValue( memset(&px, 0, sizeof(px)); px.aBlob = (u8*)sqlite3_value_blob(pValue); px.nBlob = sqlite3_value_bytes(pValue); - jsonRenderBlob(&px, 0, p); + jsonXlateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); p->eErr = JSTRING_ERR; @@ -868,11 +868,10 @@ static int jsonParseAddCleanup( } /* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. +** Translate the JsonNode pNode into a pure JSON string and +** append that string on pOut. Subsubstructure is also included. */ -static void jsonRenderNodeAsText( +static void jsonXlateNodeToText( JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ JsonString *pOut /* Write JSON here */ @@ -947,7 +946,7 @@ static void jsonRenderNodeAsText( while( j<=pNode->n ){ if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNodeAsText(pParse, &pNode[j], pOut); + jsonXlateNodeToText(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } @@ -967,9 +966,9 @@ static void jsonRenderNodeAsText( while( j<=pNode->n ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); - jsonRenderNodeAsText(pParse, &pNode[j], pOut); + jsonXlateNodeToText(pParse, &pNode[j], pOut); jsonAppendChar(pOut, ':'); - jsonRenderNodeAsText(pParse, &pNode[j+1], pOut); + jsonXlateNodeToText(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -1012,7 +1011,7 @@ static void jsonReturnNodeAsJson( if( flags & JSON_BLOB ){ JsonParse x; memset(&x, 0, sizeof(x)); - jsonRenderNodeAsBlob(pParse, pNode, &x); + jsonXlateNodeToBlob(pParse, pNode, &x); if( x.oom ){ sqlite3_result_error_nomem(pCtx); sqlite3_free(x.aBlob); @@ -1021,7 +1020,7 @@ static void jsonReturnNodeAsJson( } }else{ jsonStringInit(&s, pCtx); - jsonRenderNodeAsText(pParse, pNode, &s); + jsonXlateNodeToText(pParse, pNode, &s); if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ pParse->zAlt = sqlite3RCStrRef(s.zBuf); pParse->nAlt = s.nUsed; @@ -1072,7 +1071,7 @@ static u32 jsonHexToInt4(const char *z){ ** value returned is either RFC-8259 JSON text or a BLOB in the JSONB ** format, depending on the JSON_BLOB flag of the function user-data. */ -static void jsonReturnNodeAsSql( +static void jsonReturnFromNode( JsonParse *pParse, /* Complete JSON parse tree */ JsonNode *pNode, /* Node to return */ sqlite3_context *pCtx /* Return value for this function */ @@ -1510,8 +1509,11 @@ static const struct NanInfName { }; /* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. +** Translate a single element of JSON text beginning at pParse->zJson[i] into +** its JsonNode representation. Append the translation onto the +** pParse->aNode[] array, which is increased in size as necessary. +** Return the pJson->zJson[] index of the first character past the end of +** the element that was parsed. ** ** Special return values: ** @@ -1522,7 +1524,7 @@ static const struct NanInfName { ** -4 ',' seen / the index in zJson[] of the seen character ** -5 ':' seen / */ -static int jsonParseValueFromText(JsonParse *pParse, u32 i){ +static int jsonXlateTextToNode(JsonParse *pParse, u32 i){ char c; u32 j; int iThis; @@ -1541,7 +1543,7 @@ json_parse_restart: } for(j=i+1;;j++){ u32 nNode = pParse->nNode; - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; @@ -1584,7 +1586,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1592,7 +1594,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -1611,7 +1613,7 @@ json_parse_restart: break; } } - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -1640,7 +1642,7 @@ json_parse_restart: } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -1664,7 +1666,7 @@ json_parse_restart: break; } } - x = jsonParseValueFromText(pParse, j); + x = jsonXlateTextToNode(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -2004,9 +2006,9 @@ static int jsonParse( if( pParse->isBinary ){ pParse->aBlob = (u8*)pParse->zJson; pParse->nBlob = pParse->nJson; - i = jsonParseValueFromBlob(pParse, 0); + i = jsonXlateBlobToNode(pParse, 0); }else{ - i = jsonParseValueFromText(pParse, 0); + i = jsonXlateTextToNode(pParse, 0); } if( pParse->oom ) i = -1; if( !pParse->isBinary && i>0 ){ @@ -2679,12 +2681,12 @@ static int jsonIs4HexB(const char *z, int *pOp){ } /* -** Parse a single JSON text value which begins at pParse->zJson[i] into -** its equivalent BLOB representation in pParse->aBlob[]. The parse is -** appended to pParse->aBlob[] beginning at pParse->nBlob. The size of +** Translate a single element of JSON text at pParse->zJson[i] into +** its equivalent binary JSONB representation. Append the translation into +** pParse->aBlob[] beginning at pParse->nBlob. The size of ** pParse->aBlob[] is increased as necessary. ** -** Return the index of the first character past the end of the value parsed, +** Return the index of the first character past the end of the element parsed, ** or one of the following special result codes: ** ** 0 End of input @@ -2694,7 +2696,7 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** -4 ',' seen / the index in zJson[] of the seen character ** -5 ':' seen / */ -static int jsonTranslateTextValueToBlob(JsonParse *pParse, u32 i){ +static int jsonXlateTextToBlob(JsonParse *pParse, u32 i){ char c; u32 j; u32 iThis, iStart; @@ -2714,7 +2716,7 @@ json_parse_restart: iStart = pParse->nBlob; for(j=i+1;;j++){ u32 iBlob = pParse->nBlob; - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x<=0 ){ int op; if( x==(-2) ){ @@ -2760,7 +2762,7 @@ json_parse_restart: goto parse_object_value; } } - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x!=(-5) ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -2768,7 +2770,7 @@ json_parse_restart: j = pParse->iErr+1; } parse_object_value: - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x<=0 ){ if( x!=(-1) ) pParse->iErr = j; return -1; @@ -2787,7 +2789,7 @@ json_parse_restart: break; } } - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -2815,7 +2817,7 @@ json_parse_restart: return -1; } for(j=i+1;;j++){ - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; @@ -2839,7 +2841,7 @@ json_parse_restart: break; } } - x = jsonTranslateTextValueToBlob(pParse, j); + x = jsonXlateTextToBlob(pParse, j); if( x==(-4) ){ j = pParse->iErr; continue; @@ -3156,7 +3158,7 @@ static int jsonConvertTextToBlob( ){ int i; const char *zJson = pParse->zJson; - i = jsonTranslateTextValueToBlob(pParse, 0); + i = jsonXlateTextToBlob(pParse, 0); if( pParse->oom ) i = -1; if( i>0 ){ assert( pParse->iDepth==0 ); @@ -3194,7 +3196,7 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ memset(&px, 0, sizeof(px)); px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; - (void)jsonTranslateTextValueToBlob(&px, 0); + (void)jsonXlateTextToBlob(&px, 0); if( px.oom ){ sqlite3_free(px.aBlob); sqlite3_result_error_nomem(pStr->pCtx); @@ -3253,9 +3255,10 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ /* -** Convert the binary BLOB representation of JSON beginning at -** aBlob[0] and extending for no more than nBlob bytes into -** a pure JSON string. The string is appended to pOut. +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. ** ** If an error is detected in the BLOB input, the pOut->eErr flag ** might get set to JSTRING_MALFORMED. But not all BLOB input errors @@ -3264,7 +3267,7 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ ** ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ -static u32 jsonRenderBlob( +static u32 jsonXlateBlobToText( JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ @@ -3403,7 +3406,7 @@ static u32 jsonRenderBlob( j = i+n; iEnd = j+sz; while( j0 ) pOut->nUsed--; @@ -3416,7 +3419,7 @@ static u32 jsonRenderBlob( j = i+n; iEnd = j+sz; while( j0 ) pOut->nUsed--; @@ -3458,14 +3461,13 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ return sz+n==(u32)nBlob; } -/* Parse a single element of binary JSON into the legacy Node structure. -** Return the index of the first byte past the end of the binary JSON element. -** -** This routine is a temporary translator while the legacy Node encoding -** is still in use. It will be removed after all processing translates -** to the new BLOB encoding. +/* Translate a single element of JSONB into the JsonNode format. The +** first byte of the element to be translated is at pParse->aBlob[i]. +** Return the index in pParse->aBlob[] of the first byte past the end +** of the JSONB element. Append the JsonNode translation in +** pParse->aNode[], which is increased in size as necessary. */ -static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ +static int jsonXlateBlobToNode(JsonParse *pParse, u32 i){ u8 t; /* Node type */ u32 sz; /* Node size */ u32 x; /* Index of payload start */ @@ -3535,7 +3537,7 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); u32 j = i+x; while( joom ){ pParse->aNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; @@ -3569,10 +3571,11 @@ static int jsonParseValueFromBlob(JsonParse *pParse, u32 i){ } /* -** Convert pNode that belongs to pParse into the BLOB representation. -** The BLOB representation is added to the pOut->aBlob[] array. +** Translate pNode (which is always a node found in pParse->aNode[]) into +** the JSONB representation and append the translation onto the end of the +** pOut->aBlob[] array. */ -static void jsonRenderNodeAsBlob( +static void jsonXlateNodeToBlob( JsonParse *pParse, /* the complete parse of the JSON */ JsonNode *pNode, /* The node to render */ JsonParse *pOut /* Write the BLOB rendering of JSON here */ @@ -3661,7 +3664,7 @@ static void jsonRenderNodeAsBlob( for(;;){ while( j<=pNode->n ){ if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); + jsonXlateNodeToBlob(pParse, &pNode[j], pOut); } j += jsonNodeSize(&pNode[j]); } @@ -3682,8 +3685,8 @@ static void jsonRenderNodeAsBlob( for(;;){ while( j<=pNode->n ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonRenderNodeAsBlob(pParse, &pNode[j], pOut); - jsonRenderNodeAsBlob(pParse, &pNode[j+1], pOut); + jsonXlateNodeToBlob(pParse, &pNode[j], pOut); + jsonXlateNodeToBlob(pParse, &pNode[j+1], pOut); } j += 1 + jsonNodeSize(&pNode[j+1]); } @@ -3850,7 +3853,7 @@ static void jsonReturnTextJsonFromBlob( x.aBlob = (u8*)aBlob; x.nBlob = nBlob; jsonStringInit(&s, ctx); - jsonRenderBlob(&x, 0, &s); + jsonXlateBlobToText(&x, 0, &s); jsonReturnString(&s); } @@ -4431,13 +4434,13 @@ static void jsonExtractFunc( if( flags & JSON_JSON ){ jsonReturnNodeAsJson(p, pNode, ctx, 0); }else{ - jsonReturnNodeAsSql(p, pNode, ctx); + jsonReturnFromNode(p, pNode, ctx); sqlite3_result_subtype(ctx, 0); } } }else{ pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturnNodeAsSql(p, pNode, ctx); + if( p->nErr==0 && pNode ) jsonReturnFromNode(p, pNode, ctx); } }else{ /* Two or more PATH arguments results in a JSON array with each @@ -4451,7 +4454,7 @@ static void jsonExtractFunc( if( p->nErr ) break; jsonAppendSeparator(&jx); if( pNode ){ - jsonRenderNodeAsText(p, pNode, &jx); + jsonXlateNodeToText(p, pNode, &jx); }else{ jsonAppendRawNZ(&jx, "null", 4); } @@ -5379,7 +5382,7 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->i==0 ) break; if( p->eType==JSON_OBJECT ){ - jsonReturnNodeAsSql(&p->sParse, pThis, ctx); + jsonReturnFromNode(&p->sParse, pThis, ctx); }else if( p->eType==JSON_ARRAY ){ u32 iKey; if( p->bRecursive ){ @@ -5395,7 +5398,7 @@ static int jsonEachColumn( } case JEACH_VALUE: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturnNodeAsSql(&p->sParse, pThis, ctx); + jsonReturnFromNode(&p->sParse, pThis, ctx); break; } case JEACH_TYPE: { @@ -5406,7 +5409,7 @@ static int jsonEachColumn( case JEACH_ATOM: { if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; - jsonReturnNodeAsSql(&p->sParse, pThis, ctx); + jsonReturnFromNode(&p->sParse, pThis, ctx); break; } case JEACH_ID: { From 064c1688dade8ba7b67e9bf43833d3f799b5ad56 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Oct 2023 18:04:40 +0000 Subject: [PATCH 064/191] Fix bugs uncovered by the fuzzer. FossilOrigin-Name: c96eb7fb618dc0a5aeec8a5e85076475b77dcd56309438aba1f9bddfc8921e3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 592f7c39e5..051af18412 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-10-10T17:34:41.118 +C Fix\sbugs\suncovered\sby\sthe\sfuzzer. +D 2023-10-10T18:04:40.744 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c bbf52181d04e09852e273f11cf31544a8030ffc911359cb7f77e39f252cce7a6 +F src/json.c 7df84bb5b918dca3e9c4361989a3667bf1ca080299fdf9947e6f99b39d7523af F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db44bd1d62084ef69c808f7d07e0a25d5a089dcb8b98b1b5d026777591bbdefc 65ccf5fef812d43aed9e00af36c90e1a499d197e30148753790445e25ee1324c -R 7120be3e2682c2f2dc1a6290819c2e25 +P f93f16c94d0a58dfa90939e84aba048be958c7b085b45a6f25c2243fdf79b3aa +R 77f1fc6f77b161facff6daa6dc651655 U drh -Z a077701d7ef896cae6a827ec97befa72 +Z 2b3feb335f873a4bcfbf7b38cfcaa6e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 53e25911e6..abbf08e743 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f93f16c94d0a58dfa90939e84aba048be958c7b085b45a6f25c2243fdf79b3aa \ No newline at end of file +c96eb7fb618dc0a5aeec8a5e85076475b77dcd56309438aba1f9bddfc8921e3c \ No newline at end of file diff --git a/src/json.c b/src/json.c index c2b7e39225..cf622899fa 100644 --- a/src/json.c +++ b/src/json.c @@ -2861,6 +2861,7 @@ json_parse_restart: case '\'': { u8 opcode; char cDelim; + int nn; pParse->hasNonstd = 1; opcode = JSONB_TEXT; goto parse_string; @@ -2869,7 +2870,8 @@ json_parse_restart: opcode = JSONB_TEXT; parse_string: cDelim = z[i]; - for(j=i+1; 1; j++){ + nn = pParse->nJson; + for(j=i+1; j0 ){ jsonAppendRawNZ(pOut, zIn, k); + if( sz2<=k ) break; zIn += k; sz2 -= k; - if( sz2==0 ) break; } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ @@ -3366,8 +3368,12 @@ static u32 jsonXlateBlobToText( case 'x': jsonAppendRawNZ(pOut, "\\u00", 4); jsonAppendRawNZ(pOut, &zIn[2], 2); - zIn += 2; - sz2 -= 2; + if( sz2<2 ){ + sz2 = 0; + }else{ + zIn += 2; + sz2 -= 2; + } break; case '0': jsonAppendRawNZ(pOut, "\\u0000", 6); @@ -3391,6 +3397,7 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, zIn, 2); break; } + if( sz2<2 ) break; zIn += 2; sz2 -= 2; } From d88f378c7d316a9523485504b31705ca1040e2fe Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Oct 2023 18:32:14 +0000 Subject: [PATCH 065/191] Fix a potential buffer overrun due to corrupt JSONB. FossilOrigin-Name: 5cbb861fc6cb7421a81c957ba9d576e0ae92cb4ed725cdb539b364d66f4ee145 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 051af18412..c31090bbce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbugs\suncovered\sby\sthe\sfuzzer. -D 2023-10-10T18:04:40.744 +C Fix\sa\spotential\sbuffer\soverrun\sdue\sto\scorrupt\sJSONB. +D 2023-10-10T18:32:14.227 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7df84bb5b918dca3e9c4361989a3667bf1ca080299fdf9947e6f99b39d7523af +F src/json.c e6e3641f54366634dc10dfb3684c3643821883b533f89c207b33dd2d23499c7d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f93f16c94d0a58dfa90939e84aba048be958c7b085b45a6f25c2243fdf79b3aa -R 77f1fc6f77b161facff6daa6dc651655 +P c96eb7fb618dc0a5aeec8a5e85076475b77dcd56309438aba1f9bddfc8921e3c +R 24c9be5529f36a58a40f05b718ffa50e U drh -Z 2b3feb335f873a4bcfbf7b38cfcaa6e3 +Z 1fee0a4053cb04fe5ff07cf374f7fcdd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index abbf08e743..587d87d872 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c96eb7fb618dc0a5aeec8a5e85076475b77dcd56309438aba1f9bddfc8921e3c \ No newline at end of file +5cbb861fc6cb7421a81c957ba9d576e0ae92cb4ed725cdb539b364d66f4ee145 \ No newline at end of file diff --git a/src/json.c b/src/json.c index cf622899fa..2d68aef113 100644 --- a/src/json.c +++ b/src/json.c @@ -1124,7 +1124,7 @@ static void jsonReturnFromNode( assert( pNode->eU==1 ); to_double: z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3AtoF(z, &r, pNode->n, SQLITE_UTF8); sqlite3_result_double(pCtx, r); break; } From 59ded6b5f112793ff67957b731ab26e2eefa6dd8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Oct 2023 18:42:08 +0000 Subject: [PATCH 066/191] Improved robustness when translating corrupt JSONB into JSON text. FossilOrigin-Name: 0caa320d9099adbaf98e3719003dbdc4d158abcb3d8a1af20fbcd4c08c970f4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 22 +++++++++++++++++----- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c31090bbce..ec6701a0b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sbuffer\soverrun\sdue\sto\scorrupt\sJSONB. -D 2023-10-10T18:32:14.227 +C Improved\srobustness\swhen\stranslating\scorrupt\sJSONB\sinto\sJSON\stext. +D 2023-10-10T18:42:08.681 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e6e3641f54366634dc10dfb3684c3643821883b533f89c207b33dd2d23499c7d +F src/json.c fc5b17b021ce7699e08b8709698701d3de4128c07af53295ae1b241c38de7662 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c96eb7fb618dc0a5aeec8a5e85076475b77dcd56309438aba1f9bddfc8921e3c -R 24c9be5529f36a58a40f05b718ffa50e +P 5cbb861fc6cb7421a81c957ba9d576e0ae92cb4ed725cdb539b364d66f4ee145 +R 56495ee1952f6b69fc5e0b7ca9cd10db U drh -Z 1fee0a4053cb04fe5ff07cf374f7fcdd +Z 53d35a2e8ad88bb30ea85a59c02aea4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 587d87d872..fdb11253ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cbb861fc6cb7421a81c957ba9d576e0ae92cb4ed725cdb539b364d66f4ee145 \ No newline at end of file +0caa320d9099adbaf98e3719003dbdc4d158abcb3d8a1af20fbcd4c08c970f4a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 2d68aef113..a8f0049e7d 100644 --- a/src/json.c +++ b/src/json.c @@ -3353,7 +3353,10 @@ static u32 jsonXlateBlobToText( for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); - if( sz2<=k ) break; + if( sz2<=k ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } zIn += k; sz2 -= k; } @@ -3369,6 +3372,7 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, "\\u00", 4); jsonAppendRawNZ(pOut, &zIn[2], 2); if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; sz2 = 0; }else{ zIn += 2; @@ -3387,9 +3391,14 @@ static u32 jsonXlateBlobToText( case '\n': break; case 0xe2: - assert( sz2>=4 ); - assert( 0x80==(u8)zIn[2] ); - assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + if( sz2<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + pOut->eErr |= JSTRING_MALFORMED; + k = sz2; + break; + } zIn += 2; sz2 -= 2; break; @@ -3397,7 +3406,10 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, zIn, 2); break; } - if( sz2<2 ) break; + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } zIn += 2; sz2 -= 2; } From 6b7b23c5802128205cfcc2c02a38a9e1513a06c6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Oct 2023 18:55:29 +0000 Subject: [PATCH 067/191] Fix an off-by-one error in the changes from the previous check-in on the jsonb branch. FossilOrigin-Name: 216191b113da43516d31301fb133173add66358d503677ab997bc82cc88bfea4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ec6701a0b7..8b71972a22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srobustness\swhen\stranslating\scorrupt\sJSONB\sinto\sJSON\stext. -D 2023-10-10T18:42:08.681 +C Fix\san\soff-by-one\serror\sin\sthe\schanges\sfrom\sthe\sprevious\scheck-in\son\nthe\sjsonb\sbranch. +D 2023-10-10T18:55:29.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fc5b17b021ce7699e08b8709698701d3de4128c07af53295ae1b241c38de7662 +F src/json.c 3a0d5a277dfe592998f803b166f2bcd761259643446c4ef3b58f662f4e6e121a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5cbb861fc6cb7421a81c957ba9d576e0ae92cb4ed725cdb539b364d66f4ee145 -R 56495ee1952f6b69fc5e0b7ca9cd10db +P 0caa320d9099adbaf98e3719003dbdc4d158abcb3d8a1af20fbcd4c08c970f4a +R 2be00e287cc1d614a9c2f8c8c8493433 U drh -Z 53d35a2e8ad88bb30ea85a59c02aea4e +Z f13f9b96095bfab368044d04f5c134b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fdb11253ec..a728066e52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0caa320d9099adbaf98e3719003dbdc4d158abcb3d8a1af20fbcd4c08c970f4a \ No newline at end of file +216191b113da43516d31301fb133173add66358d503677ab997bc82cc88bfea4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a8f0049e7d..b9f13f12f1 100644 --- a/src/json.c +++ b/src/json.c @@ -3354,7 +3354,7 @@ static u32 jsonXlateBlobToText( if( k>0 ){ jsonAppendRawNZ(pOut, zIn, k); if( sz2<=k ){ - pOut->eErr |= JSTRING_MALFORMED; + if( sz2eErr |= JSTRING_MALFORMED; break; } zIn += k; From 71dc3c714b1aaabdae58391584faf790a0b5c86c Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Oct 2023 23:02:31 +0000 Subject: [PATCH 068/191] Improved robustness in the decoding of JSON5 text escape sequences found in malformed JSONB. FossilOrigin-Name: 35e0108af2bdd830375c31c525f8ed0e8df64959d89649a88402dc1a5c376612 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 47 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 8b71972a22..1d877db3b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\schanges\sfrom\sthe\sprevious\scheck-in\son\nthe\sjsonb\sbranch. -D 2023-10-10T18:55:29.425 +C Improved\srobustness\sin\sthe\sdecoding\sof\sJSON5\stext\sescape\ssequences\sfound\nin\smalformed\sJSONB. +D 2023-10-10T23:02:31.182 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3a0d5a277dfe592998f803b166f2bcd761259643446c4ef3b58f662f4e6e121a +F src/json.c d86e0d27c8d99c4a9e9bbe44401224270d667118742df371a3928e0406243647 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0caa320d9099adbaf98e3719003dbdc4d158abcb3d8a1af20fbcd4c08c970f4a -R 2be00e287cc1d614a9c2f8c8c8493433 +P 216191b113da43516d31301fb133173add66358d503677ab997bc82cc88bfea4 +R 3e258cb31e76158544ba210b63aec864 U drh -Z f13f9b96095bfab368044d04f5c134b6 +Z 26ce59215821be9c842a531e0d11f121 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a728066e52..37779a76ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -216191b113da43516d31301fb133173add66358d503677ab997bc82cc88bfea4 \ No newline at end of file +35e0108af2bdd830375c31c525f8ed0e8df64959d89649a88402dc1a5c376612 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b9f13f12f1..8f263556a7 100644 --- a/src/json.c +++ b/src/json.c @@ -577,9 +577,13 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ for(i=0; i0 ){ jsonAppendRawNZ(p, zIn, i); + if( i>=N ) break; zIn += i; N -= i; - if( N==0 ) break; + } + if( N<2 ){ + p->eErr |= JSTRING_MALFORMED; + break; } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ @@ -590,6 +594,11 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, "\\u0009", 6); break; case 'x': + if( N<4 ){ + N = 2; + p->eErr |= JSTRING_MALFORMED; + break; + } jsonAppendRawNZ(p, "\\u00", 4); jsonAppendRawNZ(p, &zIn[2], 2); zIn += 2; @@ -599,14 +608,22 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, "\\u0000", 6); break; case '\r': - if( zIn[2]=='\n' ){ + if( N>2 && zIn[2]=='\n' ){ zIn++; N--; } break; case '\n': break; - case 0xe2: + case 0xe2: /* \ followed by U+2028 or U+2029 line terminator ignored */ + if( N<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + N = 2; + p->eErr |= JSTRING_MALFORMED; + break; + } assert( N>=4 ); assert( 0x80==(u8)zIn[2] ); assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); @@ -617,6 +634,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRawNZ(p, zIn, 2); break; } + assert( N>=2 ); zIn += 2; N -= 2; } @@ -3353,13 +3371,16 @@ static u32 jsonXlateBlobToText( for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); - if( sz2<=k ){ - if( sz2eErr |= JSTRING_MALFORMED; + if( k>=sz2 ){ break; } zIn += k; sz2 -= k; } + if( sz2<2 ){ + if( sz2>0 ) pOut->eErr |= JSTRING_MALFORMED; + if( sz2==0 ) break; + } assert( zIn[0]=='\\' ); switch( (u8)zIn[1] ){ case '\'': @@ -3369,21 +3390,21 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, "\\u0009", 6); break; case 'x': - jsonAppendRawNZ(pOut, "\\u00", 4); - jsonAppendRawNZ(pOut, &zIn[2], 2); if( sz2<2 ){ pOut->eErr |= JSTRING_MALFORMED; sz2 = 0; - }else{ - zIn += 2; - sz2 -= 2; + break; } + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz2 -= 2; break; case '0': jsonAppendRawNZ(pOut, "\\u0000", 6); break; case '\r': - if( zIn[2]=='\n' ){ + if( sz2>2 && zIn[2]=='\n' ){ zIn++; sz2--; } @@ -3391,6 +3412,9 @@ static u32 jsonXlateBlobToText( case '\n': break; case 0xe2: + /* '\' followed by either U+2028 or U+2029 is ignored as + ** whitespace. Not that in UTF8, U+2028 is 0xe2 0x80 0x29. + ** U+2029 is the same except for the last byte */ if( sz2<4 || 0x80!=(u8)zIn[2] || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) @@ -3407,6 +3431,7 @@ static u32 jsonXlateBlobToText( break; } if( sz2<2 ){ + sz2 = 0; pOut->eErr |= JSTRING_MALFORMED; break; } From 2a96a1584d58a9959e1f4867de25682793637837 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Oct 2023 11:42:06 +0000 Subject: [PATCH 069/191] Fix an assertion fault in json_patch() that can occur if the patch JSON is malformed JSONB. Report the malformed JSONB instead. FossilOrigin-Name: a72d54645ca0dd80c60a5ed586049dead3ea7f5fa9ad05c6610a506242a7032a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 ++++++++++------ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1d877db3b8..6f76814908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srobustness\sin\sthe\sdecoding\sof\sJSON5\stext\sescape\ssequences\sfound\nin\smalformed\sJSONB. -D 2023-10-10T23:02:31.182 +C Fix\san\sassertion\sfault\sin\sjson_patch()\sthat\scan\soccur\sif\sthe\spatch\sJSON\nis\smalformed\sJSONB.\s\sReport\sthe\smalformed\sJSONB\sinstead. +D 2023-10-11T11:42:06.265 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d86e0d27c8d99c4a9e9bbe44401224270d667118742df371a3928e0406243647 +F src/json.c 51ae066d271f4a37643921782fbcf61cf76d7166d06e9ad4ab75a32510d8b03a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 216191b113da43516d31301fb133173add66358d503677ab997bc82cc88bfea4 -R 3e258cb31e76158544ba210b63aec864 +P 35e0108af2bdd830375c31c525f8ed0e8df64959d89649a88402dc1a5c376612 +R 574c458e7556ba233ae1bd2b81029222 U drh -Z 26ce59215821be9c842a531e0d11f121 +Z b70fdf7e40754bcaf3921ae744996dea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 37779a76ba..c04985ac0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35e0108af2bdd830375c31c525f8ed0e8df64959d89649a88402dc1a5c376612 \ No newline at end of file +a72d54645ca0dd80c60a5ed586049dead3ea7f5fa9ad05c6610a506242a7032a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8f263556a7..0ddc9c498e 100644 --- a/src/json.c +++ b/src/json.c @@ -4536,8 +4536,10 @@ static JsonNode *jsonMergePatch( for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ u32 nKey; const char *zKey; - assert( pPatch[i].eType==JSON_STRING ); - assert( pPatch[i].jnFlags & JNODE_LABEL ); + if( pPatch[i].eType!=JSON_STRING ){ + pParse->nErr = 1; + return 0; + } assert( pPatch[i].eU==1 ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; @@ -4606,13 +4608,15 @@ static void jsonPatchFunc( pX->useMod = 1; pY->useMod = 1; pResult = jsonMergePatch(pX, 0, pY->aNode); - assert( pResult!=0 || pX->oom ); - if( pResult && pX->oom==0 ){ + assert( pResult!=0 || pX->oom || pX->nErr ); + if( pX->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( pX->nErr ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else if( pResult ){ jsonDebugPrintParse(pX); jsonDebugPrintNode(pResult); jsonReturnNodeAsJson(pX, pResult, ctx, 0); - }else{ - sqlite3_result_error_nomem(ctx); } } From 33b56217cb7f1ced7553449daccc032230a8b53a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Oct 2023 12:21:29 +0000 Subject: [PATCH 070/191] Improved robustness against corrupt JSONB. FossilOrigin-Name: 0fbda92bb0eeb40f95c83f717e4e8f5bff1ac82f1c899e9f6d400d67df67214e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 8 ++------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 6f76814908..c5837f11b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassertion\sfault\sin\sjson_patch()\sthat\scan\soccur\sif\sthe\spatch\sJSON\nis\smalformed\sJSONB.\s\sReport\sthe\smalformed\sJSONB\sinstead. -D 2023-10-11T11:42:06.265 +C Improved\srobustness\sagainst\scorrupt\sJSONB. +D 2023-10-11T12:21:29.280 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 51ae066d271f4a37643921782fbcf61cf76d7166d06e9ad4ab75a32510d8b03a +F src/json.c 4130133dcd43ac5c9b71a88a47f5ba8b6453d7cfbf158ebeb57ddd89f5def919 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35e0108af2bdd830375c31c525f8ed0e8df64959d89649a88402dc1a5c376612 -R 574c458e7556ba233ae1bd2b81029222 +P a72d54645ca0dd80c60a5ed586049dead3ea7f5fa9ad05c6610a506242a7032a +R e706082c7e7e2d4bd83182da6946ab5f U drh -Z b70fdf7e40754bcaf3921ae744996dea +Z 820cb01f77053065d828d065556f10f5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c04985ac0f..aa52b1f5c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a72d54645ca0dd80c60a5ed586049dead3ea7f5fa9ad05c6610a506242a7032a \ No newline at end of file +0fbda92bb0eeb40f95c83f717e4e8f5bff1ac82f1c899e9f6d400d67df67214e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0ddc9c498e..ff735f36f3 100644 --- a/src/json.c +++ b/src/json.c @@ -981,7 +981,7 @@ static void jsonXlateNodeToText( u32 j = 1; jsonAppendChar(pOut, '{'); for(;;){ - while( j<=pNode->n ){ + while( jn ){ if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ jsonAppendSeparator(pOut); jsonXlateNodeToText(pParse, &pNode[j], pOut); @@ -1054,7 +1054,7 @@ static void jsonReturnNodeAsJson( ** character: 0..9a..fA..F */ static u8 jsonHexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); + if( !sqlite3Isxdigit(h) ) return 0; #ifdef SQLITE_EBCDIC h += 9*(1&~(h>>4)); #else @@ -1068,10 +1068,6 @@ static u8 jsonHexToInt(int h){ */ static u32 jsonHexToInt4(const char *z){ u32 v; - assert( sqlite3Isxdigit(z[0]) ); - assert( sqlite3Isxdigit(z[1]) ); - assert( sqlite3Isxdigit(z[2]) ); - assert( sqlite3Isxdigit(z[3]) ); v = (jsonHexToInt(z[0])<<12) + (jsonHexToInt(z[1])<<8) + (jsonHexToInt(z[2])<<4) From 2b7a1f5926278c2698521310d29f39ef20b90789 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Oct 2023 12:44:17 +0000 Subject: [PATCH 071/191] Fix an issue that can arise when processing corrupt JSONB. FossilOrigin-Name: e50045c22296be84c6bea82bb8b310f07bca820c84d4a7349b16da0cf5d2657c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c5837f11b5..9d433f7b1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srobustness\sagainst\scorrupt\sJSONB. -D 2023-10-11T12:21:29.280 +C Fix\san\sissue\sthat\scan\sarise\swhen\sprocessing\scorrupt\sJSONB. +D 2023-10-11T12:44:17.002 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4130133dcd43ac5c9b71a88a47f5ba8b6453d7cfbf158ebeb57ddd89f5def919 +F src/json.c d2b028758bc86b4ea3bde09effd05460945ee79286f1147bd90b233cdaec440b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a72d54645ca0dd80c60a5ed586049dead3ea7f5fa9ad05c6610a506242a7032a -R e706082c7e7e2d4bd83182da6946ab5f +P 0fbda92bb0eeb40f95c83f717e4e8f5bff1ac82f1c899e9f6d400d67df67214e +R d76df8cb5e7347049ec337effa1f07fd U drh -Z 820cb01f77053065d828d065556f10f5 +Z 97ae228d7814f1fddc06d3cfd13d0a3f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aa52b1f5c1..2d38285105 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fbda92bb0eeb40f95c83f717e4e8f5bff1ac82f1c899e9f6d400d67df67214e \ No newline at end of file +e50045c22296be84c6bea82bb8b310f07bca820c84d4a7349b16da0cf5d2657c \ No newline at end of file diff --git a/src/json.c b/src/json.c index ff735f36f3..b92db836c3 100644 --- a/src/json.c +++ b/src/json.c @@ -2209,7 +2209,7 @@ static JsonParse *jsonParseCached( ** a match. */ static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ - assert( pNode->eU==1 ); + if( pNode->eType!=JSON_STRING ) return 0; if( pNode->n!=nKey ) return 0; return strncmp(pNode->u.zJContent, zKey, nKey)==0; } From 86db4555cab9e74f9c141a151af459c0e8069f54 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Oct 2023 13:19:37 +0000 Subject: [PATCH 072/191] Fix a missing zero-terminator on a string when processing JSON aggregates into JSONB. FossilOrigin-Name: fb81d570a3d9b8f55e9f278d5240605c5d0f3c5abde51550797999d03cf193a7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9d433f7b1d..65f313ceaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sthat\scan\sarise\swhen\sprocessing\scorrupt\sJSONB. -D 2023-10-11T12:44:17.002 +C Fix\sa\smissing\szero-terminator\son\sa\sstring\swhen\sprocessing\nJSON\saggregates\sinto\sJSONB. +D 2023-10-11T13:19:37.907 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d2b028758bc86b4ea3bde09effd05460945ee79286f1147bd90b233cdaec440b +F src/json.c 6c394327f052d3be3c4c2d21d9b8874320992e7d6f8b806db73c5b834e71c7bd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0fbda92bb0eeb40f95c83f717e4e8f5bff1ac82f1c899e9f6d400d67df67214e -R d76df8cb5e7347049ec337effa1f07fd +P e50045c22296be84c6bea82bb8b310f07bca820c84d4a7349b16da0cf5d2657c +R 8c875e6758b9f56e162d8b6b955529e9 U drh -Z 97ae228d7814f1fddc06d3cfd13d0a3f +Z d51d36a4b4191be2ce04b67a30dc0525 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d38285105..bba78e7854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e50045c22296be84c6bea82bb8b310f07bca820c84d4a7349b16da0cf5d2657c \ No newline at end of file +fb81d570a3d9b8f55e9f278d5240605c5d0f3c5abde51550797999d03cf193a7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b92db836c3..3d0bba1941 100644 --- a/src/json.c +++ b/src/json.c @@ -486,6 +486,14 @@ static void jsonAppendChar(JsonString *p, char c){ } } +/* Make sure there is a zero terminator on p->zBuf[] +*/ +static void jsonStringTerminate(JsonString *p){ + if( p->nUsednAlloc || jsonStringGrow(p,1) ){ + p->zBuf[p->nUsed] = 0; + } +} + /* Try to force the string to be a zero-terminated RCStr string. ** ** Return true on success. Return false if an OOM prevents this @@ -3210,6 +3218,7 @@ static int jsonConvertTextToBlob( static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; memset(&px, 0, sizeof(px)); + jsonStringTerminate(pStr); px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; (void)jsonXlateTextToBlob(&px, 0); From 7d1c9da62dcde30f42f2a1d59aaf466207f922b7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Oct 2023 17:21:16 +0000 Subject: [PATCH 073/191] Fix the use of an uninitialized value that occurs when doing a json_insert() of a string value that contains embedded U+0000 characters. FossilOrigin-Name: fc5ee9e51ad4556af526a6cefca5ae5a3b1b7affc4edf09832491d6b4f4ba366 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 6 ++++-- test/json101.test | 3 +++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 65f313ceaa..29b386a1a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smissing\szero-terminator\son\sa\sstring\swhen\sprocessing\nJSON\saggregates\sinto\sJSONB. -D 2023-10-11T13:19:37.907 +C Fix\sthe\suse\sof\san\suninitialized\svalue\sthat\soccurs\swhen\sdoing\sa\sjson_insert()\nof\sa\sstring\svalue\sthat\scontains\sembedded\sU+0000\scharacters. +D 2023-10-11T17:21:16.637 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -674,7 +674,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6c394327f052d3be3c4c2d21d9b8874320992e7d6f8b806db73c5b834e71c7bd +F src/json.c fc8783af356d27cfdf14a583e5a7809394269763384cde979c0825a29f13e1d6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 98cfba10989b3da6f1807ad42444017742db7f100a54f1032af7a8b1295912c0 F src/main.c 618aeb399e993cf561864f4b0cf6a331ee4f355cf663635f8d9da3193a46aa40 @@ -1307,7 +1307,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test abb5a0cfde077a6f1124604e75806fbe889bc1c0acc11d32897f191e1f9c6b2c +F test/json101.test 8e7ee64714deff6deb927f417d90068cee38756cfe54375be46351ac5cb0962a F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2129,8 +2129,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e50045c22296be84c6bea82bb8b310f07bca820c84d4a7349b16da0cf5d2657c -R 8c875e6758b9f56e162d8b6b955529e9 +P fb81d570a3d9b8f55e9f278d5240605c5d0f3c5abde51550797999d03cf193a7 +R d78e19c15a2441ced442368555cb1a45 U drh -Z d51d36a4b4191be2ce04b67a30dc0525 +Z 3bf927da2cbd54464f0b6fb8e90f7a35 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bba78e7854..0fb4b428b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb81d570a3d9b8f55e9f278d5240605c5d0f3c5abde51550797999d03cf193a7 \ No newline at end of file +fc5ee9e51ad4556af526a6cefca5ae5a3b1b7affc4edf09832491d6b4f4ba366 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3d0bba1941..688ab32fea 100644 --- a/src/json.c +++ b/src/json.c @@ -4756,11 +4756,13 @@ static void jsonReplaceNode( break; } if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ - char *zCopy = sqlite3DbStrDup(0, z); + char *zCopy = sqlite3_malloc64( n+1 ); int k; if( zCopy ){ + memcpy(zCopy, z, n); + zCopy[n] = 0; jsonParseAddCleanup(p, sqlite3_free, zCopy); - }else{ + }else{ p->oom = 1; sqlite3_result_error_nomem(pCtx); } diff --git a/test/json101.test b/test/json101.test index 7445cc987c..b8c2a58d34 100644 --- a/test/json101.test +++ b/test/json101.test @@ -169,6 +169,9 @@ do_execsql_test json101-4.7 { do_execsql_test json101-4.8 { SELECT x FROM j1 WHERE json_insert(x)<>x; } {} +do_execsql_test json101-4.9 { + SELECT json_insert('{"a":1}','$.b',CAST(x'0000' AS text)); +} {{{"a":1,"b":"\u0000\u0000"}}} # json_extract(JSON,'$') will return objects and arrays without change. # From 03204e910680096d982139bda04c6f8a1b0f8f67 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 11 Oct 2023 21:08:12 +0000 Subject: [PATCH 074/191] Add the tokendata=1 option to ignore trailing token-data when querying an fts5 table. FossilOrigin-Name: 122935182ad5869ce3a4c6d796c38a0509f6f3384dd1b3e60a3f2f0f366cc5f5 --- ext/fts5/fts5Int.h | 1 + ext/fts5/fts5_config.c | 10 +++++ ext/fts5/fts5_expr.c | 41 +++++++++++-------- ext/fts5/fts5_index.c | 19 +++++++-- ext/fts5/test/fts5_common.tcl | 14 +++++++ ext/fts5/test/fts5aa.test | 68 ++++++++++++++++++++----------- ext/fts5/test/fts5origintext.test | 45 +++++++++++++++++++- manifest | 24 +++++------ manifest.uuid | 2 +- 9 files changed, 164 insertions(+), 60 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1687168d5f..4aa578559b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -196,6 +196,7 @@ struct Fts5Config { char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bTokendata; /* "tokendata=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 5d0770502e..d2e8309cd2 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -398,6 +398,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed tokendata=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bTokendata = (zArg[0]=='1'); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 745a5d9fa6..a2c6320719 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -101,7 +101,8 @@ struct Fts5ExprTerm { u8 bPrefix; /* True for a prefix term */ u8 bFirst; /* True if token must be first in column */ char *pTerm; /* Term data */ - int nTerm; /* Size of term in bytes */ + int nQueryTerm; /* Effective size of term in bytes */ + int nFullTerm; /* Size of term in bytes incl. tokendata */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; @@ -968,7 +969,7 @@ static int fts5ExprNearInitAll( p->pIter = 0; } rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->pTerm, p->nTerm, + pExpr->pIndex, p->pTerm, p->nQueryTerm, (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), pNear->pColset, @@ -1703,6 +1704,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + Fts5Config *pConfig; int rc; }; @@ -1737,7 +1739,8 @@ static int fts5ParseTokenize( }else{ memset(pSyn, 0, (size_t)nByte); pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); - pSyn->nTerm = nToken; + pSyn->nFullTerm = pSyn->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ) pSyn->nQueryTerm = strlen(pSyn->pTerm); memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; @@ -1764,7 +1767,8 @@ static int fts5ParseTokenize( pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); - pTerm->nTerm = nToken; + pTerm->nFullTerm = pTerm->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ) pTerm->nQueryTerm = strlen(pTerm->pTerm); } } @@ -1831,6 +1835,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( memset(&sCtx, 0, sizeof(TokenCtx)); sCtx.pPhrase = pAppend; + sCtx.pConfig = pConfig; rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ @@ -1880,8 +1885,7 @@ int sqlite3Fts5ExprClonePhrase( int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ - + TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ pOrig = pExpr->apExprPhrase[iPhrase]; pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); if( rc==SQLITE_OK ){ @@ -1912,11 +1916,12 @@ int sqlite3Fts5ExprClonePhrase( if( pOrig->nTerm ){ int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; for(i=0; rc==SQLITE_OK && inTerm; i++){ int tflags = 0; Fts5ExprTerm *p; for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm, p->nTerm, 0, 0); + rc = fts5ParseTokenize((void*)&sCtx, tflags, p->pTerm,p->nFullTerm,0,0); tflags = FTS5_TOKEN_COLOCATED; } if( rc==SQLITE_OK ){ @@ -2298,12 +2303,12 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( fts5ExprPhraseFree(pPhrase); }else{ Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii]; + Fts5ExprTerm *pTo = &pPhrase->aTerm[0]; pParse->apPhrase[pParse->nPhrase++] = pPhrase; pPhrase->nTerm = 1; - pPhrase->aTerm[0].pTerm = sqlite3Fts5Strndup( - &pParse->rc, p->pTerm, p->nTerm - ); - pPhrase->aTerm[0].nTerm = p->nTerm; + pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm); + pTo->nQueryTerm = p->nQueryTerm; + pTo->nFullTerm = p->nFullTerm; pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) ); @@ -2488,7 +2493,7 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += pTerm->nTerm * 2 + 3 + 2; + nByte += pTerm->nQueryTerm * 2 + 3 + 2; } zQuoted = sqlite3_malloc64(nByte); @@ -2496,7 +2501,7 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ int i = 0; for(p=pTerm; p; p=p->pSynonym){ char *zIn = p->pTerm; - char *zEnd = &zIn[p->nTerm]; + char *zEnd = &zIn[p->nQueryTerm]; zQuoted[i++] = '"'; while( zInnTerm; iTerm++){ Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", - p->nTerm, p->pTerm + p->nQueryTerm, p->pTerm ); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); @@ -2997,11 +3002,11 @@ static int fts5ExprPopulatePoslistsCb( if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ - Fts5ExprTerm *pTerm; + Fts5ExprTerm *pT; if( p->aPopulator[i].bOk==0 ) continue; - for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - if( (pTerm->nTerm==nToken || (pTerm->nTermbPrefix)) - && memcmp(pTerm->pTerm, pToken, pTerm->nTerm)==0 + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + if( (pT->nFullTerm==nToken || (pT->nFullTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nFullTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 6cf30b5a00..b90a66b88f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6037,11 +6037,12 @@ static void fts5MergePrefixLists( static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ + int bTokenscan, int iIdx, /* Index to scan for data */ u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ - Fts5Iter **ppIter /* OUT: New iterator */ + Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; @@ -6060,6 +6061,8 @@ static void fts5SetupPrefixIter( xAppend = fts5AppendPoslist; } + assert( bTokenscan==0 || iIdx==0 ); + aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); @@ -6075,6 +6078,12 @@ static void fts5SetupPrefixIter( int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); + + /* If iIdx is non-zero, then it is the number of a prefix-index for + ** prefixes 1 character longer than the prefix being queried for. That + ** index contains all the doclists required, except for the one + ** corresponding to the prefix itself. That one is extracted from the + ** main term index here. */ if( iIdx!=0 ){ int dummy = 0; const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; @@ -6110,6 +6119,7 @@ static void fts5SetupPrefixIter( assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTermnToken && pTerm[nToken]!=0x00 ) break; } if( p1->base.nData==0 ) continue; @@ -6438,7 +6448,7 @@ int sqlite3Fts5IndexQuery( } } - if( iIdx<=pConfig->nPrefix ){ + if( iIdx<=pConfig->nPrefix && (pConfig->bTokendata==0 || iIdx!=0) ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); @@ -6451,7 +6461,10 @@ int sqlite3Fts5IndexQuery( }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; - fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); + int bTokenscan = (iIdx==0); + fts5SetupPrefixIter( + p, bDesc, bTokenscan, iPrefixIdx, buf.p, nToken+1, pColset, &pRet + ); if( pRet==0 ){ assert( p->rc!=SQLITE_OK ); }else{ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 9c012932da..001cad1de2 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -438,6 +438,20 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } +proc foreach_tokenizer_mode {prefix script} { + set saved $::testprefix + foreach {d mapping} { + "" {} + "-origintext" {, tokenize="origintext unicode61", tokendata=1} + } { + set s [string map [list %TOKENIZER% $mapping] $script] + set ::testprefix "$prefix$d" + reset_db + sqlite3_fts5_register_origintext db + uplevel $s + } + set ::testprefix $saved +} #------------------------------------------------------------------------- # Convert a poslist of the type returned by fts5_test_poslist() to a diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index e1551fc516..a80a307a49 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -22,6 +22,7 @@ ifcapable !fts5 { } foreach_detail_mode $::testprefix { +foreach_tokenizer_mode $::testprefix { do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); @@ -44,7 +45,7 @@ do_execsql_test 1.1 { # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); @@ -73,8 +74,9 @@ do_execsql_test 2.4 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } foreach {i x y} { 1 {g f d b f} {h h e i a} @@ -97,8 +99,9 @@ foreach {i x y} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -121,8 +124,9 @@ foreach {i x y} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 5.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { @@ -145,8 +149,9 @@ foreach {i x y} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 6.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } @@ -181,6 +186,7 @@ do_execsql_test 6.6 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db expr srand(0) do_execsql_test 7.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y,z); @@ -222,6 +228,7 @@ for {set i 1} {$i <= 10} {incr i} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); @@ -236,6 +243,7 @@ do_execsql_test 8.1 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db expr srand(0) @@ -280,8 +288,9 @@ for {set i 1} {$i <= 10} {incr i} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 10.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } set d10 { 1 {g f d b f} {h h e i a} @@ -314,19 +323,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # do_catchsql_test 11.1 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 column name: rank}} do_catchsql_test 11.2 { - CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 table name: rank}} do_catchsql_test 11.3 { - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%); } {1 {reserved fts5 column name: rowid}} #------------------------------------------------------------------------- # do_execsql_test 12.1 { - CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%); } {} do_catchsql_test 12.2 { @@ -341,8 +350,9 @@ do_test 12.3 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 13.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); } {} @@ -365,8 +375,9 @@ do_execsql_test 13.6 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 14.1 { - CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH d(x,y) AS ( SELECT NULL, 'xyz xyz xyz xyz xyz xyz' @@ -449,8 +460,9 @@ do_catchsql_test 16.2 { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO b2 VALUES('a'); INSERT INTO b2 VALUES('b'); INSERT INTO b2 VALUES('c'); @@ -466,8 +478,9 @@ do_test 17.2 { if {[string match n* %DETAIL%]==0} { reset_db + sqlite3_fts5_register_origintext db do_execsql_test 17.3 { - CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%); INSERT INTO c2 VALUES('x x x', 'x x x'); SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; } {1} @@ -476,8 +489,9 @@ if {[string match n* %DETAIL%]==0} { #------------------------------------------------------------------------- # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 17.1 { - CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%); INSERT INTO uio VALUES(NULL); INSERT INTO uio SELECT NULL FROM uio; INSERT INTO uio SELECT NULL FROM uio; @@ -524,8 +538,8 @@ do_execsql_test 17.9 { #-------------------------------------------------------------------- # do_execsql_test 18.1 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); - CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%); + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1 VALUES('abc*', NULL); INSERT INTO t2 VALUES(1, 'abcdefg'); } @@ -540,8 +554,9 @@ do_execsql_test 18.3 { # fts5 table in the temp schema. # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 19.0 { - CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; @@ -551,8 +566,9 @@ do_execsql_test 19.0 { # Test that 6 and 7 byte varints can be read. # reset_db +sqlite3_fts5_register_origintext db do_execsql_test 20.0 { - CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%); } set ::ids [list \ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] @@ -570,7 +586,7 @@ do_test 20.1 { # do_execsql_test 21.0 { CREATE TEMP TABLE t8(a, b); - CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 21.1 { @@ -581,7 +597,7 @@ do_execsql_test 21.1 { } do_execsql_test 22.0 { - CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t9(rowid, x) VALUES(2, 'bbb'); BEGIN; INSERT INTO t9(rowid, x) VALUES(1, 'aaa'); @@ -596,7 +612,7 @@ do_execsql_test 22.1 { #------------------------------------------------------------------------- do_execsql_test 23.0 { - CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%); CREATE TABLE t11(x); } do_execsql_test 23.1 { @@ -608,7 +624,7 @@ do_execsql_test 23.2 { #------------------------------------------------------------------------- do_execsql_test 24.0 { - CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%); INSERT INTO t12 VALUES('aaaa'); } do_execsql_test 24.1 { @@ -618,6 +634,9 @@ do_execsql_test 24.1 { INSERT INTO t12 VALUES('aaaa'); END; } +execsql_pp { + SELECT rowid, hex(block) FROM t12_data +} do_execsql_test 24.2 { INSERT INTO t12(t12) VALUES('integrity-check'); } @@ -627,7 +646,7 @@ do_execsql_test 24.3 { #------------------------------------------------------------------------- do_execsql_test 25.0 { - CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%); + CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%); } do_execsql_test 25.1 { BEGIN; @@ -638,6 +657,7 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB } +} } expand_all_sql db diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 155d74c025..3fd5f17e7b 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -112,9 +112,50 @@ do_execsql_test 2.5 { INSERT INTO ft(ft) VALUES('optimize'); } -proc b {x} { string map [list "\0" "."] $x } -db func b b +#------------------------------------------------------------------------- +reset_db + +sqlite3_fts5_register_origintext db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); + + INSERT INTO ft(rowid, x) VALUES(1, 'hello'); + INSERT INTO ft(rowid, x) VALUES(2, 'Hello'); + INSERT INTO ft(rowid, x) VALUES(3, 'HELLO'); +} + +#proc b {x} { string map [list "\0" "."] $x } +#db func b b #execsql_pp { SELECT b(term) FROM vocab } +do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1 +do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2 +do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3 + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft2 USING fts5(x, + tokenize="origintext unicode61", + tokendata=1 + ); + CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance); + + INSERT INTO ft2(rowid, x) VALUES(1, 'hello'); + INSERT INTO ft2(rowid, x) VALUES(2, 'Hello'); + INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO'); + + INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo'); +} + +#proc b {x} { string map [list "\0" "."] $x } +#db func b b +#execsql_pp { SELECT b(term) FROM vocab } + +do_execsql_test 3.1.1 { SELECT rowid FROM ft2('hello') } {1 2 3} +do_execsql_test 3.1.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} +do_execsql_test 3.1.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} + +do_execsql_test 3.1.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} + finish_test diff --git a/manifest b/manifest index e0c37bf1e2..92f3bfcf68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sfts5\sexpression\sparser\smodule\sto\sallow\sembedded\s0x00\sbytes\sin\stokens. -D 2023-10-03T19:06:52.966 +C Add\sthe\stokendata=1\soption\sto\signore\strailing\stoken-data\swhen\squerying\san\sfts5\stable. +D 2023-10-11T21:08:12.656 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,13 +88,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520 -F ext/fts5/fts5Int.h 66a38b285e2b860baa29745d8eff27f5b0809268e7820498494d9acfaccf8a5c +F ext/fts5/fts5Int.h a21eb1cf036ac9eb943e45ed307762901ea86f0159bf0848baa2079a112ddc2f F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 -F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 -F ext/fts5/fts5_expr.c cc215d39714b428523d2f2ef42b713c83095a28a67bc7f6f2dc4ac036a29f460 +F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf +F ext/fts5/fts5_expr.c fd091d0558fda2517602ed5886ec615ce3e1bd76fb0bb0e5d1aa85ba8db287a8 F ext/fts5/fts5_hash.c 76765856397eff56f526b0640b23a1677d737d35e07bc00e4b4b2e0fc5fda60d -F ext/fts5/fts5_index.c e472083d371f420d52ec80445b9d2a99b16b23548205cb4064ddcd41bd79f63e +F ext/fts5/fts5_index.c 79a8e45771d0be24f0399b12268299f132ce0970ade941ba8a2d40b1d1aee4d7 F ext/fts5/fts5_main.c 799ec88d2309055f6406bddb0bd6ed80148c5da5eb14594c3c5309a6e944d489 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5 F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f @@ -106,8 +106,8 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988 F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520424dc9d6 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b -F ext/fts5/test/fts5aa.test ba5158eba7d61359becdfca895ef471072c7bf7b20e5e60dcb4d024c8419c926 +F ext/fts5/test/fts5_common.tcl 8b1848ac2baad10e444e4183034a52050b52d20b3796d9d30e78f01ab0d05583 +F ext/fts5/test/fts5aa.test 4db81519863244a3cab35795fe65ab6b592e7970c7409eba098b23ebbfc08d95 F ext/fts5/test/fts5ab.test bd932720c748383277456b81f91bc00453de2174f9762cd05f95d0495dc50390 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de @@ -187,7 +187,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 3e1ac3230f65a0d644e9bf0738bebb09b4db9d9f123e1307d8630e42269b4afb +F ext/fts5/test/fts5origintext.test 646df137f1aa5b3d7032374ebe82bfdbe88d9f825d73ce8d44bead480317a9c5 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2123,8 +2123,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e051120067fd87f57b498e505e3960cf4d14e8e33bad940618cc0823253254f7 -R 5cce41f02eae121cb66e72942ef56113 +P 342c8d0783f449817d3f565ff6b9f010a6c690beeea32f1861640810490a8b5f +R 28168382a793dd4cf732de7d6442ef72 U dan -Z 80b8e2664e768ed2ac03913fcf0180ea +Z cded15b3b6aeefc28aa13f7856e47e8d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4798938837..8bc375eea2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -342c8d0783f449817d3f565ff6b9f010a6c690beeea32f1861640810490a8b5f \ No newline at end of file +122935182ad5869ce3a4c6d796c38a0509f6f3384dd1b3e60a3f2f0f366cc5f5 \ No newline at end of file From 7b5123c7963c0b956e298e9e355ee45fdbd4de02 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 3 Nov 2023 12:09:22 +0000 Subject: [PATCH 075/191] Enhance the JSONB lookup routine with logic to apply edits. The new logic is currently unused and hence untested but does not create any regressions. FossilOrigin-Name: b12110276fc15d1b6b0cc455f89747ace7a09650d5ba433d8bb431bb4e5bc951 --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 869a09d63f..9aadbdf941 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\sfixes\sand\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-11-03T11:35:33.434 +C Enhance\sthe\sJSONB\slookup\sroutine\swith\slogic\sto\sapply\sedits.\s\sThe\snew\slogic\sis\ncurrently\sunused\sand\shence\suntested\sbut\sdoes\snot\screate\sany\sregressions. +D 2023-11-03T12:09:22.342 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -683,7 +683,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2c0599c54aef24739015a0b7f7b569adecd760e4d06afd3d9c01c8c53713d3f0 +F src/json.c e109da4d5f29e239db837dad09a886a78b6a217592ea8c71e3b584483ce9b179 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c e1bc8864834697503d370d94613be945d05ca1c5ebdda43e7d5c8ee8c48d433c @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a420a4f7ff76b0e9cf5f6d515ccfa31e526d58f4001a4015a367e2aa3c82091f 15b618e92a2708cc83256947736de8c494a9985a77e38bd5efc7e51e72cba344 -R 7bce1ab3b73fc9ca8b3b94f2470fc2d6 +P b089bf46374b374d02d7654c65eb3e75d1777638b398061e47644af0dab48c9b +R 02d78f9919449761c67674ad6f60265a U drh -Z 13124fe09b7592eb801d594e79ae949a +Z f8a451c94f8b9175a3b99dd9c9c86acd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 71f504a420..c8ced97e73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b089bf46374b374d02d7654c65eb3e75d1777638b398061e47644af0dab48c9b \ No newline at end of file +b12110276fc15d1b6b0cc455f89747ace7a09650d5ba433d8bb431bb4e5bc951 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 48d456e536..e998af5230 100644 --- a/src/json.c +++ b/src/json.c @@ -341,8 +341,19 @@ struct JsonParse { u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ u8 *aBlob; /* BLOB representation of zJson */ + /* Search and edit information. See jsonLookupBlobStep() */ + u8 eEdit; /* Edit operation to apply */ + int delta; /* Size change due to the edit */ + u32 nIns; /* Number of bytes to insert */ + u8 *aIns; /* Content to be inserted */ }; +/* Allowed values for JsonParse.eEdit */ +#define JEDIT_DEL 0x01 /* Delete if exists */ +#define JEDIT_INS 0x02 /* Insert if not exists */ +#define JEDIT_REPL 0x04 /* Overwrite if exists */ +#define JEDIT_SET 0x08 /* Insert or overwrite */ + /* ** Maximum nesting depth of JSON for this implementation. ** @@ -720,8 +731,6 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ } } - - /* ** Append an sqlite3_value (such as a function parameter) to the JSON ** string under construction in p. @@ -2558,6 +2567,31 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ return 0; } +/* +** If pParse->aBlob is not previously editable (because it is taken +** from sqlite3_value_blob(), as indicated by the fact that +** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable +** by making a copy into space obtained from malloc. +** +** Return true on success. Return false on OOM. +*/ +static int jsonBlobMakeEditable(JsonParse *pParse){ + u8 *aOld; + u32 nSize; + if( pParse->nBlobAlloc>0 ) return 1; + aOld = pParse->aBlob; + nSize = pParse->nBlob + pParse->nIns; + if( nSize>100 ) nSize -= 100; + pParse->aBlob = 0; + if( jsonBlobExpand(pParse, nSize) ){ + return 0; + } + assert( pParse->nBlobAlloc >= pParse->nBlob + pParse->nIns ); + memcpy(pParse->aBlob, aOld, pParse->nBlob); + return 1; +} + + /* Expand pParse->aBlob and append N bytes. ** ** Return the number of errors. @@ -3767,12 +3801,24 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ return k; } +/* +** Edit the size of the element at iRoot by the amount in pParse->delta. +*/ +static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ + u32 sz; + assert( pParse->delta==0 ); + (void)jsonbPayloadSize(pParse, iRoot, &sz); + sz += pParse->delta; + jsonBlobChangePayloadSize(pParse, iRoot, sz); +} + /* ** Error returns from jsonLookupBlobStep() */ #define JSON_BLOB_ERROR 0xffffffff #define JSON_BLOB_NOTFOUND 0xfffffffe #define JSON_BLOB_PATHERROR 0xfffffffd +#define JSON_BLOB_ISERROR(x) ((x)>=JSON_BLOB_PATHERROR) /* ** Search along zPath to find the Json element specified. Return an @@ -3785,12 +3831,39 @@ static u32 jsonLookupBlobStep( u32 iRoot, /* Begin the search at this element of aBlob[] */ const char *zPath /* The path to search */ ){ - u32 i, j, k, nKey, sz, n, iEnd; + u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; u8 x; - if( pParse->oom ) return 0; - if( zPath[0]==0 ) return iRoot; + if( zPath[0]==0 ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse) ){ + n = jsonbPayloadSize(pParse, iRoot, &sz); + sz += n; + if( pParse->eEdit==JEDIT_DEL ){ + memmove(&pParse->aBlob[iRoot], &pParse->aBlob[iRoot+sz], + pParse->nBlob - iRoot); + pParse->nBlob -= n; + pParse->delta = -(int)n; + }else if( pParse->eEdit==JEDIT_INS ){ + /* Already exists, so json_insert() is a no-op */ + }else{ + /* json_set() or json_replace() */ + int d = (int)pParse->nIns - (int)sz; + pParse->delta = d; + if( d!=0 ){ + if( pParse->nBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return iRoot; + } + memmove(&pParse->aBlob[iRoot+pParse->nIns], + &pParse->aBlob[iRoot+sz], + pParse->nBlob - iRoot - sz); + } + memcpy(&pParse->aBlob[iRoot], pParse->aIns, pParse->nIns); + } + } + return iRoot; + } if( zPath[0]=='.' ){ x = pParse->aBlob[iRoot]; zPath++; @@ -3828,7 +3901,9 @@ static u32 jsonLookupBlobStep( if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 || j+n+sz>iEnd ) return JSON_BLOB_ERROR; - return jsonLookupBlobStep(pParse, j, &zPath[i]); + rc = jsonLookupBlobStep(pParse, j, &zPath[i]); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } j = k+sz; if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; @@ -3872,7 +3947,9 @@ static u32 jsonLookupBlobStep( iEnd = j+sz; while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; } k--; n = jsonbPayloadSize(pParse, j, &sz); @@ -3880,9 +3957,20 @@ static u32 jsonLookupBlobStep( j += n+sz; } if( j>iEnd ) return JSON_BLOB_ERROR; + if( k>1 ) return JSON_BLOB_NOTFOUND; }else{ return JSON_BLOB_PATHERROR; } + if( pParse->eEdit==JEDIT_INS && jsonBlobMakeEditable(pParse) ){ + assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); + memmove(&pParse->aBlob[j], &pParse->aBlob[j+pParse->nIns], + pParse->nBlob - j); + memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); + pParse->delta = pParse->nIns; + pParse->nBlob += pParse->nIns; + jsonAfterEditSizeAdjust(pParse, iRoot); + return j; + } return JSON_BLOB_NOTFOUND; } From 43d05ccc314472ece8dfcb2eaac36c6e93ceee23 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Nov 2023 14:55:20 +0000 Subject: [PATCH 076/191] Add declarations for new API functions. FossilOrigin-Name: b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 --- ext/fts5/fts5.h | 5 +++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 323d73a28f..730a2fb5e3 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -298,6 +298,11 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, int iPhrase, int iToken, const char**, int*); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + int (*xPhraseToken)(Fts5Context*, Fts5PhraseIter*, int, const char**, int*); }; /* diff --git a/manifest b/manifest index 8eba96ed3c..baff022d13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\sinto\sthis\sbranch. -D 2023-11-06T19:16:38.711 +C Add\sdeclarations\sfor\snew\sAPI\sfunctions. +D 2023-11-08T14:55:20.854 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,7 +87,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 05501612cc655504c5dce8ba765ab621d50fc478490089beaa0d75e00b23e520 +F ext/fts5/fts5.h 68256dd94eaba3e874762a63578922fd62a5ca87c76a28f7636effee4d9bd781 F ext/fts5/fts5Int.h a21eb1cf036ac9eb943e45ed307762901ea86f0159bf0848baa2079a112ddc2f F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac5570614ebb77feff4943c7a78d877508e31087b550f337c11b45f1016e3c55 c2058a045b57571b2b5d342adb212fe606717c633a0422755691ae6bf5725d25 -R 5cb5a1a54d74cfe313891abc677e111f +P 3a869cf1f84b0e9bdcc4de53685430ab41eafacbba1ca7b87e727aa98811c6c5 +R 9e06a9e0b44056d476c02db915884372 U dan -Z d0201f081f93fb8379f972a79c65f95a +Z 6c0dbde9f9a0f05c4a27f8bd01924beb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 412eb8af05..cf6088cc7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a869cf1f84b0e9bdcc4de53685430ab41eafacbba1ca7b87e727aa98811c6c5 \ No newline at end of file +b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 \ No newline at end of file From e108029332f7d403961b3f5c9bbe1c8fe3a9e6d5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Nov 2023 14:29:12 +0000 Subject: [PATCH 077/191] Add new fts5 API xQueryToken(). FossilOrigin-Name: 828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 --- ext/fts5/fts5.h | 2 +- ext/fts5/fts5Int.h | 2 ++ ext/fts5/fts5_expr.c | 26 +++++++++++++++++++ ext/fts5/fts5_main.c | 16 +++++++++++- ext/fts5/fts5_tcl.c | 18 +++++++++++++ ext/fts5/test/fts5origintext.test | 43 +++++++++++++++++++++++++++++++ manifest | 22 ++++++++-------- manifest.uuid | 2 +- 8 files changed, 117 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 730a2fb5e3..5a2008882f 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -263,7 +263,7 @@ struct Fts5PhraseIter { ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4aa578559b..df25fd9ffa 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -773,6 +773,8 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); +int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a2c6320719..80387a90fd 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3145,3 +3145,29 @@ int sqlite3Fts5ExprPhraseCollist( return rc; } + +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 6e86ca5951..7ddc7b6fcf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2323,13 +2323,24 @@ static int fts5ApiPhraseFirstColumn( return rc; } +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { - 2, /* iVersion */ + 3, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -2349,6 +2360,9 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + 0, + 0 }; /* diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index fb4bea8e9e..a9390adc9e 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -244,6 +244,8 @@ static int SQLITE_TCLAPI xF5tApi( { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ + + { "xQueryToken", 2, "IPHRASE ITERM" }, /* 19 */ { 0, 0, 0} }; @@ -500,6 +502,22 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(18, "xQueryToken") { + const char *pTerm = 0; + int nTerm = 0; + int iPhrase = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 3fd5f17e7b..2c9a4ba5a3 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -157,5 +157,48 @@ do_execsql_test 3.1.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} do_execsql_test 3.1.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +proc querytoken {cmd iPhrase iToken} { + set txt [$cmd xQueryToken $iPhrase $iToken] + string map [list "\0" "."] $txt +} +sqlite3_fts5_create_function db querytoken querytoken + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize='origintext unicode61', tokendata=1 + ); + INSERT INTO ft VALUES('one two three four'); +} + +do_execsql_test 3.1 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO') +} {1 two.TwO} +do_execsql_test 3.2 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE') +} {1 one} +do_execsql_test 3.3 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') +} {1 two.TWO} +do_execsql_test 3.4 { + SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') +} {1 three.ThreE} + +do_catchsql_test 3.5 { + SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} +do_catchsql_test 3.6 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} +do_catchsql_test 3.7 { + SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} + + + + finish_test diff --git a/manifest b/manifest index baff022d13..4a539764cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdeclarations\sfor\snew\sAPI\sfunctions. -D 2023-11-08T14:55:20.854 +C Add\snew\sfts5\sAPI\sxQueryToken(). +D 2023-11-13T14:29:12.382 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,17 +87,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 68256dd94eaba3e874762a63578922fd62a5ca87c76a28f7636effee4d9bd781 -F ext/fts5/fts5Int.h a21eb1cf036ac9eb943e45ed307762901ea86f0159bf0848baa2079a112ddc2f +F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c +F ext/fts5/fts5Int.h 19b198459a2791415919428d44ebf4c830b59b2da6f27f8faaffe39a876b7ecf F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c fd091d0558fda2517602ed5886ec615ce3e1bd76fb0bb0e5d1aa85ba8db287a8 +F ext/fts5/fts5_expr.c 69c81af515ce1cedccf093c7c76f8b3b4f24bafbfb1d03a431af9f5c69a81834 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 145723e22ffee28dbe2a24933e74ad998d32419223c0ddb8506a1f0c39b952c4 -F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f +F ext/fts5/fts5_main.c ddac85dbd28167af81f64e568bfe9020bf9708d650de207d1465ed19938316d1 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d -F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f +F ext/fts5/fts5_tcl.c 71641a0c5693c64acfad9d10e64475ec92d9f464d06ba7fd350552de373586d8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13 @@ -188,7 +188,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 646df137f1aa5b3d7032374ebe82bfdbe88d9f825d73ce8d44bead480317a9c5 +F ext/fts5/test/fts5origintext.test 8296984d268d1d20f85c9de316f422ffb6ebc12020d3f8a0a18144d6ca7b347f F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a869cf1f84b0e9bdcc4de53685430ab41eafacbba1ca7b87e727aa98811c6c5 -R 9e06a9e0b44056d476c02db915884372 +P b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 +R 177f59b3c48b3865f7f595e0ec855966 U dan -Z 6c0dbde9f9a0f05c4a27f8bd01924beb +Z ac818dc8fd8dc9007ff77eae486bea2e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf6088cc7c..f6858a05cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 \ No newline at end of file +828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 \ No newline at end of file From 50b0e25a556c34115337e7f3eca85dabea6b463d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Nov 2023 11:45:19 +0000 Subject: [PATCH 078/191] Add implementation of xInstToken() API. FossilOrigin-Name: a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07 --- ext/fts5/fts5Int.h | 5 + ext/fts5/fts5_expr.c | 24 ++++ ext/fts5/fts5_index.c | 206 +++++++++++++++++++++++++++++- ext/fts5/fts5_main.c | 55 ++++++-- ext/fts5/fts5_tcl.c | 19 ++- ext/fts5/test/fts5origintext.test | 55 ++++++-- manifest | 22 ++-- manifest.uuid | 2 +- 8 files changed, 356 insertions(+), 32 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index df25fd9ffa..19d9579eb7 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -464,6 +464,10 @@ void *sqlite3Fts5StructureRef(Fts5Index*); void sqlite3Fts5StructureRelease(void*); int sqlite3Fts5StructureTest(Fts5Index*, void*); +/* +** Used by xInstToken() and xPhraseToken(). +*/ +int sqlite3Fts5IterToken(Fts5IndexIter*, int, int, const char**, int*); /* ** Insert or remove data to or from the index. Each time a document is @@ -774,6 +778,7 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); +int sqlite3Fts5ExprInstToken(Fts5Expr*, int, int, int, int, const char**, int*); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 80387a90fd..ae851a877a 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3171,3 +3171,27 @@ int sqlite3Fts5ExprQueryToken( return SQLITE_OK; } +int sqlite3Fts5ExprInstToken( + Fts5Expr *pExpr, + int iPhrase, + int iCol, + int iOff, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + Fts5IndexIter *pIter = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + pIter = pPhrase->aTerm[iToken].pIter; + + return sqlite3Fts5IterToken(pIter, iCol, iOff+iToken, ppOut, pnOut); +} + diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 6cc2c13927..2d6d561b7b 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -323,6 +323,9 @@ typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; +typedef struct Fts5TokenMapEntry Fts5TokenMapEntry; +typedef struct Fts5TokenMapToken Fts5TokenMapToken; +typedef struct Fts5TokenMap Fts5TokenMap; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ @@ -605,11 +608,34 @@ struct Fts5Iter { int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ + Fts5TokenMap *pTokenMap; i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; +struct Fts5TokenMapEntry { + i64 iRowid; + u16 iCol; + int iOff; + int iTok; /* Offset into aToken[] + 1 */ +}; + +struct Fts5TokenMapToken { + u8 *pTerm; + int nTerm; +}; + +struct Fts5TokenMap { + int nEntryAlloc; + int nEntry; + Fts5TokenMapEntry *aEntry; + + int nTokenAlloc; + int nToken; + Fts5TokenMapToken *aToken; +}; + /* ** An instance of the following type is used to iterate through the contents @@ -3023,6 +3049,20 @@ static void fts5SegIterNextFrom( }while( p->rc==SQLITE_OK ); } +/* +** Free the Fts5TokenMap object passed as the only argument. +*/ +static void fts5TokenMapFree(Fts5TokenMap *pMap){ + if( pMap ){ + int ii; + for(ii=0; iinToken; ii++){ + sqlite3_free(pMap->aToken[ii].pTerm); + } + sqlite3_free(pMap->aToken); + sqlite3_free(pMap->aEntry); + sqlite3_free(pMap); + } +} /* ** Free the iterator object passed as the second argument. @@ -3034,6 +3074,7 @@ static void fts5MultiIterFree(Fts5Iter *pIter){ fts5SegIterClear(&pIter->aSeg[i]); } fts5BufferFree(&pIter->poslist); + fts5TokenMapFree(pIter->pTokenMap); sqlite3_free(pIter); } } @@ -3847,6 +3888,7 @@ fts5MultiIterNew_post_check: static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ + Fts5TokenMap *pMap, /* Token-map, if any */ int bDesc, /* True for descending rowid order */ Fts5Iter **ppOut /* New object */ ){ @@ -3854,7 +3896,8 @@ static void fts5MultiIterNew2( pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - + pNew->pTokenMap = pMap; + pMap = 0; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -3877,6 +3920,7 @@ static void fts5MultiIterNew2( *ppOut = pNew; } + fts5TokenMapFree(pMap); fts5DataRelease(pData); } @@ -6063,6 +6107,128 @@ static void fts5MergePrefixLists( *p1 = out; } +static u8 *fts5IdxBufferDup(Fts5Index *p, const u8 *pDup, int nDup){ + u8 *pRet = fts5IdxMalloc(p, nDup+1); + if( pRet ){ + memcpy(pRet, pDup, nDup); + } + return pRet; +} + +static void fts5TokenMapTerm( + Fts5Index *p, + Fts5TokenMap *pMap, + const u8 *pTerm, + int nTerm +){ + if( p->rc==SQLITE_OK ){ + Fts5TokenMapToken *pToken = 0; + if( pMap->nToken==pMap->nTokenAlloc ){ + i64 nNew = (pMap->nTokenAlloc ? pMap->nTokenAlloc * 2 : 32); + Fts5TokenMapToken *aNew = sqlite3_realloc64( + pMap->aToken, nNew*sizeof(Fts5TokenMapToken) + ); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + pMap->nTokenAlloc = nNew; + pMap->aToken = aNew; + } + pToken = &pMap->aToken[pMap->nToken++]; + pToken->nTerm = nTerm; + pToken->pTerm = fts5IdxBufferDup(p, pTerm, nTerm); + } +} + + +static void fts5TokenMapPoslist( + Fts5Index *p, + Fts5TokenMap *pMap, + Fts5Iter *p1 +){ + if( p->rc==SQLITE_OK ){ + const u8 *a = p1->base.pData; + i64 iPos = 0; + int iOff = 0; + + while( 0==sqlite3Fts5PoslistNext64(a, p1->base.nData, &iOff, &iPos) ){ + Fts5TokenMapEntry *pEntry = 0; + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + + if( pMap->nEntry==pMap->nEntryAlloc ){ + i64 nNew = (pMap->nEntryAlloc ? pMap->nEntryAlloc * 2 : 32); + Fts5TokenMapEntry *aNew = sqlite3_realloc64( + pMap->aEntry, nNew*sizeof(Fts5TokenMapEntry) + ); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + pMap->nEntryAlloc = nNew; + pMap->aEntry = aNew; + } + pEntry = &pMap->aEntry[pMap->nEntry++]; + pEntry->iRowid = p1->base.iRowid; + pEntry->iCol = iCol; + pEntry->iOff = iTokOff; + pEntry->iTok = pMap->nToken; + } + } +} + +static int fts5TokenMapHash(i64 iRowid, int iCol, int iOff){ + return iRowid + (iRowid << 3) + (iCol << 6) + (iOff << 9); +} + +static void fts5TokenMapHashify(Fts5Index *p, Fts5TokenMap *pMap){ + int nHash = pMap->nEntry*2; + Fts5TokenMapEntry *aHash = 0; + + aHash = (Fts5TokenMapEntry*)fts5IdxMalloc(p, nHash*sizeof(Fts5TokenMapEntry)); + if( aHash ){ + int ii; + for(ii=0; iinEntry; ii++){ + Fts5TokenMapEntry *pEntry = &pMap->aEntry[ii]; + Fts5TokenMapEntry *pCopy = 0; + int iHash = fts5TokenMapHash(pEntry->iRowid, pEntry->iCol, pEntry->iOff); + + while( aHash[iHash % nHash].iTok ){ + iHash++; + } + pCopy = &aHash[iHash % nHash]; + memcpy(pCopy, pEntry, sizeof(Fts5TokenMapEntry)); + } + + sqlite3_free(pMap->aEntry); + pMap->aEntry = aHash; + pMap->nEntry = pMap->nEntryAlloc = nHash; + } +} + +static const u8 *fts5TokenMapLookup( + Fts5TokenMap *pMap, + i64 iRowid, + int iCol, + int iOff, + int *pnOut +){ + int iHash = fts5TokenMapHash(iRowid, iCol, iOff) % pMap->nEntry; + + for(; pMap->aEntry[iHash].iTok!=0; iHash = (iHash+1)%pMap->nEntry){ + Fts5TokenMapEntry *pEntry = &pMap->aEntry[iHash]; + if( pEntry->iRowid==iRowid && pEntry->iCol==iCol && pEntry->iOff==iOff ){ + *pnOut = pMap->aToken[pEntry->iTok-1].nTerm; + return pMap->aToken[pEntry->iTok-1].pTerm; + } + } + + *pnOut = 0; + return 0; +} + + static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -6077,6 +6243,7 @@ static void fts5SetupPrefixIter( Fts5Buffer *aBuf; int nBuf = 32; int nMerge = 1; + Fts5TokenMap *pMap = 0; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); @@ -6094,8 +6261,12 @@ static void fts5SetupPrefixIter( aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); + if( iIdx==0 ){ + pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); + } + assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); - if( aBuf && pStruct ){ + if( p->rc==SQLITE_OK ){ const int flags = FTS5INDEX_QUERY_SCAN | FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_NOOUTPUT; @@ -6113,6 +6284,7 @@ static void fts5SetupPrefixIter( ** index contains all the doclists required, except for the one ** corresponding to the prefix itself. That one is extracted from the ** main term index here. */ + assert( iIdx==0 || pMap==0 ); if( iIdx!=0 ){ int dummy = 0; const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; @@ -6145,6 +6317,13 @@ static void fts5SetupPrefixIter( const u8 *pTerm = pSeg->term.p; p1->xSetOutputs(p1, pSeg); + if( pMap ){ + if( bNewTerm ){ + fts5TokenMapTerm(p, pMap, &pTerm[1], nTerm-1); + } + fts5TokenMapPoslist(p, pMap, p1); + } + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTermp = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); - fts5MultiIterNew2(p, pData, bDesc, ppIter); + if( pMap ) fts5TokenMapHashify(p, pMap); + fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter); } fts5BufferFree(&doclist); } @@ -6575,6 +6755,26 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +/* +** +*/ +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + int iCol, + int iOff, + const char **ppOut, int *pnOut +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter->pTokenMap ){ + *ppOut = fts5TokenMapLookup( + pIter->pTokenMap, pIndexIter->iRowid, iCol, iOff, pnOut + ); + }else{ + *ppOut = sqlite3Fts5IterTerm(pIndexIter, pnOut); + } + return SQLITE_OK; +} + /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 7ddc7b6fcf..5ef80719ae 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2063,12 +2063,6 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; -#if 0 - }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ - *piPhrase = pCsr->aInst[iIdx*3]; - *piCol = pCsr->aInst[iIdx*3 + 2]; - *piOff = -1; -#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -2323,6 +2317,9 @@ static int fts5ApiPhraseFirstColumn( return rc; } +/* +** xQueryToken() API implemenetation. +*/ static int fts5ApiQueryToken( Fts5Context* pCtx, int iPhrase, @@ -2334,6 +2331,48 @@ static int fts5ApiQueryToken( return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); } +/* +** xInstToken() API implemenetation. +*/ +static int fts5ApiInstToken( + Fts5Context *pCtx, + int iIdx, + int iToken, + const char **ppOut, int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + int iPhrase = pCsr->aInst[iIdx*3]; + int iCol = pCsr->aInst[iIdx*3 + 1]; + int iOff = pCsr->aInst[iIdx*3 + 2]; + + rc = sqlite3Fts5ExprInstToken( + pCsr->pExpr, iPhrase, iCol, iOff, iToken, ppOut, pnOut + ); + } + } + return rc; +} + +/* +** xPhraseToken() API implemenetation. +*/ +static int fts5ApiPhraseToken( + Fts5Context *pCtx, + Fts5PhraseIter *pIter, + int iToken, + const char **ppOut, + int *pnOut +){ + return SQLITE_OK; +} + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) @@ -2361,8 +2400,8 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, fts5ApiQueryToken, - 0, - 0 + fts5ApiInstToken, + fts5ApiPhraseToken }; /* diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index a9390adc9e..853a41865e 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -245,7 +245,8 @@ static int SQLITE_TCLAPI xF5tApi( { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ - { "xQueryToken", 2, "IPHRASE ITERM" }, /* 19 */ + { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ + { "xInstToken", 2, "IDX ITERM" }, /* 19 */ { 0, 0, 0} }; @@ -518,6 +519,22 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(19, "xInstToken") { + const char *pTerm = 0; + int nTerm = 0; + int iIdx = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 2c9a4ba5a3..07e5b4fb2e 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -167,38 +167,77 @@ proc querytoken {cmd iPhrase iToken} { } sqlite3_fts5_create_function db querytoken querytoken -do_execsql_test 3.0 { +do_execsql_test 4.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize='origintext unicode61', tokendata=1 ); INSERT INTO ft VALUES('one two three four'); } -do_execsql_test 3.1 { +do_execsql_test 4.1 { SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO') } {1 two.TwO} -do_execsql_test 3.2 { +do_execsql_test 4.2 { SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE') } {1 one} -do_execsql_test 3.3 { +do_execsql_test 4.3 { SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') } {1 two.TWO} -do_execsql_test 3.4 { +do_execsql_test 4.4 { SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') } {1 three.ThreE} -do_catchsql_test 3.5 { +do_catchsql_test 4.5 { SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} -do_catchsql_test 3.6 { +do_catchsql_test 4.6 { SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} -do_catchsql_test 3.7 { +do_catchsql_test 4.7 { SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +proc insttoken {cmd iIdx iToken} { + set txt [$cmd xInstToken $iIdx $iToken] + string map [list "\0" "."] $txt +} +sqlite3_fts5_create_function db insttoken insttoken +fts5_aux_test_functions db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize='origintext unicode61', tokendata=1 + ); + INSERT INTO ft VALUES('one ONE One oNe oNE one'); +} +do_execsql_test 5.1 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft('one'); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test 5.2 { + SELECT insttoken(ft, 1, 0) FROM ft('one'); +} { + one.ONE +} + +do_execsql_test 5.3 { + SELECT fts5_test_poslist(ft) FROM ft('one'); +} { + {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5} +} finish_test diff --git a/manifest b/manifest index 4a539764cf..3477a07b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sfts5\sAPI\sxQueryToken(). -D 2023-11-13T14:29:12.382 +C Add\simplementation\sof\sxInstToken()\sAPI. +D 2023-11-15T11:45:19.681 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,16 +88,16 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c -F ext/fts5/fts5Int.h 19b198459a2791415919428d44ebf4c830b59b2da6f27f8faaffe39a876b7ecf +F ext/fts5/fts5Int.h 88ab1ee1eefa6f98e4c7fd3c96c99ef76ed2819cc3058736c87bb01e4a301628 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 69c81af515ce1cedccf093c7c76f8b3b4f24bafbfb1d03a431af9f5c69a81834 +F ext/fts5/fts5_expr.c 4b50ed0c724cb160f086e20e964ed2d57b99d0d3c1cb1b029901c0300b11bd9f F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 145723e22ffee28dbe2a24933e74ad998d32419223c0ddb8506a1f0c39b952c4 -F ext/fts5/fts5_main.c ddac85dbd28167af81f64e568bfe9020bf9708d650de207d1465ed19938316d1 +F ext/fts5/fts5_index.c 3b51c2f0554a665694e777c8f2765cb5b1283d4bc960dde350a604af3e5e5d98 +F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d -F ext/fts5/fts5_tcl.c 71641a0c5693c64acfad9d10e64475ec92d9f464d06ba7fd350552de373586d8 +F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13 @@ -188,7 +188,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 8296984d268d1d20f85c9de316f422ffb6ebc12020d3f8a0a18144d6ca7b347f +F ext/fts5/test/fts5origintext.test 908a1fb6b1106e4b6ed0f9cf683c2ad7f986cce1aea1e0a13b3309c6f568932b F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 -R 177f59b3c48b3865f7f595e0ec855966 +P 828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 +R 7870d9470a55737470bd92d95fe480a9 U dan -Z ac818dc8fd8dc9007ff77eae486bea2e +Z d10d6cf5b22c051f4553454e4a3996a4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f6858a05cd..96d818fc70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 \ No newline at end of file +a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07 \ No newline at end of file From 0399994759cef5531a528275039d20a35e13e2f7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Nov 2023 16:10:31 +0000 Subject: [PATCH 079/191] Work toward getting jsonb_remove() to work directly on JSONB blobs. FossilOrigin-Name: a79ff8e58fcaf718a6fb78e145117f1d6d40d133f31e9752bb9c6d484850a27b --- manifest | 17 ++++++----- manifest.uuid | 2 +- src/json.c | 81 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 74 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index ea4fb34420..f9398864df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\senhancements\sand\sfixes\sfrom\strunk\sinto\sthe\sjsonb\sbranch. -D 2023-11-15T13:23:40.944 +C Work\stoward\sgetting\sjsonb_remove()\sto\swork\sdirectly\son\sJSONB\sblobs. +D 2023-11-15T16:10:31.627 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -258,7 +258,7 @@ F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 482f53dfec9e3ac2a9070d3fceebd56250932aaaf7c4f5bc8de29fc011416e0c F ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b995ca412f59b631803b93aa5b3684fce62e335d1e123207084c054abfd488d4 -F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 w ext/jni/src/org/sqlite/jni/capi/ConfigSqllogCallback.java +F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 F ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b7036dcb1ef1b39f1f36ac605dde0ff1a24a9a01ade6aa1a605039443e089a61 F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 246b0e66c4603f41c567105a21189d138aaf8c58203ecd4928802333da553e7c F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 97352091abd7556167f4799076396279a51749fdae2b72a6ba61cd39b3df0359 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 01644dc0c33b331fd5f294eddb1e4e02a4057535193f5f38161ee75047dd7177 +F src/json.c 3ff1ea643cab62e3ddb8be3a64bff2dd94af37ea461ea343054796e7f11638f8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2141,8 +2141,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 091a5f058dfe2115fb9213655b34f00bcec80aebb299b571975cfe4ecd5ec206 9264955e6e47aa8fc3a6f8bed192a6c12f43de49f7fba2e0cc080e47abedde14 -R cb95d20763613c633369d4c9e1ac8345 +P ba91408f4c044feda003ef93784ccefb619f99ab64379ced481ee8e9e890fd41 +R 58cf82355c33a53c2234fea2c219b8e4 +T *branch * jsonb-remove +T *sym-jsonb-remove * +T -sym-jsonb * U drh -Z c8298398287bfd474381f2a9e0abdbd8 +Z 367d226fdddcafa052b6fb6b4d87ed0a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5cff63237a..7cf8c33cb1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba91408f4c044feda003ef93784ccefb619f99ab64379ced481ee8e9e890fd41 \ No newline at end of file +a79ff8e58fcaf718a6fb78e145117f1d6d40d133f31e9752bb9c6d484850a27b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3787167e90..e66c4b1326 100644 --- a/src/json.c +++ b/src/json.c @@ -3813,11 +3813,12 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ ** Edit the size of the element at iRoot by the amount in pParse->delta. */ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ - u32 sz; - assert( pParse->delta==0 ); + u32 sz = 0; + assert( pParse->delta!=0 ); (void)jsonbPayloadSize(pParse, iRoot, &sz); sz += pParse->delta; jsonBlobChangePayloadSize(pParse, iRoot, sz); + pParse->nBlob += pParse->delta; } /* @@ -3837,7 +3838,8 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ static u32 jsonLookupBlobStep( JsonParse *pParse, /* The JSON to search */ u32 iRoot, /* Begin the search at this element of aBlob[] */ - const char *zPath /* The path to search */ + const char *zPath, /* The path to search */ + u32 iLabel /* Label if iRoot is a value of in an object */ ){ u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; @@ -3848,10 +3850,13 @@ static u32 jsonLookupBlobStep( n = jsonbPayloadSize(pParse, iRoot, &sz); sz += n; if( pParse->eEdit==JEDIT_DEL ){ + if( iLabel>0 ){ + sz += iRoot - iLabel; + iRoot = iLabel; + } memmove(&pParse->aBlob[iRoot], &pParse->aBlob[iRoot+sz], - pParse->nBlob - iRoot); - pParse->nBlob -= n; - pParse->delta = -(int)n; + pParse->nBlob - (iRoot+sz)); + pParse->delta = -(int)sz; }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ }else{ @@ -3895,21 +3900,22 @@ static u32 jsonLookupBlobStep( } if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); - j = iRoot + n; + j = iRoot + n; /* j is the index of a label */ iEnd = j+sz; while( jaBlob[j] & 0x0f; if( xJSONB_TEXTRAW ) return JSON_BLOB_ERROR; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_BLOB_ERROR; - k = j+n; + k = j+n; /* k is the index of the label text */ if( k+sz>=iEnd ) return JSON_BLOB_ERROR; if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ - j = k+sz; - if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; - n = jsonbPayloadSize(pParse, j, &sz); - if( n==0 || j+n+sz>iEnd ) return JSON_BLOB_ERROR; - rc = jsonLookupBlobStep(pParse, j, &zPath[i]); + u32 v = k+sz; /* v is the index of the value */ + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + n = jsonbPayloadSize(pParse, v, &sz); + if( n==0 || v+n+sz>iEnd ) return JSON_BLOB_ERROR; + assert( j>0 ); + rc = jsonLookupBlobStep(pParse, v, &zPath[i], j); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -3955,7 +3961,7 @@ static u32 jsonLookupBlobStep( iEnd = j+sz; while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -3975,7 +3981,6 @@ static u32 jsonLookupBlobStep( pParse->nBlob - j); memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); pParse->delta = pParse->nIns; - pParse->nBlob += pParse->nIns; jsonAfterEditSizeAdjust(pParse, iRoot); return j; } @@ -4195,7 +4200,7 @@ static void jsonExtractFromBlob( if( px.aBlob==0 ) return; if( zPath[0]=='$' ){ zPath++; - i = jsonLookupBlobStep(&px, 0, zPath); + i = jsonLookupBlobStep(&px, 0, zPath, 0); }else if( (flags & JSON_ABPATH) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for @@ -4218,7 +4223,7 @@ static void jsonExtractFromBlob( jsonAppendChar(&jx, 0); zPath = jx.zBuf; } - i = jsonLookupBlobStep(&px, 0, zPath); + i = jsonLookupBlobStep(&px, 0, zPath, 0); jsonStringReset(&jx); }else{ sqlite3_result_error(ctx, "bad path", -1); @@ -4237,7 +4242,41 @@ static void jsonExtractFromBlob( sqlite3_free(zMsg); } } - + +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments are JSON paths of elements to be removed. Do that removal +** and return the result. +*/ +static void jsonRemoveFromBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + u32 rc; + JsonParse px; + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(argv[0]); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( px.aBlob==0 ) return; + for(i=1; i0 ? sqlite3_free : SQLITE_DYNAMIC); + return; + +jsonRemoveFromBlob_error: + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + return; + +} /**************************************************************************** ** SQL functions used for testing and debugging @@ -4780,6 +4819,12 @@ static void jsonRemoveFunc( u32 i; if( argc<1 ) return; + if( jsonFuncArgMightBeBinary(argv[0]) + && (SQLITE_PTR_TO_INT(sqlite3_user_data(ctx))&JSON_BLOB)!=0 + ){ + jsonRemoveFromBlob(ctx, argc, argv); + return; + } pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); if( pParse==0 ) return; for(i=1; i<(u32)argc; i++){ From af0c9ffb4ab9fc894398f8fae0cce8eb9fb7086f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Nov 2023 18:47:31 +0000 Subject: [PATCH 080/191] The jsonb_remove() routine now appears to be working. FossilOrigin-Name: e76d48137ea823b7810dc8c3b70eb21adabdd6cfbac36050c85d1375e94be1de --- manifest | 15 ++++++-------- manifest.uuid | 2 +- src/json.c | 55 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index f9398864df..911d7e5d49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\sgetting\sjsonb_remove()\sto\swork\sdirectly\son\sJSONB\sblobs. -D 2023-11-15T16:10:31.627 +C The\sjsonb_remove()\sroutine\snow\sappears\sto\sbe\sworking. +D 2023-11-15T18:47:31.537 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3ff1ea643cab62e3ddb8be3a64bff2dd94af37ea461ea343054796e7f11638f8 +F src/json.c a647820e1e82fd516df80328ef3372209006bf99b7008d156411c8cd0d4d9fc4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2141,11 +2141,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba91408f4c044feda003ef93784ccefb619f99ab64379ced481ee8e9e890fd41 -R 58cf82355c33a53c2234fea2c219b8e4 -T *branch * jsonb-remove -T *sym-jsonb-remove * -T -sym-jsonb * +P a79ff8e58fcaf718a6fb78e145117f1d6d40d133f31e9752bb9c6d484850a27b +R 6ab8908817380ee469413c76bca0c832 U drh -Z 367d226fdddcafa052b6fb6b4d87ed0a +Z a103fcc805c4c7ff278e886cba60f9d7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7cf8c33cb1..ebac1147c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a79ff8e58fcaf718a6fb78e145117f1d6d40d133f31e9752bb9c6d484850a27b \ No newline at end of file +e76d48137ea823b7810dc8c3b70eb21adabdd6cfbac36050c85d1375e94be1de \ No newline at end of file diff --git a/src/json.c b/src/json.c index e66c4b1326..fa7913d944 100644 --- a/src/json.c +++ b/src/json.c @@ -2465,11 +2465,21 @@ static JsonNode *jsonLookupAppend( } /* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). +** Compute the text of an error in JSON path syntax. +** +** If ctx is not NULL then push the error message into ctx and return NULL. +* If ctx is NULL, then return the text of the error message. */ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); +static char *jsonPathSyntaxError(const char *zErr, sqlite3_context *ctx){ + char *zMsg = sqlite3_mprintf("JSON path error near '%q'", zErr); + if( ctx==0 ) return zMsg; + if( zMsg==0 ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + } + return 0; } /* @@ -2490,7 +2500,6 @@ static JsonNode *jsonLookup( ){ const char *zErr = 0; JsonNode *pNode = 0; - char *zMsg; if( zPath==0 ) return 0; if( zPath[0]!='$' ){ @@ -2504,13 +2513,7 @@ static JsonNode *jsonLookup( lookup_err: pParse->nErr++; assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); - if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); - }else{ - sqlite3_result_error_nomem(pCtx); - } + jsonPathSyntaxError(zErr, pCtx); return 0; } @@ -3814,11 +3817,15 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ */ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ u32 sz = 0; + u32 nBlob; assert( pParse->delta!=0 ); + assert( pParse->nBlobAlloc >= pParse->nBlob ); + nBlob = pParse->nBlob; + pParse->nBlob = pParse->nBlobAlloc; (void)jsonbPayloadSize(pParse, iRoot, &sz); + pParse->nBlob = nBlob; sz += pParse->delta; jsonBlobChangePayloadSize(pParse, iRoot, sz); - pParse->nBlob += pParse->delta; } /* @@ -3857,6 +3864,7 @@ static u32 jsonLookupBlobStep( memmove(&pParse->aBlob[iRoot], &pParse->aBlob[iRoot+sz], pParse->nBlob - (iRoot+sz)); pParse->delta = -(int)sz; + pParse->nBlob -= sz; }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ }else{ @@ -3871,6 +3879,7 @@ static u32 jsonLookupBlobStep( memmove(&pParse->aBlob[iRoot+pParse->nIns], &pParse->aBlob[iRoot+sz], pParse->nBlob - iRoot - sz); + pParse->nBlob += d; } memcpy(&pParse->aBlob[iRoot], pParse->aIns, pParse->nIns); } @@ -3981,6 +3990,7 @@ static u32 jsonLookupBlobStep( pParse->nBlob - j); memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); pParse->delta = pParse->nIns; + pParse->nBlob += pParse->nIns; jsonAfterEditSizeAdjust(pParse, iRoot); return j; } @@ -4254,6 +4264,7 @@ static void jsonRemoveFromBlob( ){ int i; u32 rc; + const char *zPath = 0; JsonParse px; memset(&px, 0, sizeof(px)); px.nBlob = sqlite3_value_bytes(argv[0]); @@ -4261,21 +4272,25 @@ static void jsonRemoveFromBlob( if( px.aBlob==0 ) return; for(i=1; i0 ? sqlite3_free : SQLITE_DYNAMIC); + px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); return; -jsonRemoveFromBlob_error: +jsonRemoveFromBlob_patherror: if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonPathSyntaxError(zPath, ctx); return; - } /**************************************************************************** @@ -5814,7 +5829,7 @@ static int jsonEachFilter( } if( zErr ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; }else if( pNode==0 ){ From 5e3ae1ec765fb7b5eb00854e64fbc84090636a9b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Nov 2023 19:21:09 +0000 Subject: [PATCH 081/191] Some simple test cases for JSONB direct remove. FossilOrigin-Name: 8cb4d2cbfc1b5f3c612d85138e66da8735d6589c7538fdf494a761c9b8104f18 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/jsonb01.test | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 test/jsonb01.test diff --git a/manifest b/manifest index 874a64e0cc..a211f0b659 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C jsonb_remove()\snow\sworks\swithout\shaving\sto\suse\sa\sJsonNode\sparse,\sassuming\nthat\sthe\sinput\sis\sJSONB. -D 2023-11-15T18:55:03.625 +C Some\ssimple\stest\scases\sfor\sJSONB\sdirect\sremove. +D 2023-11-15T19:21:09.380 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1329,6 +1329,7 @@ F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba01143 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b +F test/jsonb01.test 830e87e012518aa0bf4358ed794fe40cbf0743247d5ade273223064547cb1ceb F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -2141,9 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba91408f4c044feda003ef93784ccefb619f99ab64379ced481ee8e9e890fd41 e76d48137ea823b7810dc8c3b70eb21adabdd6cfbac36050c85d1375e94be1de -R 6ab8908817380ee469413c76bca0c832 -T +closed e76d48137ea823b7810dc8c3b70eb21adabdd6cfbac36050c85d1375e94be1de +P 5207679e929786e577a0553d0d84dda5125456dcde80c0f3156f14f4d8c804cb +R 76eca9722da06337c910d0b8ea9ca3cc U drh -Z a599420b20eb0d182795d486baa84f09 +Z 9b402027f08ff9f924d703e81b5baa92 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a1f41dd79e..1e29b18d5e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5207679e929786e577a0553d0d84dda5125456dcde80c0f3156f14f4d8c804cb \ No newline at end of file +8cb4d2cbfc1b5f3c612d85138e66da8735d6589c7538fdf494a761c9b8104f18 \ No newline at end of file diff --git a/test/jsonb01.test b/test/jsonb01.test new file mode 100644 index 0000000000..a01a96f558 --- /dev/null +++ b/test/jsonb01.test @@ -0,0 +1,46 @@ +# 2023-11-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Test cases for JSONB +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test jsonb01-1.1 { + CREATE TABLE t1(x JSON BLOB); + INSERT INTO t1 VALUES(jsonb('{a:5,b:{x:10,y:11},c:[1,2,3,4]}')); +} +foreach {id path res} { + 1 {$.a} {{{"b":{"x":10,"y":11},"c":[1,2,3,4]}}} + 2 {$.b} {{{"a":5,"c":[1,2,3,4]}}} + 3 {$.c} {{{"a":5,"b":{"x":10,"y":11}}}} + 4 {$.d} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} + 5 {$.b.x} {{{"a":5,"b":{"y":11},"c":[1,2,3,4]}}} + 6 {$.b.y} {{{"a":5,"b":{"x":10},"c":[1,2,3,4]}}} + 7 {$.c[0]} {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}} + 8 {$.c[1]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}} + 9 {$.c[2]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}} + 10 {$.c[3]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}} + 11 {$.c[4]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} + 12 {$.c[#]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} + 13 {$.c[#-1]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3]}}} + 14 {$.c[#-2]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,4]}}} + 15 {$.c[#-3]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,3,4]}}} + 16 {$.c[#-4]} {{{"a":5,"b":{"x":10,"y":11},"c":[2,3,4]}}} + 17 {$.c[#-5]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} + 18 {$.c[#-6]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} +} { + do_execsql_test jsonb01-1.2.$id { + SELECT json(jsonb_remove(x,$path)) FROM t1; + } $res +} + +finish_test From 11e8242e2eec9172d2c47157900638f568813736 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Nov 2023 20:32:06 +0000 Subject: [PATCH 082/191] Both json_remove() jsonb_remove() work on pure JSONB as long as the input is JSONB. FossilOrigin-Name: 68d551730be0a3ea9579646ed4836c73554c83ca7f2303b69a18843f1750f1a7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 19 ++++++++++++++----- test/jsonb01.test | 5 ++++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a211f0b659..9b3331efef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Some\ssimple\stest\scases\sfor\sJSONB\sdirect\sremove. -D 2023-11-15T19:21:09.380 +C Both\sjson_remove()\sjsonb_remove()\swork\son\spure\sJSONB\sas\slong\sas\sthe\sinput\nis\sJSONB. +D 2023-11-15T20:32:06.379 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a647820e1e82fd516df80328ef3372209006bf99b7008d156411c8cd0d4d9fc4 +F src/json.c 949c4a36244d402d1e5f9c1815d9ce2cf8e94cf27ce7419e2d1b9dd5a97a2baa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1329,7 +1329,7 @@ F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba01143 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b -F test/jsonb01.test 830e87e012518aa0bf4358ed794fe40cbf0743247d5ade273223064547cb1ceb +F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5207679e929786e577a0553d0d84dda5125456dcde80c0f3156f14f4d8c804cb -R 76eca9722da06337c910d0b8ea9ca3cc +P 8cb4d2cbfc1b5f3c612d85138e66da8735d6589c7538fdf494a761c9b8104f18 +R 17bf060181dd906a53600b2c931378c9 U drh -Z 9b402027f08ff9f924d703e81b5baa92 +Z a169b712ea2dc449ca00887f2092a50b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e29b18d5e..94af86d981 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cb4d2cbfc1b5f3c612d85138e66da8735d6589c7538fdf494a761c9b8104f18 \ No newline at end of file +68d551730be0a3ea9579646ed4836c73554c83ca7f2303b69a18843f1750f1a7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index fa7913d944..89109e0039 100644 --- a/src/json.c +++ b/src/json.c @@ -4265,7 +4265,9 @@ static void jsonRemoveFromBlob( int i; u32 rc; const char *zPath = 0; + int flgs; JsonParse px; + memset(&px, 0, sizeof(px)); px.nBlob = sqlite3_value_bytes(argv[0]); px.aBlob = (u8*)sqlite3_value_blob(argv[0]); @@ -4283,8 +4285,17 @@ static void jsonRemoveFromBlob( if( rc==JSON_BLOB_NOTFOUND ) continue; if( JSON_BLOB_ISERROR(rc) ) goto jsonRemoveFromBlob_patherror; } - sqlite3_result_blob(ctx, px.aBlob, px.nBlob, - px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + sqlite3_result_blob(ctx, px.aBlob, px.nBlob, + px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&px, 0, &s); + jsonReturnString(&s); + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + } return; jsonRemoveFromBlob_patherror: @@ -4834,9 +4845,7 @@ static void jsonRemoveFunc( u32 i; if( argc<1 ) return; - if( jsonFuncArgMightBeBinary(argv[0]) - && (SQLITE_PTR_TO_INT(sqlite3_user_data(ctx))&JSON_BLOB)!=0 - ){ + if( jsonFuncArgMightBeBinary(argv[0]) ){ jsonRemoveFromBlob(ctx, argc, argv); return; } diff --git a/test/jsonb01.test b/test/jsonb01.test index a01a96f558..d1b53ae6cc 100644 --- a/test/jsonb01.test +++ b/test/jsonb01.test @@ -38,9 +38,12 @@ foreach {id path res} { 17 {$.c[#-5]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} 18 {$.c[#-6]} {{{"a":5,"b":{"x":10,"y":11},"c":[1,2,3,4]}}} } { - do_execsql_test jsonb01-1.2.$id { + do_execsql_test jsonb01-1.2.$id.1 { SELECT json(jsonb_remove(x,$path)) FROM t1; } $res + do_execsql_test jsonb01-1.2.$id.2 { + SELECT json_remove(x,$path) FROM t1; + } $res } finish_test From a0764f63a8fa984cc1c90b377916f8dbe11207ec Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Nov 2023 21:11:56 +0000 Subject: [PATCH 083/191] When querying a tokendata=1 fts5 table, do not use a prefix cursor for the case where the term has only one variant. FossilOrigin-Name: d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47 --- ext/fts5/fts5_index.c | 232 +++++++++++++++++++++-------- ext/fts5/test/fts5origintext2.test | 107 +++++++++++++ manifest | 13 +- manifest.uuid | 2 +- 4 files changed, 282 insertions(+), 72 deletions(-) create mode 100644 ext/fts5/test/fts5origintext2.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 2d6d561b7b..887bb75dac 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -368,6 +368,7 @@ struct Fts5Index { sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxProbe; int nRead; /* Total number of blocks read */ sqlite3_stmt *pDeleteFromIdx; @@ -2629,6 +2630,18 @@ static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ return p->pIdxSelect; } +static sqlite3_stmt *fts5IdxProbeStmt(Fts5Index *p){ + if( p->pIdxProbe==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxProbe, sqlite3_mprintf( + "SELECT 1 FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? AND termzDb, pConfig->zName + )); + } + return p->pIdxProbe; +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. @@ -3846,7 +3859,7 @@ static void fts5MultiIterNew( assert( iIter==nSeg ); } - /* If the above was successful, each component iterators now points + /* If the above was successful, each component iterator now points ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ @@ -6179,7 +6192,7 @@ static void fts5TokenMapPoslist( } static int fts5TokenMapHash(i64 iRowid, int iCol, int iOff){ - return iRowid + (iRowid << 3) + (iCol << 6) + (iOff << 9); + return (iRowid + (iRowid << 3) + (iCol << 6) + (iOff << 9)) & 0x7FFFFFFF; } static void fts5TokenMapHashify(Fts5Index *p, Fts5TokenMap *pMap){ @@ -6228,6 +6241,84 @@ static const u8 *fts5TokenMapLookup( return 0; } +/* +** The iterator passed as the second argument has been opened to scan and +** merge doclists for a series of tokens in tokendata=1 mode. This function +** tests whether or not, instead of using the cursor to read doclists to +** merge, it can be used directly by the upper layer. This is the case +** if the cursor currently points to the only token that corresponds to +** the queried term. i.e. if the next token that will be visited by the +** iterator does not match the query. +*/ +int fts5TokendataIterIsOk( + Fts5Index *p, + Fts5Iter *pIter, + const u8 *pToken, + int nToken +){ + int ii; + Fts5Buffer buf = {0, 0, 0}; + int bRet = 1; + Fts5Buffer *pTerm = 0; + + /* Iterator is not usable if it uses the hash table */ + if( pIter->aSeg[0].pSeg==0 ) return 0; + + for(ii=0; bRet && iinSeg; ii++){ + Fts5SegIter *pSeg = &pIter->aSeg[ii]; + Fts5Data *pLeaf = pSeg->pLeaf; + if( pLeaf ){ + + if( pTerm==0 ){ + pTerm = &pSeg->term; + }else{ + if( pSeg->term.n!=pTerm->n + || memcmp(pSeg->term.p, pTerm->p, pTerm->n) + ){ + bRet = 0; + break; + } + } + + if( pSeg->iEndofDoclistszLeaf ){ + /* Next term is on this node. Check it directly. */ + int nPrefix = 0; + fts5GetVarint32(&pLeaf->p[pSeg->iEndofDoclist], nPrefix); + if( nPrefix>=nToken ) bRet = 0; + }else{ + /* Next term is on a subsequent page. In this case query the %_idx + ** table to discover exactly what that next term is. */ + sqlite3_stmt *pProbe = fts5IdxProbeStmt(p); + if( pProbe ){ + int rc = SQLITE_OK; + if( buf.n==0 ){ + sqlite3Fts5BufferAppendBlob(&p->rc, &buf, nToken, pToken); + sqlite3Fts5BufferAppendBlob(&p->rc, &buf, 1, (const u8*)"\1"); + } + sqlite3_bind_int(pProbe, 1, pSeg->pSeg->iSegid); + sqlite3_bind_blob(pProbe,2, pSeg->term.p,pSeg->term.n, SQLITE_STATIC); + sqlite3_bind_blob(pProbe,3, buf.p, buf.n, SQLITE_STATIC); + + if( sqlite3_step(pProbe)==SQLITE_ROW ){ + bRet = 0; + } + rc = sqlite3_reset(pProbe); + if( p->rc==SQLITE_OK ) p->rc = rc; + } + } + } + } + + if( bRet ){ + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pSeg = &pIter->aSeg[ii]; + pSeg->flags |= FTS5_SEGITER_ONETERM; + } + } + + fts5BufferFree(&buf); + return bRet; +} static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ @@ -6261,9 +6352,6 @@ static void fts5SetupPrefixIter( aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); - if( iIdx==0 ){ - pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); - } assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); if( p->rc==SQLITE_OK ){ @@ -6308,79 +6396,92 @@ static void fts5SetupPrefixIter( pToken[0] = FTS5_MAIN_PREFIX + iIdx; fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - p1->xSetOutputs(p1, pSeg); - if( pMap ){ + if( bDesc==0 && bTokenscan && fts5TokendataIterIsOk(p, p1, pToken,nToken) ){ + /* In this case iterator p1 may be used as is. */ + *ppIter = p1; + }else{ + + if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); + } + assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); + + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); + + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ - fts5TokenMapTerm(p, pMap, &pTerm[1], nTerm-1); + if( nTermnToken && pTerm[nToken]!=0x00 ) break; } - fts5TokenMapPoslist(p, pMap, p1); - } - - assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( bNewTerm ){ - if( nTermnToken && pTerm[nToken]!=0x00 ) break; - } - - if( p1->base.nData==0 ) continue; - - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ - for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - int i1 = i*nMerge; - int iStore; - assert( i1+nMerge<=nBuf ); - for(iStore=i1; iStorebase.nData==0 ) continue; + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ + for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=nBuf ); for(iStore=i1; iStorebase.iRowid-(u64)iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; } - - xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - - assert( (nBuf%nMerge)==0 ); - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, nMerge, &aBuf[i]); + + assert( (nBuf%nMerge)==0 ); + for(i=0; irc==SQLITE_OK ){ + xMerge(p, &doclist, nMerge, &aBuf[i]); + } + for(iFree=i; iFreep = (u8*)&pData[1]; + pData->nn = pData->szLeaf = doclist.n; + if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + if( pMap ) fts5TokenMapHashify(p, pMap); + fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter); + pMap = 0; } + fts5BufferFree(&doclist); } - fts5MultiIterFree(p1); - - pData = fts5IdxMalloc(p, sizeof(Fts5Data)+doclist.n+FTS5_DATA_ZERO_PADDING); - if( pData ){ - pData->p = (u8*)&pData[1]; - pData->nn = pData->szLeaf = doclist.n; - if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); - if( pMap ) fts5TokenMapHashify(p, pMap); - fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter); - } - fts5BufferFree(&doclist); } + fts5TokenMapFree(pMap); fts5StructureRelease(pStruct); sqlite3_free(aBuf); } @@ -6514,6 +6615,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxProbe); sqlite3_finalize(p->pDataVersion); sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); @@ -6766,7 +6868,7 @@ int sqlite3Fts5IterToken( ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; if( pIter->pTokenMap ){ - *ppOut = fts5TokenMapLookup( + *ppOut = (const char*)fts5TokenMapLookup( pIter->pTokenMap, pIndexIter->iRowid, iCol, iOff, pnOut ); }else{ diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test new file mode 100644 index 0000000000..7cf8d80071 --- /dev/null +++ b/ext/fts5/test/fts5origintext2.test @@ -0,0 +1,107 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft VALUES('Hello'); + INSERT INTO ft VALUES('hello'); + INSERT INTO ft VALUES('HELLO'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('World'); + INSERT INTO ft VALUES('world'); + INSERT INTO ft VALUES('WORLD'); + COMMIT; +} + +do_execsql_test 1.2 { SELECT rowid FROM ft('hello'); } {1 2 3} +do_execsql_test 1.3 { SELECT rowid FROM ft('today'); } {4 5 6} +do_execsql_test 1.4 { SELECT rowid FROM ft('world'); } {7 8 9} + +do_execsql_test 1.5 { + SELECT count(*) FROM ft_data +} 3 + +do_execsql_test 1.6 { + DELETE FROM ft; + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO ft SELECT 'Hello Hello Hello Hello Hello Hello Hello' FROM s; + INSERT INTO ft VALUES ('hELLO hELLO hELLO'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('today today today today today today today'); + INSERT INTO ft VALUES('World World World World World World World'); + INSERT INTO ft VALUES('world world world world world world world'); + INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD'); + INSERT INTO ft VALUES('World World World World World World World'); + INSERT INTO ft VALUES('world world world world world world world'); + INSERT INTO ft VALUES('WORLD WORLD WORLD WORLD WORLD WORLD WORLD'); + COMMIT; +} + +do_execsql_test 1.7 { + SELECT count(*) FROM ft_data; +} 23 + +do_execsql_test 1.8 { SELECT rowid FROM ft('hello') WHERE rowid>100; } {101} + +do_execsql_test 1.9 { + DELETE FROM ft; + INSERT INTO ft(ft) VALUES('optimize'); + SELECT count(*) FROM ft_data; +} {2} +do_execsql_test 1.10 { + BEGIN; + INSERT INTO ft VALUES('Hello'); + INSERT INTO ft VALUES('hello'); + INSERT INTO ft VALUES('HELLO'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('today'); + INSERT INTO ft VALUES('World'); + INSERT INTO ft VALUES('world'); + INSERT INTO ft VALUES('WORLD'); +} + +breakpoint +do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3} +do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} +do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} + +finish_test + diff --git a/manifest b/manifest index 3477a07b6e..7763b46700 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\simplementation\sof\sxInstToken()\sAPI. -D 2023-11-15T11:45:19.681 +C When\squerying\sa\stokendata=1\sfts5\stable,\sdo\snot\suse\sa\sprefix\scursor\sfor\sthe\scase\swhere\sthe\sterm\shas\sonly\sone\svariant. +D 2023-11-16T21:11:56.608 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 4b50ed0c724cb160f086e20e964ed2d57b99d0d3c1cb1b029901c0300b11bd9f F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 3b51c2f0554a665694e777c8f2765cb5b1283d4bc960dde350a604af3e5e5d98 +F ext/fts5/fts5_index.c 70fa4a6d8a062ca4b63a62d0721d72ce2f6336413c6e8b0703881c708797d24d F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -189,6 +189,7 @@ F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca F ext/fts5/test/fts5origintext.test 908a1fb6b1106e4b6ed0f9cf683c2ad7f986cce1aea1e0a13b3309c6f568932b +F ext/fts5/test/fts5origintext2.test a654c77f1548ccd8eab7f6d07230655c0070cdf32dcd4740ccdf496f77d5282c F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2143,8 +2144,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 -R 7870d9470a55737470bd92d95fe480a9 +P a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07 +R d7c277a055a404d272fdcb5090bf371a U dan -Z d10d6cf5b22c051f4553454e4a3996a4 +Z 0e1bf556ad9eba9db356685a09c7ab31 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 96d818fc70..6373f95ef5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07 \ No newline at end of file +d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47 \ No newline at end of file From 8a4cecaea2dd09f7174a0838875a6e47c71fcf62 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Nov 2023 17:51:58 +0000 Subject: [PATCH 084/191] Make edits directly to the JSONB BLOB when the input to json_replace() is a JSONB. FossilOrigin-Name: d69c6acef54a81f46a97a05d443fe648635b4b70772069d6705ef829b718e985 --- manifest | 12 ++--- manifest.uuid | 2 +- src/json.c | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5ede8a53d8..7339368a54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\senhancements\sand\sfixes\sinto\sthe\sjsonb\sbranch. -D 2023-11-17T17:03:45.388 +C Make\sedits\sdirectly\sto\sthe\sJSONB\sBLOB\swhen\sthe\sinput\sto\sjson_replace()\nis\sa\sJSONB. +D 2023-11-21T17:51:58.335 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 949c4a36244d402d1e5f9c1815d9ce2cf8e94cf27ce7419e2d1b9dd5a97a2baa +F src/json.c 9be29c023ce9c839a507ddfd572dbca226c9707ba6b0714da004e00a7312a000 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68d551730be0a3ea9579646ed4836c73554c83ca7f2303b69a18843f1750f1a7 ce6a75622ea5bca517bc6613e738aa670c9e1dd863596220eded5c2379c616c7 -R 3f0ae99fa66c7ee3590aa56d01ed6b36 +P 162f0509ef27bcd3ec87629640281a71c773e7c3bbd2cd0df76faf481531e7f1 +R 38409c37dc02439917dd59332bdf58d4 U drh -Z c061d55cfc7406269213a15ddaf22fe9 +Z 4892279bd4eeae014e8915fe6a430955 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 475b441c30..4a62c55635 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -162f0509ef27bcd3ec87629640281a71c773e7c3bbd2cd0df76faf481531e7f1 \ No newline at end of file +d69c6acef54a81f46a97a05d443fe648635b4b70772069d6705ef829b718e985 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 89109e0039..def5e63d30 100644 --- a/src/json.c +++ b/src/json.c @@ -4304,6 +4304,143 @@ jsonRemoveFromBlob_patherror: return; } +/* +** pArg is a function argument that might be an SQL value or a JSON +** value. Figure out what it is and encode it as a JSONB blob. +** Return the results in pParse. +** +** pParse is uninitialized upon entry. This routine will handle the +** initialization of pParse. The result will be contained in +** pParse->aBlob and pParse->nBlob. pParse->aBlob might be dynamically +** allocated (if pParse->nBlobAlloc is greater than zero) in which case +** the caller is responsible for freeing the space allocated to pParse->aBlob +** when it has finished with it. Or pParse->aBlob might be a static string +** or a value obtained from sqlite3_value_blob(pArg). +** +** If the argument is a BLOB that is clearly not a JSONB, then this +** function might set an error message in ctx and return non-zero. +** It might also set an error message and return non-zero on an OOM error. +*/ +static int jsonFunctionArgToBlob( + sqlite3_context *ctx, + sqlite3_value *pArg, + JsonParse *pParse +){ + int eType = sqlite3_value_type(pArg); + static u8 aNull[] = { 0x00 }; + memset(pParse, 0, sizeof(pParse[0])); + switch( eType ){ + case SQLITE_NULL: { + pParse->aBlob = aNull; + pParse->nBlob = 1; + return 0; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(pArg) ){ + pParse->aBlob = (u8*)sqlite3_value_blob(pArg); + pParse->nBlob = sqlite3_value_bytes(pArg); + }else{ + sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); + return 1; + } + break; + } + case SQLITE_TEXT: { + const char *zJson = (const char*)sqlite3_value_text(pArg); + int nJson = sqlite3_value_bytes(pArg); + if( zJson==0 ) return 1; + if( sqlite3_value_subtype(pArg)==JSON_SUBTYPE ){ + pParse->zJson = (char*)zJson; + pParse->nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3_free(pParse->aBlob); + memset(pParse, 0, sizeof(pParse[0])); + return 1; + } + }else{ + jsonBlobAppendNodeType(pParse, JSONB_TEXTRAW, nJson); + jsonBlobAppendNBytes(pParse, (const u8*)zJson, nJson); + } + break; + } + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + int e = eType==SQLITE_INTEGER ? JSONB_INT : JSONB_FLOAT; + if( z==0 ) return 1; + jsonBlobAppendNodeType(pParse, e, n); + jsonBlobAppendNBytes(pParse, (const u8*)z, n); + break; + } + } + return 0; +} + +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments come in parse where each pair contains a JSON path and +** content to insert or set at that patch. Do the updates +** and return the result. +** +** The specific operation is determined by eEdit, which can be one +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +*/ +static void jsonInsertIntoBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv, + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ +){ + int i; + u32 rc; + const char *zPath = 0; + int flgs; + JsonParse px, ax; + + assert( (argc&1)==1 ); + memset(&px, 0, sizeof(px)); + px.nBlob = sqlite3_value_bytes(argv[0]); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( px.aBlob==0 ) return; + for(i=1; i0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&px, 0, &s); + jsonReturnString(&s); + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + } + return; + +jsonInsertIntoBlob_patherror: + if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonPathSyntaxError(zPath, ctx); + return; +} + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ @@ -4987,6 +5124,10 @@ static void jsonReplaceFunc( jsonWrongNumArgs(ctx, "replace"); return; } + if( jsonFuncArgMightBeBinary(argv[0]) && argc>=3 ){ + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); + return; + } pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); if( pParse==0 ) return; pParse->nJPRef++; From 664fe310b5749c18fceb7c249f45b2b8cfc7dc21 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Nov 2023 18:23:43 +0000 Subject: [PATCH 085/191] Fix the translation of JSON5 numeric values from BLOB into text. FossilOrigin-Name: 40c4fb441f220982e4d61fd42597cf18546791a302fbcc8eec2eed29ee15ef35 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0e2a5d70fc..7e945f8442 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\sfixes\sand\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-11-21T17:54:55.986 +C Fix\sthe\stranslation\sof\sJSON5\snumeric\svalues\sfrom\sBLOB\sinto\stext. +D 2023-11-21T18:23:43.037 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9be29c023ce9c839a507ddfd572dbca226c9707ba6b0714da004e00a7312a000 +F src/json.c c5d4b35c598f50e5be9191260ae92be4073d17bff84daa0dd64c31edbb7dce7e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d69c6acef54a81f46a97a05d443fe648635b4b70772069d6705ef829b718e985 8936daa08243729d8538bb7288bbefb43f3bd842a0d4b2e8019092f5701c2926 -R e8c6337278c34718864f8e382e5ab75f +P 6d78d50ed2357e6c943c1ef97b1d2ea0902cbadef90c2c35dccdbdc2bdf8702f +R 2ba901c1c280b3893e91bda5d71d76cb U drh -Z 78714a658254db90406116c46e75110c +Z 574e66b6116d2f9186df90de5e4b8fb3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d87fbc1eaf..7e841c3b50 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d78d50ed2357e6c943c1ef97b1d2ea0902cbadef90c2c35dccdbdc2bdf8702f \ No newline at end of file +40c4fb441f220982e4d61fd42597cf18546791a302fbcc8eec2eed29ee15ef35 \ No newline at end of file diff --git a/src/json.c b/src/json.c index def5e63d30..69950766da 100644 --- a/src/json.c +++ b/src/json.c @@ -3375,6 +3375,8 @@ static u32 jsonXlateBlobToText( if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; + }else if( zIn[0]=='+' ){ + k++; } for(; k Date: Tue, 21 Nov 2023 19:05:22 +0000 Subject: [PATCH 086/191] Correct blob-to-text rendering in some corner cases. FossilOrigin-Name: 7822e0e59f9b611fe6289cc762b0aff61f9b87c3a82c60de110f447589a2c125 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7e945f8442..ad10087655 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stranslation\sof\sJSON5\snumeric\svalues\sfrom\sBLOB\sinto\stext. -D 2023-11-21T18:23:43.037 +C Correct\sblob-to-text\srendering\sin\ssome\scorner\scases. +D 2023-11-21T19:05:22.846 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c5d4b35c598f50e5be9191260ae92be4073d17bff84daa0dd64c31edbb7dce7e +F src/json.c 54e4261d68a463bb8e8adf7ce0ba9c8ad2853870ce53e4e5f903c6a97eb8b644 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d78d50ed2357e6c943c1ef97b1d2ea0902cbadef90c2c35dccdbdc2bdf8702f -R 2ba901c1c280b3893e91bda5d71d76cb +P 40c4fb441f220982e4d61fd42597cf18546791a302fbcc8eec2eed29ee15ef35 +R e0de56e45777fbb83b14bb9ecc453e36 U drh -Z 574e66b6116d2f9186df90de5e4b8fb3 +Z 0c3d75110c96246c67d1b5a0fbc45fae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e841c3b50..0085f7c96b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -40c4fb441f220982e4d61fd42597cf18546791a302fbcc8eec2eed29ee15ef35 \ No newline at end of file +7822e0e59f9b611fe6289cc762b0aff61f9b87c3a82c60de110f447589a2c125 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 69950766da..199bc5e18d 100644 --- a/src/json.c +++ b/src/json.c @@ -3372,6 +3372,7 @@ static u32 jsonXlateBlobToText( u32 k = 2; sqlite3_uint64 u = 0; const char *zIn = (const char*)&pParse->aBlob[i+n]; + int bOverflow = 0; if( zIn[0]=='-' ){ jsonAppendChar(pOut, '-'); k++; @@ -3382,11 +3383,13 @@ static u32 jsonXlateBlobToText( if( !sqlite3Isxdigit(zIn[k]) ){ pOut->eErr |= JSTRING_MALFORMED; break; + }else if( (u>>60)!=0 ){ + bOverflow = 1; }else{ u = u*16 + sqlite3HexToInt(zIn[k]); } } - jsonPrintf(100,pOut,"%llu",u); + jsonPrintf(100,pOut,bOverflow?"9.0e999":"%llu", u); break; } case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ @@ -3519,6 +3522,7 @@ static u32 jsonXlateBlobToText( j = jsonXlateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } + if( x & 1 ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) pOut->nUsed--; jsonAppendChar(pOut, '}'); break; From 27fea97e4451ddb181a4d0cda5b6aeb1865159ea Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Nov 2023 20:13:08 +0000 Subject: [PATCH 087/191] Direct editing of JSONB using json_insert() and json_set(). FossilOrigin-Name: fffb7a9538838e26991e6f16ea3138346a30c33ea6c3d3834680ee6d1f6eece2 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 66 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index ad10087655..2fb4d4c6c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sblob-to-text\srendering\sin\ssome\scorner\scases. -D 2023-11-21T19:05:22.846 +C Direct\sediting\sof\sJSONB\susing\sjson_insert()\sand\sjson_set(). +D 2023-11-21T20:13:08.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 54e4261d68a463bb8e8adf7ce0ba9c8ad2853870ce53e4e5f903c6a97eb8b644 +F src/json.c 6530a0ad967e10973ba46db399b46e20fc243b2b495cd53226aa31a9ce1b1e46 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40c4fb441f220982e4d61fd42597cf18546791a302fbcc8eec2eed29ee15ef35 -R e0de56e45777fbb83b14bb9ecc453e36 +P 7822e0e59f9b611fe6289cc762b0aff61f9b87c3a82c60de110f447589a2c125 +R 42a0f1601a18c87c6ad9e79182203b90 U drh -Z 0c3d75110c96246c67d1b5a0fbc45fae +Z 7d1450dc466726ea7e03ad211ff3e833 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0085f7c96b..eba39323db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7822e0e59f9b611fe6289cc762b0aff61f9b87c3a82c60de110f447589a2c125 \ No newline at end of file +fffb7a9538838e26991e6f16ea3138346a30c33ea6c3d3834680ee6d1f6eece2 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 199bc5e18d..69557aa721 100644 --- a/src/json.c +++ b/src/json.c @@ -349,10 +349,10 @@ struct JsonParse { }; /* Allowed values for JsonParse.eEdit */ -#define JEDIT_DEL 0x01 /* Delete if exists */ -#define JEDIT_INS 0x02 /* Insert if not exists */ -#define JEDIT_REPL 0x04 /* Overwrite if exists */ -#define JEDIT_SET 0x08 /* Insert or overwrite */ +#define JEDIT_DEL 1 /* Delete if exists */ +#define JEDIT_REPL 2 /* Overwrite if exists */ +#define JEDIT_INS 3 /* Insert if not exists */ +#define JEDIT_SET 4 /* Insert or overwrite */ /* ** Maximum nesting depth of JSON for this implementation. @@ -2586,12 +2586,12 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ ** ** Return true on success. Return false on OOM. */ -static int jsonBlobMakeEditable(JsonParse *pParse){ +static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ u8 *aOld; u32 nSize; if( pParse->nBlobAlloc>0 ) return 1; aOld = pParse->aBlob; - nSize = pParse->nBlob + pParse->nIns; + nSize = pParse->nBlob + pParse->nIns + nExtra; if( nSize>100 ) nSize -= 100; pParse->aBlob = 0; if( jsonBlobExpand(pParse, nSize) ){ @@ -3859,7 +3859,7 @@ static u32 jsonLookupBlobStep( u8 x; if( zPath[0]==0 ){ - if( pParse->eEdit && jsonBlobMakeEditable(pParse) ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse, 0) ){ n = jsonbPayloadSize(pParse, iRoot, &sz); sz += n; if( pParse->eEdit==JEDIT_DEL ){ @@ -3941,6 +3941,32 @@ static u32 jsonLookupBlobStep( j += n+sz; } if( j>iEnd ) return JSON_BLOB_ERROR; + if( pParse->eEdit>=JEDIT_INS ){ + u32 nIns; + u8 aLabel[16]; + JsonParse ix; + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + memset(&ix, 0, sizeof(ix)); + ix.aBlob = aLabel; + ix.nBlobAlloc = sizeof(aLabel); + jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); + if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey) ){ + nIns = ix.nBlob + nKey + pParse->nIns; + assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); + memmove(&pParse->aBlob[j+nIns], &pParse->aBlob[j], + pParse->nBlob - j); + memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); + k = j + ix.nBlob; + memcpy(&pParse->aBlob[k], zKey, nKey); + k += nKey; + memcpy(&pParse->aBlob[k], pParse->aIns, pParse->nIns); + pParse->delta = nIns; + pParse->nBlob += nIns; + jsonAfterEditSizeAdjust(pParse, iRoot); + } + return j; + } }else if( zPath[0]=='[' ){ x = pParse->aBlob[iRoot] & 0x0f; if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; @@ -3987,19 +4013,21 @@ static u32 jsonLookupBlobStep( } if( j>iEnd ) return JSON_BLOB_ERROR; if( k>1 ) return JSON_BLOB_NOTFOUND; + if( pParse->eEdit>=JEDIT_INS && jsonBlobMakeEditable(pParse, 0) ){ + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); + memmove(&pParse->aBlob[j+pParse->nIns], &pParse->aBlob[j], + pParse->nBlob - j); + memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); + pParse->delta = pParse->nIns; + pParse->nBlob += pParse->nIns; + jsonAfterEditSizeAdjust(pParse, iRoot); + return j; + } }else{ return JSON_BLOB_PATHERROR; } - if( pParse->eEdit==JEDIT_INS && jsonBlobMakeEditable(pParse) ){ - assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); - memmove(&pParse->aBlob[j], &pParse->aBlob[j+pParse->nIns], - pParse->nBlob - j); - memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); - pParse->delta = pParse->nIns; - pParse->nBlob += pParse->nIns; - jsonAfterEditSizeAdjust(pParse, iRoot); - return j; - } return JSON_BLOB_NOTFOUND; } @@ -5183,6 +5211,10 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } + if( jsonFuncArgMightBeBinary(argv[0]) && argc>=3 ){ + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); + return; + } pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); if( pParse==0 ) return; pParse->nJPRef++; From cf72606a7d05c7fc3b51ba596db013d51eb73d4b Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Nov 2023 22:36:32 +0000 Subject: [PATCH 088/191] Inserts invalid JSONB should return "malformed JSON", not a json path error. FossilOrigin-Name: 306ee66fbd0231a9f5b229e5630d5cc66c9cf08b466d2d9204e79e1f44a16374 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2fb4d4c6c1..35eb7e15dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Direct\sediting\sof\sJSONB\susing\sjson_insert()\sand\sjson_set(). -D 2023-11-21T20:13:08.271 +C Inserts\sinvalid\sJSONB\sshould\sreturn\s"malformed\sJSON",\snot\sa\sjson\spath\serror. +D 2023-11-21T22:36:32.538 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -685,7 +685,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6530a0ad967e10973ba46db399b46e20fc243b2b495cd53226aa31a9ce1b1e46 +F src/json.c f8cfdf1d024a7b2c5ad3eae15903a7a83a9b2c8ab61716d219a57600544b712c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2142,8 +2142,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7822e0e59f9b611fe6289cc762b0aff61f9b87c3a82c60de110f447589a2c125 -R 42a0f1601a18c87c6ad9e79182203b90 +P fffb7a9538838e26991e6f16ea3138346a30c33ea6c3d3834680ee6d1f6eece2 +R 99205893bc97f055f5ba354365513ffb U drh -Z 7d1450dc466726ea7e03ad211ff3e833 +Z ff4b069c664b897025c4d05de8c19bcd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eba39323db..0b616dcb8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fffb7a9538838e26991e6f16ea3138346a30c33ea6c3d3834680ee6d1f6eece2 \ No newline at end of file +306ee66fbd0231a9f5b229e5630d5cc66c9cf08b466d2d9204e79e1f44a16374 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 69557aa721..54e221392e 100644 --- a/src/json.c +++ b/src/json.c @@ -4427,7 +4427,7 @@ static void jsonInsertIntoBlob( int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ ){ int i; - u32 rc; + u32 rc = 0; const char *zPath = 0; int flgs; JsonParse px, ax; @@ -4471,7 +4471,11 @@ static void jsonInsertIntoBlob( jsonInsertIntoBlob_patherror: if( px.nBlobAlloc ) sqlite3_free(px.aBlob); - jsonPathSyntaxError(zPath, ctx); + if( rc==JSON_BLOB_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + jsonPathSyntaxError(zPath, ctx); + } return; } From 5c268bbf67d8bfd5383a16c4618b9aaefd86abeb Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Nov 2023 19:02:54 +0000 Subject: [PATCH 089/191] Fix tokendata=1 and xInstToken() APIs for detail=none and detail=column tables. FossilOrigin-Name: 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86 --- ext/fts5/fts5Int.h | 7 ++++ ext/fts5/fts5_expr.c | 48 +++++++++++++++++++-- ext/fts5/fts5_index.c | 67 ++++++++++++++++++++++++++++++ ext/fts5/test/fts5origintext.test | 51 +++++++++++++++-------- ext/fts5/test/fts5origintext2.test | 2 +- ext/fts5/test/fts5origintext3.test | 60 ++++++++++++++++++++++++++ manifest | 21 +++++----- manifest.uuid | 2 +- 8 files changed, 224 insertions(+), 34 deletions(-) create mode 100644 ext/fts5/test/fts5origintext3.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 19d9579eb7..317d66db99 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -545,6 +545,13 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p); int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter*, const char*, int, int iCol, int iOff +); +int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter*); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ae851a877a..9889bccb32 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2985,6 +2985,12 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +static int fts5QueryTerm(const char *pToken, int nToken){ + int ii; + for(ii=0; iipExpr; int i; + int nQuery = nToken; UNUSED_PARAM2(iUnused1, iUnused2); - if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE; + if( pExpr->pConfig->bTokendata ){ + nQuery = fts5QueryTerm(pToken, nQuery); + } if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ Fts5ExprTerm *pT; if( p->aPopulator[i].bOk==0 ) continue; for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ - if( (pT->nFullTerm==nToken || (pT->nFullTermbPrefix)) - && memcmp(pT->pTerm, pToken, pT->nFullTerm)==0 + if( (pT->nQueryTerm==nQuery || (pT->nQueryTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); + if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){ + int iCol = p->iOff>>32; + int iTokOff = p->iOff & 0x7FFFFFFF; + rc = sqlite3Fts5IndexIterWriteTokendata( + pT->pIter, pToken, nToken, iCol, iTokOff + ); + } if( rc ) return rc; break; } @@ -3027,11 +3044,23 @@ int sqlite3Fts5ExprPopulatePoslists( const char *z, int n ){ int i; + int rc = SQLITE_OK; Fts5ExprCtx sCtx; sCtx.pExpr = pExpr; sCtx.aPopulator = aPopulator; sCtx.iOff = (((i64)iCol) << 32) - 1; + /* If this is a tokendata=1 table, clear out the hash tables of + ** full-terms. */ + if( pConfig->bTokendata ){ + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } + } + for(i=0; inPhrase; i++){ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; Fts5Colset *pColset = pNode->pNear->pColset; @@ -3044,9 +3073,20 @@ int sqlite3Fts5ExprPopulatePoslists( } } - return sqlite3Fts5Tokenize(pConfig, + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); + + if( pConfig->bTokendata ){ + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterHashifyTokendata(pT->pIter); + } + } + } + + return rc; } static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 887bb75dac..f206b8116d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6877,6 +6877,73 @@ int sqlite3Fts5IterToken( return SQLITE_OK; } +/* +** Clear any existing entries from the token-map associated with the +** iterator passed as the only argument. +*/ +void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL ); + if( pIter->pTokenMap ){ + pIter->pTokenMap->nEntry = 0; + } +} + +int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + int iCol, int iOff +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5Index *p = pIter->pIndex; + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); + if( pIter->pTokenMap==0 ){ + pIter->pTokenMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); + } + if( p->rc==SQLITE_OK ){ + Fts5TokenMap *pMap = pIter->pTokenMap; + int ii; + for(ii=0; iinToken; ii++){ + if( nToken==pMap->aToken[ii].nTerm + && 0==memcmp(pMap->aToken[ii].pTerm, pToken, nToken) + ){ + break; + } + } + if( ii==pMap->nToken ){ + fts5TokenMapTerm(p, pMap, (const u8*)pToken, nToken); + } + if( pMap->nEntry>=pMap->nEntryAlloc ){ + int nNew = pMap->nEntryAlloc ? pMap->nEntryAlloc*2 : 32; + Fts5TokenMapEntry *aNew = (Fts5TokenMapEntry*)sqlite3_realloc( + pMap->aEntry, nNew * sizeof(Fts5TokenMapEntry) + ); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + pMap->aEntry = aNew; + pMap->nEntryAlloc = nNew; + } + } + if( p->rc==SQLITE_OK ){ + Fts5TokenMapEntry *pEntry = &pMap->aEntry[pMap->nEntry++]; + pEntry->iRowid = pIndexIter->iRowid; + pEntry->iCol = iCol; + pEntry->iOff = iOff; + pEntry->iTok = ii+1; + } + } + return fts5IndexReturn(p); +} + +int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter->pTokenMap ){ + fts5TokenMapHashify(pIter->pIndex, pIter->pTokenMap); + } + return fts5IndexReturn(pIter->pIndex); +} + /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 07e5b4fb2e..845e8145db 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -21,9 +21,13 @@ ifcapable !fts5 { return } +foreach_detail_mode $testprefix { + sqlite3_fts5_register_origintext db do_execsql_test 1.0 { - CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); } @@ -85,7 +89,9 @@ db func document document sqlite3_fts5_register_origintext db do_execsql_test 2.0 { - CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); INSERT INTO ft(ft, rank) VALUES('pgsz', 128); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); } @@ -117,7 +123,9 @@ reset_db sqlite3_fts5_register_origintext db do_execsql_test 3.0 { - CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="origintext unicode61"); + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", detail=%DETAIL% + ); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); INSERT INTO ft(rowid, x) VALUES(1, 'hello'); @@ -136,7 +144,8 @@ do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3 do_execsql_test 3.0 { CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize="origintext unicode61", - tokendata=1 + tokendata=1, + detail=%DETAIL% ); CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance); @@ -169,7 +178,7 @@ sqlite3_fts5_create_function db querytoken querytoken do_execsql_test 4.0 { CREATE VIRTUAL TABLE ft USING fts5( - x, tokenize='origintext unicode61', tokendata=1 + x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% ); INSERT INTO ft VALUES('one two three four'); } @@ -183,19 +192,23 @@ do_execsql_test 4.2 { do_execsql_test 4.3 { SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') } {1 two.TWO} -do_execsql_test 4.4 { - SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') -} {1 three.ThreE} -do_catchsql_test 4.5 { - SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') -} {1 SQLITE_RANGE} -do_catchsql_test 4.6 { - SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') -} {1 SQLITE_RANGE} -do_catchsql_test 4.7 { - SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') -} {1 SQLITE_RANGE} +if {"%DETAIL%"=="full"} { + # Phrase queries are only supported for detail=full. + # + do_execsql_test 4.4 { + SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') + } {1 three.ThreE} + do_catchsql_test 4.5 { + SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} + do_catchsql_test 4.6 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} + do_catchsql_test 4.7 { + SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') + } {1 SQLITE_RANGE} +} #------------------------------------------------------------------------- # @@ -210,7 +223,7 @@ fts5_aux_test_functions db do_execsql_test 5.0 { CREATE VIRTUAL TABLE ft USING fts5( - x, tokenize='origintext unicode61', tokendata=1 + x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% ); INSERT INTO ft VALUES('one ONE One oNe oNE one'); } @@ -239,5 +252,7 @@ do_execsql_test 5.3 { {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5} } +} + finish_test diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test index 7cf8d80071..26f9864098 100644 --- a/ext/fts5/test/fts5origintext2.test +++ b/ext/fts5/test/fts5origintext2.test @@ -13,7 +13,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5origintext +set testprefix fts5origintext2 # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test new file mode 100644 index 0000000000..57b5984f4d --- /dev/null +++ b/ext/fts5/test/fts5origintext3.test @@ -0,0 +1,60 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext3 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + reset_db + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc insttoken {cmd iIdx iToken} { + set txt [$cmd xInstToken $iIdx $iToken] + string map [list "\0" "."] $txt + } + sqlite3_fts5_create_function db insttoken insttoken + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + + do_execsql_test 1.1 { + INSERT INTO ft VALUES('Hello world HELLO WORLD hello'); + } + + do_execsql_test 1.2 { + SELECT fts5_test_poslist(ft) FROM ft('hello'); + } {{0.0.0 0.0.2 0.0.4}} + + do_execsql_test 1.3 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello'); + } {hello.Hello hello.HELLO hello} + +} + +finish_test + diff --git a/manifest b/manifest index 7763b46700..e729bc7cf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\squerying\sa\stokendata=1\sfts5\stable,\sdo\snot\suse\sa\sprefix\scursor\sfor\sthe\scase\swhere\sthe\sterm\shas\sonly\sone\svariant. -D 2023-11-16T21:11:56.608 +C Fix\stokendata=1\sand\sxInstToken()\sAPIs\sfor\sdetail=none\sand\sdetail=column\stables. +D 2023-11-22T19:02:54.078 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,13 +88,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c -F ext/fts5/fts5Int.h 88ab1ee1eefa6f98e4c7fd3c96c99ef76ed2819cc3058736c87bb01e4a301628 +F ext/fts5/fts5Int.h d330c2e20051c300b26325b8ba29aa89e99d301c80e2f51092e5bb70346a17cd F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 4b50ed0c724cb160f086e20e964ed2d57b99d0d3c1cb1b029901c0300b11bd9f +F ext/fts5/fts5_expr.c 0d846134eafeeb1f0724b9c8cc02a2ef9c4082519aa3923173deadd5155910b1 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 70fa4a6d8a062ca4b63a62d0721d72ce2f6336413c6e8b0703881c708797d24d +F ext/fts5/fts5_index.c 7b87808d788238eff4a0a68728e6ed49817e71bbfb328a18050d7d8e92a5d66a F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -188,8 +188,9 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 908a1fb6b1106e4b6ed0f9cf683c2ad7f986cce1aea1e0a13b3309c6f568932b -F ext/fts5/test/fts5origintext2.test a654c77f1548ccd8eab7f6d07230655c0070cdf32dcd4740ccdf496f77d5282c +F ext/fts5/test/fts5origintext.test 7caef7634889bab8b44d145141c0d9325299398fb89b116bccd6262fde5659db +F ext/fts5/test/fts5origintext2.test 26482f4af1f2785cb01d06af9aae202289b6e8cf7b708d18aea305b459c2f302 +F ext/fts5/test/fts5origintext3.test 87a212b8235794348c56cb70f21e122d182a5af688c56057b90b7c151d0aa347 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2144,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a34b26fe7f60b74e7ae5cf64900920a3d352a20da2496401bcbc27041689cd07 -R d7c277a055a404d272fdcb5090bf371a +P d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47 +R 161c23f360cac1706a2c5f6b11155312 U dan -Z 0e1bf556ad9eba9db356685a09c7ab31 +Z 281a4d74e3cce55af028079774718a8b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6373f95ef5..38bacc7670 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47 \ No newline at end of file +37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86 \ No newline at end of file From af54826e4a76259a37d396e88dc3c582ef76664c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Nov 2023 20:02:55 +0000 Subject: [PATCH 090/191] Defer building xInstToken() hash-table until it is to be used. FossilOrigin-Name: 9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa --- ext/fts5/fts5Int.h | 1 - ext/fts5/fts5_expr.c | 14 +------------- ext/fts5/fts5_index.c | 24 ++++++++++++------------ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 22 insertions(+), 35 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 317d66db99..9d2622448d 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -550,7 +550,6 @@ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); int sqlite3Fts5IndexIterWriteTokendata( Fts5IndexIter*, const char*, int, int iCol, int iOff ); -int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter*); /* ** End of interface to code in fts5_index.c. diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9889bccb32..6589d1b3e6 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3044,7 +3044,6 @@ int sqlite3Fts5ExprPopulatePoslists( const char *z, int n ){ int i; - int rc = SQLITE_OK; Fts5ExprCtx sCtx; sCtx.pExpr = pExpr; sCtx.aPopulator = aPopulator; @@ -3073,20 +3072,9 @@ int sqlite3Fts5ExprPopulatePoslists( } } - rc = sqlite3Fts5Tokenize(pConfig, + return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); - - if( pConfig->bTokendata ){ - for(i=0; inPhrase; i++){ - Fts5ExprTerm *pT; - for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ - sqlite3Fts5IndexIterHashifyTokendata(pT->pIter); - } - } - } - - return rc; } static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index f206b8116d..a4150d3d5b 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -628,6 +628,8 @@ struct Fts5TokenMapToken { }; struct Fts5TokenMap { + int bHashed; /* True once hashed */ + int nEntryAlloc; int nEntry; Fts5TokenMapEntry *aEntry; @@ -6372,7 +6374,6 @@ static void fts5SetupPrefixIter( ** index contains all the doclists required, except for the one ** corresponding to the prefix itself. That one is extracted from the ** main term index here. */ - assert( iIdx==0 || pMap==0 ); if( iIdx!=0 ){ int dummy = 0; const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; @@ -6402,7 +6403,7 @@ static void fts5SetupPrefixIter( *ppIter = p1; }else{ - if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL && bTokenscan ){ pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); } assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); @@ -6473,7 +6474,6 @@ static void fts5SetupPrefixIter( pData->p = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); - if( pMap ) fts5TokenMapHashify(p, pMap); fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter); pMap = 0; } @@ -6867,7 +6867,15 @@ int sqlite3Fts5IterToken( const char **ppOut, int *pnOut ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenMap ){ + Fts5TokenMap *pMap = pIter->pTokenMap; + if( pMap ){ + if( pMap->bHashed==0 ){ + Fts5Index *p = pIter->pIndex; + fts5TokenMapHashify(p, pMap); + if( p->rc ){ + return fts5IndexReturn(p); + } + } *ppOut = (const char*)fts5TokenMapLookup( pIter->pTokenMap, pIndexIter->iRowid, iCol, iOff, pnOut ); @@ -6936,14 +6944,6 @@ int sqlite3Fts5IndexIterWriteTokendata( return fts5IndexReturn(p); } -int sqlite3Fts5IndexIterHashifyTokendata(Fts5IndexIter *pIndexIter){ - Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenMap ){ - fts5TokenMapHashify(pIter->pIndex, pIter->pTokenMap); - } - return fts5IndexReturn(pIter->pIndex); -} - /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). */ diff --git a/manifest b/manifest index e729bc7cf1..dfaf7bb154 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stokendata=1\sand\sxInstToken()\sAPIs\sfor\sdetail=none\sand\sdetail=column\stables. -D 2023-11-22T19:02:54.078 +C Defer\sbuilding\sxInstToken()\shash-table\suntil\sit\sis\sto\sbe\sused. +D 2023-11-22T20:02:55.862 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,13 +88,13 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c -F ext/fts5/fts5Int.h d330c2e20051c300b26325b8ba29aa89e99d301c80e2f51092e5bb70346a17cd +F ext/fts5/fts5Int.h 782151060d176be22861f57bf38e087a82cfb0dfc4b2fa6f9ccbc2641b6d01e3 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 0d846134eafeeb1f0724b9c8cc02a2ef9c4082519aa3923173deadd5155910b1 +F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 7b87808d788238eff4a0a68728e6ed49817e71bbfb328a18050d7d8e92a5d66a +F ext/fts5/fts5_index.c 710b022dcdf152eb7bbbc3f83eb662e1e67c25e0643416096ed070b10d7829fb F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d711c96ba855686d6881a50498418de3492144f005684b5ae55bca24413dce47 -R 161c23f360cac1706a2c5f6b11155312 +P 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86 +R 0d85011f1dd994f228f63f6f67d53fdc U dan -Z 281a4d74e3cce55af028079774718a8b +Z f3ab4e11005318d9a7fa0f164d5ead9f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38bacc7670..81d2b68945 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86 \ No newline at end of file +9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa \ No newline at end of file From 41c9e0b767dbd637ac8c5f7b82d9190799d24cc2 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Nov 2023 21:15:05 +0000 Subject: [PATCH 091/191] Add documentation for new fts5 auxiliary function APIs. FossilOrigin-Name: 9be8969edd49e3da96fb8ac2279aff6fe2e215d6ac55162b4734aca1b6316580 --- ext/fts5/fts5.h | 27 +++++++++++++++++++++++++-- ext/fts5/fts5_main.c | 16 +--------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 5a2008882f..9feedbba19 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -261,6 +261,27 @@ struct Fts5PhraseIter { ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ @@ -300,9 +321,11 @@ struct Fts5ExtensionApi { void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); /* Below this point are iVersion>=3 only */ - int (*xQueryToken)(Fts5Context*, int iPhrase, int iToken, const char**, int*); + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); - int (*xPhraseToken)(Fts5Context*, Fts5PhraseIter*, int, const char**, int*); }; /* diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5ef80719ae..9f19b24b88 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2360,19 +2360,6 @@ static int fts5ApiInstToken( return rc; } -/* -** xPhraseToken() API implemenetation. -*/ -static int fts5ApiPhraseToken( - Fts5Context *pCtx, - Fts5PhraseIter *pIter, - int iToken, - const char **ppOut, - int *pnOut -){ - return SQLITE_OK; -} - static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) @@ -2400,8 +2387,7 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, fts5ApiQueryToken, - fts5ApiInstToken, - fts5ApiPhraseToken + fts5ApiInstToken }; /* diff --git a/manifest b/manifest index dfaf7bb154..8adb17e981 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Defer\sbuilding\sxInstToken()\shash-table\suntil\sit\sis\sto\sbe\sused. -D 2023-11-22T20:02:55.862 +C Add\sdocumentation\sfor\snew\sfts5\sauxiliary\sfunction\sAPIs. +D 2023-11-22T21:15:05.459 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,7 +87,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c +F ext/fts5/fts5.h 5e5630fc81e212f658afaa5b2650dac939d2729d0723aef1eeaff908f1725648 F ext/fts5/fts5Int.h 782151060d176be22861f57bf38e087a82cfb0dfc4b2fa6f9ccbc2641b6d01e3 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 @@ -95,7 +95,7 @@ F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532 F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 710b022dcdf152eb7bbbc3f83eb662e1e67c25e0643416096ed070b10d7829fb -F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74 +F ext/fts5/fts5_main.c 55b53085dbd1693b5735463198a8d124dfbc27f08311c839637b44b8254ef7cb F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86 -R 0d85011f1dd994f228f63f6f67d53fdc +P 9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa +R 2f037553902ade27b74f0aa5f4f0c2dc U dan -Z f3ab4e11005318d9a7fa0f164d5ead9f +Z 56eb427eba85d076d9cd44ffbac445d7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 81d2b68945..eca2a6957f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa \ No newline at end of file +9be8969edd49e3da96fb8ac2279aff6fe2e215d6ac55162b4734aca1b6316580 \ No newline at end of file From ab702666981363147a0a3bd81545981e1220b5d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Nov 2023 14:25:56 +0000 Subject: [PATCH 092/191] Fix jsonParseReset() to properly clear the JsonParse.aBlob element. FossilOrigin-Name: ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 27 ++++++++++++++++----------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index f916388b89..90ba0620cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sprecompiled\sbinaries\sfrom\sthe\ssource\stree. -D 2023-11-24T14:03:20.819 +C Fix\sjsonParseReset()\sto\sproperly\sclear\sthe\sJsonParse.aBlob\selement. +D 2023-11-24T14:25:56.679 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c f8cfdf1d024a7b2c5ad3eae15903a7a83a9b2c8ab61716d219a57600544b712c +F src/json.c c6b23115b4561008363346a209b5c8a3538d967b35c63685b2e07dc7de1f1e31 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a838ebcbbd9f554fd84a1d3176fb572faaef7d0ec0675f1a5bf9430865dafae0 4ff103d294b79cf7734e87e94e9d88c0e9f0b087cbb352e6da2f0a3a6b268f46 -R 0a66dd6facaa7d203b266fe2d20488a5 +P 7dbc2f496d7a362460bb4c262ecafe5f30e35a8744861163d12c996365c2142f +R 1676f3ca8710b44ac390888e709fad02 U drh -Z 62c86eba16d8545370a182d5ae7c09b8 +Z 41b7b01f3bc28811bfbbf765da2aea03 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e2ff938fb7..8430a0984f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7dbc2f496d7a362460bb4c262ecafe5f30e35a8744861163d12c996365c2142f \ No newline at end of file +ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 54e221392e..0eacca3089 100644 --- a/src/json.c +++ b/src/json.c @@ -865,6 +865,12 @@ static void jsonParseReset(JsonParse *pParse){ sqlite3RCStrUnref(pParse->zAlt); pParse->zAlt = 0; } + if( pParse->nBlobAlloc ){ + sqlite3_free(pParse->aBlob); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } } /* @@ -4311,7 +4317,7 @@ static void jsonRemoveFromBlob( if( zPath==0 ) goto jsonRemoveFromBlob_patherror; if( zPath[0]!='$' ) goto jsonRemoveFromBlob_patherror; if( zPath[1]==0 ){ - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); return; /* return NULL if $ is removed */ } px.eEdit = JEDIT_DEL; @@ -4328,12 +4334,12 @@ static void jsonRemoveFromBlob( jsonStringInit(&s, ctx); jsonXlateBlobToText(&px, 0, &s); jsonReturnString(&s); - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); } return; jsonRemoveFromBlob_patherror: - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); jsonPathSyntaxError(zPath, ctx); return; } @@ -4445,14 +4451,14 @@ static void jsonInsertIntoBlob( break; } if( zPath[1]==0 ){ - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); return; /* return NULL if $ is removed */ } px.eEdit = eEdit; px.nIns = ax.nBlob; px.aIns = ax.aBlob; rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); - if( ax.nBlobAlloc ) sqlite3_free(ax.aBlob); + jsonParseReset(&ax); if( rc==JSON_BLOB_NOTFOUND ) continue; if( JSON_BLOB_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; } @@ -4465,12 +4471,12 @@ static void jsonInsertIntoBlob( jsonStringInit(&s, ctx); jsonXlateBlobToText(&px, 0, &s); jsonReturnString(&s); - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); } return; jsonInsertIntoBlob_patherror: - if( px.nBlobAlloc ) sqlite3_free(px.aBlob); + jsonParseReset(&px); if( rc==JSON_BLOB_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ @@ -4648,13 +4654,12 @@ static void jsonbFunc( x.nJson = nJson; if( jsonConvertTextToBlob(pParse, ctx) ){ sqlite3_result_error(ctx, "malformed JSON", -1); - sqlite3_free(pParse->aBlob); }else{ sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; } - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->nBlobAlloc = 0; jsonParseReset(pParse); } } From abbdbdfc1f1ea527233c4fdf711c97dc51d9a4ff Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Nov 2023 18:44:00 +0000 Subject: [PATCH 093/191] Incremental progress toward getting json_each() and json_tree() to work directly off of a JSONB blob. FossilOrigin-Name: f8cab41b3bc65af6ff34b481db693d640ea025d09463d50b8e56d855e2abc913 --- manifest | 15 +- manifest.uuid | 2 +- src/json.c | 483 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 356 insertions(+), 144 deletions(-) diff --git a/manifest b/manifest index 90ba0620cd..49e3c13317 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sjsonParseReset()\sto\sproperly\sclear\sthe\sJsonParse.aBlob\selement. -D 2023-11-24T14:25:56.679 +C Incremental\sprogress\stoward\sgetting\sjson_each()\sand\sjson_tree()\sto\swork\ndirectly\soff\sof\sa\sJSONB\sblob. +D 2023-11-24T18:44:00.124 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c6b23115b4561008363346a209b5c8a3538d967b35c63685b2e07dc7de1f1e31 +F src/json.c 1b9cc57728bc3a420bae5f4ae8233dcd2c3a337f00a2467346f7b788aedf1ba0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7dbc2f496d7a362460bb4c262ecafe5f30e35a8744861163d12c996365c2142f -R 1676f3ca8710b44ac390888e709fad02 +P ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 +R b8a3524be145a961ee4bfad02645fa25 +T *branch * jsonb-tree +T *sym-jsonb-tree * +T -sym-jsonb * U drh -Z 41b7b01f3bc28811bfbbf765da2aea03 +Z b88ae8be743d3b0432d41e7f9773a471 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8430a0984f..3afbf02fc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 \ No newline at end of file +f8cab41b3bc65af6ff34b481db693d640ea025d09463d50b8e56d855e2abc913 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0eacca3089..fdd2a21d59 100644 --- a/src/json.c +++ b/src/json.c @@ -133,6 +133,14 @@ #define JSONB_ARRAY 11 /* An array */ #define JSONB_OBJECT 12 /* An object */ +/* Human-readalbe names for the JSONB values: +*/ +static const char * const jsonbType[] = { + "null", "true", "false", "integer", "integer", + "real", "real", "text", "text", "text", + "text", "array", "object" +}; + /* ** Growing our own isspace() routine this way is twice as fast as ** the library isspace() function, resulting in a 7% overall performance @@ -4070,7 +4078,8 @@ static void jsonReturnTextJsonFromBlob( static void jsonReturnFromBlob( JsonParse *pParse, /* Complete JSON parse tree */ u32 i, /* Index of the node */ - sqlite3_context *pCtx /* Return value for this function */ + sqlite3_context *pCtx, /* Return value for this function */ + int textOnly /* return text JSON. Disregard user-data */ ){ u32 n, sz; int rc; @@ -4217,7 +4226,7 @@ static void jsonReturnFromBlob( } case JSONB_ARRAY: case JSONB_OBJECT: { - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + int flags = textOnly ? 0 : SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); if( flags & JSON_BLOB ){ sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); }else{ @@ -4280,7 +4289,7 @@ static void jsonExtractFromBlob( return; } if( idb = db; } return rc; } @@ -5628,12 +5655,14 @@ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ /* constructor for a JsonEachCursor object for json_each(). */ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachConnection *pVtab = (JsonEachConnection*)p; JsonEachCursor *pCur; UNUSED_PARAMETER(p); pCur = sqlite3_malloc( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); + pCur->db = pVtab->db; *ppCursor = &pCur->base; return SQLITE_OK; } @@ -5653,8 +5682,12 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); + sqlite3DbFree(p->db, p->aParent); p->iRowid = 0; p->i = 0; + p->aParent = 0; + p->nParent = 0; + p->nParentAlloc = 0; p->iEnd = 0; p->eType = 0; p->zJson = 0; @@ -5676,45 +5709,106 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ return p->i >= p->iEnd; } +/* +** If the cursor is currently pointing at the label of a object entry, +** then return the index of the value. For all other cases, return the +** current pointer position, which is the value. +*/ +static int jsonSkipLabel(JsonEachCursor *p){ + if( p->eType==JSONB_OBJECT ){ + u32 sz = 0; + u32 n = jsonbPayloadSize(&p->sParse, p->i, &sz); + return p->i + n + sz; + }else{ + return p->i; + } +} + /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==0 || pUp->eU==3 ); - testcase( pUp->eU==3 ); - JSON_VVA( pUp->eU = 3 ); - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; + if( p->sParse.aNode ){ + /* LEGACY */ + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->iiEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + assert( pUp->eU==0 || pUp->eU==3 ); + testcase( pUp->eU==3 ); + JSON_VVA( pUp->eU = 3 ); + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } + } + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; } } } - }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; + return SQLITE_OK; + }else if( p->bRecursive ){ + u8 x; + u8 levelChange = 0; + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + x = p->sParse.aBlob[i] & 0x0f; + n = jsonbPayloadSize(&p->sParse, p->i, &sz); + if( x==JSONB_OBJECT || x==JSONB_ARRAY ){ + JsonParent *pParent; + if( p->nParent>=p->nParentAlloc ){ + JsonParent *pNew; + u64 nNew; + nNew = p->nParentAlloc*2 + 3; + pNew = sqlite3DbRealloc(p->db, p->aParent, sizeof(JsonParent)*nNew); + if( pNew==0 ) return SQLITE_NOMEM; + p->nParentAlloc = (u32)nNew; + p->aParent = pNew; + levelChange = 1; } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; + pParent = &p->aParent[p->nParent++]; + pParent->iHead = p->i; + pParent->iEnd = p->i + n + sz; + pParent->iKey = 0; + }else{ + p->i = i + n + sz; + } + if( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ + p->nParent--; + levelChange = 1; + } + if( levelChange ){ + if( p->nParent>0 ){ + p->eType = p->sParse.aBlob[p->aParent[p->nParent-1].iHead] & 0x0f; + }else{ + p->eType = 0; } } + }else{ + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->i = i + n + sz; } return SQLITE_OK; } @@ -5722,7 +5816,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ /* Append an object label to the JSON Path being constructed ** in pStr. */ -static void jsonAppendObjectPathElement( +static void jsonAppendObjectPathElementOfNode( JsonString *pStr, JsonNode *pNode ){ @@ -5763,18 +5857,23 @@ static void jsonEachComputePath( jsonAppendChar(pStr, '$'); return; } - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==3 || (pUp->eU==0 && pUp->u.iKey==0) ); - testcase( pUp->eU==0 ); - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - jsonAppendObjectPathElement(pStr, pNode); + if( p->sParse.aNode ){ + /* LEGACY */ + assert( p->sParse.aUp ); + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + assert( pUp->eU==3 || (pUp->eU==0 && pUp->u.iKey==0) ); + testcase( pUp->eU==0 ); + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + jsonAppendObjectPathElementOfNode(pStr, pNode); + } + return; } } @@ -5782,102 +5881,200 @@ static void jsonEachComputePath( static int jsonEachColumn( sqlite3_vtab_cursor *cur, /* The cursor */ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ + int iColumn /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( i ){ - case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ + if( p->sParse.aNode!=0 ){ + /* LEGACY */ + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( iColumn ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturnFromNode(&p->sParse, pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; - }else{ - iKey = p->iRowid; - } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + break; } - break; - } - case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - break; - } - case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); - break; - } - case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - break; - } - case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); - break; - } - case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; } - break; - } - case JEACH_FULLKEY: { - JsonString x; - jsonStringInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - jsonAppendObjectPathElement(&x, pThis); - } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturnFromNode(&p->sParse, pThis, ctx, 0); + break; } - jsonReturnString(&x); - break; - } - case JEACH_PATH: { - if( p->bRecursive ){ + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { JsonString x; jsonStringInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonAppendObjectPathElementOfNode(&x, pThis); + } + } jsonReturnString(&x); break; } - /* For json_each() path and root are the same so fall through - ** into the root case */ - /* no break */ deliberate_fall_through - } - default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); - break; - } - case JEACH_JSON: { - if( p->sParse.isBinary ){ - sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); - }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonStringInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonReturnString(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + /* no break */ deliberate_fall_through + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } + break; } - break; } + }else{ + /* Blob scan */ + switch( iColumn ){ + case JEACH_KEY: { + if( p->nParent==0 ) break; + if( p->eType==JSONB_OBJECT ){ + jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); + }else{ + assert( p->eType==JSONB_ARRAY ); + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); + } + break; + } + case JEACH_VALUE: { + u32 i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, 1); + break; + } + case JEACH_TYPE: { + u32 i = jsonSkipLabel(p); + u8 eType = eType = p->sParse.aBlob[i] & 0x0f; + sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + u32 i; + if( p->eType>=JSON_ARRAY ) break; + i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, 1); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, (sqlite3_int64)p->i); + break; + } + case JEACH_PARENT: { + if( p->nParent>0 ){ + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); + } + break; + } + case JEACH_FULLKEY: { +#if 0 + JsonString x; + jsonStringInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonAppendObjectPathElementOfNode(&x, pThis); + } + } + jsonReturnString(&x); +#endif + break; + } + case JEACH_PATH: { +#if 0 + if( p->bRecursive ){ + JsonString x; + jsonStringInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonReturnString(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + /* no break */ deliberate_fall_through +#endif + break; + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } + break; + } + } + } return SQLITE_OK; } @@ -5975,16 +6172,28 @@ static int jsonEachFilter( UNUSED_PARAMETER(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; if( jsonFuncArgMightBeBinary(argv[0]) ){ - z = (const char*)sqlite3_value_blob(argv[0]); - isBinary = 1; + u32 i, n, sz; + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( p->sParse.aBlob==0 ){ + return SQLITE_NOMEM; + } + i = p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->nParent = 0; + p->sParse.isBinary = 1; + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->iEnd = i+n+sz; + return SQLITE_OK; }else{ z = (const char*)sqlite3_value_text(argv[0]); isBinary = 0; } if( z==0 ) return SQLITE_OK; - memset(&p->sParse, 0, sizeof(p->sParse)); - p->sParse.nJPRef = 1; if( sqlite3ValueIsOfClass(argv[0], sqlite3RCStrUnref) ){ p->sParse.zJson = sqlite3RCStrRef((char*)z); }else{ From 5e6500c81c717b74d3910909283fa23dec6139ee Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Nov 2023 21:57:38 +0000 Subject: [PATCH 094/191] Continuing work on json_tree() against a JSONB. FossilOrigin-Name: 3df891cb11feee65e239ee2506eda34a9688341f05210d7c2e25a05338cb71ad --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 35 +++++++++++++++++++++++++++++------ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 49e3c13317..55cf674438 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Incremental\sprogress\stoward\sgetting\sjson_each()\sand\sjson_tree()\sto\swork\ndirectly\soff\sof\sa\sJSONB\sblob. -D 2023-11-24T18:44:00.124 +C Continuing\swork\son\sjson_tree()\sagainst\sa\sJSONB. +D 2023-11-24T21:57:38.322 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 1b9cc57728bc3a420bae5f4ae8233dcd2c3a337f00a2467346f7b788aedf1ba0 +F src/json.c 699b3fa340c24a10874988b9cbf4a1635b4231569dfaef9e4fd8fb96195406aa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 -R b8a3524be145a961ee4bfad02645fa25 -T *branch * jsonb-tree -T *sym-jsonb-tree * -T -sym-jsonb * +P f8cab41b3bc65af6ff34b481db693d640ea025d09463d50b8e56d855e2abc913 +R 12dbc87edffde88122e858181ec1f996 U drh -Z b88ae8be743d3b0432d41e7f9773a471 +Z f88095c49473dd89bb79ab974b58607c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3afbf02fc8..a401927f74 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8cab41b3bc65af6ff34b481db693d640ea025d09463d50b8e56d855e2abc913 \ No newline at end of file +3df891cb11feee65e239ee2506eda34a9688341f05210d7c2e25a05338cb71ad \ No newline at end of file diff --git a/src/json.c b/src/json.c index fdd2a21d59..5823eadc88 100644 --- a/src/json.c +++ b/src/json.c @@ -5576,6 +5576,7 @@ static void jsonObjectFinal(sqlite3_context *ctx){ typedef struct JsonParent JsonParent; struct JsonParent { u32 iHead; /* Start of object or array */ + u32 iValue; /* Start of the value */ u32 iEnd; /* First byte past the end */ i64 iKey; /* Key for JSONB_ARRAY */ }; @@ -5773,7 +5774,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ u32 n, sz = 0; u32 i = jsonSkipLabel(p); x = p->sParse.aBlob[i] & 0x0f; - n = jsonbPayloadSize(&p->sParse, p->i, &sz); + n = jsonbPayloadSize(&p->sParse, i, &sz); if( x==JSONB_OBJECT || x==JSONB_ARRAY ){ JsonParent *pParent; if( p->nParent>=p->nParentAlloc ){ @@ -5784,32 +5785,41 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ if( pNew==0 ) return SQLITE_NOMEM; p->nParentAlloc = (u32)nNew; p->aParent = pNew; - levelChange = 1; } + levelChange = 1; pParent = &p->aParent[p->nParent++]; pParent->iHead = p->i; - pParent->iEnd = p->i + n + sz; - pParent->iKey = 0; + pParent->iValue = i; + pParent->iEnd = i + n + sz; + pParent->iKey = -1; + p->i = i + n; }else{ p->i = i + n + sz; } - if( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ + while( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ p->nParent--; levelChange = 1; } if( levelChange ){ if( p->nParent>0 ){ - p->eType = p->sParse.aBlob[p->aParent[p->nParent-1].iHead] & 0x0f; + JsonParent *pParent = &p->aParent[p->nParent-1]; + u32 i = pParent->iValue; + p->eType = p->sParse.aBlob[i] & 0x0f; }else{ p->eType = 0; } } + if( p->eType==JSONB_ARRAY ){ + assert( p->nParent>0 ); + p->aParent[p->nParent-1].iKey++; + } }else{ u32 n, sz = 0; u32 i = jsonSkipLabel(p); n = jsonbPayloadSize(&p->sParse, i, &sz); p->i = i + n + sz; } + p->iRowid++; return SQLITE_OK; } @@ -6022,6 +6032,19 @@ static int jsonEachColumn( break; } case JEACH_FULLKEY: { +#if 0 + u32 i; + JsonString x; + jsonStringInit(&x, ctx); + for(i=0; inParent; i++){ + jsonPrintf(200,&x,"(%u,%u,%u,%lld)", + p->aParent[i].iHead, + p->aParent[i].iValue, + p->aParent[i].iEnd, + p->aParent[i].iKey); + } + jsonReturnString(&x); +#endif #if 0 JsonString x; jsonStringInit(&x, ctx); From b7d5cb711ae2cc0906e91de380d0eef8a5b3c363 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Nov 2023 13:40:19 +0000 Subject: [PATCH 095/191] Handle the path argument to json_tree() and json_each(). FossilOrigin-Name: fded888469565b2a4687185a926bd23fccfbf167c8bebe6c10696fc7f972f41e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 55cf674438..b7f38de853 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continuing\swork\son\sjson_tree()\sagainst\sa\sJSONB. -D 2023-11-24T21:57:38.322 +C Handle\sthe\spath\sargument\sto\sjson_tree()\sand\sjson_each(). +D 2023-11-25T13:40:19.363 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 699b3fa340c24a10874988b9cbf4a1635b4231569dfaef9e4fd8fb96195406aa +F src/json.c ff48d52463493f5f40d6e1e14ac3dc9277c40290df140a52bd17e7a562f766ea F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8cab41b3bc65af6ff34b481db693d640ea025d09463d50b8e56d855e2abc913 -R 12dbc87edffde88122e858181ec1f996 +P 3df891cb11feee65e239ee2506eda34a9688341f05210d7c2e25a05338cb71ad +R 86973d4c807995bd645f2f5dbc7eac48 U drh -Z f88095c49473dd89bb79ab974b58607c +Z e2d0e1a86b5ddc3117954e8a28079ed7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a401927f74..555d0938d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3df891cb11feee65e239ee2506eda34a9688341f05210d7c2e25a05338cb71ad \ No newline at end of file +fded888469565b2a4687185a926bd23fccfbf167c8bebe6c10696fc7f972f41e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5823eadc88..503fff9368 100644 --- a/src/json.c +++ b/src/json.c @@ -353,6 +353,7 @@ struct JsonParse { u8 eEdit; /* Edit operation to apply */ int delta; /* Size change due to the edit */ u32 nIns; /* Number of bytes to insert */ + u32 iLabel; /* Location of label if search landed on an object value */ u8 *aIns; /* Content to be inserted */ }; @@ -3904,6 +3905,7 @@ static u32 jsonLookupBlobStep( memcpy(&pParse->aBlob[iRoot], pParse->aIns, pParse->nIns); } } + pParse->iLabel = iLabel; return iRoot; } if( zPath[0]=='.' ){ @@ -6015,10 +6017,10 @@ static int jsonEachColumn( break; } case JEACH_ATOM: { - u32 i; - if( p->eType>=JSON_ARRAY ) break; - i = jsonSkipLabel(p); - jsonReturnFromBlob(&p->sParse, i, ctx, 1); + u32 i = jsonSkipLabel(p); + if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); + } break; } case JEACH_ID: { @@ -6204,13 +6206,50 @@ static int jsonEachFilter( if( p->sParse.aBlob==0 ){ return SQLITE_NOMEM; } - i = p->i = 0; - p->iEnd = 0; - p->eType = 0; + if( idxNum==3 ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + if( zRoot[1]==0 ){ + i = p->i = 0; + p->eType = 0; + }else{ + i = jsonLookupBlobStep(&p->sParse, 0, p->zRoot+1, 0); + if( JSON_BLOB_ISERROR(i) ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; + return SQLITE_OK; + } + if( p->sParse.iLabel ){ + p->i = p->sParse.iLabel; + p->eType = JSONB_OBJECT; + }else{ + p->i = i; + p->eType = JSONB_ARRAY; + } + } + }else{ + i = p->i = 0; + p->eType = 0; + } p->nParent = 0; p->sParse.isBinary = 1; n = jsonbPayloadSize(&p->sParse, i, &sz); p->iEnd = i+n+sz; + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ + p->i += n; + p->eType = p->sParse.aBlob[i] & 0x0f; + } return SQLITE_OK; }else{ z = (const char*)sqlite3_value_text(argv[0]); From c2474105ca39070a40f53b3c36550c9b169c5469 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Nov 2023 18:11:11 +0000 Subject: [PATCH 096/191] Generate the fullkey and path columns of json_tree(). FossilOrigin-Name: ffaa468ab8871906121df9ee5ef3dc00129a0086ed9c18831ecda69bf7f71455 --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 104 +++++++++++++++++++++++++------------------------- 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index b7f38de853..af86c512c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\spath\sargument\sto\sjson_tree()\sand\sjson_each(). -D 2023-11-25T13:40:19.363 +C Generate\sthe\sfullkey\sand\spath\scolumns\sof\sjson_tree(). +D 2023-11-25T18:11:11.468 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c ff48d52463493f5f40d6e1e14ac3dc9277c40290df140a52bd17e7a562f766ea +F src/json.c 9a5ff28f7b55e83b92eeab9285eb4b0132f11526620565e2d5b81fb84d4e8611 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3df891cb11feee65e239ee2506eda34a9688341f05210d7c2e25a05338cb71ad -R 86973d4c807995bd645f2f5dbc7eac48 +P fded888469565b2a4687185a926bd23fccfbf167c8bebe6c10696fc7f972f41e +R 90c664426338898baffedf8dcba41bea U drh -Z e2d0e1a86b5ddc3117954e8a28079ed7 +Z e9b6e925dfee6d388855b55f2b458a4c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 555d0938d3..acd1f46af1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fded888469565b2a4687185a926bd23fccfbf167c8bebe6c10696fc7f972f41e \ No newline at end of file +ffaa468ab8871906121df9ee5ef3dc00129a0086ed9c18831ecda69bf7f71455 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 503fff9368..c8269436eb 100644 --- a/src/json.c +++ b/src/json.c @@ -796,7 +796,6 @@ static void jsonAppendSqlValue( } } - /* Make the text in p (which is probably a generated JSON text string) ** the result of the SQL function. ** @@ -5580,6 +5579,7 @@ struct JsonParent { u32 iHead; /* Start of object or array */ u32 iValue; /* Start of the value */ u32 iEnd; /* First byte past the end */ + u32 nPath; /* Length of path */ i64 iKey; /* Key for JSONB_ARRAY */ }; @@ -5598,6 +5598,7 @@ struct JsonEachCursor { char *zJson; /* Input JSON */ char *zRoot; /* Path by which to filter zJson */ sqlite3 *db; /* Database connection */ + JsonString path; /* Current path */ JsonParse sParse; /* Parse of the input JSON */ }; typedef struct JsonEachConnection JsonEachConnection; @@ -5666,6 +5667,7 @@ static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = pVtab->db; + jsonStringZero(&pCur->path); *ppCursor = &pCur->base; return SQLITE_OK; } @@ -5685,6 +5687,7 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); + jsonStringReset(&p->path); sqlite3DbFree(p->db, p->aParent); p->iRowid = 0; p->i = 0; @@ -5701,6 +5704,7 @@ static void jsonEachCursorReset(JsonEachCursor *p){ static int jsonEachClose(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; jsonEachCursorReset(p); + sqlite3_free(cur); return SQLITE_OK; } @@ -5727,6 +5731,39 @@ static int jsonSkipLabel(JsonEachCursor *p){ } } +/* +** Append the path name for the current element. +*/ +static void jsonAppendPathName(JsonEachCursor *p){ + assert( p->nParent>0 ); + assert( p->eType==JSONB_ARRAY || p->eType==JSONB_OBJECT ); + if( p->eType==JSONB_ARRAY ){ + jsonPrintf(30, &p->path, "[%lld]", p->aParent[p->nParent-1].iKey); + }else{ + u32 n, sz = 0, k, i; + const char *z; + int needQuote = 0; + n = jsonbPayloadSize(&p->sParse, p->i, &sz); + k = p->i + n; + z = (const char*)&p->sParse.aBlob[k]; + if( sz==0 || !sqlite3Isalpha(z[0]) ){ + needQuote = 1; + }else{ + for(i=0; ipath,".\"%.*s\"", sz, z); + }else{ + jsonPrintf(sz+2,&p->path,".%.*s", sz, z); + } + } +} + /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; @@ -5789,17 +5826,21 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ p->aParent = pNew; } levelChange = 1; - pParent = &p->aParent[p->nParent++]; + pParent = &p->aParent[p->nParent]; pParent->iHead = p->i; pParent->iValue = i; pParent->iEnd = i + n + sz; pParent->iKey = -1; + pParent->nPath = (u32)p->path.nUsed; + if( p->eType && p->nParent ) jsonAppendPathName(p); + p->nParent++; p->i = i + n; }else{ p->i = i + n + sz; } while( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ p->nParent--; + p->path.nUsed = p->aParent[p->nParent].nPath; levelChange = 1; } if( levelChange ){ @@ -6034,53 +6075,16 @@ static int jsonEachColumn( break; } case JEACH_FULLKEY: { -#if 0 - u32 i; - JsonString x; - jsonStringInit(&x, ctx); - for(i=0; inParent; i++){ - jsonPrintf(200,&x,"(%u,%u,%u,%lld)", - p->aParent[i].iHead, - p->aParent[i].iValue, - p->aParent[i].iEnd, - p->aParent[i].iKey); - } - jsonReturnString(&x); -#endif -#if 0 - JsonString x; - jsonStringInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - jsonAppendObjectPathElementOfNode(&x, pThis); - } - } - jsonReturnString(&x); -#endif + u64 nBase = p->path.nUsed; + if( p->nParent ) jsonAppendPathName(p); + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + p->path.nUsed = nBase; break; } case JEACH_PATH: { -#if 0 - if( p->bRecursive ){ - JsonString x; - jsonStringInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonReturnString(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - /* no break */ deliberate_fall_through -#endif + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); break; } default: { @@ -6209,10 +6213,6 @@ static int jsonEachFilter( if( idxNum==3 ){ zRoot = (const char*)sqlite3_value_text(argv[1]); if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); @@ -6223,7 +6223,7 @@ static int jsonEachFilter( i = p->i = 0; p->eType = 0; }else{ - i = jsonLookupBlobStep(&p->sParse, 0, p->zRoot+1, 0); + i = jsonLookupBlobStep(&p->sParse, 0, zRoot+1, 0); if( JSON_BLOB_ISERROR(i) ){ p->i = 0; p->eType = 0; @@ -6238,9 +6238,11 @@ static int jsonEachFilter( p->eType = JSONB_ARRAY; } } + jsonAppendRaw(&p->path, zRoot, sqlite3Strlen30(zRoot)); }else{ i = p->i = 0; p->eType = 0; + jsonAppendRaw(&p->path, "$", 1); } p->nParent = 0; p->sParse.isBinary = 1; From aea2d2312103e2c63c6433977e6e06a32e2244a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Nov 2023 19:28:44 +0000 Subject: [PATCH 097/191] Almost working. Path is still not exactly right when Root is defined on json_tree(). FossilOrigin-Name: 92258246916a9c0d72785964513113848a850dec78bdade8b3f274e410df4e7e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index af86c512c2..8851c060f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generate\sthe\sfullkey\sand\spath\scolumns\sof\sjson_tree(). -D 2023-11-25T18:11:11.468 +C Almost\sworking.\s\sPath\sis\sstill\snot\sexactly\sright\swhen\sRoot\sis\sdefined\son\njson_tree(). +D 2023-11-25T19:28:44.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9a5ff28f7b55e83b92eeab9285eb4b0132f11526620565e2d5b81fb84d4e8611 +F src/json.c 617eaedbe44c139f0144f04ecbfa240721462f45d76e715eaacc99e172bf43b0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fded888469565b2a4687185a926bd23fccfbf167c8bebe6c10696fc7f972f41e -R 90c664426338898baffedf8dcba41bea +P ffaa468ab8871906121df9ee5ef3dc00129a0086ed9c18831ecda69bf7f71455 +R b82a578143acb9c5e1a084b85e2e99b7 U drh -Z e9b6e925dfee6d388855b55f2b458a4c +Z 1f6ed13fcc3b9ce0d1d8c376c125c482 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index acd1f46af1..78ef7d1413 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffaa468ab8871906121df9ee5ef3dc00129a0086ed9c18831ecda69bf7f71455 \ No newline at end of file +92258246916a9c0d72785964513113848a850dec78bdade8b3f274e410df4e7e \ No newline at end of file diff --git a/src/json.c b/src/json.c index c8269436eb..2de4c61a12 100644 --- a/src/json.c +++ b/src/json.c @@ -6249,8 +6249,16 @@ static int jsonEachFilter( n = jsonbPayloadSize(&p->sParse, i, &sz); p->iEnd = i+n+sz; if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ - p->i += n; + p->i = i + n; p->eType = p->sParse.aBlob[i] & 0x0f; + p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); + if( p->aParent==0 ) return SQLITE_NOMEM; + p->nParent = 1; + p->nParentAlloc = 1; + p->aParent[0].iKey = 0; + p->aParent[0].iEnd = p->iEnd; + p->aParent[0].iHead = p->i; + p->aParent[0].iValue = i; } return SQLITE_OK; }else{ From 796abda53828a776e0d6057406656e4295446002 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Nov 2023 20:59:03 +0000 Subject: [PATCH 098/191] Remove the vestigal JsonNode logic from json_tree() and json_each(). FossilOrigin-Name: 66c2ab9ebbf90477742e6be0d30e061d827c409de038f2a5b73479ed9448c4a6 --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 552 +++++++++++--------------------------------------- 3 files changed, 124 insertions(+), 442 deletions(-) diff --git a/manifest b/manifest index 8851c060f1..1f18d63513 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Almost\sworking.\s\sPath\sis\sstill\snot\sexactly\sright\swhen\sRoot\sis\sdefined\son\njson_tree(). -D 2023-11-25T19:28:44.393 +C Remove\sthe\svestigal\sJsonNode\slogic\sfrom\sjson_tree()\sand\sjson_each(). +D 2023-11-25T20:59:03.008 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 617eaedbe44c139f0144f04ecbfa240721462f45d76e715eaacc99e172bf43b0 +F src/json.c 211c4e1b8fd886c97d3224105bfef9d4cc04db2e3f1daca1e95c52343df4e85c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffaa468ab8871906121df9ee5ef3dc00129a0086ed9c18831ecda69bf7f71455 -R b82a578143acb9c5e1a084b85e2e99b7 +P 92258246916a9c0d72785964513113848a850dec78bdade8b3f274e410df4e7e +R 43fba60a847876455ec8abde78205ef3 U drh -Z 1f6ed13fcc3b9ce0d1d8c376c125c482 +Z d0660c039b12ecd1fe4ffd828711a405 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 78ef7d1413..3b93842124 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92258246916a9c0d72785964513113848a850dec78bdade8b3f274e410df4e7e \ No newline at end of file +66c2ab9ebbf90477742e6be0d30e061d827c409de038f2a5b73479ed9448c4a6 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 2de4c61a12..cc4eb83c5b 100644 --- a/src/json.c +++ b/src/json.c @@ -329,7 +329,6 @@ struct JsonParse { JsonNode *aNode; /* Array of nodes containing the parse */ char *zJson; /* Original JSON string (before edits) */ char *zAlt; /* Revised and/or mimified JSON */ - u32 *aUp; /* Index of parent of each node */ JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ @@ -860,10 +859,6 @@ static void jsonParseReset(JsonParse *pParse){ } pParse->nNode = 0; pParse->nAlloc = 0; - if( pParse->aUp ){ - sqlite3_free(pParse->aUp); - pParse->aUp = 0; - } if( pParse->bJsonIsRCStr ){ sqlite3RCStrUnref(pParse->zJson); pParse->zJson = 0; @@ -2013,33 +2008,6 @@ json_parse_restart: } /* End switch(z[i]) */ } -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - /* ** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB ** blob) into the JsonNode representation. @@ -2090,21 +2058,6 @@ static int jsonParse( return 0; } -/* -** Compute the parentage of all nodes in a completed parse. -*/ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; - } - jsonParseFillInParentage(pParse, 0, 0); - return SQLITE_OK; -} - /* ** Magic number used for the JSON parse cache in sqlite3_get_auxdata() */ @@ -3257,7 +3210,7 @@ static int jsonConvertTextToBlob( } } if( i<=0 ){ - if( ALWAYS(pCtx!=0) ){ + if( pCtx!=0 ){ if( pParse->oom ){ sqlite3_result_error_nomem(pCtx); }else{ @@ -5767,47 +5720,7 @@ static void jsonAppendPathName(JsonEachCursor *p){ /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->sParse.aNode ){ - /* LEGACY */ - if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==0 || pUp->eU==3 ); - testcase( pUp->eU==3 ); - JSON_VVA( pUp->eU = 3 ); - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; - } - } - } - }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } - } - } - return SQLITE_OK; - }else if( p->bRecursive ){ + if( p->bRecursive ){ u8 x; u8 levelChange = 0; u32 n, sz = 0; @@ -5866,70 +5779,6 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ return SQLITE_OK; } -/* Append an object label to the JSON Path being constructed -** in pStr. -*/ -static void jsonAppendObjectPathElementOfNode( - JsonString *pStr, - JsonNode *pNode -){ - int nn; - const char *z; - int bNeedQuote = 0; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - nn = pNode->n; - if( pNode->jnFlags & JNODE_RAW ){ - /* no-op */ - }else if( nn==0 || !sqlite3Isalpha(z[0]) ){ - bNeedQuote = 1; - }else{ - int jj; - for(jj=1; jjsParse.aNode ){ - /* LEGACY */ - assert( p->sParse.aUp ); - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - assert( pUp->eU==3 || (pUp->eU==0 && pUp->u.iKey==0) ); - testcase( pUp->eU==0 ); - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - jsonAppendObjectPathElementOfNode(pStr, pNode); - } - return; - } -} - /* Return the value of a column */ static int jsonEachColumn( sqlite3_vtab_cursor *cur, /* The cursor */ @@ -5937,173 +5786,73 @@ static int jsonEachColumn( int iColumn /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->sParse.aNode!=0 ){ - /* LEGACY */ - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( iColumn ){ - case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ - jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; - }else{ - iKey = p->iRowid; - } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); - } - break; - } - case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - break; - } - case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); - break; - } - case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturnFromNode(&p->sParse, pThis, ctx, 0); - break; - } - case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); - break; - } - case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); - } - break; - } - case JEACH_FULLKEY: { - JsonString x; - jsonStringInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - jsonAppendObjectPathElementOfNode(&x, pThis); - } - } - jsonReturnString(&x); - break; - } - case JEACH_PATH: { - if( p->bRecursive ){ - JsonString x; - jsonStringInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonReturnString(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - /* no break */ deliberate_fall_through - } - default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); - break; - } - case JEACH_JSON: { - if( p->sParse.isBinary ){ - sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); - }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); - } - break; + switch( iColumn ){ + case JEACH_KEY: { + if( p->nParent==0 ) break; + if( p->eType==JSONB_OBJECT ){ + jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); + }else{ + assert( p->eType==JSONB_ARRAY ); + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); } + break; } - }else{ - /* Blob scan */ - switch( iColumn ){ - case JEACH_KEY: { - if( p->nParent==0 ) break; - if( p->eType==JSONB_OBJECT ){ - jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); - }else{ - assert( p->eType==JSONB_ARRAY ); - sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); - } - break; - } - case JEACH_VALUE: { - u32 i = jsonSkipLabel(p); + case JEACH_VALUE: { + u32 i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, 1); + break; + } + case JEACH_TYPE: { + u32 i = jsonSkipLabel(p); + u8 eType = eType = p->sParse.aBlob[i] & 0x0f; + sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + u32 i = jsonSkipLabel(p); + if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); - break; - } - case JEACH_TYPE: { - u32 i = jsonSkipLabel(p); - u8 eType = eType = p->sParse.aBlob[i] & 0x0f; - sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); - break; - } - case JEACH_ATOM: { - u32 i = jsonSkipLabel(p); - if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); - } - break; - } - case JEACH_ID: { - sqlite3_result_int64(ctx, (sqlite3_int64)p->i); - break; - } - case JEACH_PARENT: { - if( p->nParent>0 ){ - sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); - } - break; - } - case JEACH_FULLKEY: { - u64 nBase = p->path.nUsed; - if( p->nParent ) jsonAppendPathName(p); - sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, - SQLITE_TRANSIENT, SQLITE_UTF8); - p->path.nUsed = nBase; - break; - } - case JEACH_PATH: { - sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, - SQLITE_TRANSIENT, SQLITE_UTF8); - break; - } - default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); - break; - } - case JEACH_JSON: { - if( p->sParse.isBinary ){ - sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, - SQLITE_STATIC); - }else{ - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); - } - break; } + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, (sqlite3_int64)p->i); + break; + } + case JEACH_PARENT: { + if( p->nParent>0 ){ + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); + } + break; + } + case JEACH_FULLKEY: { + u64 nBase = p->path.nUsed; + if( p->nParent ) jsonAppendPathName(p); + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + p->path.nUsed = nBase; + break; + } + case JEACH_PATH: { + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + break; + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + if( p->sParse.isBinary ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + } + break; } - } return SQLITE_OK; } @@ -6192,10 +5941,8 @@ static int jsonEachFilter( int argc, sqlite3_value **argv ){ JsonEachCursor *p = (JsonEachCursor*)cur; - const char *z; const char *zRoot = 0; - sqlite3_int64 n; - int isBinary; + u32 i, n, sz; UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(argc); @@ -6204,136 +5951,71 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; if( jsonFuncArgMightBeBinary(argv[0]) ){ - u32 i, n, sz; p->sParse.nBlob = sqlite3_value_bytes(argv[0]); p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); if( p->sParse.aBlob==0 ){ return SQLITE_NOMEM; } - if( idxNum==3 ){ - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - if( zRoot[0]!='$' ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); - jsonEachCursorReset(p); - return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else{ + p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); + p->sParse.nJson = sqlite3_value_bytes(argv[0]); + if( p->sParse.zJson==0 || jsonConvertTextToBlob(&p->sParse, 0) ){ + if( p->sParse.oom ){ + return SQLITE_NOMEM; } - if( zRoot[1]==0 ){ - i = p->i = 0; - p->eType = 0; - }else{ - i = jsonLookupBlobStep(&p->sParse, 0, zRoot+1, 0); - if( JSON_BLOB_ISERROR(i) ){ - p->i = 0; - p->eType = 0; - p->iEnd = 0; - return SQLITE_OK; - } - if( p->sParse.iLabel ){ - p->i = p->sParse.iLabel; - p->eType = JSONB_OBJECT; - }else{ - p->i = i; - p->eType = JSONB_ARRAY; - } - } - jsonAppendRaw(&p->path, zRoot, sqlite3Strlen30(zRoot)); - }else{ + p->i = p->iEnd = 0; + return SQLITE_OK; + } + } + if( idxNum==3 ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + if( zRoot[1]==0 ){ i = p->i = 0; p->eType = 0; - jsonAppendRaw(&p->path, "$", 1); - } - p->nParent = 0; - p->sParse.isBinary = 1; - n = jsonbPayloadSize(&p->sParse, i, &sz); - p->iEnd = i+n+sz; - if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ - p->i = i + n; - p->eType = p->sParse.aBlob[i] & 0x0f; - p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); - if( p->aParent==0 ) return SQLITE_NOMEM; - p->nParent = 1; - p->nParentAlloc = 1; - p->aParent[0].iKey = 0; - p->aParent[0].iEnd = p->iEnd; - p->aParent[0].iHead = p->i; - p->aParent[0].iValue = i; - } - return SQLITE_OK; - }else{ - z = (const char*)sqlite3_value_text(argv[0]); - isBinary = 0; - } - if( z==0 ) return SQLITE_OK; - if( sqlite3ValueIsOfClass(argv[0], sqlite3RCStrUnref) ){ - p->sParse.zJson = sqlite3RCStrRef((char*)z); - }else{ - n = sqlite3_value_bytes(argv[0]); - p->sParse.zJson = sqlite3RCStrNew( n+1 ); - if( p->sParse.zJson==0 ) return SQLITE_NOMEM; - memcpy(p->sParse.zJson, z, (size_t)n+(isBinary==0)); - p->sParse.nJson = n; - } - p->sParse.bJsonIsRCStr = 1; - p->sParse.isBinary = isBinary; - p->zJson = p->sParse.zJson; - if( jsonParse(&p->sParse, 0) ){ - int rc = SQLITE_NOMEM; - if( p->sParse.oom==0 ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; - } - jsonEachCursorReset(p); - return rc; - }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ - jsonEachCursorReset(p); - return SQLITE_NOMEM; - }else{ - JsonNode *pNode = 0; - if( idxNum==3 ){ - const char *zErr = 0; - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); - if( zRoot[0]!='$' ){ - zErr = zRoot; - }else{ - pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); - } - if( zErr ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr, 0); - jsonEachCursorReset(p); - return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; - }else if( pNode==0 ){ + }else{ + i = jsonLookupBlobStep(&p->sParse, 0, zRoot+1, 0); + if( JSON_BLOB_ISERROR(i) ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; return SQLITE_OK; } - }else{ - pNode = p->sParse.aNode; - } - p->iBegin = p->i = (int)(pNode - p->sParse.aNode); - p->eType = pNode->eType; - if( p->eType>=JSON_ARRAY ){ - assert( pNode->eU==0 ); - JSON_VVA( pNode->eU = 3 ); - pNode->u.iKey = 0; - p->iEnd = p->i + pNode->n + 1; - if( p->bRecursive ){ - p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; - if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ - p->i--; - } + if( p->sParse.iLabel ){ + p->i = p->sParse.iLabel; + p->eType = JSONB_OBJECT; }else{ - p->i++; + p->i = i; + p->eType = JSONB_ARRAY; } - }else{ - p->iEnd = p->i+1; } + jsonAppendRaw(&p->path, zRoot, sqlite3Strlen30(zRoot)); + }else{ + i = p->i = 0; + p->eType = 0; + jsonAppendRaw(&p->path, "$", 1); + } + p->nParent = 0; + p->sParse.isBinary = 1; + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->iEnd = i+n+sz; + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ + p->i = i + n; + p->eType = p->sParse.aBlob[i] & 0x0f; + p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); + if( p->aParent==0 ) return SQLITE_NOMEM; + p->nParent = 1; + p->nParentAlloc = 1; + p->aParent[0].iKey = 0; + p->aParent[0].iEnd = p->iEnd; + p->aParent[0].iHead = p->i; + p->aParent[0].iValue = i; } return SQLITE_OK; } From e09a38c2e830b0311405cb6b442aa6f4d07583fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Nov 2023 23:00:50 +0000 Subject: [PATCH 099/191] Remove unused elements from the json_tree() cursor. FossilOrigin-Name: 914a50117d477b2cd30d58388fb8d1b71ff7ff6842ba025f38efc6e9647d06d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 14 ++++---------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 1f18d63513..17f760ca91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\svestigal\sJsonNode\slogic\sfrom\sjson_tree()\sand\sjson_each(). -D 2023-11-25T20:59:03.008 +C Remove\sunused\selements\sfrom\sthe\sjson_tree()\scursor. +D 2023-11-25T23:00:50.622 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 211c4e1b8fd886c97d3224105bfef9d4cc04db2e3f1daca1e95c52343df4e85c +F src/json.c 7bd0d3809b31e32ffa053ba68cf03f67121f6bdd0be7031e8d4c761bef9b6aee F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92258246916a9c0d72785964513113848a850dec78bdade8b3f274e410df4e7e -R 43fba60a847876455ec8abde78205ef3 +P 66c2ab9ebbf90477742e6be0d30e061d827c409de038f2a5b73479ed9448c4a6 +R 79cb22ebedffb4f5b0a0989ee5c920ed U drh -Z d0660c039b12ecd1fe4ffd828711a405 +Z b95c9f50e6b9041cf41818958255b068 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3b93842124..a22ec07c45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66c2ab9ebbf90477742e6be0d30e061d827c409de038f2a5b73479ed9448c4a6 \ No newline at end of file +914a50117d477b2cd30d58388fb8d1b71ff7ff6842ba025f38efc6e9647d06d0 \ No newline at end of file diff --git a/src/json.c b/src/json.c index cc4eb83c5b..1a2a5bd9ef 100644 --- a/src/json.c +++ b/src/json.c @@ -5540,16 +5540,14 @@ typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ u32 iRowid; /* The rowid */ - u32 iBegin; /* The first node of the scan */ u32 i; /* Index in sParse.aBlob[] of current row */ u32 iEnd; /* EOF when i equals or exceeds this value */ + u32 nRoot; /* Size of the root argument in bytes */ u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ u32 nParent; /* Current nesting depth */ u32 nParentAlloc; /* Space allocated for aParent[] */ JsonParent *aParent; /* Parent elements of i */ - char *zJson; /* Input JSON */ - char *zRoot; /* Path by which to filter zJson */ sqlite3 *db; /* Database connection */ JsonString path; /* Current path */ JsonParse sParse; /* Parse of the input JSON */ @@ -5638,7 +5636,6 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); jsonStringReset(&p->path); sqlite3DbFree(p->db, p->aParent); @@ -5649,8 +5646,6 @@ static void jsonEachCursorReset(JsonEachCursor *p){ p->nParentAlloc = 0; p->iEnd = 0; p->eType = 0; - p->zJson = 0; - p->zRoot = 0; } /* Destructor for a jsonEachCursor object */ @@ -5839,9 +5834,7 @@ static int jsonEachColumn( break; } default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_STATIC); break; } case JEACH_JSON: { @@ -5956,6 +5949,7 @@ static int jsonEachFilter( if( p->sParse.aBlob==0 ){ return SQLITE_NOMEM; } + p->sParse.isBinary = 1; }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); @@ -6001,8 +5995,8 @@ static int jsonEachFilter( p->eType = 0; jsonAppendRaw(&p->path, "$", 1); } + p->nRoot = p->path.nUsed; p->nParent = 0; - p->sParse.isBinary = 1; n = jsonbPayloadSize(&p->sParse, i, &sz); p->iEnd = i+n+sz; if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ From 50b37832b250acb3fc55b8f177935ab7af4eef48 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Nov 2023 00:48:37 +0000 Subject: [PATCH 100/191] Same results as the legacy JsonNode implementation on a small set of test cases. FossilOrigin-Name: c3da4b079a1a15a4c0b1a6e71f876648b1d9eb32eddc67b9946c2475c7b6d085 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 62 ++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 17f760ca91..4ae8c18715 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\selements\sfrom\sthe\sjson_tree()\scursor. -D 2023-11-25T23:00:50.622 +C Same\sresults\sas\sthe\slegacy\sJsonNode\simplementation\son\sa\ssmall\sset\sof\stest\scases. +D 2023-11-26T00:48:37.511 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7bd0d3809b31e32ffa053ba68cf03f67121f6bdd0be7031e8d4c761bef9b6aee +F src/json.c b93f282f032f904118121ae64552972f078ac13adcd7fc982d20f596f97a424a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 66c2ab9ebbf90477742e6be0d30e061d827c409de038f2a5b73479ed9448c4a6 -R 79cb22ebedffb4f5b0a0989ee5c920ed +P 914a50117d477b2cd30d58388fb8d1b71ff7ff6842ba025f38efc6e9647d06d0 +R d2ae12035201ed7d53848ee421edc8df U drh -Z b95c9f50e6b9041cf41818958255b068 +Z e4ce705fdf38aad5873f97f479b0490a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a22ec07c45..a5cbb721e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -914a50117d477b2cd30d58388fb8d1b71ff7ff6842ba025f38efc6e9647d06d0 \ No newline at end of file +c3da4b079a1a15a4c0b1a6e71f876648b1d9eb32eddc67b9946c2475c7b6d085 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1a2a5bd9ef..14fe274146 100644 --- a/src/json.c +++ b/src/json.c @@ -5542,7 +5542,7 @@ struct JsonEachCursor { u32 iRowid; /* The rowid */ u32 i; /* Index in sParse.aBlob[] of current row */ u32 iEnd; /* EOF when i equals or exceeds this value */ - u32 nRoot; /* Size of the root argument in bytes */ + u32 nRoot; /* Size of the root path in bytes */ u8 eType; /* Type of the container for element i */ u8 bRecursive; /* True for json_tree(). False for json_each() */ u32 nParent; /* Current nesting depth */ @@ -5760,20 +5760,44 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ p->eType = 0; } } - if( p->eType==JSONB_ARRAY ){ - assert( p->nParent>0 ); - p->aParent[p->nParent-1].iKey++; - } }else{ u32 n, sz = 0; u32 i = jsonSkipLabel(p); n = jsonbPayloadSize(&p->sParse, i, &sz); p->i = i + n + sz; } + if( p->eType==JSONB_ARRAY ){ + assert( p->nParent>0 ); + p->aParent[p->nParent-1].iKey++; + } p->iRowid++; return SQLITE_OK; } +/* Length of the path for rowid==0 in bRecursive mode. +*/ +static int jsonEachPathLength(JsonEachCursor *p){ + u32 n = p->path.nUsed; + if( p->iRowid==0 && p->bRecursive && n>1 ){ + if( p->path.zBuf[n-1]==']' ){ + do{ + n--; + assert( n>0 ); + }while( p->path.zBuf[n]!='[' ); + }else{ + u32 sz = 0; + jsonbPayloadSize(&p->sParse, p->i, &sz); + if( p->path.zBuf[n-1]=='"' ) sz += 2; + n -= sz; + while( p->path.zBuf[n]!='.' ){ + n--; + assert( n>0 ); + } + } + } + return n; +} + /* Return the value of a column */ static int jsonEachColumn( sqlite3_vtab_cursor *cur, /* The cursor */ @@ -5783,7 +5807,23 @@ static int jsonEachColumn( JsonEachCursor *p = (JsonEachCursor*)cur; switch( iColumn ){ case JEACH_KEY: { - if( p->nParent==0 ) break; + if( p->nParent==0 ){ + u32 n, j; + assert( p->iRowid==0 && p->bRecursive ); + if( p->nRoot==1 ) break; + j = jsonEachPathLength(p); + n = p->nRoot - j; + if( p->path.zBuf[j]=='[' ){ + i64 x; + sqlite3Atoi64(&p->path.zBuf[j+1], &x, n-1, SQLITE_UTF8); + sqlite3_result_int64(ctx, x); + }else if( p->path.zBuf[j+1]=='"' ){ + sqlite3_result_text(ctx, &p->path.zBuf[j+2], n-3, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(ctx, &p->path.zBuf[j+1], n-1, SQLITE_TRANSIENT); + } + break; + } if( p->eType==JSONB_OBJECT ){ jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); }else{ @@ -5815,7 +5855,7 @@ static int jsonEachColumn( break; } case JEACH_PARENT: { - if( p->nParent>0 ){ + if( p->nParent>0 && p->bRecursive ){ sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); } break; @@ -5829,7 +5869,8 @@ static int jsonEachColumn( break; } case JEACH_PATH: { - sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + u32 n = jsonEachPathLength(p); + sqlite3_result_text64(ctx, p->path.zBuf, n, SQLITE_TRANSIENT, SQLITE_UTF8); break; } @@ -5970,6 +6011,7 @@ static int jsonEachFilter( jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } + p->nRoot = sqlite3_value_bytes(argv[1]); if( zRoot[1]==0 ){ i = p->i = 0; p->eType = 0; @@ -5989,13 +6031,13 @@ static int jsonEachFilter( p->eType = JSONB_ARRAY; } } - jsonAppendRaw(&p->path, zRoot, sqlite3Strlen30(zRoot)); + jsonAppendRaw(&p->path, zRoot, p->nRoot); }else{ i = p->i = 0; p->eType = 0; + p->nRoot = 1; jsonAppendRaw(&p->path, "$", 1); } - p->nRoot = p->path.nUsed; p->nParent = 0; n = jsonbPayloadSize(&p->sParse, i, &sz); p->iEnd = i+n+sz; From 15c0b03c5dcccd8a5c3282b7b961dbfc9fda7760 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Nov 2023 00:56:40 +0000 Subject: [PATCH 101/191] Fix corner-case error conditions. FossilOrigin-Name: ec23d34ab75e1d7e9366e59c633e0d30def8759f6d4717583ebeb4c90aeccf0d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4ae8c18715..7736b5bcaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Same\sresults\sas\sthe\slegacy\sJsonNode\simplementation\son\sa\ssmall\sset\sof\stest\scases. -D 2023-11-26T00:48:37.511 +C Fix\scorner-case\serror\sconditions. +D 2023-11-26T00:56:40.131 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c b93f282f032f904118121ae64552972f078ac13adcd7fc982d20f596f97a424a +F src/json.c 59dd8bf951f0c7b2e1004f406f8690d9743146f176d675da06427f99c75c78f7 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 914a50117d477b2cd30d58388fb8d1b71ff7ff6842ba025f38efc6e9647d06d0 -R d2ae12035201ed7d53848ee421edc8df +P c3da4b079a1a15a4c0b1a6e71f876648b1d9eb32eddc67b9946c2475c7b6d085 +R 6a83cb0634a1b413d8dd9f72a2caf803 U drh -Z e4ce705fdf38aad5873f97f479b0490a +Z cf069d3ef952e41c66c8bcc7a7e83b20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a5cbb721e5..90bef2caad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3da4b079a1a15a4c0b1a6e71f876648b1d9eb32eddc67b9946c2475c7b6d085 \ No newline at end of file +ec23d34ab75e1d7e9366e59c633e0d30def8759f6d4717583ebeb4c90aeccf0d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 14fe274146..9317608267 100644 --- a/src/json.c +++ b/src/json.c @@ -416,7 +416,7 @@ static void jsonStringReset(JsonString *p){ */ static void jsonStringOom(JsonString *p){ p->eErr |= JSTRING_OOM; - sqlite3_result_error_nomem(p->pCtx); + if( p->pCtx ) sqlite3_result_error_nomem(p->pCtx); jsonStringReset(p); } @@ -5766,8 +5766,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ n = jsonbPayloadSize(&p->sParse, i, &sz); p->i = i + n + sz; } - if( p->eType==JSONB_ARRAY ){ - assert( p->nParent>0 ); + if( p->eType==JSONB_ARRAY && p->nParent ){ p->aParent[p->nParent-1].iKey++; } p->iRowid++; @@ -5809,7 +5808,6 @@ static int jsonEachColumn( case JEACH_KEY: { if( p->nParent==0 ){ u32 n, j; - assert( p->iRowid==0 && p->bRecursive ); if( p->nRoot==1 ) break; j = jsonEachPathLength(p); n = p->nRoot - j; From b4e5bc6cdb3fb282e735706dbceb888ac500ac23 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Nov 2023 12:30:55 +0000 Subject: [PATCH 102/191] All tests passing. FossilOrigin-Name: b5a5660ca22437640c9bf32c44d92c76a7293dafcbaf4fa6a4c171128d64871d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 38 ++++++++++++++++++++++++++++---------- test/json101.test | 8 ++++---- 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 7736b5bcaf..8e410950f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scorner-case\serror\sconditions. -D 2023-11-26T00:56:40.131 +C All\stests\spassing. +D 2023-11-27T12:30:55.729 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 59dd8bf951f0c7b2e1004f406f8690d9743146f176d675da06427f99c75c78f7 +F src/json.c efd9ec1cb0f0cbee3e43ebaba69469b3d3f44ee623b53b1ce72361da42e05fbe F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1325,7 +1325,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 2420fbed1d2dddb9562f8b9d0ac540849b99deba74e869560f94f674ef24171c +F test/json101.test 19f9abc77c33151bd7fdb7850fb473cb0154f229b863b740590ad9de807a032d F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3da4b079a1a15a4c0b1a6e71f876648b1d9eb32eddc67b9946c2475c7b6d085 -R 6a83cb0634a1b413d8dd9f72a2caf803 +P ec23d34ab75e1d7e9366e59c633e0d30def8759f6d4717583ebeb4c90aeccf0d +R 6545b38a1b858ef8dae597ee252577f2 U drh -Z cf069d3ef952e41c66c8bcc7a7e83b20 +Z b98cdf654b65fc79957da28b6036dc53 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 90bef2caad..050c01db3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec23d34ab75e1d7e9366e59c633e0d30def8759f6d4717583ebeb4c90aeccf0d \ No newline at end of file +b5a5660ca22437640c9bf32c44d92c76a7293dafcbaf4fa6a4c171128d64871d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9317608267..0e3f6fe575 100644 --- a/src/json.c +++ b/src/json.c @@ -5715,6 +5715,7 @@ static void jsonAppendPathName(JsonEachCursor *p){ /* Advance the cursor to the next element for json_tree() */ static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; + int rc = SQLITE_OK; if( p->bRecursive ){ u8 x; u8 levelChange = 0; @@ -5740,7 +5741,10 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ pParent->iEnd = i + n + sz; pParent->iKey = -1; pParent->nPath = (u32)p->path.nUsed; - if( p->eType && p->nParent ) jsonAppendPathName(p); + if( p->eType && p->nParent ){ + jsonAppendPathName(p); + if( p->path.eErr ) rc = SQLITE_NOMEM; + } p->nParent++; p->i = i + n; }else{ @@ -5770,7 +5774,7 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ p->aParent[p->nParent-1].iKey++; } p->iRowid++; - return SQLITE_OK; + return rc; } /* Length of the path for rowid==0 in bRecursive mode. @@ -5811,7 +5815,9 @@ static int jsonEachColumn( if( p->nRoot==1 ) break; j = jsonEachPathLength(p); n = p->nRoot - j; - if( p->path.zBuf[j]=='[' ){ + if( n==0 ){ + break; + }else if( p->path.zBuf[j]=='[' ){ i64 x; sqlite3Atoi64(&p->path.zBuf[j+1], &x, n-1, SQLITE_UTF8); sqlite3_result_int64(ctx, x); @@ -5992,12 +5998,18 @@ static int jsonEachFilter( }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); - if( p->sParse.zJson==0 || jsonConvertTextToBlob(&p->sParse, 0) ){ + if( p->sParse.zJson==0 ){ + p->i = p->iEnd = 0; + return SQLITE_OK; + } + if( jsonConvertTextToBlob(&p->sParse, 0) ){ if( p->sParse.oom ){ return SQLITE_NOMEM; } - p->i = p->iEnd = 0; - return SQLITE_OK; + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } } if( idxNum==3 ){ @@ -6016,10 +6028,16 @@ static int jsonEachFilter( }else{ i = jsonLookupBlobStep(&p->sParse, 0, zRoot+1, 0); if( JSON_BLOB_ISERROR(i) ){ - p->i = 0; - p->eType = 0; - p->iEnd = 0; - return SQLITE_OK; + if( i==JSON_BLOB_NOTFOUND ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; + return SQLITE_OK; + } + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } if( p->sParse.iLabel ){ p->i = p->sParse.iLabel; diff --git a/test/json101.test b/test/json101.test index 426746377c..280d467393 100644 --- a/test/json101.test +++ b/test/json101.test @@ -925,16 +925,16 @@ do_execsql_test json101-14.170 { # do_execsql_test json101-15.100 { SELECT * FROM JSON_EACH('{"a":1, "b":2}'); -} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}} do_execsql_test json101-15.110 { SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz; -} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}} do_execsql_test json101-15.120 { SELECT * FROM (JSON_EACH('{"a":1, "b":2}')); -} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}} do_execsql_test json101-15.130 { SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz; -} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +} {a 1 integer 1 1 {} {$.a} {$} b 2 integer 2 5 {} {$.b} {$}} # 2019-11-10 # Mailing list bug report on the handling of surrogate pairs From 821a4c9f8c681faaa45c131ae2c9be455dd3097e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Nov 2023 15:57:11 +0000 Subject: [PATCH 103/191] Give the json_valid() function an optional second argument that determines what is meant by "valid". FossilOrigin-Name: a4e19ad43dac81e7655ec03ff69bb99d1d02b0c227034c90fb41415fd4793fe3 --- manifest | 15 +++---- manifest.uuid | 2 +- src/json.c | 111 +++++++++++++++++++++++++++++++++++++--------- test/json101.test | 2 +- 4 files changed, 98 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 17d296c97f..188b5f5277 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\sjson_tree()\sand\sjson_each()\svirtual\stables\sover\sto\suse\sJSONB\shas\ntheir\sinternal\srepresentation. -D 2023-11-27T12:36:29.017 +C Give\sthe\sjson_valid()\sfunction\san\soptional\ssecond\sargument\sthat\sdetermines\nwhat\sis\smeant\sby\s"valid". +D 2023-11-27T15:57:11.976 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c efd9ec1cb0f0cbee3e43ebaba69469b3d3f44ee623b53b1ce72361da42e05fbe +F src/json.c 6c1de0d7802a15669a78008125b409b67aa23eefcacfc52436d713419d09be9e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1325,7 +1325,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 19f9abc77c33151bd7fdb7850fb473cb0154f229b863b740590ad9de807a032d +F test/json101.test 643936557daeb2eeeb839e9d87a682c7d7c6f6aee66045eedd8613cb8471d8d8 F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2145,9 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab2644aacf4757a51cf62e05cff6711a0a3605d60502a3dd310887df1b993545 b5a5660ca22437640c9bf32c44d92c76a7293dafcbaf4fa6a4c171128d64871d -R 6545b38a1b858ef8dae597ee252577f2 -T +closed b5a5660ca22437640c9bf32c44d92c76a7293dafcbaf4fa6a4c171128d64871d +P ec18caa3f7790b780dde66c1ccbb6eb09d2f1507629cc45955fc1b08380b4017 +R ae458da46f597856d9232f7ad8297973 U drh -Z 9c0c7326e55f087530f0f53e8712d89c +Z 6c8b34340e171672e2ebcad369756a23 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 189e2ccc7a..c418cd9780 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec18caa3f7790b780dde66c1ccbb6eb09d2f1507629cc45955fc1b08380b4017 \ No newline at end of file +a4e19ad43dac81e7655ec03ff69bb99d1d02b0c227034c90fb41415fd4793fe3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0e3f6fe575..1107a24f4c 100644 --- a/src/json.c +++ b/src/json.c @@ -5241,15 +5241,60 @@ static void jsonTypeFunc( /* ** json_valid(JSON) +** json_valid(JSON, FLAGS) ** -** Return 1 if the argument is one of: +** Check the JSON argument to see if it is well-formed. The FLAGS argument +** encodes the various constraints on what is meant by "well-formed": ** -** * A well-formed canonical JSON string according to RFC-8259 -** (without JSON5 enhancements), or +** 0x01 Canonical RFC-8259 JSON text +** 0x02 JSON text with optional JSON-5 extensions +** 0x04 Superficially appears to be JSONB +** 0x08 Strictly well-formed JSONB ** -** * A BLOB that plausibly could be a JSONB value. +** If the FLAGS argument is omitted, it defaults to 1. Useful values for +** FLAGS include: ** -** Return 0 otherwise. +** 1 Strict canonical JSON text +** 2 JSON text perhaps with JSON-5 extensions +** 4 Superficially appears to be JSONB +** 5 Canonical JSON text or superficial JSONB +** 6 JSON-5 text or superficial JSONB +** 8 Strict JSONB +** 9 Canonical JSON text or strict JSONB +** 10 JSON-5 text or strict JSONB +** +** Other flag combinations are redundant. For example, every canonical +** JSON text is also well-formed JSON-5 text, so FLAG values 2 and 3 +** are the same. Similarly, any input that passes a strict JSONB validation +** will also pass the superficial validation so 12 through 15 are the same +** as 8 through 11 respectively. +** +** This routine runs in linear time to validate text and when doing strict +** JSONB validation. Superficial JSONB validation is constant time, +** assuming the BLOB is already in memory. The performance advantage +** of superficial JSONB validation is why that option is provided. +** Application developers can choose to do fast superficial validation or +** slower strict validation, according to their specific needs. +** +** Only the lower four bits of the FLAGS argument are currently used. +** Higher bits are reserved for future expansion. To facilitate +** compatibility, the current implementation raises an error if any bit +** in FLAGS is set other than the lower four bits. +** +** The original circa 2015 implementation of the JSON routines in +** SQLite only supported canonical RFC-8259 JSON text and the json_valid() +** function only accepted one argument. That is why the default value +** for the FLAGS argument is 1, since FLAGS=1 causes this routine to only +** recognize canonical RFC-8259 JSON text as valid. The extra FLAGS +** argument was added when the JSON routines were extended to support +** JSON5-like extensions and binary JSONB stored in BLOBs. +** +** Return Values: +** +** * Raise an error if FLAGS is outside the range of 1 to 15. +** * Return NULL if the input is NULL +** * Return 1 if the input is well-formed. +** * Return 0 if the input is not well-formed. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -5257,26 +5302,47 @@ static void jsonValidFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + u8 flags = 1; + u8 res = 0; + if( argc==2 ){ + i64 f = sqlite3_value_int64(argv[1]); + if( f<1 || f>15 ){ + sqlite3_result_error(ctx, "FLAGS parameter to json_valid() must be" + " between 1 and 15", -1); + return; + } + flags = f & 0x0f; + } + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_NULL: { #ifdef SQLITE_LEGACY_JSON_VALID - /* Incorrect legacy behavior was to return FALSE for a NULL input */ - sqlite3_result_int(ctx, 0); + /* Incorrect legacy behavior was to return FALSE for a NULL input */ + sqlite3_result_int(ctx, 0); #endif - return; - } - if( jsonFuncArgMightBeBinary(argv[0]) ){ - sqlite3_result_int(ctx, 1); - return; - } - p = jsonParseCached(ctx, argv[0], 0, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); - }else{ - sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod)); - if( p->nErr ) jsonParseFree(p); + return; + } + case SQLITE_BLOB: { + if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ + /* TO-DO: strict checking if flags & 0x08 */ + res = 1; + } + break; + } + default: { + if( (flags & 0x3)==0 ) break; + p = jsonParseCached(ctx, argv[0], 0, 0); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else if( p->nErr ){ + jsonParseFree(p); + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 || p->useMod ){ + res = 1; + } + break; + } } + sqlite3_result_int(ctx, res); } /* @@ -6171,6 +6237,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1,1,0, 0,0,0, jsonTypeFunc), JFUNCTION(json_type, 2,1,0, 0,0,0, jsonTypeFunc), JFUNCTION(json_valid, 1,1,0, 0,0,0, jsonValidFunc), + JFUNCTION(json_valid, 2,1,0, 0,0,0, jsonValidFunc), #if SQLITE_DEBUG JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), JFUNCTION(json_test1, 1,1,0, 1,0,0, jsonTest1Func), diff --git a/test/json101.test b/test/json101.test index 280d467393..ebaf0f3b34 100644 --- a/test/json101.test +++ b/test/json101.test @@ -324,7 +324,7 @@ do_execsql_test json101-5.2 { SELECT id, json_valid(json), json_type(json), '|' FROM j2 ORDER BY id; } {1 1 object | 2 1 object | 3 1 array |} do_execsql_test json101-5.2b { - SELECT id, json_valid(json), json_type(json), '|' FROM j2b ORDER BY id; + SELECT id, json_valid(json,5), json_type(json), '|' FROM j2b ORDER BY id; } {1 1 object | 2 1 object | 3 1 array |} ifcapable !vtab { From 8a3034add8dac022f58706bbcdbd076c624f8d11 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Nov 2023 17:13:18 +0000 Subject: [PATCH 104/191] Enhance the (SQLITE_DEBUG-only) json_parse() routine so that it shows a decoding of JSONB when given a BLOB argument. FossilOrigin-Name: af267868562e0799ad691dccad05f17afbc34d609eede8c55f57d209290246ef --- manifest | 12 +++---- manifest.uuid | 2 +- src/json.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 188b5f5277..0989187da0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Give\sthe\sjson_valid()\sfunction\san\soptional\ssecond\sargument\sthat\sdetermines\nwhat\sis\smeant\sby\s"valid". -D 2023-11-27T15:57:11.976 +C Enhance\sthe\s(SQLITE_DEBUG-only)\sjson_parse()\sroutine\sso\sthat\sit\sshows\sa\s\ndecoding\sof\sJSONB\swhen\sgiven\sa\sBLOB\sargument. +D 2023-11-27T17:13:18.242 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6c1de0d7802a15669a78008125b409b67aa23eefcacfc52436d713419d09be9e +F src/json.c aefd3bf7c225ad03e25680febb2659779a99b227cd98222270e16719b1251bdb F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec18caa3f7790b780dde66c1ccbb6eb09d2f1507629cc45955fc1b08380b4017 -R ae458da46f597856d9232f7ad8297973 +P a4e19ad43dac81e7655ec03ff69bb99d1d02b0c227034c90fb41415fd4793fe3 +R 2b39f71294c83f1ae33c269e29617f78 U drh -Z 6c8b34340e171672e2ebcad369756a23 +Z ba17737c2ff9c3cb6da75654e948a46b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c418cd9780..5a42393c10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e19ad43dac81e7655ec03ff69bb99d1d02b0c227034c90fb41415fd4793fe3 \ No newline at end of file +af267868562e0799ad691dccad05f17afbc34d609eede8c55f57d209290246ef \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1107a24f4c..e672921521 100644 --- a/src/json.c +++ b/src/json.c @@ -4490,6 +4490,84 @@ static void jsonDebugPrintNodeEntries( } #endif /* SQLITE_DEBUG */ +#if SQLITE_DEBUG +/* +** Decode JSONB bytes in aBlob[] starting at iStart through but not +** including iEnd. Indent the +** content by nIndent spaces. +*/ +static void jsonDebugPrintBlob( + JsonParse *pParse, /* JSON content */ + u32 iStart, /* Start rendering here */ + u32 iEnd, /* Do not render this byte or any byte after this one */ + int nIndent /* Indent by this many spaces */ +){ + while( iStartaBlob[iStart] & 0x0f; + printf("%5d:%*s", iStart, nIndent, ""); + nn = n = jsonbPayloadSize(pParse, iStart, &sz); + if( nn==0 ) nn = 1; + if( sz>0 && xaBlob[iStart+i]); + if( n==0 || iStart+n+sz>iEnd ){ + printf(" ERROR invalid node size\n"); + iStart = n==0 ? iStart+1 : iEnd; + continue; + } + printf(" <-- "); + switch( x ){ + case JSONB_NULL: printf("null"); break; + case JSONB_TRUE: printf("true"); break; + case JSONB_FALSE: printf("false"); break; + case JSONB_INT: printf("int"); break; + case JSONB_INT5: printf("int5"); break; + case JSONB_FLOAT: printf("float"); break; + case JSONB_FLOAT5: printf("float5"); break; + case JSONB_TEXT: printf("text"); break; + case JSONB_TEXTJ: printf("textj"); break; + case JSONB_TEXT5: printf("text5"); break; + case JSONB_TEXTRAW: printf("textraw"); break; + case JSONB_ARRAY: { + printf("array, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2); + showContent = 0; + break; + } + case JSONB_OBJECT: { + printf("object, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2); + showContent = 0; + break; + } + default: { + printf("ERROR: unknown node type\n"); + showContent = 0; + break; + } + } + if( showContent ){ + if( sz==0 && x<=JSON_FALSE ){ + printf("\n"); + }else{ + u32 i; + printf(": \""); + for(i=iStart+n; iaBlob[i]; + if( c<0x20 || c>=0x7f ) c = '.'; + putchar(c); + } + printf("\"\n"); + } + } + iStart += n + sz; + } +} +#endif /* SQLITE_DEBUG */ + #if 0 /* 1 for debugging. 0 normally. Requires -DSQLITE_DEBUG too */ static void jsonDebugPrintParse(JsonParse *p){ @@ -4520,6 +4598,14 @@ static void jsonParseFunc( JsonParse *p; /* The parse */ assert( argc==1 ); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + JsonParse x; + memset(&x, 0, sizeof(x)); + x.nBlob = sqlite3_value_bytes(argv[0]); + x.aBlob = (u8*)sqlite3_value_blob(argv[0]); + jsonDebugPrintBlob(&x, 0, x.nBlob, 0); + return; + } p = jsonParseCached(ctx, argv[0], ctx, 0); if( p==0 ) return; printf("nNode = %u\n", p->nNode); From eb04a0bb7b7bb111adb0ab2554445998244b7002 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Nov 2023 23:46:12 +0000 Subject: [PATCH 105/191] Add untested (#ifdefed-out) code for the MergePatch algorithm against JSONB. Add (and test) the jsonBlobEdit() routine that is needed by the new MergePatch. FossilOrigin-Name: 4d353387fc10e1038cfdd86e66007bf728c231a928e588897bbee0fbfe76f225 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 240 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 224 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 0989187da0..6ae9cbf521 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s(SQLITE_DEBUG-only)\sjson_parse()\sroutine\sso\sthat\sit\sshows\sa\s\ndecoding\sof\sJSONB\swhen\sgiven\sa\sBLOB\sargument. -D 2023-11-27T17:13:18.242 +C Add\suntested\s(#ifdefed-out)\scode\sfor\sthe\sMergePatch\salgorithm\sagainst\sJSONB.\nAdd\s(and\stest)\sthe\sjsonBlobEdit()\sroutine\sthat\sis\sneeded\sby\sthe\snew\sMergePatch. +D 2023-11-27T23:46:12.954 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c aefd3bf7c225ad03e25680febb2659779a99b227cd98222270e16719b1251bdb +F src/json.c 7d6387920736cd9b78a4211771265c4c574284613d8e63f43b3ce7bc160da498 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4e19ad43dac81e7655ec03ff69bb99d1d02b0c227034c90fb41415fd4793fe3 -R 2b39f71294c83f1ae33c269e29617f78 +P af267868562e0799ad691dccad05f17afbc34d609eede8c55f57d209290246ef +R 47ffcc493583e45ee7703e45cf1805e3 U drh -Z ba17737c2ff9c3cb6da75654e948a46b +Z 2cedbcb9affb9f675b8d832bfea283a4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5a42393c10..a4814c0752 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af267868562e0799ad691dccad05f17afbc34d609eede8c55f57d209290246ef \ No newline at end of file +4d353387fc10e1038cfdd86e66007bf728c231a928e588897bbee0fbfe76f225 \ No newline at end of file diff --git a/src/json.c b/src/json.c index e672921521..99205decf1 100644 --- a/src/json.c +++ b/src/json.c @@ -3801,6 +3801,41 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ jsonBlobChangePayloadSize(pParse, iRoot, sz); } +/* +** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of +** content beginning at iDel, and replacing them with nIns bytes of +** content given by aIns. +** +** nDel may be zero, in which case no bytes are removed. But iDel is +** still important as new bytes will be insert beginning at iDel. +** +** nIns may be zero, in which case no new bytes are inserted. aIns might +** be a NULL pointer in this case. +** +** Set pParse->oom if an OOM occurs. +*/ +static void jsonBlobEdit( + JsonParse *pParse, /* The JSONB to be modified is in pParse->aBlob */ + u32 iDel, /* First byte to be removed */ + u32 nDel, /* Number of bytes to remove */ + const u8 *aIns, /* Content to insert */ + u32 nIns /* Bytes of content to insert */ +){ + i64 d = (i64)nIns - (i64)nDel; + if( d!=0 ){ + if( pParse->nBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return; + } + memmove(&pParse->aBlob[iDel+nIns], + &pParse->aBlob[iDel+nDel], + pParse->nBlob - (iDel+nDel)); + pParse->nBlob += d; + pParse->delta += d; + } + if( nIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); +} + /* ** Error returns from jsonLookupBlobStep() */ @@ -3834,27 +3869,12 @@ static u32 jsonLookupBlobStep( sz += iRoot - iLabel; iRoot = iLabel; } - memmove(&pParse->aBlob[iRoot], &pParse->aBlob[iRoot+sz], - pParse->nBlob - (iRoot+sz)); - pParse->delta = -(int)sz; - pParse->nBlob -= sz; + jsonBlobEdit(pParse, iRoot, sz, 0, 0); }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ }else{ /* json_set() or json_replace() */ - int d = (int)pParse->nIns - (int)sz; - pParse->delta = d; - if( d!=0 ){ - if( pParse->nBlob + d > pParse->nBlobAlloc ){ - jsonBlobExpand(pParse, pParse->nBlob+d); - if( pParse->oom ) return iRoot; - } - memmove(&pParse->aBlob[iRoot+pParse->nIns], - &pParse->aBlob[iRoot+sz], - pParse->nBlob - iRoot - sz); - pParse->nBlob += d; - } - memcpy(&pParse->aBlob[iRoot], pParse->aIns, pParse->nIns); + jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); } } pParse->iLabel = iLabel; @@ -3920,6 +3940,8 @@ static u32 jsonLookupBlobStep( ix.nBlobAlloc = sizeof(aLabel); jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey) ){ + /* This is similar to jsonBlobEdit() except that the inserted + ** bytes come from two different places, ix.aBlob and pParse->aBlob. */ nIns = ix.nBlob + nKey + pParse->nIns; assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); memmove(&pParse->aBlob[j+nIns], &pParse->aBlob[j], @@ -3984,12 +4006,7 @@ static u32 jsonLookupBlobStep( if( pParse->eEdit>=JEDIT_INS && jsonBlobMakeEditable(pParse, 0) ){ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); - assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); - memmove(&pParse->aBlob[j+pParse->nIns], &pParse->aBlob[j], - pParse->nBlob - j); - memcpy(&pParse->aBlob[j], pParse->aIns, pParse->nIns); - pParse->delta = pParse->nIns; - pParse->nBlob += pParse->nIns; + jsonBlobEdit(pParse, j, 0, pParse->aIns, pParse->nIns); jsonAfterEditSizeAdjust(pParse, iRoot); return j; } @@ -4284,6 +4301,7 @@ static void jsonRemoveFromBlob( return; /* return NULL if $ is removed */ } px.eEdit = JEDIT_DEL; + px.delta = 0; rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); if( rc==JSON_BLOB_NOTFOUND ) continue; if( JSON_BLOB_ISERROR(rc) ) goto jsonRemoveFromBlob_patherror; @@ -4420,6 +4438,7 @@ static void jsonInsertIntoBlob( px.eEdit = eEdit; px.nIns = ax.nBlob; px.aIns = ax.aBlob; + px.delta = 0; rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); jsonParseReset(&ax); if( rc==JSON_BLOB_NOTFOUND ) continue; @@ -4979,6 +4998,181 @@ static JsonNode *jsonMergePatch( return pTarget; } + +#if 0 +/* +** Return codes for jsonMergePatchBlob() +*/ +#define JSON_MERGE_OK 0 /* Success */ +#define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ +#define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ +#define JSON_MERGE_OOM 3 /* Out-of-memory condition */ + +/* +** RFC-7396 MergePatch for two JSONB blobs. +** +** pTarget is the target. pPatch is the patch. The target is updated +** in place. The patch is read-only. +** +** The original RFC-7396 algorithm is this: +** +** define MergePatch(Target, Patch): +** if Patch is an Object: +** if Target is not an Object: +** Target = {} # Ignore the contents and set it to an empty Object +** for each Name/Value pair in Patch: +** if Value is null: +** if Name exists in Target: +** remove the Name/Value pair from Target +** else: +** Target[Name] = MergePatch(Target[Name], Value) +** return Target +** else: +** return Patch +** +** Here is the same algorithm restrictured to show the actual +** implementation: +** +** 01 define MergePatch(Target, Patch): +** 02 if Patch is not an Object: +** 03 return Patch +** 04 else: // if Patch is an Object: +** 05 if Target is not an Object: +** 06 Target = {} +** 07 for each Name/Value pair in Patch: +** 08 if Name exists in Target: +** 09 if Value is null: +** 10 remove the Name/Value pair from Target +** 11 else +** 12 Target[name] = MergePatch(Target[Name], Value) +** 13 else if Value is not NULL: +** 14 Target[name] = RemoveNullVAlues(Value) +** 15 return Target +** | +** ^---- Line numbers referenced in comments in the implementation +*/ +static int jsonMergePatchBlob( + JsonParse *pTarget, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ + const JsonParse *pPatch /* The PATCH */ + u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ +){ + u8 x; /* Type of a single node */ + u32 n, sz=0; /* Return values from jsonbPayloadSize() */ + u32 iTCursor; /* Cursor position while scanning the target object */ + u32 iTStart; /* First label in the target object */ + u32 iTEndBE; /* Original first byte past end of target, before edit */ + u32 iTEnd; /* Current first byte past end of target */ + u32 iTLabel; /* Index of the label */ + u32 nTLabel; /* Header size in bytes for the target label */ + u32 szTLabel; /* Size of the target label payload */ + u32 iTValue; /* Index of the target value */ + u32 nTValue; /* Header size of the target value */ + u32 szTValue; /* Payload size for the target value */ + + u32 iPCursor; /* Cursor position while scanning the patch */ + u32 iPEnd; /* First byte past the end of the patch */ + u32 iPLabel; /* Start of patch label */ + u32 nPLabel; /* Size of header on the patch label */ + u32 szPLabel; /* Payload size of the patch label */ + u32 iPValue; /* Start of patch value */ + u32 nPValue; /* Header size for the patch value */ + u32 szPValue; /* Payload size of the patch value */ + + assert( iTarget>=0 && iTargetnBlob ); + assert( iPatch>=0 && iPatchnBlob ); + x = pPatch->aBlob[iPatch] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 02 */ + u32 szPatch; /* Total size of the patch, header+payload */ + u32 szTarget; /* Total size of the target, header+payload */ + n = jsonbPayloadSize(pPatch, iPatch, &sz); + szPatch = n+sz; + sz = 0; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + szTarget = n+sz; + jsonBlobEdit(pTarget, iTarget, szTarget, pPatch->aBlob+iPatch, szPatch); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; /* Line 03 */ + } + x = pTarget->aBlob[iTarget] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ + static const u8 emptyObject = { JSONB_OBJECT }; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + jsonBlobEdit(pTarget, iTarget, szTarget, emptyObject, 1); /* Line 06 */ + } + n = jsonbPayloadSize(pPatch, iPatch, &sz); + if( n==0 ) return JSON_MERGE_BADPATCH; + iPCursor = iPatch+n; + iPEnd = iPCursor+sz; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + if( n==0 ) return JSON_MERGE_BADTARGET; + iTStart = iTarget+n; + iTEndBE = iTStart+sz; + + while( iPCursoraBlob[iPCursor] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_MERGE_BADPATCH; + nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); + if( nPLabel==0 ) return JSON_MERGE_BADPATCH; + iPValue = iPCursor + nPLabel + szPLabel; + if( iPCursor>=iPEnd ) return JSON_MERGE_BADPATCH; + nPValue = jsonbPayloadSize(pPatch, iPValue, &szPValue); + if( nPValue==0 ) return JSON_MERGE_BADPATCH; + iPCursor = iPValue + nPValue + szPValue; + if( iPCursor>iPEnd ) return JSON_MERGE_BADPATCH; + + iTCursor = iTStart; + iTEnd = iTEndBE + pTarget->delta; + while( iTCursoraBlob[iTCursor] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_MERGE_BADTARGET; + nTLabel = jsonbPayloadSize(pTarget, iTCursor, &szTLabel); + if( nTLabel==0 ) return JSON_MERGE_BADTARGET; + iTValue = iTLabel + nTLabel + szTLabel; + if( iTValue>=iTEnd ) return JSON_MERGE_BADTARGET; + nTValue = jsonbPayloadSize(pTarget, iTValue, &szTValue); + if( nTValue==0 ) return JSON_MERGE_BADTARGET; + if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; + if( eTLabel==ePLabel ){ + if( szTLabel==szPLabel + && memcmp(&pTarget->aBlob[iTLabel+nTLabel], + &pPatch->aBlob[iPLabel+nPLabel], szTLabel)==0 + ){ + break; /* Labels match. */ + } + }else{ + if( jsonLabelEqual(pTarget, iTLabel, pPatch, iPLabel) ) break; + } + iTCursor = iTValue + nTValue + szTValue; + } + x = pPatch->aBlob[iPValue] & 0x0f; + if( iTCursoroom ) return JSON_MERGE_OOM; + }else{ + /* Algorithm line 12 */ + int rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, pPValue); + if( rc ) return rc; + } + }else if( x>0 ){ /* Algorithm line 13 */ + /* No match and patch value is not NULL */ + jsonBlobEdit(pTarget, iTEnd, 0, + pPatch->aBlob+iPValue, szPValue+nPValue); + if( pTarget->oom ) return JSON_MERGE_OOM; + jsonBlobRemoveNullsFromObject(pTarget, iTEnd); + } + } + jsonAfterEditSizeAdjust(pTarget, iTarget); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; +} +#endif + + /* ** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON ** object that is the result of running the RFC 7396 MergePatch() algorithm From f46f89df975100808dc096be69c577206a587123 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 00:27:58 +0000 Subject: [PATCH 106/191] More aggressive use of jsonBlobEdit(). Improvements to the MergePatch implementation sketch. FossilOrigin-Name: fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 41 ++++++++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 6ae9cbf521..eb129dd02f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\suntested\s(#ifdefed-out)\scode\sfor\sthe\sMergePatch\salgorithm\sagainst\sJSONB.\nAdd\s(and\stest)\sthe\sjsonBlobEdit()\sroutine\sthat\sis\sneeded\sby\sthe\snew\sMergePatch. -D 2023-11-27T23:46:12.954 +C More\saggressive\suse\sof\sjsonBlobEdit().\s\sImprovements\sto\sthe\sMergePatch\nimplementation\ssketch. +D 2023-11-28T00:27:58.169 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 7d6387920736cd9b78a4211771265c4c574284613d8e63f43b3ce7bc160da498 +F src/json.c a91647a3c56298ff94bb2d5a989e8e329c4435e3a0ae9760b04b46c7b2083541 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P af267868562e0799ad691dccad05f17afbc34d609eede8c55f57d209290246ef -R 47ffcc493583e45ee7703e45cf1805e3 +P 4d353387fc10e1038cfdd86e66007bf728c231a928e588897bbee0fbfe76f225 +R 42f4ec3bb9c19f358b256294966169e5 U drh -Z 2cedbcb9affb9f675b8d832bfea283a4 +Z 551901455bde8067c499befcab52664b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a4814c0752..6460236f38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d353387fc10e1038cfdd86e66007bf728c231a928e588897bbee0fbfe76f225 \ No newline at end of file +fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 99205decf1..0138db5183 100644 --- a/src/json.c +++ b/src/json.c @@ -3809,8 +3809,8 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ ** nDel may be zero, in which case no bytes are removed. But iDel is ** still important as new bytes will be insert beginning at iDel. ** -** nIns may be zero, in which case no new bytes are inserted. aIns might -** be a NULL pointer in this case. +** aIns may be zero, in which case space is created to hold nIns bytes +** beginning at iDel, but that space is uninitialized. ** ** Set pParse->oom if an OOM occurs. */ @@ -3833,7 +3833,7 @@ static void jsonBlobEdit( pParse->nBlob += d; pParse->delta += d; } - if( nIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); + if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); } /* @@ -3940,19 +3940,14 @@ static u32 jsonLookupBlobStep( ix.nBlobAlloc = sizeof(aLabel); jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey) ){ - /* This is similar to jsonBlobEdit() except that the inserted - ** bytes come from two different places, ix.aBlob and pParse->aBlob. */ nIns = ix.nBlob + nKey + pParse->nIns; + jsonBlobEdit(pParse, j, 0, 0, nIns); assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); - memmove(&pParse->aBlob[j+nIns], &pParse->aBlob[j], - pParse->nBlob - j); memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); k += nKey; memcpy(&pParse->aBlob[k], pParse->aIns, pParse->nIns); - pParse->delta = nIns; - pParse->nBlob += nIns; jsonAfterEditSizeAdjust(pParse, iRoot); } return j; @@ -5030,13 +5025,13 @@ static JsonNode *jsonMergePatch( ** else: ** return Patch ** -** Here is the same algorithm restrictured to show the actual +** Here is an equivalent algorithm restructured to show the actual ** implementation: ** ** 01 define MergePatch(Target, Patch): ** 02 if Patch is not an Object: ** 03 return Patch -** 04 else: // if Patch is an Object: +** 04 else: // if Patch is an Object ** 05 if Target is not an Object: ** 06 Target = {} ** 07 for each Name/Value pair in Patch: @@ -5046,8 +5041,11 @@ static JsonNode *jsonMergePatch( ** 11 else ** 12 Target[name] = MergePatch(Target[Name], Value) ** 13 else if Value is not NULL: -** 14 Target[name] = RemoveNullVAlues(Value) -** 15 return Target +** 14 if Value is not an Object: +** 15 Target[name] = Value +** 16 else: +** 17 Target[name] = MergePatch('{}',value) +** 18 return Target ** | ** ^---- Line numbers referenced in comments in the implementation */ @@ -5161,10 +5159,19 @@ static int jsonMergePatchBlob( } }else if( x>0 ){ /* Algorithm line 13 */ /* No match and patch value is not NULL */ - jsonBlobEdit(pTarget, iTEnd, 0, - pPatch->aBlob+iPValue, szPValue+nPValue); - if( pTarget->oom ) return JSON_MERGE_OOM; - jsonBlobRemoveNullsFromObject(pTarget, iTEnd); + if( pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ + jsonBlobEdit(pTarget, iTEnd, 0, + pPatch->aBlob+iPValue, szPValue+nPValue, 0, 0); + if( pTarget->oom ) return JSON_MERGE_OOM; + }else{ + u32 szNew = szPLabel+nPLabel; + jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + pTarget->aBlob[iTEnd+szNew] = 0x00; + rc = jsonMergePatch(pTarget, iTEnd+szNew, pPatch, iPValue); + if( rc ) return rc; + } } } jsonAfterEditSizeAdjust(pTarget, iTarget); From 5026ddb83d04aeca4f4ec487c572e27b2c87d6b7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 12:28:28 +0000 Subject: [PATCH 107/191] The json_patch() code for JSONB compiles and works sometimes, but there are still issues. Incremental check-in. FossilOrigin-Name: e0099464a0045a04f4ccf29bc2b8325fc8c7f39ccf4847e74818f928c9153588 --- manifest | 15 ++++--- manifest.uuid | 2 +- src/json.c | 106 +++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 93 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index eb129dd02f..3e1841dc0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\saggressive\suse\sof\sjsonBlobEdit().\s\sImprovements\sto\sthe\sMergePatch\nimplementation\ssketch. -D 2023-11-28T00:27:58.169 +C The\sjson_patch()\scode\sfor\sJSONB\scompiles\sand\sworks\ssometimes,\sbut\sthere\sare\nstill\sissues.\s\sIncremental\scheck-in. +D 2023-11-28T12:28:28.510 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a91647a3c56298ff94bb2d5a989e8e329c4435e3a0ae9760b04b46c7b2083541 +F src/json.c 2dfbca7572f686feea28665d524a34cdde23cf3dcb15cfa79cabe77f8032524f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d353387fc10e1038cfdd86e66007bf728c231a928e588897bbee0fbfe76f225 -R 42f4ec3bb9c19f358b256294966169e5 +P fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 +R 74a90e7c3058a227c8ae29f97ecb7181 +T *branch * jsonb-patch +T *sym-jsonb-patch * +T -sym-jsonb * U drh -Z 551901455bde8067c499befcab52664b +Z 73c44f2da8d785efd9b8e3ab2061af48 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6460236f38..477eefc522 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 \ No newline at end of file +e0099464a0045a04f4ccf29bc2b8325fc8c7f39ccf4847e74818f928c9153588 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0138db5183..6f5021aafe 100644 --- a/src/json.c +++ b/src/json.c @@ -380,7 +380,7 @@ static int jsonParseAddNode(JsonParse*,u32,u32,const char*); static int jsonXlateBlobToNode(JsonParse *pParse, u32 i); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); -static u32 jsonXlateBlobToText(JsonParse*,u32,JsonString*); +static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -2558,13 +2558,12 @@ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ u32 nSize; if( pParse->nBlobAlloc>0 ) return 1; aOld = pParse->aBlob; - nSize = pParse->nBlob + pParse->nIns + nExtra; - if( nSize>100 ) nSize -= 100; + nSize = pParse->nBlob + nExtra; pParse->aBlob = 0; if( jsonBlobExpand(pParse, nSize) ){ return 0; } - assert( pParse->nBlobAlloc >= pParse->nBlob + pParse->nIns ); + assert( pParse->nBlobAlloc >= pParse->nBlob + nExtra ); memcpy(pParse->aBlob, aOld, pParse->nBlob); return 1; } @@ -3248,7 +3247,7 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ ** payload size in to *pSz. It returns the offset from i to the ** beginning of the payload. Return 0 on error. */ -static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ +static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; @@ -3306,7 +3305,7 @@ static u32 jsonbPayloadSize(JsonParse *pParse, u32 i, u32 *pSz){ ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ static u32 jsonXlateBlobToText( - JsonParse *pParse, /* the complete parse of the JSON */ + const JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ @@ -3861,7 +3860,7 @@ static u32 jsonLookupBlobStep( u8 x; if( zPath[0]==0 ){ - if( pParse->eEdit && jsonBlobMakeEditable(pParse, 0) ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ n = jsonbPayloadSize(pParse, iRoot, &sz); sz += n; if( pParse->eEdit==JEDIT_DEL ){ @@ -3939,7 +3938,7 @@ static u32 jsonLookupBlobStep( ix.aBlob = aLabel; ix.nBlobAlloc = sizeof(aLabel); jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); - if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey) ){ + if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+pParse->nIns) ){ nIns = ix.nBlob + nKey + pParse->nIns; jsonBlobEdit(pParse, j, 0, 0, nIns); assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); @@ -3998,7 +3997,9 @@ static u32 jsonLookupBlobStep( } if( j>iEnd ) return JSON_BLOB_ERROR; if( k>1 ) return JSON_BLOB_NOTFOUND; - if( pParse->eEdit>=JEDIT_INS && jsonBlobMakeEditable(pParse, 0) ){ + if( pParse->eEdit>=JEDIT_INS + && jsonBlobMakeEditable(pParse, pParse->nIns) + ){ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); jsonBlobEdit(pParse, j, 0, pParse->aIns, pParse->nIns); @@ -4994,7 +4995,6 @@ static JsonNode *jsonMergePatch( } -#if 0 /* ** Return codes for jsonMergePatchBlob() */ @@ -5052,7 +5052,7 @@ static JsonNode *jsonMergePatch( static int jsonMergePatchBlob( JsonParse *pTarget, /* The JSON parser that contains the TARGET */ u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ - const JsonParse *pPatch /* The PATCH */ + const JsonParse *pPatch, /* The PATCH */ u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ ){ u8 x; /* Type of a single node */ @@ -5061,6 +5061,7 @@ static int jsonMergePatchBlob( u32 iTStart; /* First label in the target object */ u32 iTEndBE; /* Original first byte past end of target, before edit */ u32 iTEnd; /* Current first byte past end of target */ + u8 eTLabel; /* Node type of the target label */ u32 iTLabel; /* Index of the label */ u32 nTLabel; /* Header size in bytes for the target label */ u32 szTLabel; /* Size of the target label payload */ @@ -5070,6 +5071,7 @@ static int jsonMergePatchBlob( u32 iPCursor; /* Cursor position while scanning the patch */ u32 iPEnd; /* First byte past the end of the patch */ + u8 ePLabel; /* Node type of the patch label */ u32 iPLabel; /* Start of patch label */ u32 nPLabel; /* Size of header on the patch label */ u32 szPLabel; /* Payload size of the patch label */ @@ -5093,9 +5095,9 @@ static int jsonMergePatchBlob( } x = pTarget->aBlob[iTarget] & 0x0f; if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ - static const u8 emptyObject = { JSONB_OBJECT }; + static const u8 emptyObject[] = { JSONB_OBJECT }; n = jsonbPayloadSize(pTarget, iTarget, &sz); - jsonBlobEdit(pTarget, iTarget, szTarget, emptyObject, 1); /* Line 06 */ + jsonBlobEdit(pTarget, iTarget, sz, emptyObject, 1); /* Line 06 */ } n = jsonbPayloadSize(pPatch, iPatch, &sz); if( n==0 ) return JSON_MERGE_BADPATCH; @@ -5108,8 +5110,10 @@ static int jsonMergePatchBlob( while( iPCursoraBlob[iPCursor] & 0x0f; - if( xJSONB_TEXTRAW ) return JSON_MERGE_BADPATCH; + ePLabel = pPatch->aBlob[iPCursor] & 0x0f; + if( ePLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADPATCH; + } nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); if( nPLabel==0 ) return JSON_MERGE_BADPATCH; iPValue = iPCursor + nPLabel + szPLabel; @@ -5123,8 +5127,10 @@ static int jsonMergePatchBlob( iTEnd = iTEndBE + pTarget->delta; while( iTCursoraBlob[iTCursor] & 0x0f; - if( xJSONB_TEXTRAW ) return JSON_MERGE_BADTARGET; + eTLabel = pTarget->aBlob[iTCursor] & 0x0f; + if( eTLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADTARGET; + } nTLabel = jsonbPayloadSize(pTarget, iTCursor, &szTLabel); if( nTLabel==0 ) return JSON_MERGE_BADTARGET; iTValue = iTLabel + nTLabel + szTLabel; @@ -5133,6 +5139,7 @@ static int jsonMergePatchBlob( if( nTValue==0 ) return JSON_MERGE_BADTARGET; if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; if( eTLabel==ePLabel ){ + /* Common case */ if( szTLabel==szPLabel && memcmp(&pTarget->aBlob[iTLabel+nTLabel], &pPatch->aBlob[iPLabel+nPLabel], szTLabel)==0 @@ -5140,7 +5147,23 @@ static int jsonMergePatchBlob( break; /* Labels match. */ } }else{ - if( jsonLabelEqual(pTarget, iTLabel, pPatch, iPLabel) ) break; + /* Should rarely happen */ + JsonString s1, s2; + int isEqual; + jsonStringInit(&s1, 0); + jsonXlateBlobToText(pTarget, iTLabel, &s1); + if( s1.eErr ) return JSON_MERGE_OOM; + jsonStringInit(&s2, 0); + jsonXlateBlobToText(pPatch, iPLabel, &s2); + if( s2.eErr ) return JSON_MERGE_OOM; + if( s1.nUsed==s2.nUsed && memcmp(s1.zBuf, s2.zBuf, s1.nUsed)==0 ){ + isEqual = 1; + }else{ + isEqual = 0; + } + jsonStringReset(&s1); + jsonStringReset(&s2); + if( isEqual ) break; } iTCursor = iTValue + nTValue + szTValue; } @@ -5154,22 +5177,23 @@ static int jsonMergePatchBlob( if( pTarget->oom ) return JSON_MERGE_OOM; }else{ /* Algorithm line 12 */ - int rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, pPValue); + int rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, iPValue); if( rc ) return rc; } }else if( x>0 ){ /* Algorithm line 13 */ /* No match and patch value is not NULL */ - if( pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ + if( (pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ /* Line 14 */ jsonBlobEdit(pTarget, iTEnd, 0, - pPatch->aBlob+iPValue, szPValue+nPValue, 0, 0); + pPatch->aBlob+iPValue, szPValue+nPValue); if( pTarget->oom ) return JSON_MERGE_OOM; }else{ + int rc; u32 szNew = szPLabel+nPLabel; jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); if( pTarget->oom ) return JSON_MERGE_OOM; memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); pTarget->aBlob[iTEnd+szNew] = 0x00; - rc = jsonMergePatch(pTarget, iTEnd+szNew, pPatch, iPValue); + rc = jsonMergePatchBlob(pTarget, iTEnd+szNew,pPatch,iPValue); if( rc ) return rc; } } @@ -5177,7 +5201,6 @@ static int jsonMergePatchBlob( jsonAfterEditSizeAdjust(pTarget, iTarget); return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; } -#endif /* @@ -5195,6 +5218,43 @@ static void jsonPatchFunc( JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); + if( jsonFuncArgMightBeBinary(argv[0]) + && jsonFuncArgMightBeBinary(argv[1]) + ){ + JsonParse target, patch; + sqlite3 *db; + int rc; + memset(&target, 0, sizeof(target)); + memset(&patch, 0, sizeof(patch)); + target.aBlob = (u8*)sqlite3_value_blob(argv[0]); + target.nBlob = sqlite3_value_bytes(argv[0]); + patch.aBlob = (u8*)sqlite3_value_blob(argv[1]); + patch.nBlob = sqlite3_value_bytes(argv[1]); + db = sqlite3_context_db_handle(ctx); + if( db->mallocFailed ) return; + if( !jsonBlobMakeEditable(&target, patch.nBlob) ) return; + rc = jsonMergePatchBlob(&target, 0, &patch, 0); + if( rc ){ + if( rc==JSON_MERGE_OOM ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + jsonParseReset(&target); + }else{ + int flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + sqlite3_result_blob(ctx, target.aBlob, target.nBlob, SQLITE_DYNAMIC); + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(&target, 0, &s); + jsonReturnString(&s); + jsonParseReset(&target); + } + } + return; + } pX = jsonParseCached(ctx, argv[0], ctx, 1); if( pX==0 ) return; assert( pX->hasMod==0 ); From ec1f59f0cdf988c2b4d66d0c8662832a64a60dfa Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 13:35:53 +0000 Subject: [PATCH 108/191] All legacy tests are passing. FossilOrigin-Name: 2c436806b8d5f57de99c00f6154b038454fb9ae427d00d7b4a46ab9c7c69bcb9 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3e1841dc0f..0d87ba4a36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_patch()\scode\sfor\sJSONB\scompiles\sand\sworks\ssometimes,\sbut\sthere\sare\nstill\sissues.\s\sIncremental\scheck-in. -D 2023-11-28T12:28:28.510 +C All\slegacy\stests\sare\spassing. +D 2023-11-28T13:35:53.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2dfbca7572f686feea28665d524a34cdde23cf3dcb15cfa79cabe77f8032524f +F src/json.c fa3a1b6f2432848dc03448d359a07040f7e4a44d868e9e16512c6066b65aeef9 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 -R 74a90e7c3058a227c8ae29f97ecb7181 -T *branch * jsonb-patch -T *sym-jsonb-patch * -T -sym-jsonb * +P e0099464a0045a04f4ccf29bc2b8325fc8c7f39ccf4847e74818f928c9153588 +R e5b38e7d70999e4556fdfbbe91a82c08 U drh -Z 73c44f2da8d785efd9b8e3ab2061af48 +Z 686f56079312574c393270803643d299 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 477eefc522..7c28bb5a17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0099464a0045a04f4ccf29bc2b8325fc8c7f39ccf4847e74818f928c9153588 \ No newline at end of file +2c436806b8d5f57de99c00f6154b038454fb9ae427d00d7b4a46ab9c7c69bcb9 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6f5021aafe..bdc9087323 100644 --- a/src/json.c +++ b/src/json.c @@ -5177,24 +5177,32 @@ static int jsonMergePatchBlob( if( pTarget->oom ) return JSON_MERGE_OOM; }else{ /* Algorithm line 12 */ - int rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, iPValue); + int rc, savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, iPValue); if( rc ) return rc; + pTarget->delta += savedDelta; } }else if( x>0 ){ /* Algorithm line 13 */ /* No match and patch value is not NULL */ + u32 szNew = szPLabel+nPLabel; if( (pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ /* Line 14 */ - jsonBlobEdit(pTarget, iTEnd, 0, - pPatch->aBlob+iPValue, szPValue+nPValue); + jsonBlobEdit(pTarget, iTEnd, 0, 0, szPValue+nPValue+szNew); if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + memcpy(&pTarget->aBlob[iTEnd+szNew], + &pPatch->aBlob[iPValue], szPValue+nPValue); }else{ - int rc; - u32 szNew = szPLabel+nPLabel; + int rc, savedDelta; jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); if( pTarget->oom ) return JSON_MERGE_OOM; memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); pTarget->aBlob[iTEnd+szNew] = 0x00; + savedDelta = pTarget->delta; + pTarget->delta = 0; rc = jsonMergePatchBlob(pTarget, iTEnd+szNew,pPatch,iPValue); if( rc ) return rc; + pTarget->delta += savedDelta; } } } From ef97f8360a7063ea2191824cccd0414d86b607a1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 18:16:02 +0000 Subject: [PATCH 109/191] The json_remove() function now uses only JSONB, never JsonNodes, internally. FossilOrigin-Name: b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 --- manifest | 15 +- manifest.uuid | 2 +- src/json.c | 354 ++++++++++++++++++++++++---------------------- test/json501.test | 6 +- 4 files changed, 193 insertions(+), 184 deletions(-) diff --git a/manifest b/manifest index af38aa5b5a..510a2234e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Activate\sthe\sability\sof\sjson_patch()\sto\swork\son\sJSONB. -D 2023-11-28T13:38:22.987 +C The\sjson_remove()\sfunction\snow\suses\sonly\sJSONB,\snever\sJsonNodes,\sinternally. +D 2023-11-28T18:16:02.028 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fa3a1b6f2432848dc03448d359a07040f7e4a44d868e9e16512c6066b65aeef9 +F src/json.c 442ff1ce9ebb82728aca5ddfcab21af46c3230e9f672307b91489f43cd6d6fdc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1330,7 +1330,7 @@ F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d -F test/json501.test c419deb835b70c1a2c8532936927bcc1146730328edd2052276715bfd209724d +F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff @@ -2145,9 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fbca9570fd2e1465739e4d3a8d9bb40fad594fd78ab49b2cb34efa27ebdd8361 2c436806b8d5f57de99c00f6154b038454fb9ae427d00d7b4a46ab9c7c69bcb9 -R e5b38e7d70999e4556fdfbbe91a82c08 -T +closed 2c436806b8d5f57de99c00f6154b038454fb9ae427d00d7b4a46ab9c7c69bcb9 +P 11aba347ff7c639500eec904e212eabe889b077351b946cfeac2b74b9703672a +R 82f8a4e62603e32381a0117063382c7a U drh -Z 1523e6413d446a9df2c7c937b4cb4eb3 +Z 37d5559550d9d10448cc0117446ead8c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b1acbb8add..a64ec6f678 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11aba347ff7c639500eec904e212eabe889b077351b946cfeac2b74b9703672a \ No newline at end of file +b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 \ No newline at end of file diff --git a/src/json.c b/src/json.c index bdc9087323..4baffa7fa3 100644 --- a/src/json.c +++ b/src/json.c @@ -885,11 +885,13 @@ static void jsonParseReset(JsonParse *pParse){ ** that this recursive destructor sequence terminates harmlessly. */ static void jsonParseFree(JsonParse *pParse){ - if( pParse->nJPRef>1 ){ - pParse->nJPRef--; - }else{ - jsonParseReset(pParse); - sqlite3_free(pParse); + if( pParse ){ + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3_free(pParse); + } } } @@ -2636,7 +2638,7 @@ static void jsonBlobAppendNodeType( /* Change the payload size for the node at index i to be szPayload. */ -static void jsonBlobChangePayloadSize( +static int jsonBlobChangePayloadSize( JsonParse *pParse, u32 i, u32 szPayload @@ -2645,8 +2647,8 @@ static void jsonBlobChangePayloadSize( u8 szType; u8 nExtra; u8 nNeeded; - i8 delta; - if( pParse->oom ) return; + int delta; + if( pParse->oom ) return 0; a = &pParse->aBlob[i]; szType = a[0]>>4; if( szType<=11 ){ @@ -2672,7 +2674,7 @@ static void jsonBlobChangePayloadSize( u32 newSize = pParse->nBlob + delta; if( delta>0 ){ if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ - return; /* OOM error. Error state recorded in pParse->oom. */ + return 0; /* OOM error. Error state recorded in pParse->oom. */ } a = &pParse->aBlob[i]; memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); @@ -2697,6 +2699,7 @@ static void jsonBlobChangePayloadSize( a[3] = (szPayload >> 8) & 0xff; a[4] = szPayload & 0xff; } + return delta; } /* @@ -3376,13 +3379,13 @@ static u32 jsonXlateBlobToText( } break; } - case JSONB_TEXT: case JSONB_TEXTJ: { jsonAppendChar(pOut, '"'); jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); jsonAppendChar(pOut, '"'); break; } + case JSONB_TEXT: case JSONB_TEXT5: { const char *zIn; u32 k; @@ -3390,7 +3393,7 @@ static u32 jsonXlateBlobToText( zIn = (const char*)&pParse->aBlob[i+n]; jsonAppendChar(pOut, '"'); while( sz2>0 ){ - for(k=0; k0 ){ jsonAppendRawNZ(pOut, zIn, k); if( k>=sz2 ){ @@ -3399,6 +3402,12 @@ static u32 jsonXlateBlobToText( zIn += k; sz2 -= k; } + if( zIn[0]=='"' ){ + jsonAppendRawNZ(pOut, "\\\"", 2); + zIn++; + sz2--; + continue; + } if( sz2<2 ){ if( sz2>0 ) pOut->eErr |= JSTRING_MALFORMED; if( sz2==0 ) break; @@ -3514,9 +3523,9 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ int nBlob; JsonParse s; if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + aBlob = sqlite3_value_blob(pJson); nBlob = sqlite3_value_bytes(pJson); if( nBlob<1 ) return 0; - aBlob = sqlite3_value_blob(pJson); if( aBlob==0 || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; memset(&s, 0, sizeof(s)); s.aBlob = (u8*)aBlob; @@ -3797,7 +3806,7 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ (void)jsonbPayloadSize(pParse, iRoot, &sz); pParse->nBlob = nBlob; sz += pParse->delta; - jsonBlobChangePayloadSize(pParse, iRoot, sz); + pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); } /* @@ -4268,58 +4277,6 @@ static void jsonExtractFromBlob( sqlite3_free(zMsg); } } - -/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent -** arguments are JSON paths of elements to be removed. Do that removal -** and return the result. -*/ -static void jsonRemoveFromBlob( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - int i; - u32 rc; - const char *zPath = 0; - int flgs; - JsonParse px; - - memset(&px, 0, sizeof(px)); - px.nBlob = sqlite3_value_bytes(argv[0]); - px.aBlob = (u8*)sqlite3_value_blob(argv[0]); - if( px.aBlob==0 ) return; - for(i=1; i0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); - }else{ - JsonString s; - jsonStringInit(&s, ctx); - jsonXlateBlobToText(&px, 0, &s); - jsonReturnString(&s); - jsonParseReset(&px); - } - return; - -jsonRemoveFromBlob_patherror: - jsonParseReset(&px); - jsonPathSyntaxError(zPath, ctx); - return; -} /* ** pArg is a function argument that might be an SQL value or a JSON @@ -4463,48 +4420,119 @@ jsonInsertIntoBlob_patherror: return; } +/* +** Allowed values for the flgs argument to jsonParseFuncArg(); +*/ +#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ + +/* +** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, +** from the SQL function argument pArg. Return a pointer to the new +** JsonParse object. +** +** Ownership of the new JsonParse object is passed to the caller. The +** caller should invoke jsonParseFree() on the return value when it +** has finished using it. +** +** If any errors are detected, an appropriate error messages is set +** using sqlite3_result_error() or the equivalent and this routine +** returns NULL. This routine also returns NULL if the pArg argument +** is an SQL NULL value, but no error message is set in that case. This +** is so that SQL functions that are given NULL arguments will return +** a NULL value. +*/ +static JsonParse *jsonParseFuncArg( + sqlite3_context *ctx, + sqlite3_value *pArg, + u32 flgs +){ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + + assert( ctx!=0 ); + eType = sqlite3_value_type(pArg); + if( eType==SQLITE_NULL ){ + return 0; + } + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) goto json_pfa_oom; + memset(p, 0, sizeof(*p)); + if( eType==SQLITE_BLOB ){ + u32 n, sz = 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob==0 ){ + goto json_pfa_malformed; + } + if( p->aBlob==0 ){ + goto json_pfa_oom; + } + if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){ + goto json_pfa_malformed; + } + n = jsonbPayloadSize(p, 0, &sz); + if( n==0 + || sz+n!=p->nBlob + || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0) + || sz+n!=p->nBlob + ){ + goto json_pfa_malformed; + } + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; + } + /* TODO: Check in the cache */ + p->zJson = (char*)sqlite3_value_text(pArg); + p->nJson = sqlite3_value_bytes(pArg); + if( p->nJson==0 ) goto json_pfa_malformed; + if( p->zJson==0 ) goto json_pfa_oom; + if( jsonConvertTextToBlob(p, ctx) ){ + jsonParseFree(p); + return 0; + } + return p; + +json_pfa_malformed: + jsonParseFree(p); + sqlite3_result_error(ctx, "malformed JSON", -1); + return 0; + +json_pfa_oom: + jsonParseFree(p); + sqlite3_result_error_nomem(ctx); + return 0; +} + +/* +** Make the return value of a JSON function either the raw JSONB blob +** or make it JSON text, depending on whether the JSON_BLOB flag is +** set on the function. +*/ +static void jsonReturnParse( + sqlite3_context *ctx, + JsonParse *p +){ + int flgs; + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, + p->nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); + p->nBlobAlloc = 0; + }else{ + JsonString s; + jsonStringInit(&s, ctx); + jsonXlateBlobToText(p, 0, &s); + jsonReturnString(&s); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } +} + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ -#if SQLITE_DEBUG -/* -** Print N node entries. -*/ -static void jsonDebugPrintNodeEntries( - JsonNode *aNode, /* First node entry to print */ - int N /* Number of node entries to print */ -){ - int i; - for(i=0; inBlob); + } + jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0); +} #endif /* SQLITE_DEBUG */ - -#if 0 /* 1 for debugging. 0 normally. Requires -DSQLITE_DEBUG too */ -static void jsonDebugPrintParse(JsonParse *p){ - jsonDebugPrintNodeEntries(p->aNode, p->nNode); -} -static void jsonDebugPrintNode(JsonNode *pNode){ - jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode)); -} -#else - /* The usual case */ -# define jsonDebugPrintNode(X) -# define jsonDebugPrintParse(X) -#endif - #ifdef SQLITE_DEBUG /* ** SQL function: json_parse(JSON) ** -** Parse JSON using jsonParseCached(). Then print a dump of that -** parse on standard output. Return the mimified JSON result, just -** like the json() function. +** Parse JSON using jsonParseFuncArg(). Then print a dump of that +** parse on standard output. */ static void jsonParseFunc( sqlite3_context *ctx, @@ -4613,30 +4635,9 @@ static void jsonParseFunc( JsonParse *p; /* The parse */ assert( argc==1 ); - if( jsonFuncArgMightBeBinary(argv[0]) ){ - JsonParse x; - memset(&x, 0, sizeof(x)); - x.nBlob = sqlite3_value_bytes(argv[0]); - x.aBlob = (u8*)sqlite3_value_blob(argv[0]); - jsonDebugPrintBlob(&x, 0, x.nBlob, 0); - return; - } - p = jsonParseCached(ctx, argv[0], ctx, 0); - if( p==0 ) return; - printf("nNode = %u\n", p->nNode); - printf("nAlloc = %u\n", p->nAlloc); - printf("nJson = %d\n", p->nJson); - printf("nAlt = %d\n", p->nAlt); - printf("nErr = %u\n", p->nErr); - printf("oom = %u\n", p->oom); - printf("hasNonstd = %u\n", p->hasNonstd); - printf("useMod = %u\n", p->useMod); - printf("hasMod = %u\n", p->hasMod); - printf("nJPRef = %u\n", p->nJPRef); - printf("iSubst = %u\n", p->iSubst); - printf("iHold = %u\n", p->iHold); - jsonDebugPrintNodeEntries(p->aNode, p->nNode); - jsonReturnNodeAsJson(p, p->aNode, ctx, 1, 0); + p = jsonParseFuncArg(ctx, argv[0], 0); + jsonShowParse(p); + jsonParseFree(p); } /* @@ -5278,8 +5279,6 @@ static void jsonPatchFunc( }else if( pX->nErr ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else if( pResult ){ - jsonDebugPrintParse(pX); - jsonDebugPrintNode(pResult); jsonReturnNodeAsJson(pX, pResult, ctx, 0, 0); } } @@ -5337,34 +5336,47 @@ static void jsonRemoveFunc( int argc, sqlite3_value **argv ){ - JsonParse *pParse; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; + JsonParse *p; /* The parse */ + const char *zPath = 0; /* Path of element to be removed */ + u32 i; /* Loop counter */ + int rc; /* Subroutine return code */ if( argc<1 ) return; - if( jsonFuncArgMightBeBinary(argv[0]) ){ - jsonRemoveFromBlob(ctx, argc, argv); - return; - } - pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); - if( pParse==0 ) return; - for(i=1; i<(u32)argc; i++){ - zPath = (const char*)sqlite3_value_text(argv[i]); - if( zPath==0 ) goto remove_done; - pNode = jsonLookup(pParse, zPath, 0, ctx); - if( pParse->nErr ) goto remove_done; - if( pNode ){ - pNode->jnFlags |= JNODE_REMOVE; - pParse->hasMod = 1; - pParse->useMod = 1; + p = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + if( p==0 ) return; + for(i=1; ieEdit = JEDIT_DEL; + p->delta = 0; + rc = jsonLookupBlobStep(p, 0, zPath+1, 0); + if( rc==JSON_BLOB_NOTFOUND ) continue; + if( JSON_BLOB_ISERROR(rc) ) goto json_remove_patherror; } - if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); - } -remove_done: - jsonDebugPrintParse(p); + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +json_remove_patherror: + jsonParseFree(p); + jsonPathSyntaxError(zPath, ctx); + return; + +json_remove_return_null: + jsonParseFree(p); + return; } /* @@ -5503,7 +5515,6 @@ static void jsonReplaceFunc( } jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); replace_err: - jsonDebugPrintParse(pParse); jsonParseFree(pParse); } @@ -5559,7 +5570,6 @@ static void jsonSetFunc( jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); } } - jsonDebugPrintParse(pParse); jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); jsonSetDone: jsonParseFree(pParse); diff --git a/test/json501.test b/test/json501.test index 3318eea7f2..40b3b563db 100644 --- a/test/json501.test +++ b/test/json501.test @@ -252,13 +252,13 @@ do_execsql_test 8.11 { do_execsql_test 9.1 { WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c; -} {Inf {{"x":9.0e999}}} +} {Inf {{"x":9e999}}} do_execsql_test 9.2 { WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c; -} {-Inf {{"x":-9.0e999}}} +} {-Inf {{"x":-9e999}}} do_execsql_test 9.3 { WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c; -} {Inf {{"x":9.0e999}}} +} {Inf {{"x":9e999}}} do_execsql_test 9.4 { WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c; } {{} {{"x":null}}} From 1cab41e2906130ec64af0381d5a31f017133b248 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 20:25:23 +0000 Subject: [PATCH 110/191] Attempt to get json_extract() working with pure JSONB only, and without the use of JsonNode. Mostly working, but there are some differences from legacy in corner cases. FossilOrigin-Name: 8c324af1eca27e86adc45622af4f3b06a67a3f968596ac58aa7434b1f6f05f3c --- manifest | 19 +-- manifest.uuid | 2 +- src/json.c | 336 +++++++++++++--------------------------------- test/json101.test | 2 +- test/json105.test | 10 +- 5 files changed, 112 insertions(+), 257 deletions(-) diff --git a/manifest b/manifest index 510a2234e3..8d932ccf28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_remove()\sfunction\snow\suses\sonly\sJSONB,\snever\sJsonNodes,\sinternally. -D 2023-11-28T18:16:02.028 +C Attempt\sto\sget\sjson_extract()\sworking\swith\spure\sJSONB\sonly,\sand\swithout\nthe\suse\sof\sJsonNode.\s\sMostly\sworking,\sbut\sthere\sare\ssome\sdifferences\sfrom\nlegacy\sin\scorner\scases. +D 2023-11-28T20:25:23.336 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 442ff1ce9ebb82728aca5ddfcab21af46c3230e9f672307b91489f43cd6d6fdc +F src/json.c bbe9cb34174bfbc0e067a480e431ffc8f0d6cc2cfcad8a1001472bbd00ca620a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1325,11 +1325,11 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 643936557daeb2eeeb839e9d87a682c7d7c6f6aee66045eedd8613cb8471d8d8 +F test/json101.test 8f5d1a3350c36945c8b58f538e1c92cfd87fd50ab6f5e3d5f4cf3cdb03b9546d F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d +F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 @@ -2145,8 +2145,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11aba347ff7c639500eec904e212eabe889b077351b946cfeac2b74b9703672a -R 82f8a4e62603e32381a0117063382c7a +P b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 +R ccd1a96290081a160a58be4f68f67cce +T *branch * jsonb-extract +T *sym-jsonb-extract * +T -sym-jsonb * U drh -Z 37d5559550d9d10448cc0117446ead8c +Z fece822412df4bcd481f5f301c9018f9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a64ec6f678..2cee28775a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 \ No newline at end of file +8c324af1eca27e86adc45622af4f3b06a67a3f968596ac58aa7434b1f6f05f3c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 4baffa7fa3..c259ae1d87 100644 --- a/src/json.c +++ b/src/json.c @@ -1110,180 +1110,6 @@ static u32 jsonHexToInt4(const char *z){ return v; } -/* -** Make the return value from an SQL function be the SQL value of -** JsonNode pNode. -** -** If pNode is an atom (not an array or object) then the value returned -** is a pure SQL value - an SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT, or -** SQLITE_NULL. However, if pNode is a JSON array or object, then the -** value returned is either RFC-8259 JSON text or a BLOB in the JSONB -** format, depending on the JSON_BLOB flag of the function user-data. -*/ -static void jsonReturnFromNode( - JsonParse *pParse, /* Complete JSON parse tree */ - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - int omitSubtype /* Do not call sqlite3_result_subtype() */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - int rc; - int bNeg = 0; - const char *z; - char *zz; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( pNode->eU==1 ); - zz = sqlite3DbStrNDup(db, pNode->u.zJContent, pNode->n); - if( zz==0 ){ - sqlite3_result_error_nomem(pCtx); - return; - } - z = zz; - if( z[0]=='-' ){ z++; bNeg = 1; } - else if( z[0]=='+' ){ z++; } - rc = sqlite3DecOrHexToI64(z, &i); - sqlite3DbFree(db, zz); - if( rc<=1 ){ - sqlite3_result_int64(pCtx, bNeg ? -i : i); - }else if( rc==3 && bNeg ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - }else{ - goto to_double; - } - break; - } - case JSON_REAL: { - double r; - const char *z; - assert( pNode->eU==1 ); - to_double: - z = pNode->u.zJContent; - sqlite3AtoF(z, &r, pNode->n, SQLITE_UTF8); - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - assert( pNode->eU==1 ); - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z; - char *zOut; - u32 j; - u32 nOut = n; - assert( pNode->eU==1 ); - z = pNode->u.zJContent; - zOut = sqlite3_malloc( nOut+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } - for(i=0, j=0; i>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - u32 vlo; - if( (v&0xfc00)==0xd800 - && i>18); - zOut[j++] = 0x80 | ((v>>12)&0x3f); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = 0xe0 | (v>>12); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - } - continue; - }else if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - }else if( c=='v' ){ - c = '\v'; - }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ - /* pass through unchanged */ - }else if( c=='0' ){ - c = 0; - }else if( c=='x' ){ - c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); - i += 2; - }else if( c=='\r' && z[i+1]=='\n' ){ - i++; - continue; - }else if( 0xe2==(u8)c ){ - assert( 0x80==(u8)z[i+1] ); - assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); - i += 2; - continue; - }else{ - continue; - } - } /* end if( c=='\\' ) */ - zOut[j++] = c; - } /* end for() */ - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); - } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnNodeAsJson(pParse, pNode, pCtx, 0, omitSubtype); - break; - } - } -} - /* ** A macro to hint to the compiler that a function should not be ** inlined. @@ -4217,6 +4043,7 @@ static void jsonReturnFromBlob( } } +#if 0 /* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. */ static void jsonExtractFromBlob( @@ -4277,6 +4104,7 @@ static void jsonExtractFromBlob( sqlite3_free(zMsg); } } +#endif /* ** pArg is a function argument that might be an SQL value or a JSON @@ -4819,6 +4647,19 @@ static void jsonArrayLengthFunc( sqlite3_result_int64(ctx, n); } +/* +** Generate a bad path error for json_extract() +*/ +static void jsonExtractBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ +){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + char *zMsg = sqlite3MPrintf(db, "bad JSON path: %Q", zPath); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3DbFree(db, zMsg); +} + /* ** json_extract(JSON, PATH, ...) ** "->"(JSON,PATH) @@ -4844,83 +4685,94 @@ static void jsonExtractFunc( int argc, sqlite3_value **argv ){ - JsonParse *p; /* The parse */ - JsonNode *pNode; - const char *zPath; - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - JsonString jx; + JsonParse *p = 0; /* The parse */ + int flags; /* Flags associated with the function */ + int i; /* Loop counter */ + JsonString jx; /* String for array result */ if( argc<2 ) return; - if( jsonFuncArgMightBeBinary(argv[0]) && argc==2 ){ - jsonExtractFromBlob(ctx, argv[0], argv[1], flags); - return; - } - p = jsonParseCached(ctx, argv[0], ctx, 0); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; - if( argc==2 ){ - /* With a single PATH argument */ - zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return; - if( flags & JSON_ABPATH ){ - if( zPath[0]!='$' || (zPath[1]!='.' && zPath[1]!='[' && zPath[1]!=0) ){ - /* The -> and ->> operators accept abbreviated PATH arguments. This - ** is mostly for compatibility with PostgreSQL, but also for - ** convenience. - ** - ** NUMBER ==> $[NUMBER] // PG compatible - ** LABEL ==> $.LABEL // PG compatible - ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience - */ - jsonStringInit(&jx, ctx); - if( sqlite3Isdigit(zPath[0]) ){ - jsonAppendRawNZ(&jx, "$[", 2); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendRawNZ(&jx, "]", 2); - }else{ - jsonAppendRawNZ(&jx, "$.", 1 + (zPath[0]!='[')); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendChar(&jx, 0); - } - pNode = jx.eErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); - jsonStringReset(&jx); - }else{ - pNode = jsonLookup(p, zPath, 0, ctx); - } - if( pNode ){ - if( flags & JSON_JSON ){ - jsonReturnNodeAsJson(p, pNode, ctx, 0, 0); - }else{ - jsonReturnFromNode(p, pNode, ctx, 1); - } - } - }else{ - pNode = jsonLookup(p, zPath, 0, ctx); - if( p->nErr==0 && pNode ) jsonReturnFromNode(p, pNode, ctx, 0); - } - }else{ - /* Two or more PATH arguments results in a JSON array with each - ** element of the array being the value selected by one of the PATHs */ - int i; - jsonStringInit(&jx, ctx); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + jsonStringInit(&jx, ctx); + if( argc>2 ){ jsonAppendChar(&jx, '['); - for(i=1; inErr ) break; - jsonAppendSeparator(&jx); - if( pNode ){ - jsonXlateNodeToText(p, pNode, &jx); + } + for(i=1; i and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + */ + jsonStringInit(&jx, ctx); + if( sqlite3Isdigit(zPath[0]) ){ + jsonAppendRawNZ(&jx, "[", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendRawNZ(&jx, "]", 2); + }else if( zPath[0]!='[' ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendChar(&jx, 0); + } + jsonStringTerminate(&jx); + j = jsonLookupBlobStep(p, 0, jx.zBuf, 0); + jsonStringReset(&jx); + }else{ + jsonExtractBadPathError(ctx, zPath); + goto json_extract_error; + } + if( jnBlob ){ + if( argc==2 ){ + if( flags & JSON_JSON ){ + jsonStringInit(&jx, ctx); + jsonXlateBlobToText(p, j, &jx); + jsonReturnString(&jx); + jsonStringReset(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + }else{ + jsonReturnFromBlob(p, j, ctx, 0); + if( (flags & JSON_SQL)==0 && (p->aBlob[j]&0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + } }else{ + jsonAppendSeparator(&jx); + jsonXlateBlobToText(p, j, &jx); + } + }else if( j==JSON_BLOB_NOTFOUND ){ + if( argc==2 ){ + goto json_extract_error; /* Return NULL if not found */ + }else{ + jsonAppendSeparator(&jx); jsonAppendRawNZ(&jx, "null", 4); } + }else if( j==JSON_BLOB_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + goto json_extract_error; + }else{ + jsonExtractBadPathError(ctx, zPath); + goto json_extract_error; } - if( i==argc ){ - jsonAppendChar(&jx, ']'); - jsonReturnString(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); - } - jsonStringReset(&jx); } + if( argc>2 ){ + jsonAppendChar(&jx, ']'); + jsonReturnString(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } +json_extract_error: + jsonStringReset(&jx); + jsonParseFree(p); + return; } /* This is the RFC 7396 MergePatch algorithm. diff --git a/test/json101.test b/test/json101.test index ebaf0f3b34..3c3b1a5ec0 100644 --- a/test/json101.test +++ b/test/json101.test @@ -974,7 +974,7 @@ do_execsql_test json101-18.4 { } {6} do_catchsql_test json101-18.5 { SELECT json_extract('{"":8}', '$.'); -} {1 {JSON path error near ''}} +} {1 {bad JSON path: '$.'}} # 2022-08-29 https://sqlite.org/forum/forumpost/9b9e4716c0d7bbd1 # This is not a problem specifically with JSON functions. It is diff --git a/test/json105.test b/test/json105.test index 83face188e..5592191c22 100644 --- a/test/json105.test +++ b/test/json105.test @@ -96,18 +96,18 @@ json_replace_test 80 {'$.b[#-1]','AAA','$.b[#-1]','BBB'} \ do_catchsql_test json105-6.10 { SELECT json_extract(j, '$.b[#-]') FROM t1; -} {1 {JSON path error near '[#-]'}} +} {1 {bad JSON path: '$.b[#-]'}} do_catchsql_test json105-6.20 { SELECT json_extract(j, '$.b[#9]') FROM t1; -} {1 {JSON path error near '[#9]'}} +} {1 {bad JSON path: '$.b[#9]'}} do_catchsql_test json105-6.30 { SELECT json_extract(j, '$.b[#+2]') FROM t1; -} {1 {JSON path error near '[#+2]'}} +} {1 {bad JSON path: '$.b[#+2]'}} do_catchsql_test json105-6.40 { SELECT json_extract(j, '$.b[#-1') FROM t1; -} {1 {JSON path error near '[#-1'}} +} {1 {bad JSON path: '$.b[#-1'}} do_catchsql_test json105-6.50 { SELECT json_extract(j, '$.b[#-1x]') FROM t1; -} {1 {JSON path error near '[#-1x]'}} +} {1 {bad JSON path: '$.b[#-1x]'}} finish_test From 1ef232c0e1e10d17f58bfea678c0c2774e7f29ab Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Nov 2023 20:33:20 +0000 Subject: [PATCH 111/191] Preserve flexibility in the format of the RHS of -> and ->> operators found in legacy. FossilOrigin-Name: 6231ec43adb7436195eb1497de39a6c13c6b4f1c5032e6ea52515d214e61fdbc --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 7 +++++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 8d932ccf28..aa8750da24 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sget\sjson_extract()\sworking\swith\spure\sJSONB\sonly,\sand\swithout\nthe\suse\sof\sJsonNode.\s\sMostly\sworking,\sbut\sthere\sare\ssome\sdifferences\sfrom\nlegacy\sin\scorner\scases. -D 2023-11-28T20:25:23.336 +C Preserve\sflexibility\sin\sthe\sformat\sof\sthe\sRHS\sof\s->\sand\s->>\soperators\sfound\nin\slegacy. +D 2023-11-28T20:33:20.843 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c bbe9cb34174bfbc0e067a480e431ffc8f0d6cc2cfcad8a1001472bbd00ca620a +F src/json.c 4dfa914bc57559ba1ee7e67211b7146d6ea522630fa2b51ddbc9244097d50b2b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 -R ccd1a96290081a160a58be4f68f67cce -T *branch * jsonb-extract -T *sym-jsonb-extract * -T -sym-jsonb * +P 8c324af1eca27e86adc45622af4f3b06a67a3f968596ac58aa7434b1f6f05f3c +R 8969d2e433c13563f53a82b455df93bc U drh -Z fece822412df4bcd481f5f301c9018f9 +Z 4af78bee74fbed5f3138fbbbd5c5ff0e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2cee28775a..8d4b8f7819 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c324af1eca27e86adc45622af4f3b06a67a3f968596ac58aa7434b1f6f05f3c \ No newline at end of file +6231ec43adb7436195eb1497de39a6c13c6b4f1c5032e6ea52515d214e61fdbc \ No newline at end of file diff --git a/src/json.c b/src/json.c index c259ae1d87..6d1cf0e40b 100644 --- a/src/json.c +++ b/src/json.c @@ -4701,6 +4701,7 @@ static void jsonExtractFunc( for(i=1; i Date: Tue, 28 Nov 2023 23:18:04 +0000 Subject: [PATCH 112/191] Do not set the J subtype when the output is JSONB. FossilOrigin-Name: 4f106b64fe8988435872806bd0a6c223b61f53af0dd1c47c847bb4eec4e03e27 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/json.c | 72 +++++---------------------------------------------- 3 files changed, 14 insertions(+), 72 deletions(-) diff --git a/manifest b/manifest index aa8750da24..9f81009143 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preserve\sflexibility\sin\sthe\sformat\sof\sthe\sRHS\sof\s->\sand\s->>\soperators\sfound\nin\slegacy. -D 2023-11-28T20:33:20.843 +C Do\snot\sset\sthe\sJ\ssubtype\swhen\sthe\soutput\sis\sJSONB. +D 2023-11-28T23:18:04.442 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4dfa914bc57559ba1ee7e67211b7146d6ea522630fa2b51ddbc9244097d50b2b +F src/json.c dd5c3f52877448cc9644887e83a2ff33ab0be87e158de7d5a86db12f90b28045 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c324af1eca27e86adc45622af4f3b06a67a3f968596ac58aa7434b1f6f05f3c -R 8969d2e433c13563f53a82b455df93bc +P 6231ec43adb7436195eb1497de39a6c13c6b4f1c5032e6ea52515d214e61fdbc +R e440543ab70160e0cdb3ca23863ac507 U drh -Z 4af78bee74fbed5f3138fbbbd5c5ff0e +Z d79a3adcec115b27b232bb847ce59659 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8d4b8f7819..2e041f86bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6231ec43adb7436195eb1497de39a6c13c6b4f1c5032e6ea52515d214e61fdbc \ No newline at end of file +4f106b64fe8988435872806bd0a6c223b61f53af0dd1c47c847bb4eec4e03e27 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6d1cf0e40b..1a2af6f8f1 100644 --- a/src/json.c +++ b/src/json.c @@ -4043,69 +4043,6 @@ static void jsonReturnFromBlob( } } -#if 0 -/* Do a JSON_EXTRACT(JSON, PATH) on a when JSON is a BLOB. -*/ -static void jsonExtractFromBlob( - sqlite3_context *ctx, - sqlite3_value *pJson, - sqlite3_value *pPath, - int flags -){ - const char *zPath = (const char*)sqlite3_value_text(pPath); - u32 i = 0; - JsonParse px; - if( zPath==0 ) return; - memset(&px, 0, sizeof(px)); - px.nBlob = sqlite3_value_bytes(pJson); - px.aBlob = (u8*)sqlite3_value_blob(pJson); - if( px.aBlob==0 ) return; - if( zPath[0]=='$' ){ - zPath++; - i = jsonLookupBlobStep(&px, 0, zPath, 0); - }else if( (flags & JSON_ABPATH) ){ - /* The -> and ->> operators accept abbreviated PATH arguments. This - ** is mostly for compatibility with PostgreSQL, but also for - ** convenience. - ** - ** NUMBER ==> $[NUMBER] // PG compatible - ** LABEL ==> $.LABEL // PG compatible - ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience - */ - JsonString jx; - jsonStringInit(&jx, ctx); - if( sqlite3Isdigit(zPath[0]) ){ - jsonAppendRawNZ(&jx, "[", 1); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendRawNZ(&jx, "]", 2); - zPath = jx.zBuf; - }else if( zPath[0]!='[' ){ - jsonAppendRawNZ(&jx, ".", 1); - jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendChar(&jx, 0); - zPath = jx.zBuf; - } - i = jsonLookupBlobStep(&px, 0, zPath, 0); - jsonStringReset(&jx); - }else{ - sqlite3_result_error(ctx, "bad path", -1); - return; - } - if( iaBlob[j]&0x0f)>=JSONB_ARRAY ){ + if( (flags & (JSON_SQL|JSON_BLOB))==0 + && (p->aBlob[j]&0x0f)>=JSONB_ARRAY + ){ sqlite3_result_subtype(ctx, JSON_SUBTYPE); } } @@ -4770,7 +4710,9 @@ static void jsonExtractFunc( if( argc>2 ){ jsonAppendChar(&jx, ']'); jsonReturnString(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); + if( (flags & JSON_BLOB)==0 ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } } json_extract_error: jsonStringReset(&jx); From 4b54d6cdafacfab6f84f4fd2055739d69e34a954 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Nov 2023 01:38:15 +0000 Subject: [PATCH 113/191] Convert the json_array_length() function to use JSONB instead of JsonNodes. FossilOrigin-Name: 5ab790736d943e08f097efcee5cfbf0d83c65b0a53f273060330ba719affa5e5 --- manifest | 13 ++++---- manifest.uuid | 2 +- src/json.c | 89 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 59 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 8499e1f61f..0134cc2e4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sall\sknown\sproblems\swith\sJSONB\sjson_extract(). -D 2023-11-28T23:26:55.773 +C Convert\sthe\sjson_array_length()\sfunction\sto\suse\sJSONB\sinstead\sof\sJsonNodes. +D 2023-11-29T01:38:15.043 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c dd5c3f52877448cc9644887e83a2ff33ab0be87e158de7d5a86db12f90b28045 +F src/json.c aebeb6c1beea6a143223139e9b168b4187aff02c60a49daa6bcafadafcb6ee15 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,9 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b69786e746ae2b927b64d9871fd120b7f8f06cc53739fd46a4da51aa16cf8576 4f106b64fe8988435872806bd0a6c223b61f53af0dd1c47c847bb4eec4e03e27 -R e440543ab70160e0cdb3ca23863ac507 -T +closed 4f106b64fe8988435872806bd0a6c223b61f53af0dd1c47c847bb4eec4e03e27 +P d5f48c57e975ac468cf29a43a5d0b56ef6d06cf35a8b0bddf87ec1c0fc7ae028 +R 4f247130b18247c9fbdce9034c7ad3a3 U drh -Z c2c6abd51345cdb21ffc83ec27ff2d5d +Z 81615ca9c91f916f71f930ae045acac7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a99a8156d..539a333b70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5f48c57e975ac468cf29a43a5d0b56ef6d06cf35a8b0bddf87ec1c0fc7ae028 \ No newline at end of file +5ab790736d943e08f097efcee5cfbf0d83c65b0a53f273060330ba719affa5e5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1a2af6f8f1..d210e5e986 100644 --- a/src/json.c +++ b/src/json.c @@ -4538,6 +4538,18 @@ static void jsonArrayFunc( sqlite3_result_subtype(ctx, JSON_SUBTYPE); } +/* +** Generate a bad path error for json_extract() +*/ +static void jsonBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ +){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + char *zMsg = sqlite3MPrintf(db, "bad JSON path: %Q", zPath); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3DbFree(db, zMsg); +} /* ** json_array_length(JSON) @@ -4552,49 +4564,52 @@ static void jsonArrayLengthFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - sqlite3_int64 n = 0; + sqlite3_int64 cnt = 0; u32 i; - JsonNode *pNode; + u8 eErr = 0; - p = jsonParseCached(ctx, argv[0], ctx, 0); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; - assert( p->nNode ); if( argc==2 ){ const char *zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); - }else{ - pNode = p->aNode; - } - if( pNode==0 ){ - return; - } - if( pNode->eType==JSON_ARRAY ){ - while( 1 /*exit-by-break*/ ){ - i = 1; - while( i<=pNode->n ){ - if( (pNode[i].jnFlags & JNODE_REMOVE)==0 ) n++; - i += jsonNodeSize(&pNode[i]); + if( zPath==0 ){ + jsonParseFree(p); + return; + } + i = jsonLookupBlobStep(p, 0, zPath[0]=='$' ? zPath+1 : "@", 0); + if( JSON_BLOB_ISERROR(i) ){ + if( i==JSON_BLOB_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_BLOB_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( p->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &p->aNode[pNode->u.iAppend]; + eErr = 1; + i = 0; + } + }else{ + i = 0; + } + if( (p->aBlob[i] & 0x0f)==JSONB_ARRAY ){ + u32 n, sz = 0, iEnd; + n = jsonbPayloadSize(p, i, &sz); + if( n==0 ) eErr = 2; + iEnd = i+n+sz; + i += n; + while( eErr==0 && inBlob ){ @@ -4703,7 +4718,7 @@ static void jsonExtractFunc( sqlite3_result_error(ctx, "malformed JSON", -1); goto json_extract_error; }else{ - jsonExtractBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath); goto json_extract_error; } } From 2ba5534101c28d1edcc1f1c6e70a2bfb193b818b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Nov 2023 02:45:09 +0000 Subject: [PATCH 114/191] The assertion change at check-in [7946c79567b0ccd3] is insufficient to fix the problem of a Table object being deleted out from under the OP_VCheck opcode. We need to reference count the Table, which is accomplished here. FossilOrigin-Name: cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db --- manifest | 21 ++++++++++----------- manifest.uuid | 2 +- src/pragma.c | 3 ++- src/vdbe.c | 5 ++--- src/vdbe.h | 1 + src/vdbeaux.c | 4 ++++ 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 6390c0d3c4..d8076c2105 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds,\suse\sblocking\slocks\sin\splace\sof\ssleep()\scalls\swhen\sopening\sa\sread-transaction. -D 2023-11-28T17:12:42.879 +C The\sassertion\schange\sat\scheck-in\s[7946c79567b0ccd3]\sis\sinsufficient\sto\sfix\nthe\sproblem\sof\sa\sTable\sobject\sbeing\sdeleted\sout\sfrom\sunder\sthe\sOP_VCheck\nopcode.\s\sWe\sneed\sto\sreference\scount\sthe\sTable,\swhich\sis\saccomplished\shere. +D 2023-11-29T02:45:09.518 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -721,7 +721,7 @@ F src/parse.y 020d80386eb216ec9520549106353c517d2bbc89be28752ffdca649a9eaf56ec F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 -F src/pragma.c b3b4ad9c0298d63098a067acca613c21a5f56b4d176d5842922bcd0b07b7164e +F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750 @@ -798,11 +798,11 @@ F src/upsert.c fa125a8d3410ce9a97b02cb50f7ae68a2476c405c76aa692d3acf6b8586e9242 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c b22cc9f203a8c0b9ee5338a67f8860347d14845864c10248bebe84518a781677 F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 319af2cf092d20e233e8ad4267ae49bfe33c50ac4db4ee7e47af898f824c2368 -F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 +F src/vdbe.c c7a5a1b27c6567faeeb5f41c72a4f7f81ee34f8ec0449c3dd9ae25b8a28bdccd +F src/vdbe.h 88e19a982df9027ec1c177c793d1a5d34dc23d8f06e3b2d997f43688b05ee0eb F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c F src/vdbeapi.c b07df805110dc6e81f2a3f9cd4e83f56ea523277a59bcec489a12b740c1079e7 -F src/vdbeaux.c b34dfbc09403ccb676608da16ff0780d23d466470563d24fdf6350b8d2271d5e +F src/vdbeaux.c c5a471b34e9c4cfc0295a3e10734fd197670ffaebcb742f284c8e17e8026ceea F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5 F src/vdbemem.c 0012d5f01cc866833847c2f3ae4c318ac53a1cb3d28acad9c35e688039464cf0 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547 @@ -2143,9 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0c87ae9d3db914d18e2c8811db0d0ae3ad7b15c63de84fa975efce28bace27e 5fbf3906d272df3eb981f67455eb35f649ad2774cba9fc3f077b28d9bef3f0cb -R a3e95520ff5d52f362b8ffc8bb5b50fd -T +closed 5fbf3906d272df3eb981f67455eb35f649ad2774cba9fc3f077b28d9bef3f0cb -U dan -Z fe1d300d9f65afce84bb98eff88fc681 +P 4c055b7a6e4533e1e571773456226ca7038ce372df3eedbbbcd9a81e8652a6cf +R b802cc74f401db4801f35603d41bed2d +U drh +Z fcd4003bc0cfee07fff53afb131d1b17 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a36bb8e2e7..47522a290b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c055b7a6e4533e1e571773456226ca7038ce372df3eedbbbcd9a81e8652a6cf \ No newline at end of file +cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 90076b0c26..4c90574182 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1779,7 +1779,8 @@ void sqlite3Pragma( if( pVTab->pModule->iVersion<4 ) continue; if( pVTab->pModule->xIntegrity==0 ) continue; sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); - sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, a1); diff --git a/src/vdbe.c b/src/vdbe.c index 38155a170c..0387f21972 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -8180,9 +8180,10 @@ case OP_VCheck: { /* out2 */ pOut = &aMem[pOp->p2]; sqlite3VdbeMemSetNull(pOut); /* Innocent until proven guilty */ - assert( pOp->p4type==P4_TABLE ); + assert( pOp->p4type==P4_TABLEREF ); pTab = pOp->p4.pTab; assert( pTab!=0 ); + assert( pTab->nTabRef>0 ); assert( IsVirtual(pTab) ); if( pTab->u.vtab.p==0 ) break; pVtab = pTab->u.vtab.p->pVtab; @@ -8191,13 +8192,11 @@ case OP_VCheck: { /* out2 */ assert( pModule!=0 ); assert( pModule->iVersion>=4 ); assert( pModule->xIntegrity!=0 ); - pTab->nTabRef++; sqlite3VtabLock(pTab->u.vtab.p); assert( pOp->p1>=0 && pOp->p1nDb ); rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName, pOp->p3, &zErr); sqlite3VtabUnlock(pTab->u.vtab.p); - sqlite3DeleteTable(db, pTab); if( rc ){ sqlite3_free(zErr); goto abort_due_to_error; diff --git a/src/vdbe.h b/src/vdbe.h index f44f24f93e..25bda6be7a 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -126,6 +126,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ #define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ #define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 114a759c9a..420365e930 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1400,6 +1400,10 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } + case P4_TABLEREF: { + if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); + break; + } } } From ed088f5f8d09d98f0df704e1e56c05d4d42e78b2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Nov 2023 13:47:46 +0000 Subject: [PATCH 115/191] In the recovery extension, if a payload size is unreasonably large, it is probably corrupt, so truncate it. FossilOrigin-Name: 988c3179e978a3a6d42541e9c7a2ab98150383671810926503376ed808f150ff --- ext/recover/dbdata.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index 25a6e9fd6a..b6cb26ecd3 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -582,6 +582,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ bNextPage = 1; }else{ iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload); + if( nPayload>0x7fffff00 ) nPayload &= 0x3fff; } /* If this is a leaf intkey cell, load the rowid */ diff --git a/manifest b/manifest index d8076c2105..0c8757df1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sassertion\schange\sat\scheck-in\s[7946c79567b0ccd3]\sis\sinsufficient\sto\sfix\nthe\sproblem\sof\sa\sTable\sobject\sbeing\sdeleted\sout\sfrom\sunder\sthe\sOP_VCheck\nopcode.\s\sWe\sneed\sto\sreference\scount\sthe\sTable,\swhich\sis\saccomplished\shere. -D 2023-11-29T02:45:09.518 +C In\sthe\srecovery\sextension,\sif\sa\spayload\ssize\sis\sunreasonably\slarge,\sit\sis\nprobably\scorrupt,\sso\struncate\sit. +D 2023-11-29T13:47:46.693 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -460,7 +460,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c d4ddf8f0e93772556e452a6c2814063cf47efb760a0834391a9d0cd9859fa4b9 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c fc7147a68422cbbbaa481ee92ae1752cc25f5a24302bece1c70dcb76345bd736 +F ext/recover/dbdata.c b7746c2ec801b453840831311be4b31f8c8f9dd97791060a69bbf12392c78949 F ext/recover/recover1.test c484d01502239f11b61f23c1cee9f5dd19fa17617f8974e42e74d64639c524cf F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4c055b7a6e4533e1e571773456226ca7038ce372df3eedbbbcd9a81e8652a6cf -R b802cc74f401db4801f35603d41bed2d +P cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db +R 4a3a6ee7f774b8b1af828ef09f5272a7 U drh -Z fcd4003bc0cfee07fff53afb131d1b17 +Z 25e836823c87bebe85ade3cc7e139b02 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 47522a290b..6a5cca1b23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db \ No newline at end of file +988c3179e978a3a6d42541e9c7a2ab98150383671810926503376ed808f150ff \ No newline at end of file From 89fcfbb424b1c36b97acd151267adb7c574f8240 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 29 Nov 2023 16:22:39 +0000 Subject: [PATCH 116/191] Fix signed integer overflow in fts5. FossilOrigin-Name: 60e46c7ec68fd8caaed960ca06d98fb06855b2d0bb860dd2fb7b5e89a5e9c7b4 --- ext/fts5/fts5_index.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 993069f490..4b7c8d3335 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4276,7 +4276,7 @@ static void fts5WriteDlidxAppend( } if( pDlidx->bPrevValid ){ - iVal = iRowid - pDlidx->iPrev; + iVal = (u64)iRowid - (u64)pDlidx->iPrev; }else{ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); diff --git a/manifest b/manifest index e69bda2b13..13a7da6a20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\supdates\sinto\sthis\sbranch. -D 2023-11-28T19:43:08.965 +C Fix\ssigned\sinteger\soverflow\sin\sfts5. +D 2023-11-29T16:22:39.960 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c de7bc351d9b7e0c8891db93b211fd9e137edef2cec832dcedfb32d4a2929c0d2 +F ext/fts5/fts5_index.c bafdef8be40a20bb86a131af4f09b56919f416fa13d1a86af1bf92bad9a2870d F ext/fts5/fts5_main.c 55b53085dbd1693b5735463198a8d124dfbc27f08311c839637b44b8254ef7cb F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -264,7 +264,7 @@ F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 482f53dfec9e3ac2a9070d3fceebd56250932aaaf7c4f5bc8de29fc011416e0c F ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b995ca412f59b631803b93aa5b3684fce62e335d1e123207084c054abfd488d4 -F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 w ext/jni/src/org/sqlite/jni/capi/ConfigSqllogCallback.java +F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 F ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b7036dcb1ef1b39f1f36ac605dde0ff1a24a9a01ade6aa1a605039443e089a61 F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 246b0e66c4603f41c567105a21189d138aaf8c58203ecd4928802333da553e7c F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 97352091abd7556167f4799076396279a51749fdae2b72a6ba61cd39b3df0359 @@ -2146,8 +2146,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9be8969edd49e3da96fb8ac2279aff6fe2e215d6ac55162b4734aca1b6316580 4c055b7a6e4533e1e571773456226ca7038ce372df3eedbbbcd9a81e8652a6cf -R ec1a7fd48afc240687b4ab2330f993e9 +P 554fc13f2ca5f2ebd9ad0206034c25b556ff40db3106051c5e539f2e142e88ea +R 5c7c1632cd09d2b131adef895eab90a8 U dan -Z 65a112da054f34cd19f6d265591002ed +Z 009b023eba91b6acc35beb7d051a677a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ca4cbb4d0..df2c2789ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -554fc13f2ca5f2ebd9ad0206034c25b556ff40db3106051c5e539f2e142e88ea \ No newline at end of file +60e46c7ec68fd8caaed960ca06d98fb06855b2d0bb860dd2fb7b5e89a5e9c7b4 \ No newline at end of file From da2578391d5ae7cc8ba97f00994d14cdf6ab4a1f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Nov 2023 17:36:54 +0000 Subject: [PATCH 117/191] The json_patch() function now operates exclusively on JSONB. This patch also includes improvements to JSONB debug printing routines. FossilOrigin-Name: fee19d0098242110d2c44ec7b9620c1210ef3f87913305f66ec85d277dd96ab6 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 205 +++++++++++--------------------------------------- 3 files changed, 51 insertions(+), 168 deletions(-) diff --git a/manifest b/manifest index 275eb80263..a779c2b472 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\senhancements\sinto\sthe\sjsonb\sbranch. -D 2023-11-29T12:18:02.265 +C The\sjson_patch()\sfunction\snow\soperates\sexclusively\son\sJSONB.\s\sThis\spatch\nalso\sincludes\simprovements\sto\sJSONB\sdebug\sprinting\sroutines. +D 2023-11-29T17:36:54.076 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c aebeb6c1beea6a143223139e9b168b4187aff02c60a49daa6bcafadafcb6ee15 +F src/json.c 00b55c9e54047e1412fb7303e954e996c84bb8b06d3e376b12f49178c4fd7b69 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ab790736d943e08f097efcee5cfbf0d83c65b0a53f273060330ba719affa5e5 cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db -R e74b99bb550123c4a0166b07666ffd15 +P 1a59fcab2179cc3b52ecd3de7d2018db96ac149aaff521959773a517b8d9ac3e +R 883e1938b9afc8e6538b46e875ce6c0e U drh -Z 2ea38ce368a7d99793b6d4f302fe78c3 +Z 19b3c18296db473321d95b26a8cafb5e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2b29025123..f95e15f1a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a59fcab2179cc3b52ecd3de7d2018db96ac149aaff521959773a517b8d9ac3e \ No newline at end of file +fee19d0098242110d2c44ec7b9620c1210ef3f87913305f66ec85d277dd96ab6 \ No newline at end of file diff --git a/src/json.c b/src/json.c index d210e5e986..c0fca583a4 100644 --- a/src/json.c +++ b/src/json.c @@ -2033,15 +2033,6 @@ static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ if( pNode->n!=nKey ) return 0; return strncmp(pNode->u.zJContent, zKey, nKey)==0; } -static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ - if( p1->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p2, p1->u.zJContent, p1->n); - }else if( p2->jnFlags & JNODE_RAW ){ - return jsonLabelCompare(p1, p2->u.zJContent, p2->n); - }else{ - return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; - } -} /* ** Search along zPath to find the node specified. Return a pointer @@ -2327,25 +2318,6 @@ static void jsonWrongNumArgs( sqlite3_free(zMsg); } -/* -** Mark all NULL entries in the Object passed in as JNODE_REMOVE. -*/ -static void jsonRemoveAllNulls(JsonNode *pNode){ - int i, n; - assert( pNode->eType==JSON_OBJECT ); - n = pNode->n; - for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ - switch( pNode[i].eType ){ - case JSON_NULL: - pNode[i].jnFlags |= JNODE_REMOVE; - break; - case JSON_OBJECT: - jsonRemoveAllNulls(&pNode[i]); - break; - } - } -} - /**************************************************************************** ** Utility routines for dealing with the binary BLOB representation of JSON ****************************************************************************/ @@ -3111,7 +3083,9 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; n = 5; } - if( i+sz+n>pParse->nBlob ){ + if( i+sz+n > pParse->nBlob + && i+sz+n > pParse->nBlob-pParse->delta + ){ sz = 0; n = 0; } @@ -4314,18 +4288,33 @@ static void jsonDebugPrintBlob( u32 i, n, nn, sz = 0; int showContent = 1; u8 x = pParse->aBlob[iStart] & 0x0f; + u32 savedNBlob = pParse->nBlob; printf("%5d:%*s", iStart, nIndent, ""); + if( pParse->nBlobAlloc>pParse->nBlob ){ + pParse->nBlob = pParse->nBlobAlloc; + } nn = n = jsonbPayloadSize(pParse, iStart, &sz); if( nn==0 ) nn = 1; if( sz>0 && xaBlob[iStart+i]); - if( n==0 || iStart+n+sz>iEnd ){ + if( n==0 ){ printf(" ERROR invalid node size\n"); iStart = n==0 ? iStart+1 : iEnd; continue; } + pParse->nBlob = savedNBlob; + if( iStart+n+sz>iEnd ){ + iEnd = iStart+n+sz; + if( iEnd>pParse->nBlob ){ + if( pParse->nBlobAlloc>0 && iEnd>pParse->nBlobAlloc ){ + iEnd = pParse->nBlobAlloc; + }else{ + iEnd = pParse->nBlob; + } + } + } printf(" <-- "); switch( x ){ case JSONB_NULL: printf("null"); break; @@ -4379,7 +4368,11 @@ static void jsonShowParse(JsonParse *pParse){ printf("NULL pointer\n"); return; }else{ - printf("%u byte JSONB:\n", pParse->nBlob); + printf("nBlobAlloc = %u\n", pParse->nBlobAlloc); + printf("nBlob = %u\n", pParse->nBlob); + printf("delta = %d\n", pParse->delta); + if( pParse->nBlob==0 ) return; + printf("content (bytes 0..%u):\n", pParse->nBlob-1); } jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0); } @@ -4735,79 +4728,6 @@ json_extract_error: return; } -/* This is the RFC 7396 MergePatch algorithm. -*/ -static JsonNode *jsonMergePatch( - JsonParse *pParse, /* The JSON parser that contains the TARGET */ - u32 iTarget, /* Node of the TARGET in pParse */ - JsonNode *pPatch /* The PATCH */ -){ - u32 i, j; - u32 iRoot; - JsonNode *pTarget; - if( pPatch->eType!=JSON_OBJECT ){ - return pPatch; - } - assert( iTargetnNode ); - pTarget = &pParse->aNode[iTarget]; - assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); - if( pTarget->eType!=JSON_OBJECT ){ - jsonRemoveAllNulls(pPatch); - return pPatch; - } - iRoot = iTarget; - for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ - u32 nKey; - const char *zKey; - if( pPatch[i].eType!=JSON_STRING ){ - pParse->nErr = 1; - return 0; - } - assert( pPatch[i].eU==1 ); - nKey = pPatch[i].n; - zKey = pPatch[i].u.zJContent; - for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ - assert( pTarget[j].eType==JSON_STRING ); - assert( pTarget[j].jnFlags & JNODE_LABEL ); - if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break; - if( pPatch[i+1].eType==JSON_NULL ){ - pTarget[j+1].jnFlags |= JNODE_REMOVE; - }else{ - JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); - if( pNew==0 ) return 0; - if( pNew!=&pParse->aNode[iTarget+j+1] ){ - jsonParseAddSubstNode(pParse, iTarget+j+1); - jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew)); - } - pTarget = &pParse->aNode[iTarget]; - } - break; - } - } - if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart; - JsonNode *pApnd; - u32 nApnd; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - pApnd = &pPatch[i+1]; - if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd); - nApnd = jsonNodeSize(pApnd); - jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd)); - if( pParse->oom ) return 0; - pParse->aNode[iStart].n = 1+nApnd; - pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; - pParse->aNode[iRoot].u.iAppend = iStart; - JSON_VVA( pParse->aNode[iRoot].eU = 2 ); - iRoot = iStart; - pTarget = &pParse->aNode[iTarget]; - } - } - return pTarget; -} - - /* ** Return codes for jsonMergePatchBlob() */ @@ -4908,9 +4828,10 @@ static int jsonMergePatchBlob( } x = pTarget->aBlob[iTarget] & 0x0f; if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ - static const u8 emptyObject[] = { JSONB_OBJECT }; n = jsonbPayloadSize(pTarget, iTarget, &sz); - jsonBlobEdit(pTarget, iTarget, sz, emptyObject, 1); /* Line 06 */ + jsonBlobEdit(pTarget, iTarget+n, sz, 0, 0); + x = pTarget->aBlob[iTarget]; + pTarget->aBlob[iTarget] = (x & 0xf0) | JSONB_OBJECT; } n = jsonbPayloadSize(pPatch, iPatch, &sz); if( n==0 ) return JSON_MERGE_BADPATCH; @@ -5019,7 +4940,7 @@ static int jsonMergePatchBlob( } } } - jsonAfterEditSizeAdjust(pTarget, iTarget); + if( pTarget->delta ) jsonAfterEditSizeAdjust(pTarget, iTarget); return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; } @@ -5034,65 +4955,27 @@ static void jsonPatchFunc( int argc, sqlite3_value **argv ){ - JsonParse *pX; /* The JSON that is being patched */ - JsonParse *pY; /* The patch */ - JsonNode *pResult; /* The result of the merge */ + JsonParse *pTarget; /* The TARGET */ + JsonParse *pPatch; /* The PATCH */ + int rc; /* Result code */ UNUSED_PARAMETER(argc); - if( jsonFuncArgMightBeBinary(argv[0]) - && jsonFuncArgMightBeBinary(argv[1]) - ){ - JsonParse target, patch; - sqlite3 *db; - int rc; - memset(&target, 0, sizeof(target)); - memset(&patch, 0, sizeof(patch)); - target.aBlob = (u8*)sqlite3_value_blob(argv[0]); - target.nBlob = sqlite3_value_bytes(argv[0]); - patch.aBlob = (u8*)sqlite3_value_blob(argv[1]); - patch.nBlob = sqlite3_value_bytes(argv[1]); - db = sqlite3_context_db_handle(ctx); - if( db->mallocFailed ) return; - if( !jsonBlobMakeEditable(&target, patch.nBlob) ) return; - rc = jsonMergePatchBlob(&target, 0, &patch, 0); - if( rc ){ - if( rc==JSON_MERGE_OOM ){ - sqlite3_result_error_nomem(ctx); - }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); - } - jsonParseReset(&target); + assert( argc==2 ); + pTarget = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + if( pTarget==0 ) return; + pPatch = jsonParseFuncArg(ctx, argv[1], 0); + if( pPatch ){ + rc = jsonMergePatchBlob(pTarget, 0, pPatch, 0); + if( rc==JSON_MERGE_OK ){ + jsonReturnParse(ctx, pTarget); + }else if( rc==JSON_MERGE_OOM ){ + sqlite3_result_error_nomem(ctx); }else{ - int flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - if( flgs & JSON_BLOB ){ - sqlite3_result_blob(ctx, target.aBlob, target.nBlob, SQLITE_DYNAMIC); - }else{ - JsonString s; - jsonStringInit(&s, ctx); - jsonXlateBlobToText(&target, 0, &s); - jsonReturnString(&s); - jsonParseReset(&target); - } + sqlite3_result_error(ctx, "malformed JSON", -1); } - return; - } - pX = jsonParseCached(ctx, argv[0], ctx, 1); - if( pX==0 ) return; - assert( pX->hasMod==0 ); - pX->hasMod = 1; - pY = jsonParseCached(ctx, argv[1], ctx, 1); - if( pY==0 ) return; - pX->useMod = 1; - pY->useMod = 1; - pResult = jsonMergePatch(pX, 0, pY->aNode); - assert( pResult!=0 || pX->oom || pX->nErr ); - if( pX->oom ){ - sqlite3_result_error_nomem(ctx); - }else if( pX->nErr ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - }else if( pResult ){ - jsonReturnNodeAsJson(pX, pResult, ctx, 0, 0); + jsonParseFree(pPatch); } + jsonParseFree(pTarget); } From 694beecf2bef064f08426a23fd7c8097285c83c5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Nov 2023 20:06:49 +0000 Subject: [PATCH 118/191] Convert the json_error_position() routine to use only JSONB internally. FossilOrigin-Name: e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad --- manifest | 12 +++++------ manifest.uuid | 2 +- src/json.c | 55 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a779c2b472..393b7c556d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_patch()\sfunction\snow\soperates\sexclusively\son\sJSONB.\s\sThis\spatch\nalso\sincludes\simprovements\sto\sJSONB\sdebug\sprinting\sroutines. -D 2023-11-29T17:36:54.076 +C Convert\sthe\sjson_error_position()\sroutine\sto\suse\sonly\sJSONB\sinternally. +D 2023-11-29T20:06:49.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 00b55c9e54047e1412fb7303e954e996c84bb8b06d3e376b12f49178c4fd7b69 +F src/json.c d93f5e7d0e0a50c5842657439c3e820802f862da0d2aaf28df08cb3528acc0aa F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a59fcab2179cc3b52ecd3de7d2018db96ac149aaff521959773a517b8d9ac3e -R 883e1938b9afc8e6538b46e875ce6c0e +P fee19d0098242110d2c44ec7b9620c1210ef3f87913305f66ec85d277dd96ab6 +R de34ae25adb13e4614fb6787c64e8886 U drh -Z 19b3c18296db473321d95b26a8cafb5e +Z beaa9ff647aa958b069175a31fff1f35 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f95e15f1a1..0114d98f13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fee19d0098242110d2c44ec7b9620c1210ef3f87913305f66ec85d277dd96ab6 \ No newline at end of file +e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad \ No newline at end of file diff --git a/src/json.c b/src/json.c index c0fca583a4..ba15e12e17 100644 --- a/src/json.c +++ b/src/json.c @@ -5438,25 +5438,48 @@ static void jsonErrorFunc( int argc, sqlite3_value **argv ){ - JsonParse *p; /* The parse */ + i64 iErrPos = 0; /* Error position to be returned */ + JsonParse s; + + assert( argc==1 ); UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - p = jsonParseCached(ctx, argv[0], 0, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); - }else if( p->nErr==0 ){ - sqlite3_result_int(ctx, 0); - }else{ - int n = 1; - u32 i; - const char *z = (const char*)sqlite3_value_text(argv[0]); - for(i=0; iiErr && ALWAYS(z[i]); i++){ - if( (z[i]&0xc0)!=0x80 ) n++; + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_NULL: { + return; + } + case SQLITE_BLOB: { + if( !jsonFuncArgMightBeBinary(argv[0]) ) iErrPos = 1; + break; + } + default: { + memset(&s, 0, sizeof(s)); + s.zJson = (char*)sqlite3_value_text(argv[0]); + s.nJson = sqlite3_value_bytes(argv[0]); + if( s.nJson==0 ){ + iErrPos = 1; + break; + } + if( s.zJson==0 ){ + sqlite3_result_error_nomem(ctx); + return; + } + if( jsonConvertTextToBlob(&s,0) ){ + u32 k; + if( s.oom ){ + sqlite3_result_error_nomem(ctx); + jsonParseReset(&s); + return; + } + /* Convert byte-offset s.iErr into a character offset */ + for(k=0; k Date: Thu, 30 Nov 2023 00:52:33 +0000 Subject: [PATCH 119/191] Convert json_insert(), json_replace(), json_set() to use JSONB internally. Mostly working, but some corner cases are still not quite right. FossilOrigin-Name: 99c8f6bd5c9a31b6d00f92e383bec8a8235ed553916ad59adbb1b7663f6ebff1 --- manifest | 15 +- manifest.uuid | 2 +- src/json.c | 784 ++++---------------------------------------------- 3 files changed, 72 insertions(+), 729 deletions(-) diff --git a/manifest b/manifest index 393b7c556d..d50bd996f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\sjson_error_position()\sroutine\sto\suse\sonly\sJSONB\sinternally. -D 2023-11-29T20:06:49.393 +C Convert\sjson_insert(),\sjson_replace(),\sjson_set()\sto\suse\sJSONB\sinternally.\nMostly\sworking,\sbut\ssome\scorner\scases\sare\sstill\snot\squite\sright. +D 2023-11-30T00:52:33.320 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d93f5e7d0e0a50c5842657439c3e820802f862da0d2aaf28df08cb3528acc0aa +F src/json.c 8b509dead923cd44f31e4866d1024f1b602bdf01c9d41a08cc36aa5f1203fdf8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fee19d0098242110d2c44ec7b9620c1210ef3f87913305f66ec85d277dd96ab6 -R de34ae25adb13e4614fb6787c64e8886 +P e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad +R 4c74c3d174d84fc0b9e05e4cabac42e8 +T *branch * jsonb-insert +T *sym-jsonb-insert * +T -sym-jsonb * U drh -Z beaa9ff647aa958b069175a31fff1f35 +Z 396365e2a8dc6b109097922e3ec543f6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0114d98f13..bed72fa4ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad \ No newline at end of file +99c8f6bd5c9a31b6d00f92e383bec8a8235ed553916ad59adbb1b7663f6ebff1 \ No newline at end of file diff --git a/src/json.c b/src/json.c index ba15e12e17..b3b4f993e6 100644 --- a/src/json.c +++ b/src/json.c @@ -371,16 +371,22 @@ struct JsonParse { */ #define JSON_MAX_DEPTH 1000 +/* +** Allowed values for the flgs argument to jsonParseFuncArg(); +*/ +#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ + /************************************************************************** ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static void jsonXlateNodeToBlob(JsonParse*,JsonNode*,JsonParse*); static int jsonParseAddNode(JsonParse*,u32,u32,const char*); static int jsonXlateBlobToNode(JsonParse *pParse, u32 i); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); +static void jsonReturnParse(sqlite3_context*,JsonParse*); +static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -592,159 +598,6 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ assert( p->nUsednAlloc ); } -/* -** The zIn[0..N] string is a JSON5 string literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. -*/ -static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ - u32 i; - jsonAppendChar(p, '"'); - while( N>0 ){ - for(i=0; i0 ){ - jsonAppendRawNZ(p, zIn, i); - if( i>=N ) break; - zIn += i; - N -= i; - } - if( N<2 ){ - p->eErr |= JSTRING_MALFORMED; - break; - } - if( zIn[0]=='"' ){ - jsonAppendRawNZ(p, "\\\"", 2); - zIn++; - N--; - continue; - } - assert( zIn[0]=='\\' ); - switch( (u8)zIn[1] ){ - case '\'': - jsonAppendChar(p, '\''); - break; - case 'v': - jsonAppendRawNZ(p, "\\u0009", 6); - break; - case 'x': - if( N<4 ){ - N = 2; - p->eErr |= JSTRING_MALFORMED; - break; - } - jsonAppendRawNZ(p, "\\u00", 4); - jsonAppendRawNZ(p, &zIn[2], 2); - zIn += 2; - N -= 2; - break; - case '0': - jsonAppendRawNZ(p, "\\u0000", 6); - break; - case '\r': - if( N>2 && zIn[2]=='\n' ){ - zIn++; - N--; - } - break; - case '\n': - break; - case 0xe2: /* \ followed by U+2028 or U+2029 line terminator ignored */ - if( N<4 - || 0x80!=(u8)zIn[2] - || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) - ){ - N = 2; - p->eErr |= JSTRING_MALFORMED; - break; - } - assert( N>=4 ); - assert( 0x80==(u8)zIn[2] ); - assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); - zIn += 2; - N -= 2; - break; - default: - jsonAppendRawNZ(p, zIn, 2); - break; - } - assert( N>=2 ); - zIn += 2; - N -= 2; - } - jsonAppendChar(p, '"'); -} - -/* -** The zIn[0..N] string is a JSON5 integer literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. -*/ -static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ - char *zBuf = sqlite3_malloc64( N+1 ); - if( zBuf==0 ){ - p->eErr |= JSTRING_OOM; - return; - } - memcpy(zBuf, zIn, N); - zBuf[N] = 0; - zIn = zBuf; - if( zIn[0]=='+' ){ - zIn++; - N--; - }else if( zIn[0]=='-' ){ - jsonAppendChar(p, '-'); - zIn++; - N--; - } - if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){ - sqlite3_int64 i = 0; - int rc = sqlite3DecOrHexToI64(zIn, &i); - if( rc<=1 ){ - jsonPrintf(100,p,"%lld",i); - }else{ - assert( rc==2 ); - jsonAppendRawNZ(p, "9.0e999", 7); - } - }else{ - assert( N>0 ); - jsonAppendRawNZ(p, zIn, N); - } - sqlite3_free(zBuf); -} - -/* -** The zIn[0..N] string is a JSON5 real literal. Append to p a translation -** of the string literal that standard JSON and that omits all JSON5 -** features. -*/ -static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( zIn[0]=='+' ){ - zIn++; - N--; - }else if( zIn[0]=='-' ){ - jsonAppendChar(p, '-'); - zIn++; - N--; - } - if( zIn[0]=='.' ){ - jsonAppendChar(p, '0'); - } - for(i=0; i0 ){ - jsonAppendRawNZ(p, zIn, N); - } -} - /* ** Append an sqlite3_value (such as a function parameter) to the JSON ** string under construction in p. @@ -895,194 +748,6 @@ static void jsonParseFree(JsonParse *pParse){ } } -/* -** Add a cleanup task to the JsonParse object. -** -** If an OOM occurs, the cleanup operation happens immediately -** and this function returns SQLITE_NOMEM. -*/ -static int jsonParseAddCleanup( - JsonParse *pParse, /* Add the cleanup task to this parser */ - void(*xOp)(void*), /* The cleanup task */ - void *pArg /* Argument to the cleanup */ -){ - JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) ); - if( pTask==0 ){ - pParse->oom = 1; - xOp(pArg); - return SQLITE_ERROR; - } - pTask->pJCNext = pParse->pClup; - pParse->pClup = pTask; - pTask->xOp = xOp; - pTask->pArg = pArg; - return SQLITE_OK; -} - -/* -** Translate the JsonNode pNode into a pure JSON string and -** append that string on pOut. Subsubstructure is also included. -*/ -static void jsonXlateNodeToText( - JsonParse *pParse, /* the complete parse of the JSON */ - JsonNode *pNode, /* The node to render */ - JsonString *pOut /* Write JSON here */ -){ - assert( pNode!=0 ); - while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ - u32 idx = (u32)(pNode - pParse->aNode); - u32 i = pParse->iSubst; - while( 1 /*exit-by-break*/ ){ - assert( inNode ); - assert( pParse->aNode[i].eType==JSON_SUBST ); - assert( pParse->aNode[i].eU==4 ); - assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ - pNode = &pParse->aNode[i+1]; - break; - } - i = pParse->aNode[i].u.iPrev; - } - } - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonAppendRawNZ(pOut, "null", 4); - break; - } - case JSON_TRUE: { - jsonAppendRawNZ(pOut, "true", 4); - break; - } - case JSON_FALSE: { - jsonAppendRawNZ(pOut, "false", 5); - break; - } - case JSON_STRING: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - }else if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); - }else{ - jsonAppendChar(pOut, '"'); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - jsonAppendChar(pOut, '"'); - } - break; - } - case JSON_REAL: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); - }else{ - assert( pNode->n>0 ); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_INT: { - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); - }else{ - assert( pNode->n>0 ); - jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); - } - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonAppendSeparator(pOut); - jsonXlateNodeToText(pParse, &pNode[j], pOut); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( jn ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonAppendSeparator(pOut); - jsonXlateNodeToText(pParse, &pNode[j], pOut); - jsonAppendChar(pOut, ':'); - jsonXlateNodeToText(pParse, &pNode[j+1], pOut); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; - } - } -} - -/* -** Make the return value of an SQL function be the JSON encoded by pNode. -** -** By default, the node is rendered as RFC-8259 JSON text (canonical -** JSON text without any JSON-5 enhancements). However if the -** JSON_BLOB flag is set in the user-data for the function, then the -** node is rendered into the JSONB format and returned as a BLOB. -*/ -static void jsonReturnNodeAsJson( - JsonParse *pParse, /* The complete JSON */ - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - int bGenerateAlt, /* Also store the rendered text in zAlt */ - int omitSubtype /* Do not call sqlite3_result_subtype() */ -){ - int flags; - JsonString s; - if( pParse->oom ){ - sqlite3_result_error_nomem(pCtx); - return; - } - if( pParse->nErr ){ - return; - } - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); - if( flags & JSON_BLOB ){ - JsonParse x; - memset(&x, 0, sizeof(x)); - jsonXlateNodeToBlob(pParse, pNode, &x); - if( x.oom ){ - sqlite3_result_error_nomem(pCtx); - sqlite3_free(x.aBlob); - }else{ - sqlite3_result_blob(pCtx, x.aBlob, x.nBlob, sqlite3_free); - } - }else{ - jsonStringInit(&s, pCtx); - jsonXlateNodeToText(pParse, pNode, &s); - if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ - pParse->zAlt = sqlite3RCStrRef(s.zBuf); - pParse->nAlt = s.nUsed; - } - jsonReturnString(&s); - if( !omitSubtype ) sqlite3_result_subtype(pCtx, JSON_SUBTYPE); - } -} - /* ** Translate a single byte of Hex into an integer. ** This routine only works if h really is a valid hexadecimal @@ -1178,52 +843,6 @@ static int jsonParseAddNode( return pParse->nNode++; } -/* -** Add an array of new nodes to the current pParse->aNode array. -** Return the index of the first node added. -** -** If an OOM error occurs, set pParse->oom. -*/ -static void jsonParseAddNodeArray( - JsonParse *pParse, /* Append the node to this object */ - JsonNode *aNode, /* Array of nodes to add */ - u32 nNode /* Number of elements in aNew */ -){ - assert( aNode!=0 ); - assert( nNode>=1 ); - if( pParse->nNode + nNode > pParse->nAlloc ){ - u32 nNew = pParse->nNode + nNode; - JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode)); - if( aNew==0 ){ - pParse->oom = 1; - return; - } - pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode); - pParse->aNode = aNew; - } - memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode)); - pParse->nNode += nNode; -} - -/* -** Add a new JSON_SUBST node. The node immediately following -** this new node will be the substitute content for iNode. -*/ -static int jsonParseAddSubstNode( - JsonParse *pParse, /* Add the JSON_SUBST here */ - u32 iNode /* References this node */ -){ - int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); - if( pParse->oom ) return -1; - pParse->aNode[iNode].jnFlags |= JNODE_REPLACE; - pParse->aNode[idx].eU = 4; - pParse->aNode[idx].u.iPrev = pParse->iSubst; - pParse->iSubst = idx; - pParse->hasMod = 1; - pParse->useMod = 1; - return idx; -} - /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -3446,138 +3065,6 @@ static int jsonXlateBlobToNode(JsonParse *pParse, u32 i){ return i+x+sz; } -/* -** Translate pNode (which is always a node found in pParse->aNode[]) into -** the JSONB representation and append the translation onto the end of the -** pOut->aBlob[] array. -*/ -static void jsonXlateNodeToBlob( - JsonParse *pParse, /* the complete parse of the JSON */ - JsonNode *pNode, /* The node to render */ - JsonParse *pOut /* Write the BLOB rendering of JSON here */ -){ - assert( pNode!=0 ); - while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ - u32 idx = (u32)(pNode - pParse->aNode); - u32 i = pParse->iSubst; - while( 1 /*exit-by-break*/ ){ - assert( inNode ); - assert( pParse->aNode[i].eType==JSON_SUBST ); - assert( pParse->aNode[i].eU==4 ); - assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ - pNode = &pParse->aNode[i+1]; - break; - } - i = pParse->aNode[i].u.iPrev; - } - } - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonBlobAppendNodeType(pOut, JSONB_NULL, 0); - break; - } - case JSON_TRUE: { - jsonBlobAppendNodeType(pOut, JSONB_TRUE, 0); - break; - } - case JSON_FALSE: { - jsonBlobAppendNodeType(pOut, JSONB_FALSE, 0); - break; - } - case JSON_STRING: { - int op; - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_RAW ){ - if( memchr(pNode->u.zJContent, '"', pNode->n)==0 - && memchr(pNode->u.zJContent, '\\', pNode->n)==0 - ){ - op = JSONB_TEXT; - }else{ - op = JSONB_TEXTRAW; - } - }else if( pNode->jnFlags & JNODE_JSON5 ){ - op = JSONB_TEXT5; - }else{ - op = JSONB_TEXTJ; - } - jsonBlobAppendNodeType(pOut, op, pNode->n); - jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); - break; - } - case JSON_REAL: { - int op; - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - op = JSONB_FLOAT5; - }else{ - assert( pNode->n>0 ); - op = JSONB_FLOAT; - } - jsonBlobAppendNodeType(pOut, op, pNode->n); - jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); - break; - } - case JSON_INT: { - int op; - assert( pNode->eU==1 ); - if( pNode->jnFlags & JNODE_JSON5 ){ - op = JSONB_INT5; - }else{ - assert( pNode->n>0 ); - op = JSONB_INT; - } - jsonBlobAppendNodeType(pOut, op, pNode->n); - jsonBlobAppendNBytes(pOut, (const u8*)pNode->u.zJContent, pNode->n); - break; - } - case JSON_ARRAY: { - u32 j = 1; - u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_ARRAY, pParse->nJson*2); - iStart = pOut->nBlob; - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonXlateNodeToBlob(pParse, &pNode[j], pOut); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); - break; - } - case JSON_OBJECT: { - u32 j = 1; - u32 iStart, iThis = pOut->nBlob; - jsonBlobAppendNodeType(pOut, JSONB_OBJECT, pParse->nJson*2); - iStart = pOut->nBlob; - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - jsonXlateNodeToBlob(pParse, &pNode[j], pOut); - jsonXlateNodeToBlob(pParse, &pNode[j+1], pOut); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pNode->eU==2 ); - pNode = &pParse->aNode[pNode->u.iAppend]; - j = 1; - } - jsonBlobChangePayloadSize(pOut, iThis, pOut->nBlob - iStart); - break; - } - } -} - /* ** Given that a JSONB_ARRAY object starts at offset i, return ** the number of entries in that array. @@ -3750,13 +3237,12 @@ static u32 jsonLookupBlobStep( if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+pParse->nIns) ){ nIns = ix.nBlob + nKey + pParse->nIns; jsonBlobEdit(pParse, j, 0, 0, nIns); - assert( pParse->nBlob + pParse->nIns <= pParse->nBlobAlloc ); memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); k += nKey; memcpy(&pParse->aBlob[k], pParse->aIns, pParse->nIns); - jsonAfterEditSizeAdjust(pParse, iRoot); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); } return j; } @@ -3805,14 +3291,14 @@ static u32 jsonLookupBlobStep( j += n+sz; } if( j>iEnd ) return JSON_BLOB_ERROR; - if( k>1 ) return JSON_BLOB_NOTFOUND; + if( k>0 ) return JSON_BLOB_NOTFOUND; if( pParse->eEdit>=JEDIT_INS && jsonBlobMakeEditable(pParse, pParse->nIns) ){ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); jsonBlobEdit(pParse, j, 0, pParse->aIns, pParse->nIns); - jsonAfterEditSizeAdjust(pParse, iRoot); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return j; } }else{ @@ -4088,9 +3574,27 @@ static int jsonFunctionArgToBlob( break; } } - return 0; + if( pParse->oom ){ + sqlite3_result_error_nomem(ctx); + return 1; + }else{ + return 0; + } } - + +/* +** Generate a bad path error for json_extract() +*/ +static void jsonBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ +){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + char *zMsg = sqlite3MPrintf(db, "bad JSON path: %Q", zPath); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3DbFree(db, zMsg); +} + /* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent ** arguments come in parse where each pair contains a JSON path and ** content to insert or set at that patch. Do the updates @@ -4109,61 +3613,57 @@ static void jsonInsertIntoBlob( u32 rc = 0; const char *zPath = 0; int flgs; - JsonParse px, ax; + JsonParse *p; + JsonParse ax; assert( (argc&1)==1 ); - memset(&px, 0, sizeof(px)); - px.nBlob = sqlite3_value_bytes(argv[0]); - px.aBlob = (u8*)sqlite3_value_blob(argv[0]); - if( px.aBlob==0 ) return; + flgs = argc==1 ? 0 : JSON_EDITABLE; + p = jsonParseFuncArg(ctx, argv[0], flgs); + if( p==0 ) return; for(i=1; inBlob, ax.aBlob, ax.nBlob); + } + rc = 0; + }else{ + p->eEdit = eEdit; + p->nIns = ax.nBlob; + p->aIns = ax.aBlob; + p->delta = 0; + rc = jsonLookupBlobStep(p, 0, zPath+1, 0); } - px.eEdit = eEdit; - px.nIns = ax.nBlob; - px.aIns = ax.aBlob; - px.delta = 0; - rc = jsonLookupBlobStep(&px, 0, zPath+1, 0); jsonParseReset(&ax); if( rc==JSON_BLOB_NOTFOUND ) continue; if( JSON_BLOB_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; } - flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - if( flgs & JSON_BLOB ){ - sqlite3_result_blob(ctx, px.aBlob, px.nBlob, - px.nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); - }else{ - JsonString s; - jsonStringInit(&s, ctx); - jsonXlateBlobToText(&px, 0, &s); - jsonReturnString(&s); - jsonParseReset(&px); - } + jsonReturnParse(ctx, p); + jsonParseFree(p); return; jsonInsertIntoBlob_patherror: - jsonParseReset(&px); + jsonParseFree(p); if( rc==JSON_BLOB_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ - jsonPathSyntaxError(zPath, ctx); + jsonBadPathError(ctx, zPath); } return; } -/* -** Allowed values for the flgs argument to jsonParseFuncArg(); -*/ -#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ - /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new @@ -4531,19 +4031,6 @@ static void jsonArrayFunc( sqlite3_result_subtype(ctx, JSON_SUBTYPE); } -/* -** Generate a bad path error for json_extract() -*/ -static void jsonBadPathError( - sqlite3_context *ctx, /* The function call containing the error */ - const char *zPath /* The path with the problem */ -){ - sqlite3 *db = sqlite3_context_db_handle(ctx); - char *zMsg = sqlite3MPrintf(db, "bad JSON path: %Q", zPath); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3DbFree(db, zMsg); -} - /* ** json_array_length(JSON) ** json_array_length(JSON, PATH) @@ -5074,103 +4561,6 @@ json_remove_return_null: return; } -/* -** Substitute the value at iNode with the pValue parameter. -*/ -static void jsonReplaceNode( - sqlite3_context *pCtx, - JsonParse *p, - int iNode, - sqlite3_value *pValue -){ - int idx = jsonParseAddSubstNode(p, iNode); - if( idx<=0 ){ - assert( p->oom ); - return; - } - switch( sqlite3_value_type(pValue) ){ - case SQLITE_NULL: { - jsonParseAddNode(p, JSON_NULL, 0, 0); - break; - } - case SQLITE_FLOAT: { - char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue)); - int n; - if( z==0 ){ - p->oom = 1; - break; - } - n = sqlite3Strlen30(z); - jsonParseAddNode(p, JSON_REAL, n, z); - jsonParseAddCleanup(p, sqlite3_free, z); - break; - } - case SQLITE_INTEGER: { - char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue)); - int n; - if( z==0 ){ - p->oom = 1; - break; - } - n = sqlite3Strlen30(z); - jsonParseAddNode(p, JSON_INT, n, z); - jsonParseAddCleanup(p, sqlite3_free, z); - - break; - } - case SQLITE_TEXT: { - const char *z = (const char*)sqlite3_value_text(pValue); - u32 n = (u32)sqlite3_value_bytes(pValue); - if( z==0 ){ - p->oom = 1; - break; - } - if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ - char *zCopy = sqlite3_malloc64( n+1 ); - int k; - if( zCopy ){ - memcpy(zCopy, z, n); - zCopy[n] = 0; - jsonParseAddCleanup(p, sqlite3_free, zCopy); - }else{ - p->oom = 1; - sqlite3_result_error_nomem(pCtx); - } - k = jsonParseAddNode(p, JSON_STRING, n, zCopy); - assert( k>0 || p->oom ); - if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; - break; - } - replace_with_json: - { - JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); - if( pPatch==0 ){ - p->oom = 1; - break; - } - jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode); - /* The nodes copied out of pPatch and into p likely contain - ** u.zJContent pointers into pPatch->zJson. So preserve the - ** content of pPatch until p is destroyed. */ - assert( pPatch->nJPRef>=1 ); - pPatch->nJPRef++; - jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); - } - break; - } - default: { - if( jsonFuncArgMightBeBinary(pValue) ){ - goto replace_with_json; - }else{ - jsonParseAddNode(p, JSON_NULL, 0, 0); - sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); - p->nErr++; - } - break; - } - } -} - /* ** json_replace(JSON, PATH, VALUE, ...) ** @@ -5182,35 +4572,12 @@ static void jsonReplaceFunc( int argc, sqlite3_value **argv ){ - JsonParse *pParse; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - if( argc<1 ) return; if( (argc&1)==0 ) { jsonWrongNumArgs(ctx, "replace"); return; } - if( jsonFuncArgMightBeBinary(argv[0]) && argc>=3 ){ - jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); - return; - } - pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); - if( pParse==0 ) return; - pParse->nJPRef++; - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - pParse->useMod = 1; - pNode = jsonLookup(pParse, zPath, 0, ctx); - if( pParse->nErr ) goto replace_err; - if( pNode ){ - jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); - } - } - jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); -replace_err: - jsonParseFree(pParse); + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); } @@ -5231,11 +4598,7 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - JsonParse *pParse; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - int bApnd; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); int bIsSet = (flags&JSON_ISSET)!=0; @@ -5244,30 +4607,7 @@ static void jsonSetFunc( jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - if( jsonFuncArgMightBeBinary(argv[0]) && argc>=3 ){ - jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); - return; - } - pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); - if( pParse==0 ) return; - pParse->nJPRef++; - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - bApnd = 0; - pParse->useMod = 1; - pNode = jsonLookup(pParse, zPath, &bApnd, ctx); - if( pParse->oom ){ - sqlite3_result_error_nomem(ctx); - goto jsonSetDone; - }else if( pParse->nErr ){ - goto jsonSetDone; - }else if( pNode && (bApnd || bIsSet) ){ - jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); - } - } - jsonReturnNodeAsJson(pParse, pParse->aNode, ctx, 1, 0); -jsonSetDone: - jsonParseFree(pParse); + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); } /* From bac2fe9f44aa3540cb4589e006ff17ed8bc49860 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 30 Nov 2023 10:00:25 +0000 Subject: [PATCH 120/191] Update some OPFS-related help text in WASM tests. Minor cleanups in speedtest1-worker.js. FossilOrigin-Name: 263f6d3a7784ef7d032dbf7a3265aca8dd70bf50797f28f6b2e8ddb6a301f83a --- ext/wasm/index-dist.html | 9 +++------ ext/wasm/index.html | 9 +++------ ext/wasm/speedtest1-worker.js | 23 +++++------------------ manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 21 insertions(+), 40 deletions(-) diff --git a/ext/wasm/index-dist.html b/ext/wasm/index-dist.html index 36e21117e3..f5bcdc1cb2 100644 --- a/ext/wasm/index-dist.html +++ b/ext/wasm/index-dist.html @@ -46,6 +46,9 @@ file:// URLs.
  • Any OPFS-related pages or tests require:
      +
    • An OPFS-capable browser released after February + 2023. Some tests will work with Chromium-based browsers + going back to around v102.
    • That the web server emit the so-called COOP and @@ -53,12 +56,6 @@ headers. althttpd requires the -enable-sab flag for that.
      • -
    • A very recent version of a Chromium-based browser - (v102 at least, possibly newer). OPFS support in the - other major browsers is pending. Development and testing - is currently done against a dev-channel release of - Chrome (v111 as of 2023-02-10). -
    • diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 70ce0441e0..ebbfd6763d 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -31,6 +31,9 @@ file:// URLs.
  • Any OPFS-related pages or tests require:
      +
    • An OPFS-capable browser released after February + 2023. Some tests will work with Chromium-based browsers + going back to around v102.
    • That the web server emit the so-called COOP and @@ -38,12 +41,6 @@ headers. althttpd requires the -enable-sab flag for that.
      • -
    • A very recent version of a - Chromium-based browser (v102 at least, possibly newer). OPFS - support in the other major browsers is pending. Development - and testing is currently done against a dev-channel release - of Chrome (v111 as of 2023-02-10). -
    • diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index dedc320292..3abc589b59 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -7,9 +7,9 @@ } importScripts(speedtestJs); /** - If this environment contains OPFS, this function initializes it and - returns the name of the dir on which OPFS is mounted, else it returns - an empty string. + If this build includes WASMFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it + returns an empty string. */ const wasmfsDir = function f(wasmUtil){ if(undefined !== f._) return f._; @@ -47,6 +47,7 @@ }; const log = (...args)=>logMsg('stdout',args); const logErr = (...args)=>logMsg('stderr',args); + const realSahName = 'opfs-sahpool-speedtest1'; const runSpeedtest = async function(cliFlagsArray){ const scope = App.wasm.scopedAllocPush(); @@ -57,7 +58,6 @@ ]; App.logBuffer.length = 0; const ndxSahPool = argv.indexOf('opfs-sahpool'); - const realSahName = 'opfs-sahpool-speedtest1'; if(ndxSahPool>0){ argv[ndxSahPool] = realSahName; log("Updated argv for opfs-sahpool: --vfs",realSahName); @@ -73,7 +73,7 @@ clearOnInit: true, verbosity: 2 }).then(PoolUtil=>{ - log("opfs-sahpool successfully installed as",realSahName); + log("opfs-sahpool successfully installed as",PoolUtil.vfsName); App.sqlite3.$SAHPoolUtil = PoolUtil; //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb); }); @@ -102,19 +102,6 @@ } }; - const sahpSanityChecks = function(sqlite3){ - log("Attempting OpfsSAHPoolDb sanity checks..."); - const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db'); - const fn = db.filename; - db.exec([ - 'create table t(a);', - 'insert into t(a) values(1),(2),(3);' - ]); - db.close(); - sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn); - log("SAH sanity checks done."); - }; - const EmscriptenModule = { print: log, printErr: logErr, diff --git a/manifest b/manifest index 0c8757df1c..766de1838f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\srecovery\sextension,\sif\sa\spayload\ssize\sis\sunreasonably\slarge,\sit\sis\nprobably\scorrupt,\sso\struncate\sit. -D 2023-11-29T13:47:46.693 +C Update\ssome\sOPFS-related\shelp\stext\sin\sWASM\stests.\sMinor\scleanups\sin\sspeedtest1-worker.js. +D 2023-11-30T10:00:25.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,8 +621,8 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695 F ext/wasm/fiddle/fiddle-worker.js e0153f9af6500805c6f09c0b3cfdb7d857e9d6863dbee9d50d1628fccf5f4b4d F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 -F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b +F ext/wasm/index-dist.html e91d76e4581185238fd3d42ed86ec600f7023ed3e3a944c5c356f25304bf1263 +F ext/wasm/index.html b31ce41c0da476d5ffcef23069b9d3415b419d65af5779096ebcfbcbade453a9 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 @@ -631,7 +631,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c +F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cad269d5e274443c39203a56603b991accc0399135d436996fc039d1d28ec9db -R 4a3a6ee7f774b8b1af828ef09f5272a7 -U drh -Z 25e836823c87bebe85ade3cc7e139b02 +P 988c3179e978a3a6d42541e9c7a2ab98150383671810926503376ed808f150ff +R 73ee9ef0435c8cda72f87d3ec03cb69f +U stephan +Z 88450718d2dd1d2993dee724cc33772c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6a5cca1b23..af5689a923 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -988c3179e978a3a6d42541e9c7a2ab98150383671810926503376ed808f150ff \ No newline at end of file +263f6d3a7784ef7d032dbf7a3265aca8dd70bf50797f28f6b2e8ddb6a301f83a \ No newline at end of file From ad27c437fff38bd5ef06e609b9aa2eb457c4fc2a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 12:04:14 +0000 Subject: [PATCH 121/191] New test cases for insert/set/replace with paths that indicate substructure that does not yet exist. FossilOrigin-Name: 146c717c51940b2139befc45ac74e7a1c36ef3c32fd3cfe35b334488eebe6298 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/json101.test | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d50bd996f4..c9a863d174 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sjson_insert(),\sjson_replace(),\sjson_set()\sto\suse\sJSONB\sinternally.\nMostly\sworking,\sbut\ssome\scorner\scases\sare\sstill\snot\squite\sright. -D 2023-11-30T00:52:33.320 +C New\stest\scases\sfor\sinsert/set/replace\swith\spaths\sthat\sindicate\ssubstructure\nthat\sdoes\snot\syet\sexist. +D 2023-11-30T12:04:14.610 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1325,7 +1325,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 8f5d1a3350c36945c8b58f538e1c92cfd87fd50ab6f5e3d5f4cf3cdb03b9546d +F test/json101.test 861a5d75c296709471e88aa48d82095ed600dedd23ba29b904b91e64d319b7c3 F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad -R 4c74c3d174d84fc0b9e05e4cabac42e8 -T *branch * jsonb-insert -T *sym-jsonb-insert * -T -sym-jsonb * +P 99c8f6bd5c9a31b6d00f92e383bec8a8235ed553916ad59adbb1b7663f6ebff1 +R 757617988b02e4847aadc75efb2ede4b U drh -Z 396365e2a8dc6b109097922e3ec543f6 +Z 50c123f2db65971d190ff942903bde3b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bed72fa4ce..01f2a5b9d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99c8f6bd5c9a31b6d00f92e383bec8a8235ed553916ad59adbb1b7663f6ebff1 \ No newline at end of file +146c717c51940b2139befc45ac74e7a1c36ef3c32fd3cfe35b334488eebe6298 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 3c3b1a5ec0..d7b301e6df 100644 --- a/test/json101.test +++ b/test/json101.test @@ -1139,4 +1139,29 @@ do_execsql_test json101-23.2 { FROM (SELECT json_set('[]','$[#]',0,'$[#]',1) AS j); } {{[0,1]} 0 1} +# Insert/Set/Replace where the path specifies substructure that +# does not yet exist +# +proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]} +foreach {id start path ins set repl} { + 1 {{}} {$.a.b.c} ('a':('b':('c':'NEW'))) ('a':('b':('c':'NEW'))) () + 2 {{a:4}} {$.a.b.c} ('a':4) ('a':4) ('a':4) + 3 {{a:{}}} {$.a.b.c} ('a':('b':('c':'NEW'))) ('a':('b':('c':'NEW'))) ('a':()) + 4 {[0,1,2]} {$[3].a[0].b} <0,1,2,('a':<('b':'NEW')>)> <0,1,2,('a':<('b':'NEW')>)> <0,1,2> + 5 {[0,1,2]} {$[1].a[0].b} <0,1,2> <0,1,2> <0,1,2> + 6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':'NEW')>),2> <0,('a':<('b':'NEW')>),2> <0,(),2> + 7 {[0,1,2]} {$[3][0].b} <0,1,2,<('b':'NEW')> <0,1,2,<('b':'NEW')> <0,1,2> + 8 {[0,1,2]} {$[1][0].b} <0,1,2> <0,1,2> <0,1,2> +} { + do_execsql_test json101-24.$id.insert { + SELECT json_insert($start,$path,'NEW'); + } [list [tx $ins]] + do_execsql_test json101-24.$id.set { + SELECT json_set($start,$path,'NEW'); + } [list [tx $set]] + do_execsql_test json101-24.$id.replace { + SELECT json_replace($start,$path,'NEW'); + } [list [tx $repl]] +} + finish_test From 48222bef664ad4b51cdadccfc44426c69a66a5f9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 16:16:10 +0000 Subject: [PATCH 122/191] New JSON test cases showing insert or set with missing substructure. FossilOrigin-Name: 6802b6459d0d16c961ff41d240a6c88287f197d8f609090f79308707490a49c2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json101.test | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 766de1838f..9f44093bfe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\sOPFS-related\shelp\stext\sin\sWASM\stests.\sMinor\scleanups\sin\sspeedtest1-worker.js. -D 2023-11-30T10:00:25.069 +C New\sJSON\stest\scases\sshowing\sinsert\sor\sset\swith\smissing\ssubstructure. +D 2023-11-30T16:16:10.846 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1324,7 +1324,7 @@ F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x -F test/json101.test bc05d2476fd6f7ead31ec05b43d1b24b2b193ae112fd8f0d2ed56d9a904f9fa5 +F test/json101.test 4d8faf6a77152edadaf615bdfac4fb5556d131230cc2558f0912bed00a54943b F test/json102.test 4c69694773a470f1fda34e5f4ba24920b35184fb66050b450fc2ef9ab5ad310b F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 988c3179e978a3a6d42541e9c7a2ab98150383671810926503376ed808f150ff -R 73ee9ef0435c8cda72f87d3ec03cb69f -U stephan -Z 88450718d2dd1d2993dee724cc33772c +P 263f6d3a7784ef7d032dbf7a3265aca8dd70bf50797f28f6b2e8ddb6a301f83a +R 0e9b3f433035365e3e741a55f5ab0daf +U drh +Z 72d2be7125d7371db00777b78c0fb9f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af5689a923..fcbc7b3b7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -263f6d3a7784ef7d032dbf7a3265aca8dd70bf50797f28f6b2e8ddb6a301f83a \ No newline at end of file +6802b6459d0d16c961ff41d240a6c88287f197d8f609090f79308707490a49c2 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index c62991bbbf..5745e9d4cd 100644 --- a/test/json101.test +++ b/test/json101.test @@ -1054,6 +1054,30 @@ do_execsql_test json101-23.2 { FROM (SELECT json_set('[]','$[#]',0,'$[#]',1) AS j); } {{[0,1]} 0 1} +# Insert/Set/Replace where the path specifies substructure that +# does not yet exist +# +proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]} +foreach {id start path ins set repl} { + 1 {{}} {$.a.b.c} ('a':('b':('c':9))) ('a':('b':('c':9))) () + 2 {{a:4}} {$.a.b.c} ('a':4) ('a':4) ('a':4) + 3 {{a:{}}} {$.a.b.c} ('a':('b':('c':9))) ('a':('b':('c':9))) ('a':()) + 4 {[0,1,2]} {$[3].a[0].b} <0,1,2,('a':<('b':9)>)> <0,1,2,('a':<('b':9)>)> <0,1,2> + 5 {[0,1,2]} {$[1].a[0].b} <0,1,2> <0,1,2> <0,1,2> + 6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':9)>),2> <0,('a':<('b':9)>),2> <0,(),2> + 7 {[0,1,2]} {$[3][0].b} <0,1,2,<('b':9)>> <0,1,2,<('b':9)>> <0,1,2> + 8 {[0,1,2]} {$[1][0].b} <0,1,2> <0,1,2> <0,1,2> +} { + do_execsql_test json101-24.$id.insert { + SELECT json_insert($start,$path,9); + } [list [tx $ins]] + do_execsql_test json101-24.$id.set { + SELECT json_set($start,$path,9); + } [list [tx $set]] + do_execsql_test json101-24.$id.replace { + SELECT json_replace($start,$path,9); + } [list [tx $repl]] +} finish_test From 7394a6ef571081ed4f8bd88f77121b0b9338fa70 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 16:17:09 +0000 Subject: [PATCH 123/191] Simplification of the new JSON insert/set test cases. FossilOrigin-Name: 04c0d5644372446c924a2e31a26edf51ddc563a1990d170b0ed4739e3e8b239b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json101.test | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index c9a863d174..2c2e1cf626 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sinsert/set/replace\swith\spaths\sthat\sindicate\ssubstructure\nthat\sdoes\snot\syet\sexist. -D 2023-11-30T12:04:14.610 +C Simplification\sof\sthe\snew\sJSON\sinsert/set\stest\scases. +D 2023-11-30T16:17:09.157 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1325,7 +1325,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x -F test/json101.test 861a5d75c296709471e88aa48d82095ed600dedd23ba29b904b91e64d319b7c3 +F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547 F test/json102.test 557a46e16df1aa9bdbc4076a71a45814ea0e7503d6621d87d42a8c04cbc2b0ef F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 99c8f6bd5c9a31b6d00f92e383bec8a8235ed553916ad59adbb1b7663f6ebff1 -R 757617988b02e4847aadc75efb2ede4b +P 146c717c51940b2139befc45ac74e7a1c36ef3c32fd3cfe35b334488eebe6298 +R 8e1036414b3f5ed40f0248ecb25d6019 U drh -Z 50c123f2db65971d190ff942903bde3b +Z df0518617355b4fc840dfe4615c492b7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01f2a5b9d8..b89e5d463a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -146c717c51940b2139befc45ac74e7a1c36ef3c32fd3cfe35b334488eebe6298 \ No newline at end of file +04c0d5644372446c924a2e31a26edf51ddc563a1990d170b0ed4739e3e8b239b \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index d7b301e6df..bae68e7344 100644 --- a/test/json101.test +++ b/test/json101.test @@ -1144,23 +1144,23 @@ do_execsql_test json101-23.2 { # proc tx x {return [string map [list ( \173 ) \175 ' \042 < \133 > \135] $x]} foreach {id start path ins set repl} { - 1 {{}} {$.a.b.c} ('a':('b':('c':'NEW'))) ('a':('b':('c':'NEW'))) () - 2 {{a:4}} {$.a.b.c} ('a':4) ('a':4) ('a':4) - 3 {{a:{}}} {$.a.b.c} ('a':('b':('c':'NEW'))) ('a':('b':('c':'NEW'))) ('a':()) - 4 {[0,1,2]} {$[3].a[0].b} <0,1,2,('a':<('b':'NEW')>)> <0,1,2,('a':<('b':'NEW')>)> <0,1,2> - 5 {[0,1,2]} {$[1].a[0].b} <0,1,2> <0,1,2> <0,1,2> - 6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':'NEW')>),2> <0,('a':<('b':'NEW')>),2> <0,(),2> - 7 {[0,1,2]} {$[3][0].b} <0,1,2,<('b':'NEW')> <0,1,2,<('b':'NEW')> <0,1,2> - 8 {[0,1,2]} {$[1][0].b} <0,1,2> <0,1,2> <0,1,2> + 1 {{}} {$.a.b.c} ('a':('b':('c':9))) ('a':('b':('c':9))) () + 2 {{a:4}} {$.a.b.c} ('a':4) ('a':4) ('a':4) + 3 {{a:{}}} {$.a.b.c} ('a':('b':('c':9))) ('a':('b':('c':9))) ('a':()) + 4 {[0,1,2]} {$[3].a[0].b} <0,1,2,('a':<('b':9)>)> <0,1,2,('a':<('b':9)>)> <0,1,2> + 5 {[0,1,2]} {$[1].a[0].b} <0,1,2> <0,1,2> <0,1,2> + 6 {[0,{},2]} {$[1].a[0].b} <0,('a':<('b':9)>),2> <0,('a':<('b':9)>),2> <0,(),2> + 7 {[0,1,2]} {$[3][0].b} <0,1,2,<('b':9)>> <0,1,2,<('b':9)>> <0,1,2> + 8 {[0,1,2]} {$[1][0].b} <0,1,2> <0,1,2> <0,1,2> } { do_execsql_test json101-24.$id.insert { - SELECT json_insert($start,$path,'NEW'); + SELECT json_insert($start,$path,9); } [list [tx $ins]] do_execsql_test json101-24.$id.set { - SELECT json_set($start,$path,'NEW'); + SELECT json_set($start,$path,9); } [list [tx $set]] do_execsql_test json101-24.$id.replace { - SELECT json_replace($start,$path,'NEW'); + SELECT json_replace($start,$path,9); } [list [tx $repl]] } From 66795962b59af6deab38b178a8a3b03f223c309a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 19:06:27 +0000 Subject: [PATCH 124/191] Enhance json_set() and json_insert() so that they create missing substructure. FossilOrigin-Name: cc7a641ab5ae739d31c24f0ad0caeb15a481a63fa8f13720718ea922c25862ff --- manifest | 12 +++++----- manifest.uuid | 2 +- src/json.c | 63 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 2c2e1cf626..dd2fca3cd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sof\sthe\snew\sJSON\sinsert/set\stest\scases. -D 2023-11-30T16:17:09.157 +C Enhance\sjson_set()\sand\sjson_insert()\sso\sthat\sthey\screate\smissing\nsubstructure. +D 2023-11-30T19:06:27.003 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 8b509dead923cd44f31e4866d1024f1b602bdf01c9d41a08cc36aa5f1203fdf8 +F src/json.c 92cc05dcf87cdace6fbf2a76e0cb2b030f04ddb1819e024af4ce8885261b4cef F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 146c717c51940b2139befc45ac74e7a1c36ef3c32fd3cfe35b334488eebe6298 -R 8e1036414b3f5ed40f0248ecb25d6019 +P 04c0d5644372446c924a2e31a26edf51ddc563a1990d170b0ed4739e3e8b239b +R 4a2861d7ab7860f80e426ce44e32c50f U drh -Z df0518617355b4fc840dfe4615c492b7 +Z 1fffc27f10fd5ca429d5618946ebb513 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b89e5d463a..7e844fc2a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04c0d5644372446c924a2e31a26edf51ddc563a1990d170b0ed4739e3e8b239b \ No newline at end of file +cc7a641ab5ae739d31c24f0ad0caeb15a481a63fa8f13720718ea922c25862ff \ No newline at end of file diff --git a/src/json.c b/src/json.c index b3b4f993e6..b8b01bf452 100644 --- a/src/json.c +++ b/src/json.c @@ -3154,6 +3154,7 @@ static u32 jsonLookupBlobStep( u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; u8 x; + static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; if( zPath[0]==0 ){ if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ @@ -3225,25 +3226,44 @@ static u32 jsonLookupBlobStep( } if( j>iEnd ) return JSON_BLOB_ERROR; if( pParse->eEdit>=JEDIT_INS ){ - u32 nIns; - u8 aLabel[16]; - JsonParse ix; + u32 nIns; /* Total bytes to insert (label+value) */ + JsonParse v; /* BLOB encoding of the value to be inserted */ + JsonParse ix; /* Header of the label to be inserted */ + u8 aLabel[16]; /* Buffer space for ix */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); ix.aBlob = aLabel; ix.nBlobAlloc = sizeof(aLabel); jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); - if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+pParse->nIns) ){ - nIns = ix.nBlob + nKey + pParse->nIns; + memset(&v, 0, sizeof(v)); + if( zPath[i]==0 ){ + v.nBlob = pParse->nIns; + v.aBlob = pParse->aIns; + }else{ + v.nBlob = 1; + v.aBlob = (u8*)&emptyObject[zPath[i]=='.']; + v.eEdit = pParse->eEdit; + v.nIns = pParse->nIns; + v.aIns = pParse->aIns; + rc = jsonLookupBlobStep(&v, 0, &zPath[i], 0); + if( JSON_BLOB_ISERROR(rc) || v.oom ){ + pParse->oom |= v.oom; + jsonParseReset(&v); + return rc; + } + } + if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) ){ + nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); k += nKey; - memcpy(&pParse->aBlob[k], pParse->aIns, pParse->nIns); + memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); } + jsonParseReset(&v); return j; } }else if( zPath[0]=='[' ){ @@ -3292,12 +3312,31 @@ static u32 jsonLookupBlobStep( } if( j>iEnd ) return JSON_BLOB_ERROR; if( k>0 ) return JSON_BLOB_NOTFOUND; - if( pParse->eEdit>=JEDIT_INS - && jsonBlobMakeEditable(pParse, pParse->nIns) - ){ + if( pParse->eEdit>=JEDIT_INS ){ + JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); - jsonBlobEdit(pParse, j, 0, pParse->aIns, pParse->nIns); + memset(&v, 0, sizeof(v)); + if( zPath[i+1]==0 ){ + v.aBlob = pParse->aIns; + v.nBlob = pParse->nIns; + }else{ + v.nBlob = 1; + v.aBlob = (u8*)&emptyObject[zPath[i+1]=='.']; + v.eEdit = pParse->eEdit; + v.nIns = pParse->nIns; + v.aIns = pParse->aIns; + rc = jsonLookupBlobStep(&v, 0, &zPath[i+1], 0); + if( JSON_BLOB_ISERROR(rc) || v.oom ){ + pParse->oom |= v.oom; + jsonParseReset(&v); + return rc; + } + } + if( jsonBlobMakeEditable(pParse, v.nBlob) ){ + jsonBlobEdit(pParse, j, 0, v.aBlob, v.nBlob); + } + jsonParseReset(&v); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return j; } @@ -3754,6 +3793,10 @@ static void jsonReturnParse( JsonParse *p ){ int flgs; + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + return; + } flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( flgs & JSON_BLOB ){ sqlite3_result_blob(ctx, p->aBlob, p->nBlob, From 276042bd23b7dab88ca6029177225404f4f7f973 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 19:29:56 +0000 Subject: [PATCH 125/191] Convert json_type() to use JSONB internally. FossilOrigin-Name: 83074835b900ce85cf67059e674ce959801505c37592671af25ca0af7ed483f1 --- manifest | 13 ++++----- manifest.uuid | 2 +- src/json.c | 77 ++++++++++++++++++++++----------------------------- 3 files changed, 40 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 0b047f5b82..38eb4c4f95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sjson_insert(),\sjson_replace(),\sand\sjson_set()\sover\sto\susing\sonly\nJSONB\sinternally. -D 2023-11-30T19:11:14.034 +C Convert\sjson_type()\sto\suse\sJSONB\sinternally. +D 2023-11-30T19:29:56.759 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 92cc05dcf87cdace6fbf2a76e0cb2b030f04ddb1819e024af4ce8885261b4cef +F src/json.c 3d897ec27a8640c94e5d8293f8aa64226205140150639571c6f745c42d545e3e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,9 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e7a8ba35bff6fde55827f978de5b343b6c134c7fa53827f5c63915a9dc2598ad cc7a641ab5ae739d31c24f0ad0caeb15a481a63fa8f13720718ea922c25862ff -R 4a2861d7ab7860f80e426ce44e32c50f -T +closed cc7a641ab5ae739d31c24f0ad0caeb15a481a63fa8f13720718ea922c25862ff +P 4e2083e86f19ef7634f0b253fb924e52014b43ed0ce8acc51c36f3c5682180a6 +R 4b874649acdc6c51075a8e11e402ee44 U drh -Z 60442442b5f0c2166085d00e76659699 +Z 2246ad2d1a16f9f5b929395a88459b9a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 24560706af..cd1d889728 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e2083e86f19ef7634f0b253fb924e52014b43ed0ce8acc51c36f3c5682180a6 \ No newline at end of file +83074835b900ce85cf67059e674ce959801505c37592671af25ca0af7ed483f1 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b8b01bf452..4f47b7a758 100644 --- a/src/json.c +++ b/src/json.c @@ -1887,42 +1887,6 @@ static char *jsonPathSyntaxError(const char *zErr, sqlite3_context *ctx){ return 0; } -/* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. -** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. -** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. -*/ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ -){ - const char *zErr = 0; - JsonNode *pNode = 0; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; - -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - jsonPathSyntaxError(zErr, pCtx); - return 0; -} - - /* ** Report the wrong number of arguments for json_insert(), json_replace() ** or json_set(). @@ -4666,20 +4630,45 @@ static void jsonTypeFunc( sqlite3_value **argv ){ JsonParse *p; /* The parse */ - const char *zPath; - JsonNode *pNode; + const char *zPath = 0; + u32 i; - p = jsonParseCached(ctx, argv[0], ctx, 0); + p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; if( argc==2 ){ + if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ + goto json_type_done; + } + if( sqlite3_value_bytes(argv[1])==0 ){ + jsonBadPathError(ctx, ""); + goto json_type_done; + } zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); + if( zPath==0 ){ + sqlite3_result_error_nomem(ctx); + goto json_type_done; + } + if( zPath[0]!='$' ){ + jsonBadPathError(ctx, zPath); + goto json_type_done; + } + i = jsonLookupBlobStep(p, 0, zPath+1, 0); + if( JSON_BLOB_ISERROR(i) ){ + if( i==JSON_BLOB_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_BLOB_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_type_done; + } }else{ - pNode = p->aNode; - } - if( pNode ){ - sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + i = 0; } + sqlite3_result_text(ctx, jsonbType[p->aBlob[i]&0x0f], -1, SQLITE_STATIC); +json_type_done: + jsonParseFree(p); } /* From 68003d9f1807195955716abf3454a3bccd6d82a4 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 30 Nov 2023 20:34:24 +0000 Subject: [PATCH 126/191] Add a basic batch-mode SQL runner for the SAH Pool VFS, for use in comparing it against WebSQL. Bring the WebSQL batch runner up to date, noting that it cannot run without addition of an "origin trial" activation key from Google because that's now the only way to enable WebSQL in Chrome (that part is not checked in because that key is private). Minor code-adjacent cleanups. FossilOrigin-Name: 883990e7938c1f63906300a6113f0fadce143913b7c384e8aeb5f886f0be7c62 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/batch-runner-sahpool.html | 86 ++++++++ ext/wasm/batch-runner-sahpool.js | 341 +++++++++++++++++++++++++++++ ext/wasm/batch-runner.js | 24 +- ext/wasm/demo-123.js | 15 +- ext/wasm/speedtest1-worker.html | 2 +- manifest | 22 +- manifest.uuid | 2 +- 8 files changed, 470 insertions(+), 24 deletions(-) create mode 100644 ext/wasm/batch-runner-sahpool.html create mode 100644 ext/wasm/batch-runner-sahpool.js diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index c0cab212d8..3ef478612c 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -854,7 +854,7 @@ dir.sql := sql speedtest1 := ../../speedtest1 speedtest1.c := ../../test/speedtest1.c speedtest1.sql := $(dir.sql)/speedtest1.sql -speedtest1.cliflags := --size 25 --big-transactions +speedtest1.cliflags := --size 10 --big-transactions $(speedtest1): $(MAKE) -C ../.. speedtest1 $(speedtest1.sql): $(speedtest1) $(MAKEFILE) diff --git a/ext/wasm/batch-runner-sahpool.html b/ext/wasm/batch-runner-sahpool.html new file mode 100644 index 0000000000..ad7e7b5408 --- /dev/null +++ b/ext/wasm/batch-runner-sahpool.html @@ -0,0 +1,86 @@ + + + + + + + + sqlite3-api batch SQL runner for the SAHPool VFS + + +
    sqlite3-api batch SQL runner for the SAHPool VFS
    +
    + + + + +
    +
    + + + + diff --git a/ext/wasm/batch-runner-sahpool.js b/ext/wasm/batch-runner-sahpool.js new file mode 100644 index 0000000000..dfa5044a9e --- /dev/null +++ b/ext/wasm/batch-runner-sahpool.js @@ -0,0 +1,341 @@ +/* + 2023-11-30 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic batch SQL runner for the SAHPool VFS. This file must be run in + a worker thread. This is not a full-featured app, just a way to get some + measurements for batch execution of SQL for the OPFS SAH Pool VFS. +*/ +'use strict'; + +const wMsg = function(msgType,...args){ + postMessage({ + type: msgType, + data: args + }); +}; +const toss = function(...args){throw new Error(args.join(' '))}; +const warn = (...args)=>{ wMsg('warn',...args); }; +const error = (...args)=>{ wMsg('error',...args); }; +const log = (...args)=>{ wMsg('stdout',...args); } +let sqlite3; +const urlParams = new URL(globalThis.location.href).searchParams; +const cacheSize = (()=>{ + if(urlParams.has('cachesize')) return +urlParams.get('cachesize'); + return 200; +})(); + + +/** Throws if the given sqlite3 result code is not 0. */ +const checkSqliteRc = (dbh,rc)=>{ + if(rc) toss("Prepare failed:",sqlite3.capi.sqlite3_errmsg(dbh)); +}; + +const sqlToDrop = [ + "SELECT type,name FROM sqlite_schema ", + "WHERE name NOT LIKE 'sqlite\\_%' escape '\\' ", + "AND name NOT LIKE '\\_%' escape '\\'" +].join(''); + +const clearDbSqlite = function(db){ + // This would be SO much easier with the oo1 API, but we specifically want to + // inject metrics we can't get via that API, and we cannot reliably (OPFS) + // open the same DB twice to clear it using that API, so... + const rc = sqlite3.wasm.exports.sqlite3_wasm_db_reset(db.handle); + log("reset db rc =",rc,db.id, db.filename); +}; + +const App = { + db: undefined, + cache:Object.create(null), + log: log, + warn: warn, + error: error, + metrics: { + fileCount: 0, + runTimeMs: 0, + prepareTimeMs: 0, + stepTimeMs: 0, + stmtCount: 0, + strcpyMs: 0, + sqlBytes: 0 + }, + fileList: undefined, + execSql: async function(name,sql){ + const db = this.db; + const banner = "========================================"; + this.log(banner, + "Running",name,'('+sql.length,'bytes)'); + const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm; + let pStmt = 0, pSqlBegin; + const metrics = db.metrics = Object.create(null); + metrics.prepTotal = metrics.stepTotal = 0; + metrics.stmtCount = 0; + metrics.malloc = 0; + metrics.strcpy = 0; + if(this.gotErr){ + this.error("Cannot run SQL: error cleanup is pending."); + return; + } + // Run this async so that the UI can be updated for the above header... + const endRun = ()=>{ + metrics.evalSqlEnd = performance.now(); + metrics.evalTimeTotal = (metrics.evalSqlEnd - metrics.evalSqlStart); + this.log("metrics:",JSON.stringify(metrics, undefined, ' ')); + this.log("prepare() count:",metrics.stmtCount); + this.log("Time in prepare_v2():",metrics.prepTotal,"ms", + "("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())"); + this.log("Time in step():",metrics.stepTotal,"ms", + "("+(metrics.stepTotal / metrics.stmtCount),"ms per step())"); + this.log("Total runtime:",metrics.evalTimeTotal,"ms"); + this.log("Overhead (time - prep - step):", + (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); + this.log(banner,"End of",name); + this.metrics.prepareTimeMs += metrics.prepTotal; + this.metrics.stepTimeMs += metrics.stepTotal; + this.metrics.stmtCount += metrics.stmtCount; + this.metrics.strcpyMs += metrics.strcpy; + this.metrics.sqlBytes += sql.length; + }; + + const runner = function(resolve, reject){ + ++this.metrics.fileCount; + metrics.evalSqlStart = performance.now(); + const stack = wasm.scopedAllocPush(); + try { + let t, rc; + let sqlByteLen = sql.byteLength; + const [ppStmt, pzTail] = wasm.scopedAllocPtr(2); + t = performance.now(); + pSqlBegin = wasm.scopedAlloc( sqlByteLen + 1/*SQL + NUL*/) || toss("alloc(",sqlByteLen,") failed"); + metrics.malloc = performance.now() - t; + metrics.byteLength = sqlByteLen; + let pSql = pSqlBegin; + const pSqlEnd = pSqlBegin + sqlByteLen; + t = performance.now(); + wasm.heap8().set(sql, pSql); + wasm.poke(pSql + sqlByteLen, 0); + //log("SQL:",wasm.cstrToJs(pSql)); + metrics.strcpy = performance.now() - t; + let breaker = 0; + while(pSql && wasm.peek8(pSql)){ + wasm.pokePtr(ppStmt, 0); + wasm.pokePtr(pzTail, 0); + t = performance.now(); + rc = capi.sqlite3_prepare_v2( + db.handle, pSql, sqlByteLen, ppStmt, pzTail + ); + metrics.prepTotal += performance.now() - t; + checkSqliteRc(db.handle, rc); + pStmt = wasm.peekPtr(ppStmt); + pSql = wasm.peekPtr(pzTail); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue/*empty statement*/; + ++metrics.stmtCount; + t = performance.now(); + rc = capi.sqlite3_step(pStmt); + capi.sqlite3_finalize(pStmt); + pStmt = 0; + metrics.stepTotal += performance.now() - t; + switch(rc){ + case capi.SQLITE_ROW: + case capi.SQLITE_DONE: break; + default: checkSqliteRc(db.handle, rc); toss("Not reached."); + } + } + resolve(this); + }catch(e){ + if(pStmt) capi.sqlite3_finalize(pStmt); + this.gotErr = e; + reject(e); + }finally{ + capi.sqlite3_exec(db.handle,"rollback;",0,0,0); + wasm.scopedAllocPop(stack); + } + }.bind(this); + const p = new Promise(runner); + return p.catch( + (e)=>this.error("Error via execSql("+name+",...):",e.message) + ).finally(()=>{ + endRun(); + }); + }, + + /** + Loads batch-runner.list and populates the selection list from + it. Returns a promise which resolves to nothing in particular + when it completes. Only intended to be run once at the start + of the app. + */ + loadSqlList: async function(){ + const infile = 'batch-runner.list'; + this.log("Loading list of SQL files:", infile); + let txt; + try{ + const r = await fetch(infile); + if(404 === r.status){ + toss("Missing file '"+infile+"'."); + } + if(!r.ok) toss("Loading",infile,"failed:",r.statusText); + txt = await r.text(); + }catch(e){ + this.error(e.message); + throw e; + } + App.fileList = txt.split(/\n+/).filter(x=>!!x); + this.log("Loaded",infile); + }, + + /** Fetch ./fn and return its contents as a Uint8Array. */ + fetchFile: async function(fn, cacheIt=false){ + if(cacheIt && this.cache[fn]) return this.cache[fn]; + this.log("Fetching",fn,"..."); + let sql; + try { + const r = await fetch(fn); + if(!r.ok) toss("Fetch failed:",r.statusText); + sql = new Uint8Array(await r.arrayBuffer()); + }catch(e){ + this.error(e.message); + throw e; + } + this.log("Fetched",sql.length,"bytes from",fn); + if(cacheIt) this.cache[fn] = sql; + return sql; + }/*fetchFile()*/, + + /** + Converts this.metrics() to a form which is suitable for easy conversion to + CSV. It returns an array of arrays. The first sub-array is the column names. + The 2nd and subsequent are the values, one per test file (only the most recent + metrics are kept for any given file). + */ + metricsToArrays: function(){ + const rc = []; + Object.keys(this.dbs).sort().forEach((k)=>{ + const d = this.dbs[k]; + const m = d.metrics; + delete m.evalSqlStart; + delete m.evalSqlEnd; + const mk = Object.keys(m).sort(); + if(!rc.length){ + rc.push(['db', ...mk]); + } + const row = [k.split('/').pop()/*remove dir prefix from filename*/]; + rc.push(row); + row.push(...mk.map((kk)=>m[kk])); + }); + return rc; + }, + + metricsToBlob: function(colSeparator='\t'){ + const ar = [], ma = this.metricsToArrays(); + if(!ma.length){ + this.error("Metrics are empty. Run something."); + return; + } + ma.forEach(function(row){ + ar.push(row.join(colSeparator),'\n'); + }); + return new Blob(ar); + }, + + /** + Fetch file fn and eval it as an SQL blob. This is an async + operation and returns a Promise which resolves to this + object on success. + */ + evalFile: async function(fn){ + const sql = await this.fetchFile(fn); + return this.execSql(fn,sql); + }/*evalFile()*/, + + /** + Fetches the handle of the db associated with + this.e.selImpl.value, opening it if needed. + */ + initDb: function(){ + const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm; + const stack = wasm.scopedAllocPush(); + let pDb = 0; + const d = Object.create(null); + d.filename = "/batch.db"; + try{ + const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + const ppDb = wasm.scopedAllocPtr(); + const rc = capi.sqlite3_open_v2(d.filename, ppDb, oFlags, this.PoolUtil.vfsName); + pDb = wasm.peekPtr(ppDb) + if(rc) toss("sqlite3_open_v2() failed with code",rc); + capi.sqlite3_exec(pDb, "PRAGMA cache_size="+cacheSize, 0, 0, 0); + this.log("cache_size =",cacheSize); + }catch(e){ + if(pDb) capi.sqlite3_close_v2(pDb); + throw e; + }finally{ + wasm.scopedAllocPop(stack); + } + d.handle = pDb; + this.log("Opened db:",d.filename,'@',d.handle); + return d; + }, + + closeDb: function(){ + if(this.db.handle){ + this.sqlite3.capi.sqlite3_close_v2(this.db.handle); + this.db.handle = undefined; + } + }, + + run: async function(sqlite3){ + delete this.run; + this.sqlite3 = sqlite3; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + this.log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + this.log("WASM heap size =",wasm.heap8().length); + let timeStart; + sqlite3.installOpfsSAHPoolVfs({ + clearOnInit: true, initialCapacity: 4, + name: 'batch-sahpool', + verbosity: 2 + }).then(PoolUtil=>{ + App.PoolUtil = PoolUtil; + App.db = App.initDb(); + }) + .then(async ()=>this.loadSqlList()) + .then(async ()=>{ + timeStart = performance.now(); + for(let i = 0; i < App.fileList.length; ++i){ + const fn = App.fileList[i]; + await App.evalFile(fn); + if(App.gotErr) throw App.gotErr; + } + }) + .then(()=>{ + App.metrics.runTimeMs = performance.now() - timeStart; + App.log("total metrics:",JSON.stringify(App.metrics, undefined, ' ')); + App.log("Reload the page to run this again."); + App.closeDb(); + App.PoolUtil.removeVfs(); + }) + .catch(e=>this.error("ERROR:",e)); + }/*run()*/ +}/*App*/; + +let sqlite3Js = 'sqlite3.js'; +if(urlParams.has('sqlite3.dir')){ + sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; +} +importScripts(sqlite3Js); +globalThis.sqlite3InitModule().then(async function(sqlite3_){ + log("Done initializing. Running batch runner..."); + sqlite3 = sqlite3_; + App.run(sqlite3_); +}); diff --git a/ext/wasm/batch-runner.js b/ext/wasm/batch-runner.js index ff287a66e7..e7a322b7f0 100644 --- a/ext/wasm/batch-runner.js +++ b/ext/wasm/batch-runner.js @@ -72,7 +72,6 @@ App.logHtml("reset db rc =",rc,db.id, db.filename); }; - const E = (s)=>document.querySelector(s); const App = { e: { @@ -91,6 +90,15 @@ db: Object.create(null), dbs: Object.create(null), cache:{}, + metrics: { + fileCount: 0, + runTimeMs: 0, + prepareTimeMs: 0, + stepTimeMs: 0, + stmtCount: 0, + strcpyMs: 0, + sqlBytes: 0 + }, log: console.log.bind(console), warn: console.warn.bind(console), cls: function(){this.e.output.innerHTML = ''}, @@ -117,7 +125,6 @@ "Running",name,'('+sql.length,'bytes) using',db.id); const capi = this.sqlite3.capi, wasm = this.sqlite3.wasm; let pStmt = 0, pSqlBegin; - const stack = wasm.scopedAllocPush(); const metrics = db.metrics = Object.create(null); metrics.prepTotal = metrics.stepTotal = 0; metrics.stmtCount = 0; @@ -142,6 +149,11 @@ this.logHtml("Overhead (time - prep - step):", (metrics.evalTimeTotal - metrics.prepTotal - metrics.stepTotal)+"ms"); this.logHtml(banner,"End of",name); + this.metrics.prepareTimeMs += metrics.prepTotal; + this.metrics.stepTimeMs += metrics.stepTotal; + this.metrics.stmtCount += metrics.stmtCount; + this.metrics.strcpyMs += metrics.strcpy; + this.metrics.sqlBytes += sql.length; }; let runner; @@ -214,7 +226,9 @@ }.bind(this); }else{/*sqlite3 db...*/ runner = function(resolve, reject){ + ++this.metrics.fileCount; metrics.evalSqlStart = performance.now(); + const stack = wasm.scopedAllocPush(); try { let t; let sqlByteLen = sql.byteLength; @@ -269,7 +283,7 @@ let p; if(1){ p = new Promise(function(res,rej){ - setTimeout(()=>runner(res, rej), 50)/*give UI a chance to output the "running" banner*/; + setTimeout(()=>runner(res, rej), 0)/*give UI a chance to output the "running" banner*/; }); }else{ p = new Promise(runner); @@ -401,7 +415,7 @@ }); return new Blob(ar); }, - + downloadMetrics: function(){ const b = this.metricsToBlob(); if(!b) return; @@ -576,6 +590,8 @@ const timeTotal = performance.now() - timeStart; who.logHtml("Run-remaining time:",timeTotal,"ms ("+(timeTotal/1000/60)+" minute(s))"); who.clearStorage(); + App.metrics.runTimeMs = timeTotal; + who.logHtml("Total metrics:",JSON.stringify(App.metrics,undefined,' ')); }, false); }/*run()*/ }/*App*/; diff --git a/ext/wasm/demo-123.js b/ext/wasm/demo-123.js index 8e03ee80fa..9f90ca7568 100644 --- a/ext/wasm/demo-123.js +++ b/ext/wasm/demo-123.js @@ -20,7 +20,7 @@ the main (UI) thread. */ let logHtml; - if(self.window === self /* UI thread */){ + if(globalThis.window === globalThis /* UI thread */){ console.log("Running demo from main UI thread."); logHtml = function(cssClass,...args){ const ln = document.createElement('div'); @@ -250,7 +250,7 @@ }/*demo1()*/; log("Loading and initializing sqlite3 module..."); - if(self.window!==self) /*worker thread*/{ + if(globalThis.window!==globalThis) /*worker thread*/{ /* If sqlite3.js is in a directory other than this script, in order to get sqlite3.js to resolve sqlite3.wasm properly, we have to @@ -262,19 +262,20 @@ that's not needed. URL arguments passed as part of the filename via importScripts() - are simply lost, and such scripts see the self.location of + are simply lost, and such scripts see the globalThis.location of _this_ script. */ let sqlite3Js = 'sqlite3.js'; - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; } importScripts(sqlite3Js); } - self.sqlite3InitModule({ - // We can redirect any stdout/stderr from the module - // like so... + globalThis.sqlite3InitModule({ + /* We can redirect any stdout/stderr from the module like so, but + note that doing so makes use of Emscripten-isms, not + well-defined sqlite APIs. */ print: log, printErr: error }).then(function(sqlite3){ diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 3adf8f2ee9..8c9a77dc5e 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -350,7 +350,7 @@ eControls.classList.remove('hidden'); break; case 'stdout': log(msg.data); break; - case 'stdout': logErr(msg.data); break; + case 'stderr': logErr(msg.data); break; case 'run-start': eControls.disabled = true; log("Running speedtest1 with argv =",msg.data.join(' ')); diff --git a/manifest b/manifest index 9f44093bfe..d6a02fa193 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sJSON\stest\scases\sshowing\sinsert\sor\sset\swith\smissing\ssubstructure. -D 2023-11-30T16:16:10.846 +C Add\sa\sbasic\sbatch-mode\sSQL\srunner\sfor\sthe\sSAH\sPool\sVFS,\sfor\suse\sin\scomparing\sit\sagainst\sWebSQL.\sBring\sthe\sWebSQL\sbatch\srunner\sup\sto\sdate,\snoting\sthat\sit\scannot\srun\swithout\saddition\sof\san\s"origin\strial"\sactivation\skey\sfrom\sGoogle\sbecause\sthat's\snow\sthe\sonly\sway\sto\senable\sWebSQL\sin\sChrome\s(that\spart\sis\snot\schecked\sin\sbecause\sthat\skey\sis\sprivate).\sMinor\scode-adjacent\scleanups. +D 2023-11-30T20:34:24.440 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -568,7 +568,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 0e362f3fc04eab6628cbe4f1e35f4ab4a200881f6b5f753b27fb45eabeddd9d2 +F ext/wasm/GNUmakefile 57439eec2b8b4d4074e5d861e93aab3f32bb0f44339a2b472c23f4e638b7e8a3 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md a8a2962c3aebdf8d2104a9102e336c5554e78fc6072746e5daf9c61514e7d193 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -598,8 +598,10 @@ F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 46c4afa6c50d7369252c104f274ad977a97e91cc F ext/wasm/api/sqlite3-wasm.c d0e09eb5ed3743c00294e30019e591c3aa150572ae7ffe8a8994568a7377589f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 569d4e859968e65f55dec5fac0b879828a23c8b179162cc7812fec19f844dd21 F ext/wasm/api/sqlite3-worker1.c-pp.js a541112aa51e16705f13a99bb943c64efe178aa28c86704a955f8fd9afe4ba37 +F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 +F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 -F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e +F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 @@ -607,7 +609,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7 F ext/wasm/common/whwasmutil.js 4c64594eecc7af4ae64259e95a71ba2a7edf118881aaff0bba86d0c7164e78e4 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 -F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5 +F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f @@ -630,7 +632,7 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe -F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 +F ext/wasm/speedtest1-worker.html 864b65ed78ce24847a348c180e7f267621a02ca027068a1863ec1c90187c1852 F ext/wasm/speedtest1-worker.js 4d2ea70a3c24e05bdca78025202841f33d298c4fa9541a0070c3228661f89ecd F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x @@ -2143,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 263f6d3a7784ef7d032dbf7a3265aca8dd70bf50797f28f6b2e8ddb6a301f83a -R 0e9b3f433035365e3e741a55f5ab0daf -U drh -Z 72d2be7125d7371db00777b78c0fb9f8 +P 6802b6459d0d16c961ff41d240a6c88287f197d8f609090f79308707490a49c2 +R 0685d189d0b0c5921f2e4eda3787211e +U stephan +Z 5692f67161061876d9b3bcabcb3928cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fcbc7b3b7f..98d7aac12d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6802b6459d0d16c961ff41d240a6c88287f197d8f609090f79308707490a49c2 \ No newline at end of file +883990e7938c1f63906300a6113f0fadce143913b7c384e8aeb5f886f0be7c62 \ No newline at end of file From 38aeb97f27045f8978ce14e33155cb4a45e2483c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 20:57:48 +0000 Subject: [PATCH 127/191] Convert json_valid() over to using only JSONB as its internal format. FossilOrigin-Name: 7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 792 +++----------------------------------------------- 3 files changed, 43 insertions(+), 763 deletions(-) diff --git a/manifest b/manifest index 38eb4c4f95..0d5accceac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sjson_type()\sto\suse\sJSONB\sinternally. -D 2023-11-30T19:29:56.759 +C Convert\sjson_valid()\sover\sto\susing\sonly\sJSONB\sas\sits\sinternal\sformat. +D 2023-11-30T20:57:48.608 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3d897ec27a8640c94e5d8293f8aa64226205140150639571c6f745c42d545e3e +F src/json.c 04044605d6eb39c6c8a4cdd42a01c256d10eed88421770c7e92d391164ed5a5b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e2083e86f19ef7634f0b253fb924e52014b43ed0ce8acc51c36f3c5682180a6 -R 4b874649acdc6c51075a8e11e402ee44 +P 83074835b900ce85cf67059e674ce959801505c37592671af25ca0af7ed483f1 +R 2a1e3393332b0e1afccf3cc9eb62658b U drh -Z 2246ad2d1a16f9f5b929395a88459b9a +Z 8664a63b705df80c188b26ef2d7670eb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cd1d889728..0476c44651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83074835b900ce85cf67059e674ce959801505c37592671af25ca0af7ed483f1 \ No newline at end of file +7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 4f47b7a758..f9e2ee5aa9 100644 --- a/src/json.c +++ b/src/json.c @@ -375,13 +375,13 @@ struct JsonParse { ** Allowed values for the flgs argument to jsonParseFuncArg(); */ #define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ +#define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */ /************************************************************************** ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonParseAddNode(JsonParse*,u32,u32,const char*); -static int jsonXlateBlobToNode(JsonParse *pParse, u32 i); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); @@ -1003,645 +1003,12 @@ static const struct NanInfName { { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, }; -/* -** Translate a single element of JSON text beginning at pParse->zJson[i] into -** its JsonNode representation. Append the translation onto the -** pParse->aNode[] array, which is increased in size as necessary. -** Return the pJson->zJson[] index of the first character past the end of -** the element that was parsed. -** -** Special return values: -** -** 0 End of input -** -1 Syntax error -** -2 '}' seen \ -** -3 ']' seen \___ For these returns, pParse->iErr is set to -** -4 ',' seen / the index in zJson[] of the seen character -** -5 ':' seen / -*/ -static int jsonXlateTextToNode(JsonParse *pParse, u32 i){ - char c; - u32 j; - int iThis; - int x; - JsonNode *pNode; - const char *z = pParse->zJson; -json_parse_restart: - switch( (u8)z[i] ){ - case '{': { - /* Parse object */ - iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; - if( ++pParse->iDepth > JSON_MAX_DEPTH ){ - pParse->iErr = i; - return -1; - } - for(j=i+1;;j++){ - u32 nNode = pParse->nNode; - x = jsonXlateTextToNode(pParse, j); - if( x<=0 ){ - if( x==(-2) ){ - j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; - break; - } - j += json5Whitespace(&z[j]); - if( sqlite3JsonId1(z[j]) - || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) - ){ - int k = j+1; - while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) - || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) - ){ - k++; - } - jsonParseAddNode(pParse, JSON_STRING, k-j, &z[j]); - pParse->hasNonstd = 1; - x = k; - }else{ - if( x!=-1 ) pParse->iErr = j; - return -1; - } - } - if( pParse->oom ) return -1; - pNode = &pParse->aNode[nNode]; - if( pNode->eType!=JSON_STRING ){ - pParse->iErr = j; - return -1; - } - pNode->jnFlags |= JNODE_LABEL; - j = x; - if( z[j]==':' ){ - j++; - }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); - if( z[j]==':' ){ - j++; - goto parse_object_value; - } - } - x = jsonXlateTextToNode(pParse, j); - if( x!=(-5) ){ - if( x!=(-1) ) pParse->iErr = j; - return -1; - } - j = pParse->iErr+1; - } - parse_object_value: - x = jsonXlateTextToNode(pParse, j); - if( x<=0 ){ - if( x!=(-1) ) pParse->iErr = j; - return -1; - } - j = x; - if( z[j]==',' ){ - continue; - }else if( z[j]=='}' ){ - break; - }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); - if( z[j]==',' ){ - continue; - }else if( z[j]=='}' ){ - break; - } - } - x = jsonXlateTextToNode(pParse, j); - if( x==(-4) ){ - j = pParse->iErr; - continue; - } - if( x==(-2) ){ - j = pParse->iErr; - break; - } - } - pParse->iErr = j; - return -1; - } - if( !pParse->oom ){ - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - } - pParse->iDepth--; - return j+1; - } - case '[': { - /* Parse array */ - iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; - if( ++pParse->iDepth > JSON_MAX_DEPTH ){ - pParse->iErr = i; - return -1; - } - memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); - for(j=i+1;;j++){ - x = jsonXlateTextToNode(pParse, j); - if( x<=0 ){ - if( x==(-3) ){ - j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; - break; - } - if( x!=(-1) ) pParse->iErr = j; - return -1; - } - j = x; - if( z[j]==',' ){ - continue; - }else if( z[j]==']' ){ - break; - }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); - if( z[j]==',' ){ - continue; - }else if( z[j]==']' ){ - break; - } - } - x = jsonXlateTextToNode(pParse, j); - if( x==(-4) ){ - j = pParse->iErr; - continue; - } - if( x==(-3) ){ - j = pParse->iErr; - break; - } - } - pParse->iErr = j; - return -1; - } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - pParse->iDepth--; - return j+1; - } - case '\'': { - u8 jnFlags; - char cDelim; - pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; - goto parse_string; - case '"': - /* Parse string */ - jnFlags = 0; - parse_string: - cDelim = z[i]; - for(j=i+1; 1; j++){ - if( jsonIsOk[(unsigned char)z[j]] ) continue; - c = z[j]; - if( c==cDelim ){ - break; - }else if( c=='\\' ){ - c = z[++j]; - if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' - || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(&z[j+1])) ){ - jnFlags |= JNODE_ESCAPE; - }else if( c=='\'' || c=='0' || c=='v' || c=='\n' - || (0xe2==(u8)c && 0x80==(u8)z[j+1] - && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) - || (c=='x' && jsonIs2Hex(&z[j+1])) ){ - jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); - pParse->hasNonstd = 1; - }else if( c=='\r' ){ - if( z[j+1]=='\n' ) j++; - jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); - pParse->hasNonstd = 1; - }else{ - pParse->iErr = j; - return -1; - } - }else if( c<=0x1f ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; - } - } - jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j-1-i, &z[i+1]); - return j+1; - } - case 't': { - if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - } - pParse->iErr = i; - return -1; - } - case 'f': { - if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - } - pParse->iErr = i; - return -1; - } - case '+': { - u8 seenDP, seenE, jnFlags; - pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; - goto parse_number; - case '.': - if( sqlite3Isdigit(z[i+1]) ){ - pParse->hasNonstd = 1; - jnFlags = JNODE_JSON5; - seenE = 0; - seenDP = JSON_REAL; - goto parse_number_2; - } - pParse->iErr = i; - return -1; - case '-': - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - /* Parse number */ - jnFlags = 0; - parse_number: - seenDP = JSON_INT; - seenE = 0; - assert( '-' < '0' ); - assert( '+' < '0' ); - assert( '.' < '0' ); - c = z[i]; - - if( c<='0' ){ - if( c=='0' ){ - if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ - assert( seenDP==JSON_INT ); - pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; - for(j=i+3; sqlite3Isxdigit(z[j]); j++){} - goto parse_number_finish; - }else if( sqlite3Isdigit(z[i+1]) ){ - pParse->iErr = i+1; - return -1; - } - }else{ - if( !sqlite3Isdigit(z[i+1]) ){ - /* JSON5 allows for "+Infinity" and "-Infinity" using exactly - ** that case. SQLite also allows these in any case and it allows - ** "+inf" and "-inf". */ - if( (z[i+1]=='I' || z[i+1]=='i') - && sqlite3StrNICmp(&z[i+1], "inf",3)==0 - ){ - pParse->hasNonstd = 1; - if( z[i]=='-' ){ - jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); - }else{ - jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); - } - return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); - } - if( z[i+1]=='.' ){ - pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; - goto parse_number_2; - } - pParse->iErr = i; - return -1; - } - if( z[i+1]=='0' ){ - if( sqlite3Isdigit(z[i+2]) ){ - pParse->iErr = i+1; - return -1; - }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ - pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; - for(j=i+4; sqlite3Isxdigit(z[j]); j++){} - goto parse_number_finish; - } - } - } - } - parse_number_2: - for(j=i+1;; j++){ - c = z[j]; - if( sqlite3Isdigit(c) ) continue; - if( c=='.' ){ - if( seenDP==JSON_REAL ){ - pParse->iErr = j; - return -1; - } - seenDP = JSON_REAL; - continue; - } - if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') - && ALWAYS(j-2>=i) - && sqlite3Isdigit(z[j-2]) - ){ - pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; - }else{ - pParse->iErr = j; - return -1; - } - } - if( seenE ){ - pParse->iErr = j; - return -1; - } - seenDP = JSON_REAL; - seenE = 1; - c = z[j+1]; - if( c=='+' || c=='-' ){ - j++; - c = z[j+1]; - } - if( c<'0' || c>'9' ){ - pParse->iErr = j; - return -1; - } - continue; - } - break; - } - if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ - pParse->hasNonstd = 1; - jnFlags |= JNODE_JSON5; - }else{ - pParse->iErr = j; - return -1; - } - } - parse_number_finish: - jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); - return j; - } - case '}': { - pParse->iErr = i; - return -2; /* End of {...} */ - } - case ']': { - pParse->iErr = i; - return -3; /* End of [...] */ - } - case ',': { - pParse->iErr = i; - return -4; /* List separator */ - } - case ':': { - pParse->iErr = i; - return -5; /* Object label/value separator */ - } - case 0: { - return 0; /* End of file */ - } - case 0x09: - case 0x0a: - case 0x0d: - case 0x20: { - do{ - i++; - }while( fast_isspace(z[i]) ); - goto json_parse_restart; - } - case 0x0b: - case 0x0c: - case '/': - case 0xc2: - case 0xe1: - case 0xe2: - case 0xe3: - case 0xef: { - j = json5Whitespace(&z[i]); - if( j>0 ){ - i += j; - pParse->hasNonstd = 1; - goto json_parse_restart; - } - pParse->iErr = i; - return -1; - } - case 'n': { - if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - } - /* fall-through into the default case that checks for NaN */ - } - default: { - u32 k; - int nn; - c = z[i]; - for(k=0; khasNonstd = 1; - return i + nn; - } - pParse->iErr = i; - return -1; /* Syntax error */ - } - } /* End switch(z[i]) */ -} - -/* -** Parse JSON (either pure RFC-8259 JSON text, or JSON-5 text, or a JSONB -** blob) into the JsonNode representation. -** -** Return 0 on success or non-zero if there are any errors. -** If an error occurs, free all memory held by pParse, but not pParse itself. -** -** pParse must be initialized with pParse->zJson set to the input text or -** blob prior to calling this routine. -*/ -static int jsonParse( - JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx /* Report errors here */ -){ - int i; - const char *zJson = pParse->zJson; - if( pParse->isBinary ){ - pParse->aBlob = (u8*)pParse->zJson; - pParse->nBlob = pParse->nJson; - i = jsonXlateBlobToNode(pParse, 0); - }else{ - i = jsonXlateTextToNode(pParse, 0); - } - if( pParse->oom ) i = -1; - if( !pParse->isBinary && i>0 ){ - assert( pParse->iDepth==0 ); - while( fast_isspace(zJson[i]) ) i++; - if( zJson[i] ){ - i += json5Whitespace(&zJson[i]); - if( zJson[i] ){ - jsonParseReset(pParse); - return 1; - } - pParse->hasNonstd = 1; - } - } - if( i<=0 ){ - if( pCtx!=0 ){ - if( pParse->oom ){ - sqlite3_result_error_nomem(pCtx); - }else{ - sqlite3_result_error(pCtx, "malformed JSON", -1); - } - } - jsonParseReset(pParse); - return 1; - } - return 0; -} - /* ** Magic number used for the JSON parse cache in sqlite3_get_auxdata() */ #define JSON_CACHE_ID (-429938) /* First cache entry */ #define JSON_CACHE_SZ 4 /* Max number of cache entries */ -/* -** Obtain a complete parse of the JSON found in the pJson argument -** -** Use the sqlite3_get_auxdata() cache to find a preexisting parse -** if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse. -** Also register the new parse so that it will be available for -** future sqlite3_get_auxdata() calls. -** -** If an error occurs and pErrCtx!=0 then report the error on pErrCtx -** and return NULL. -** -** The returned pointer (if it is not NULL) is owned by the cache in -** most cases, not the caller. The caller does NOT need to invoke -** jsonParseFree(), in most cases. -** -** Except, if an error occurs and pErrCtx==0 then return the JsonParse -** object with JsonParse.nErr non-zero and the caller will own the JsonParse -** object. In that case, it will be the responsibility of the caller to -** invoke jsonParseFree(). To summarize: -** -** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the -** cache. Call does not need to -** free it. -** -** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller -** and so the caller must free it. -*/ -static JsonParse *jsonParseCached( - sqlite3_context *pCtx, /* Context to use for cache search */ - sqlite3_value *pJson, /* Function param containing JSON text */ - sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ - int bUnedited /* No prior edits allowed */ -){ - char *zJson; - int nJson; - JsonParse *p; - JsonParse *pMatch = 0; - int iKey; - int iMinKey = 0; - u32 iMinHold = 0xffffffff; - u32 iMaxHold = 0; - int bJsonRCStr; - int isBinary; - - if( jsonFuncArgMightBeBinary(pJson) ){ - zJson = (char*)sqlite3_value_blob(pJson); - isBinary = 1; - }else{ - zJson = (char*)sqlite3_value_text(pJson); - isBinary = 0; - } - nJson = sqlite3_value_bytes(pJson); - - if( zJson==0 ) return 0; - for(iKey=0; iKeynJson==nJson - && (p->hasMod==0 || bUnedited==0) - && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) - ){ - p->nErr = 0; - p->useMod = 0; - pMatch = p; - }else - if( pMatch==0 - && p->zAlt!=0 - && bUnedited==0 - && p->nAlt==nJson - && memcmp(p->zAlt, zJson, nJson)==0 - ){ - p->nErr = 0; - p->useMod = 1; - pMatch = p; - }else if( p->iHoldiHold; - iMinKey = iKey; - } - if( p->iHold>iMaxHold ){ - iMaxHold = p->iHold; - } - } - if( pMatch ){ - /* The input JSON text was found in the cache. Use the preexisting - ** parse of this JSON */ - pMatch->nErr = 0; - pMatch->iHold = iMaxHold+1; - assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ - return pMatch; - } - - /* The input JSON was not found anywhere in the cache. We will need - ** to parse it ourselves and generate a new JsonParse object. - */ - bJsonRCStr = sqlite3ValueIsOfClass(pJson,sqlite3RCStrUnref); - p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); - if( p==0 ){ - sqlite3_result_error_nomem(pCtx); - return 0; - } - memset(p, 0, sizeof(*p)); - if( bJsonRCStr ){ - p->zJson = sqlite3RCStrRef(zJson); - p->bJsonIsRCStr = 1; - }else{ - p->zJson = (char*)&p[1]; - memcpy(p->zJson, zJson, nJson+(isBinary==0)); - } - p->nJPRef = 1; - p->isBinary = isBinary; - p->nJson = nJson; - if( jsonParse(p, pErrCtx) ){ - if( pErrCtx==0 ){ - p->nErr = 1; - assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ - return p; - } - jsonParseFree(p); - return 0; - } - p->iHold = iMaxHold+1; - /* Transfer ownership of the new JsonParse to the cache */ - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, - (void(*)(void*))jsonParseFree); - return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); -} /* ** Compare the OBJECT label at pNode against zKey,nKey. Return true on @@ -2289,7 +1656,11 @@ json_parse_restart: parse_string: cDelim = z[i]; nn = pParse->nJson; - for(j=i+1; j=nn ){ + pParse->iErr = j; + return -1; + } if( jsonIsOk[(unsigned char)z[j]] ) continue; c = z[j]; if( c==cDelim ){ @@ -2920,115 +2291,6 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ return sz+n==(u32)nBlob; } -/* Translate a single element of JSONB into the JsonNode format. The -** first byte of the element to be translated is at pParse->aBlob[i]. -** Return the index in pParse->aBlob[] of the first byte past the end -** of the JSONB element. Append the JsonNode translation in -** pParse->aNode[], which is increased in size as necessary. -*/ -static int jsonXlateBlobToNode(JsonParse *pParse, u32 i){ - u8 t; /* Node type */ - u32 sz; /* Node size */ - u32 x; /* Index of payload start */ - - const char *zPayload; - x = jsonbPayloadSize(pParse, i, &sz); - if( x==0 ) return -1; - t = pParse->zJson[i] & 0x0f; - zPayload = &pParse->zJson[i+x]; - switch( t ){ - case JSONB_NULL: { - if( sz>0 ) return -1; - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - break; - } - case JSONB_TRUE: { - if( sz>0 ) return -1; - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - break; - } - case JSONB_FALSE: { - if( sz>0 ) return -1; - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - break; - } - case JSONB_INT: { - if( sz==0 ) return -1; - jsonParseAddNode(pParse, JSON_INT, sz, zPayload); - break; - } - case JSONB_INT5: { - if( sz==0 ) return -1; - pParse->hasNonstd = 1; - jsonParseAddNode(pParse, JSON_INT | (JNODE_JSON5<<8), sz, zPayload); - break; - } - case JSONB_FLOAT: { - if( sz==0 ) return -1; - jsonParseAddNode(pParse, JSON_REAL, sz, zPayload); - break; - } - case JSONB_FLOAT5: { - if( sz==0 ) return -1; - pParse->hasNonstd = 1; - jsonParseAddNode(pParse, JSON_REAL | (JNODE_JSON5<<8), sz, zPayload); - break; - } - case JSONB_TEXTRAW: { - jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), sz, zPayload); - break; - } - case JSONB_TEXT: { - jsonParseAddNode(pParse, JSON_STRING, sz, zPayload); - break; - } - case JSONB_TEXTJ: { - jsonParseAddNode(pParse, JSON_STRING | (JNODE_ESCAPE<<8), sz, zPayload); - break; - } - case JSONB_TEXT5: { - pParse->hasNonstd = 1; - jsonParseAddNode(pParse, JSON_STRING | ((JNODE_ESCAPE|JNODE_JSON5)<<8), - sz, zPayload); - break; - } - case JSONB_ARRAY: { - int iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - u32 j = i+x; - while( joom ){ - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - } - break; - } - case JSONB_OBJECT: { - int iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - u32 j = i+x, k = 0; - while( joom ){ - pParse->aNode[pParse->nNode-1].jnFlags |= JNODE_LABEL; - } - j = (u32)r; - } - if( !pParse->oom ){ - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - } - if( k&1 ) return -1; - break; - } - default: { - return -1; - } - } - return i+x+sz; -} - /* ** Given that a JSONB_ARRAY object starts at offset i, return ** the number of entries in that array. @@ -3730,16 +2992,27 @@ static JsonParse *jsonParseFuncArg( p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( p->zJson==0 ) goto json_pfa_oom; + if( flgs & JSON_KEEPERROR ) ctx = 0; if( jsonConvertTextToBlob(p, ctx) ){ - jsonParseFree(p); - return 0; + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + return 0; + } } return p; json_pfa_malformed: - jsonParseFree(p); - sqlite3_result_error(ctx, "malformed JSON", -1); - return 0; + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + sqlite3_result_error(ctx, "malformed JSON", -1); + return 0; + } json_pfa_oom: jsonParseFree(p); @@ -4761,15 +4034,22 @@ static void jsonValidFunc( break; } default: { + JsonParse px; if( (flags & 0x3)==0 ) break; - p = jsonParseCached(ctx, argv[0], 0, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); - }else if( p->nErr ){ + memset(&px, 0, sizeof(px)); + + p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); + if( p ){ + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( p->nErr ){ + /* no-op */ + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 || p->useMod ){ + res = 1; + } jsonParseFree(p); - }else if( (flags & 0x02)!=0 || p->hasNonstd==0 || p->useMod ){ - res = 1; + }else{ + sqlite3_result_error_nomem(ctx); } break; } From 4b9ed1b25657801bc43840bcf6cfde5cd6b69c33 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Nov 2023 23:36:14 +0000 Subject: [PATCH 128/191] Remove all trace of JsonNode from the JSON implementation. The JSONB format is used as the internal binary encoding for searching and editing. FossilOrigin-Name: 11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35 --- manifest | 12 +- manifest.uuid | 2 +- src/json.c | 477 ++++---------------------------------------------- 3 files changed, 43 insertions(+), 448 deletions(-) diff --git a/manifest b/manifest index 0d5accceac..0f5a595ba2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sjson_valid()\sover\sto\susing\sonly\sJSONB\sas\sits\sinternal\sformat. -D 2023-11-30T20:57:48.608 +C Remove\sall\strace\sof\sJsonNode\sfrom\sthe\sJSON\simplementation.\s\sThe\sJSONB\sformat\nis\sused\sas\sthe\sinternal\sbinary\sencoding\sfor\ssearching\sand\sediting. +D 2023-11-30T23:36:14.577 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 04044605d6eb39c6c8a4cdd42a01c256d10eed88421770c7e92d391164ed5a5b +F src/json.c c4f3602115334c210fbcdfb9565ba9fd322d29f7327b598fc0dd5dea4cb1b068 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 83074835b900ce85cf67059e674ce959801505c37592671af25ca0af7ed483f1 -R 2a1e3393332b0e1afccf3cc9eb62658b +P 7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b +R 38620fda20feff15c621a1f3bd3d0184 U drh -Z 8664a63b705df80c188b26ef2d7670eb +Z 66786c8d6e89beb8e06f2e4708cb9069 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0476c44651..be5cc905b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b \ No newline at end of file +11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f9e2ee5aa9..713dc76da9 100644 --- a/src/json.c +++ b/src/json.c @@ -20,7 +20,7 @@ ** All generated JSON text still conforms strictly to RFC-8259, but text ** with JSON-5 extensions is accepted as input. ** -** Beginning with version 3.44.0 (pending), these routines also accept +** Beginning with version 3.45.0 (pending), these routines also accept ** BLOB values that have JSON encoded using a binary representation we ** call JSONB. The name JSONB comes from PostgreSQL, however the on-disk ** format SQLite JSONB is completely different and incompatible with @@ -133,7 +133,7 @@ #define JSONB_ARRAY 11 /* An array */ #define JSONB_OBJECT 12 /* An object */ -/* Human-readalbe names for the JSONB values: +/* Human-readable names for the JSONB values: */ static const char * const jsonbType[] = { "null", "true", "false", "integer", "integer", @@ -201,9 +201,7 @@ static const char jsonIsOk[256] = { /* Objects */ typedef struct JsonString JsonString; -typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; -typedef struct JsonCleanup JsonCleanup; /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator @@ -224,49 +222,11 @@ struct JsonString { #define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ #define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ -/* A deferred cleanup task. A list of JsonCleanup objects might be -** run when the JsonParse object is destroyed. -*/ -struct JsonCleanup { - JsonCleanup *pJCNext; /* Next in a list */ - void (*xOp)(void*); /* Routine to run */ - void *pArg; /* Argument to xOp() */ -}; - -/* JSON type values for JsonNode.eType -*/ -#define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ -#define JSON_NULL 1 -#define JSON_TRUE 2 -#define JSON_FALSE 3 -#define JSON_INT 4 -#define JSON_REAL 5 -#define JSON_STRING 6 -#define JSON_ARRAY 7 -#define JSON_OBJECT 8 - -/* Human-readalbe names for the JsonNode types: -*/ -static const char * const jsonType[] = { - "subst", - "null", "true", "false", "integer", "real", "text", "array", "object" -}; - /* The "subtype" set for text JSON values passed through using ** sqlite3_result_subtype() and sqlite3_value_subtype(). */ #define JSON_SUBTYPE 74 /* Ascii for "J" */ -/* Bit values for the JsonNode.jnFlag field -*/ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x20 /* Is a label of an object */ -#define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ - /* ** Bit values for the flags passed into jsonExtractFunc() or ** jsonSetFunc() via the user-data value. @@ -278,76 +238,35 @@ static const char * const jsonType[] = { #define JSON_BLOB 0x08 /* Use the BLOB output format */ -/* A single node of parsed JSON. An array of these nodes describes -** a parse of JSON + edits. +/* A parsed JSON value. Lifecycle: ** -** Use the json_parse() SQL function (available when compiled with -** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including -** a complete listing and decoding of the array of JsonNodes. -*/ -struct JsonNode { - u8 eType; /* One of the JSON_ type values */ - u8 jnFlags; /* JNODE flags */ - u8 eU; /* Which union element to use */ - u32 n; /* Bytes of content for INT, REAL or STRING - ** Number of sub-nodes for ARRAY and OBJECT - ** Node that SUBST applies to */ - union { - const char *zJContent; /* 1: Content for INT, REAL, and STRING */ - u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ - u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ - u32 iPrev; /* 4: Previous SUBST node, or 0 */ - } u; -}; - - -/* A parsed and possibly edited JSON string. Lifecycle: +** 1. JSON comes in and is parsed into a JSONB value in aBlob. The +** original text is stored in zJson. ** -** 1. JSON comes in and is parsed into an array aNode[]. The original -** JSON text is stored in zJson. +** 2. The aBlob is searched using the JSON path notation, if needed. +** +** 3. Zero or more changes are made to aBlob (via json_remove() or +** json_replace() or similar). ** -** 2. Zero or more changes are made (via json_remove() or json_replace() -** or similar) to the aNode[] array. +** 4. New JSON text is generated from the aBlob for output. ** -** 3. A new, edited and mimified JSON string is generated from aNode -** and stored in zAlt. The JsonParse object always owns zAlt. -** -** Step 1 always happens. Step 2 and 3 may or may not happen, depending -** on the operation. -** -** aNode[].u.zJContent entries typically point into zJson. Hence zJson -** must remain valid for the lifespan of the parse. For edits, -** aNode[].u.zJContent might point to malloced space other than zJson. -** Entries in pClup are responsible for freeing that extra malloced space. -** -** When walking the parse tree in aNode[], edits are ignored if useMod is -** false. +** Step 1 is omitted if the input is a BLOB in the JSONB format. Step 4 +** is omitted if the output is JSONB or some other value that is not +** JSON text. */ struct JsonParse { - u32 nNode; /* Number of slots of aNode[] used */ - u32 nAlloc; /* Number of slots of aNode[] allocated */ - JsonNode *aNode; /* Array of nodes containing the parse */ + u8 *aBlob; /* JSONB representation of JSON value */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Original JSON string (before edits) */ - char *zAlt; /* Revised and/or mimified JSON */ - JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ - u8 useMod; /* Actually use the edits contain inside aNode */ - u8 hasMod; /* aNode contains edits from the original zJson */ - u8 isBinary; /* True if zJson is the binary encoding */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ - int nAlt; /* Length of alternative JSON string zAlt, in bytes */ u32 iErr; /* Error location in zJson[] */ - u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ - u32 iHold; /* Age of this entry in the cache for LRU replacement */ - /* Storage for the binary JSONB format */ - u32 nBlob; /* Bytes of aBlob[] actually used */ - u32 nBlobAlloc; /* Bytes allocated to aBlob[] */ - u8 *aBlob; /* BLOB representation of zJson */ /* Search and edit information. See jsonLookupBlobStep() */ u8 eEdit; /* Edit operation to apply */ int delta; /* Size change due to the edit */ @@ -381,9 +300,7 @@ struct JsonParse { ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); @@ -678,49 +595,20 @@ static void jsonReturnString(JsonString *p){ } /************************************************************************** -** Utility routines for dealing with JsonNode and JsonParse objects +** Utility routines for dealing with JsonParse objects **************************************************************************/ -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonNodeSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} - /* ** Reclaim all memory allocated by a JsonParse object. But do not ** delete the JsonParse object itself. */ static void jsonParseReset(JsonParse *pParse){ - while( pParse->pClup ){ - JsonCleanup *pTask = pParse->pClup; - pParse->pClup = pTask->pJCNext; - pTask->xOp(pTask->pArg); - sqlite3_free(pTask); - } assert( pParse->nJPRef<=1 ); - if( pParse->aNode ){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - } - pParse->nNode = 0; - pParse->nAlloc = 0; if( pParse->bJsonIsRCStr ){ sqlite3RCStrUnref(pParse->zJson); pParse->zJson = 0; pParse->bJsonIsRCStr = 0; } - if( pParse->zAlt ){ - sqlite3RCStrUnref(pParse->zAlt); - pParse->zAlt = 0; - } if( pParse->nBlobAlloc ){ sqlite3_free(pParse->aBlob); pParse->aBlob = 0; @@ -788,61 +676,6 @@ static u32 jsonHexToInt4(const char *z){ #endif -/* -** Add a single node to pParse->aNode after first expanding the -** size of the aNode array. Return the index of the new node. -** -** If an OOM error occurs, set pParse->oom and return -1. -*/ -static JSON_NOINLINE int jsonParseAddNodeExpand( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - u32 nNew; - JsonNode *pNew; - assert( pParse->nNode>=pParse->nAlloc ); - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; - } - pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode); - pParse->aNode = pNew; - assert( pParse->nNodenAlloc ); - return jsonParseAddNode(pParse, eType, n, zContent); -} - -/* -** Create a new JsonNode instance based on the arguments and append that -** instance to the JsonParse. Return the index in pParse->aNode[] of the -** new node, or -1 if a memory allocation fails. -*/ -static int jsonParseAddNode( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - JsonNode *p; - assert( pParse->aNode!=0 || pParse->nNode>=pParse->nAlloc ); - if( pParse->nNode>=pParse->nAlloc ){ - return jsonParseAddNodeExpand(pParse, eType, n, zContent); - } - assert( pParse->aNode!=0 ); - p = &pParse->aNode[pParse->nNode]; - assert( p!=0 ); - p->eType = (u8)(eType & 0xff); - p->jnFlags = (u8)(eType >> 8); - JSON_VVA( p->eU = zContent ? 1 : 0 ); - p->n = n; - p->u.zJContent = zContent; - return pParse->nNode++; -} - /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -996,11 +829,11 @@ static const struct NanInfName { char *zMatch; char *zRepl; } aNanInfName[] = { - { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" }, - { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" }, - { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, - { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, - { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, + { 'i', 'I', 3, JSONB_FLOAT, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSONB_FLOAT, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSONB_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSONB_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" }, }; /* @@ -1010,232 +843,6 @@ static const struct NanInfName { #define JSON_CACHE_SZ 4 /* Max number of cache entries */ -/* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. -*/ -static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ - if( pNode->eType!=JSON_STRING ) return 0; - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; -} - -/* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. -** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. -*/ -static JsonNode *jsonLookupStep( - JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ -){ - u32 i, j, nKey; - const char *zKey; - JsonNode *pRoot; - if( pParse->oom ) return 0; - pRoot = &pParse->aNode[iRoot]; - if( pRoot->jnFlags & (JNODE_REPLACE|JNODE_REMOVE) && pParse->useMod ){ - while( (pRoot->jnFlags & JNODE_REPLACE)!=0 ){ - u32 idx = (u32)(pRoot - pParse->aNode); - i = pParse->iSubst; - while( 1 /*exit-by-break*/ ){ - assert( inNode ); - assert( pParse->aNode[i].eType==JSON_SUBST ); - assert( pParse->aNode[i].eU==4 ); - assert( pParse->aNode[i].u.iPrevaNode[i].n==idx ){ - pRoot = &pParse->aNode[i+1]; - iRoot = i+1; - break; - } - i = pParse->aNode[i].u.iPrev; - } - } - if( pRoot->jnFlags & JNODE_REMOVE ){ - return 0; - } - } - if( zPath[0]==0 ) return pRoot; - if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; - zPath++; - if( zPath[0]=='"' ){ - zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} - nKey = i-1; - if( zPath[i] ){ - i++; - }else{ - *pzErr = zPath; - return 0; - } - testcase( nKey==0 ); - }else{ - zKey = zPath; - for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} - nKey = i; - if( nKey==0 ){ - *pzErr = zPath; - return 0; - } - } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); - } - j++; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pRoot->eU==2 ); - iRoot = pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - assert( pParse->useMod ); - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart; - pRoot->jnFlags |= JNODE_APPEND; - JSON_VVA( pRoot->eU = 2 ); - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; - } - }else if( zPath[0]=='[' ){ - i = 0; - j = 1; - while( sqlite3Isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; - } - if( j<2 || zPath[j]!=']' ){ - if( zPath[1]=='#' ){ - JsonNode *pBase = pRoot; - int iBase = iRoot; - if( pRoot->eType!=JSON_ARRAY ) return 0; - for(;;){ - while( j<=pBase->n ){ - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ - i++; - } - j += jsonNodeSize(&pBase[j]); - } - if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pBase->eU==2 ); - iBase = pBase->u.iAppend; - pBase = &pParse->aNode[iBase]; - j = 1; - } - j = 2; - if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int x = 0; - j = 3; - do{ - x = x*10 + zPath[j] - '0'; - j++; - }while( sqlite3Isdigit(zPath[j]) ); - if( x>i ) return 0; - i -= x; - } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; - } - }else{ - *pzErr = zPath; - return 0; - } - } - if( pRoot->eType!=JSON_ARRAY ) return 0; - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n - && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) - ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( i==0 && j<=pRoot->n ) break; - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - if( pParse->useMod==0 ) break; - assert( pRoot->eU==2 ); - iRoot = pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - assert( pParse->useMod ); - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - assert( pRoot->eU==0 ); - pRoot->u.iAppend = iStart; - pRoot->jnFlags |= JNODE_APPEND; - JSON_VVA( pRoot->eU = 2 ); - } - return pNode; - } - }else{ - *pzErr = zPath; - } - return 0; -} - -/* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. -*/ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ -){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; - } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; - } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); -} - /* ** Compute the text of an error in JSON path syntax. ** @@ -1919,7 +1526,7 @@ json_parse_restart: continue; } if( sqlite3Isalnum(z[i+nn]) ) continue; - if( aNanInfName[k].eType==JSON_REAL ){ + if( aNanInfName[k].eType==JSONB_FLOAT ){ jsonBlobAppendOneByte(pParse, JSONB_FLOAT | 0x50); jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); }else{ @@ -3127,7 +2734,7 @@ static void jsonDebugPrintBlob( } } if( showContent ){ - if( sz==0 && x<=JSON_FALSE ){ + if( sz==0 && x<=JSONB_FALSE ){ printf("\n"); }else{ u32 i; @@ -4044,7 +3651,7 @@ static void jsonValidFunc( sqlite3_result_error_nomem(ctx); }else if( p->nErr ){ /* no-op */ - }else if( (flags & 0x02)!=0 || p->hasNonstd==0 || p->useMod ){ + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 ){ res = 1; } jsonParseFree(p); @@ -4060,30 +3667,19 @@ static void jsonValidFunc( /* ** json_error_position(JSON) ** -** If the argument is not an interpretable JSON string, then return the 1-based -** character position at which the parser first recognized that the input -** was in error. The left-most character is 1. If the string is valid -** JSON, then return 0. +** If the argument is NULL, return NULL ** -** Note that json_valid() is only true for strictly conforming canonical JSON. -** But this routine returns zero if the input contains extension. Thus: +** If the argument is TEXT then try to interpret that text as JSON and +** return the 1-based character position for where the parser first recognized +** that the input was not valid JSON, or return 0 if the input text looks +** ok. JSON-5 extensions are accepted. ** -** (1) If the input X is strictly conforming canonical JSON: +** If the argument is a BLOB then try to interpret the blob as a JSONB +** and return the 1-based byte offset of the first position that is +** misformatted. Return 0 if the input BLOB seems to be well-formed. ** -** json_valid(X) returns true -** json_error_position(X) returns 0 -** -** (2) If the input X is JSON but it includes extension (such as JSON5) that -** are not part of RFC-8259: -** -** json_valid(X) returns false -** json_error_position(X) return 0 -** -** (3) If the input X cannot be interpreted as JSON even taking extensions -** into account: -** -** json_valid(X) return false -** json_error_position(X) returns 1 or more +** Numeric inputs are converted into text, which is usually valid +** JSON-5, so they should return 0. */ static void jsonErrorFunc( sqlite3_context *ctx, @@ -4684,7 +4280,7 @@ static int jsonEachColumn( break; } case JEACH_JSON: { - if( p->sParse.isBinary ){ + if( p->sParse.zJson==0 ){ sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, SQLITE_STATIC); }else{ @@ -4795,7 +4391,6 @@ static int jsonEachFilter( if( p->sParse.aBlob==0 ){ return SQLITE_NOMEM; } - p->sParse.isBinary = 1; }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); From ca1ce7773c42cb128b424c77755f4413663bd81c Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Dec 2023 12:57:12 +0000 Subject: [PATCH 129/191] First attempt to get the JSON text-to-binary cache working. All test cases pass, but the cache seems not to help much. FossilOrigin-Name: 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c --- manifest | 16 +-- manifest.uuid | 2 +- src/json.c | 267 ++++++++++++++++++++++++++++++++++++------------ src/printf.c | 2 +- src/sqliteInt.h | 3 + 5 files changed, 215 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index 0f5a595ba2..3cac80e1a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sall\strace\sof\sJsonNode\sfrom\sthe\sJSON\simplementation.\s\sThe\sJSONB\sformat\nis\sused\sas\sthe\sinternal\sbinary\sencoding\sfor\ssearching\sand\sediting. -D 2023-11-30T23:36:14.577 +C First\sattempt\sto\sget\sthe\sJSON\stext-to-binary\scache\sworking.\s\sAll\stest\scases\npass,\sbut\sthe\scache\sseems\snot\sto\shelp\smuch. +D 2023-12-01T12:57:12.578 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c4f3602115334c210fbcdfb9565ba9fd322d29f7327b598fc0dd5dea4cb1b068 +F src/json.c 9a96faf916f572deb94b6cc7673ba6e299c48d153c18218e104a2c6ae37c2c8e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -724,7 +724,7 @@ F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00 F src/pragma.c b5b4cff830575e6188cd56a295a57448d2b9dbc53f0dae58e22b97354cda3781 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c 371f6115cb69286ebc12c6f2d7511279c2e47d9f54f475d46a554d687a3b312c -F src/printf.c 9da63b9ae1c14789bcae12840f5d800fd9302500cd2d62733fac77f0041b4750 +F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 @@ -733,7 +733,7 @@ F src/shell.c.in 7bb83293775e1a5586d65212997442bc7acc70a2f1b781745da64ec3c2e4ea9 F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 6b82eb99a9d2887e873fb29e56befb7c50cf4624df615d23a28f071dc8abf5f6 +F src/sqliteInt.h aab66d149269f15f6f1011081b389f001f00b84045c69a4f9ec96dd68cc3a7d7 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7b5756fa6d00b093bf083a8d7a5ef5485f7a09e4eac473785c8380688f861a1b -R 38620fda20feff15c621a1f3bd3d0184 +P 11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35 +R 18f6bb3dc7a73112aa5c773b9941447e U drh -Z 66786c8d6e89beb8e06f2e4708cb9069 +Z 9a005b4c0b238db6966807971445cc41 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index be5cc905b1..298549f7a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35 \ No newline at end of file +25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c \ No newline at end of file diff --git a/src/json.c b/src/json.c index 713dc76da9..44fa5bf0b2 100644 --- a/src/json.c +++ b/src/json.c @@ -165,7 +165,7 @@ static const char jsonIsSpace[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; -#define fast_isspace(x) (jsonIsSpace[(unsigned char)x]) +#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) /* ** Characters that are special to JSON. Control charaters, @@ -200,9 +200,35 @@ static const char jsonIsOk[256] = { #endif /* Objects */ +typedef struct JsonCache JsonCache; +typedef struct JsonCacheLine JsonCacheLine; typedef struct JsonString JsonString; typedef struct JsonParse JsonParse; + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* Cache entry */ +#define JSON_CACHE_SIZE 4 /* Max number of cache entries */ + +/* A cache mapping JSON text into JSONB blobs. +** +** All content, both JSON text and the JSONB blobs, is stored as RCStr +** objects. +*/ +struct JsonCacheLine { + u32 nJson; /* Size of the JSON text, in bytes */ + u32 nBlob; /* Size of the corresponding JSONB, in bytes */ + char *zJson; /* RCStr holding the JSON text */ + char *aBlob; /* RCStr holding the corresponding JSONB */ +}; +struct JsonCache { + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonCacheLine a[JSON_CACHE_SIZE]; /* One line for each cache entry */ +}; + /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. @@ -258,11 +284,11 @@ struct JsonParse { u8 *aBlob; /* JSONB representation of JSON value */ u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ - char *zJson; /* Original JSON string (before edits) */ + char *zJson; /* Json text used for parsing */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ - u8 bJsonIsRCStr; /* True if zJson is an RCStr */ + u8 bBlobIsRCStr; /* True if aBlob is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ u32 nJPRef; /* Number of references to this object */ int nJson; /* Length of the zJson string in bytes */ @@ -304,6 +330,113 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +/************************************************************************** +** Utility routines for dealing with JsonCache objects +**************************************************************************/ + +/* +** Free a JsonCache object. +*/ +static void jsonCacheDelete(JsonCache *p){ + int i; + for(i=0; inUsed; i++){ + sqlite3RCStrUnref(p->a[i].zJson); + sqlite3RCStrUnref(p->a[i].aBlob); + } + sqlite3DbFree(p->db, p); +} +static void jsonCacheDeleteGeneric(void *p){ + jsonCacheDelete((JsonCache*)p); +} + +/* +** Insert a new entry into the cache. If the cache is full, expell +** the least recently used entry. Return SQLITE_OK on success or a +** result code otherwise. +** +** Both the input JSON and JSONB must be RCStr objects. +*/ +static int jsonCacheInsert( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + char *zJson, /* The key. Must be an RCStr! */ + u32 nJson, /* Number of bytes in zJson */ + char *aBlob, /* The value. Not an RCStr */ + u32 nBlob /* Number of bytes in aBlob */ +){ + JsonCache *p; + char *aRCBlob = 0; + + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->db = db; + sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ) return SQLITE_NOMEM; + } + aRCBlob = sqlite3RCStrNew( nBlob ); + if( aRCBlob==0 ) return SQLITE_NOMEM; + memcpy(aRCBlob, aBlob, nBlob); + if( p->nUsed >= JSON_CACHE_SIZE ){ + sqlite3RCStrUnref(p->a[0].zJson); + sqlite3RCStrUnref(p->a[0].aBlob); + memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); + p->nUsed = JSON_CACHE_SIZE-1; + } + p->a[p->nUsed].nJson = nJson; + p->a[p->nUsed].nBlob = nBlob; + p->a[p->nUsed].zJson = sqlite3RCStrRef(zJson); + p->a[p->nUsed].aBlob = aRCBlob; + p->nUsed++; + return SQLITE_OK; +} + +/* +** Search for a cached translation of zJson (size: nJson bytes) into +** JSONB. Return it if found. +** +** The returned value is an RCStr object if it is not NULL. +** The caller is responsible for incrementing the reference count. +*/ +static u8 *jsonCacheSearch( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + char *zJson, /* The key. Might or might not be an RCStr */ + u32 nJson, /* Size of the key in bytes */ + u32 *pnBlob /* OUT: Size of the result in bytes */ +){ + JsonCache *p; + int i; + + assert( pnBlob!=0 ); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + *pnBlob = 0; + return 0; + } + for(i=0; inUsed; i++){ + if( p->a[i].zJson==zJson ) break; + } + if( i>=p->nUsed ){ + for(i=0; inUsed; i++){ + if( p->a[i].nJson!=nJson ) continue; + if( memcmp(p->a[i].zJson, zJson, nJson)==0 ) break; + } + } + if( inUsed ){ + if( inUsed-1 ){ + JsonCacheLine tmp = p->a[i]; + memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); + p->a[p->nUsed-1] = tmp; + } + *pnBlob = p->a[i].nBlob; + return (u8*)p->a[i].aBlob; + }else{ + *pnBlob = 0; + return 0; + } +} /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -569,8 +702,18 @@ static void jsonAppendSqlValue( ** the result of the SQL function. ** ** The JsonString is reset. +** +** If pParse and ctx are both non-NULL and if pParse->aBlob is valid +** then an attempt is made to cache the translation from JSON text into +** the blob. */ -static void jsonReturnString(JsonString *p){ +static void jsonReturnString( + JsonString *p, /* String to return */ + JsonParse *pParse, /* JSONB source or NULL */ + sqlite3_context *ctx /* Where to cache */ +){ + assert( (pParse!=0)==(ctx!=0) ); + assert( ctx==0 || ctx==p->pCtx ); if( p->eErr==0 ){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); if( flags & JSON_BLOB ){ @@ -580,6 +723,16 @@ static void jsonReturnString(JsonString *p){ SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ sqlite3RCStrRef(p->zBuf); + if( pParse ){ + int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, + (char*)pParse->aBlob, pParse->nBlob); + if( rc==SQLITE_NOMEM ){ + sqlite3RCStrUnref(p->zBuf); + sqlite3_result_error_nomem(ctx); + jsonStringReset(p); + return; + } + } sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); @@ -604,10 +757,12 @@ static void jsonReturnString(JsonString *p){ */ static void jsonParseReset(JsonParse *pParse){ assert( pParse->nJPRef<=1 ); - if( pParse->bJsonIsRCStr ){ - sqlite3RCStrUnref(pParse->zJson); - pParse->zJson = 0; - pParse->bJsonIsRCStr = 0; + if( pParse->bBlobIsRCStr ){ + assert( pParse->nBlobAlloc==0 ); + sqlite3RCStrUnref((char*)pParse->aBlob); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->bBlobIsRCStr = 0; } if( pParse->nBlobAlloc ){ sqlite3_free(pParse->aBlob); @@ -636,46 +791,18 @@ static void jsonParseFree(JsonParse *pParse){ } } -/* -** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F -*/ -static u8 jsonHexToInt(int h){ - if( !sqlite3Isxdigit(h) ) return 0; -#ifdef SQLITE_EBCDIC - h += 9*(1&~(h>>4)); -#else - h += 9*(1&(h>>6)); -#endif - return (u8)(h & 0xf); -} - /* ** Convert a 4-byte hex string into an integer */ static u32 jsonHexToInt4(const char *z){ u32 v; - v = (jsonHexToInt(z[0])<<12) - + (jsonHexToInt(z[1])<<8) - + (jsonHexToInt(z[2])<<4) - + jsonHexToInt(z[3]); + v = (sqlite3HexToInt(z[0])<<12) + + (sqlite3HexToInt(z[1])<<8) + + (sqlite3HexToInt(z[2])<<4) + + sqlite3HexToInt(z[3]); return v; } -/* -** A macro to hint to the compiler that a function should not be -** inlined. -*/ -#if defined(__GNUC__) -# define JSON_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define JSON_NOINLINE __declspec(noinline) -#else -# define JSON_NOINLINE -#endif - - /* ** Return true if z[] begins with 2 (or more) hexadecimal digits */ @@ -836,12 +963,6 @@ static const struct NanInfName { { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" }, }; -/* -** Magic number used for the JSON parse cache in sqlite3_get_auxdata() -*/ -#define JSON_CACHE_ID (-429938) /* First cache entry */ -#define JSON_CACHE_SZ 4 /* Max number of cache entries */ - /* ** Compute the text of an error in JSON path syntax. @@ -1147,8 +1268,8 @@ json_parse_restart: if( z[j]==':' ){ j++; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + do{ j++; }while( jsonIsspace(z[j]) ); if( z[j]==':' ){ j++; goto parse_object_value; @@ -1173,8 +1294,8 @@ json_parse_restart: }else if( z[j]=='}' ){ break; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + do{ j++; }while( jsonIsspace(z[j]) ); if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ @@ -1225,8 +1346,8 @@ json_parse_restart: }else if( z[j]==']' ){ break; }else{ - if( fast_isspace(z[j]) ){ - do{ j++; }while( fast_isspace(z[j]) ); + if( jsonIsspace(z[j]) ){ + do{ j++; }while( jsonIsspace(z[j]) ); if( z[j]==',' ){ continue; }else if( z[j]==']' ){ @@ -1488,7 +1609,7 @@ json_parse_restart: case 0x20: { do{ i++; - }while( fast_isspace(z[i]) ); + }while( jsonIsspace(z[i]) ); goto json_parse_restart; } case 0x0b: @@ -1560,7 +1681,7 @@ static int jsonConvertTextToBlob( if( pParse->oom ) i = -1; if( i>0 ){ assert( pParse->iDepth==0 ); - while( fast_isspace(zJson[i]) ) i++; + while( jsonIsspace(zJson[i]) ) i++; if( zJson[i] ){ i += json5Whitespace(&zJson[i]); if( zJson[i] ){ @@ -2197,7 +2318,7 @@ static void jsonReturnTextJsonFromBlob( x.nBlob = nBlob; jsonStringInit(&s, ctx); jsonXlateBlobToText(&x, 0, &s); - jsonReturnString(&s); + jsonReturnString(&s, 0, 0); } @@ -2338,7 +2459,7 @@ static void jsonReturnFromBlob( }else if( c=='0' ){ c = 0; }else if( c=='x' ){ - c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); + c = (sqlite3HexToInt(z[iIn+1])<<4) | sqlite3HexToInt(z[iIn+2]); iIn += 2; }else if( c=='\r' && z[i+1]=='\n' ){ iIn++; @@ -2455,7 +2576,7 @@ static int jsonFunctionArgToBlob( } /* -** Generate a bad path error for json_extract() +** Generate a bad path error. */ static void jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ @@ -2599,6 +2720,22 @@ static JsonParse *jsonParseFuncArg( p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( p->zJson==0 ) goto json_pfa_oom; + + p->aBlob = jsonCacheSearch(ctx, p->zJson, p->nJson, &p->nBlob); + if( p->aBlob ){ + if( flgs & JSON_EDITABLE ){ + u8 *pNew = sqlite3_malloc64( p->nBlob ); + if( pNew==0 ) goto json_pfa_oom; + memcpy(pNew, p->aBlob, p->nBlob); + p->aBlob = pNew; + p->nBlobAlloc = p->nBlob; + }else{ + sqlite3RCStrRef((char*)p->aBlob); + p->bBlobIsRCStr = 1; + } + return p; + } + if( flgs & JSON_KEEPERROR ) ctx = 0; if( jsonConvertTextToBlob(p, ctx) ){ if( flgs & JSON_KEEPERROR ){ @@ -2650,7 +2787,7 @@ static void jsonReturnParse( JsonString s; jsonStringInit(&s, ctx); jsonXlateBlobToText(p, 0, &s); - jsonReturnString(&s); + jsonReturnString(&s, p, ctx); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } } @@ -2890,7 +3027,7 @@ static void jsonQuoteFunc( jsonStringInit(&jx, ctx); jsonAppendSqlValue(&jx, argv[0]); - jsonReturnString(&jx); + jsonReturnString(&jx, 0, 0); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -2914,7 +3051,7 @@ static void jsonArrayFunc( jsonAppendSqlValue(&jx, argv[i]); } jsonAppendChar(&jx, ']'); - jsonReturnString(&jx); + jsonReturnString(&jx, 0, 0); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -3058,7 +3195,7 @@ static void jsonExtractFunc( if( flags & JSON_JSON ){ jsonStringInit(&jx, ctx); jsonXlateBlobToText(p, j, &jx); - jsonReturnString(&jx); + jsonReturnString(&jx, 0, 0); jsonStringReset(&jx); assert( (flags & JSON_BLOB)==0 ); sqlite3_result_subtype(ctx, JSON_SUBTYPE); @@ -3091,7 +3228,7 @@ static void jsonExtractFunc( } if( argc>2 ){ jsonAppendChar(&jx, ']'); - jsonReturnString(&jx); + jsonReturnString(&jx, 0, 0); if( (flags & JSON_BLOB)==0 ){ sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -3389,7 +3526,7 @@ static void jsonObjectFunc( jsonAppendSqlValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); - jsonReturnString(&jx); + jsonReturnString(&jx, 0, 0); sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -3767,7 +3904,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ jsonAppendChar(pStr, ']'); flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ - jsonReturnString(pStr); + jsonReturnString(pStr, 0, 0); return; }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); @@ -3887,7 +4024,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ pStr->pCtx = ctx; flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ - jsonReturnString(pStr); + jsonReturnString(pStr, 0, 0); return; }else if( flags & JSON_BLOB ){ jsonReturnStringAsBlob(pStr); diff --git a/src/printf.c b/src/printf.c index 3c0b182d39..c6b3803ca9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1369,7 +1369,7 @@ void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ /***************************************************************************** -** Reference counted string storage +** Reference counted string/blob storage *****************************************************************************/ /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9e57354db8..c7ced86999 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4116,6 +4116,9 @@ struct sqlite3_str { ** ** 3. Make a (read-only) copy of a read-only RCStr string using ** sqlite3RCStrRef(). +** +** "String" is in the name, but an RCStr object can also be used to hold +** binary data. */ struct RCStr { u64 nRCRef; /* Number of references */ From 5bfa7e65d10aa3fc9ad8c1a8a139ea516f11e4b7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Dec 2023 13:28:13 +0000 Subject: [PATCH 130/191] Cache is working better, but does not preserve the hasJson5 flag. FossilOrigin-Name: a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/json.c | 32 ++++++++++++++++++++++++++------ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 3cac80e1a7..8aa6ceaf3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sattempt\sto\sget\sthe\sJSON\stext-to-binary\scache\sworking.\s\sAll\stest\scases\npass,\sbut\sthe\scache\sseems\snot\sto\shelp\smuch. -D 2023-12-01T12:57:12.578 +C Cache\sis\sworking\sbetter,\sbut\sdoes\snot\spreserve\sthe\shasJson5\sflag. +D 2023-12-01T13:28:13.897 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9a96faf916f572deb94b6cc7673ba6e299c48d153c18218e104a2c6ae37c2c8e +F src/json.c 32e3741e5ff9f8380ebf84dcd6d3c28ddcd7f6a2923b72a05165ad108ca0c278 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11ebb5f712cc7a515e2e0f2be8c1d71de20c97fe5b74c4f4d72c84fd21182d35 -R 18f6bb3dc7a73112aa5c773b9941447e +P 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c +R b145e990ca1c690b2dc7d50ed4bd4abc +T *branch * jsonb-cache +T *sym-jsonb-cache * +T -sym-jsonb * U drh -Z 9a005b4c0b238db6966807971445cc41 +Z 663fd17309f10179421e079add21165c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 298549f7a8..697f12705c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c \ No newline at end of file +a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 44fa5bf0b2..5f02a1c8ed 100644 --- a/src/json.c +++ b/src/json.c @@ -358,13 +358,15 @@ static void jsonCacheDeleteGeneric(void *p){ */ static int jsonCacheInsert( sqlite3_context *ctx, /* The SQL statement context holding the cache */ - char *zJson, /* The key. Must be an RCStr! */ + char *zJson, /* The key. */ u32 nJson, /* Number of bytes in zJson */ - char *aBlob, /* The value. Not an RCStr */ + int bJsonIsRCStr, /* True if zJson is already an RCStr */ + u8 *aBlob, /* The value. */ u32 nBlob /* Number of bytes in aBlob */ ){ JsonCache *p; - char *aRCBlob = 0; + char *aRCBlob; + char *zRCJson; p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ @@ -379,6 +381,17 @@ static int jsonCacheInsert( aRCBlob = sqlite3RCStrNew( nBlob ); if( aRCBlob==0 ) return SQLITE_NOMEM; memcpy(aRCBlob, aBlob, nBlob); + if( bJsonIsRCStr ){ + zRCJson = sqlite3RCStrRef(zJson); + }else{ + zRCJson = sqlite3RCStrNew( nJson ); + if( zRCJson==0 ){ + sqlite3RCStrUnref(aRCBlob); + return SQLITE_NOMEM; + } + memcpy(zRCJson, zJson, nJson); + zRCJson[nJson] = 0; + } if( p->nUsed >= JSON_CACHE_SIZE ){ sqlite3RCStrUnref(p->a[0].zJson); sqlite3RCStrUnref(p->a[0].aBlob); @@ -387,7 +400,7 @@ static int jsonCacheInsert( } p->a[p->nUsed].nJson = nJson; p->a[p->nUsed].nBlob = nBlob; - p->a[p->nUsed].zJson = sqlite3RCStrRef(zJson); + p->a[p->nUsed].zJson = zRCJson; p->a[p->nUsed].aBlob = aRCBlob; p->nUsed++; return SQLITE_OK; @@ -429,6 +442,7 @@ static u8 *jsonCacheSearch( JsonCacheLine tmp = p->a[i]; memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); p->a[p->nUsed-1] = tmp; + i = p->nUsed - 1; } *pnBlob = p->a[i].nBlob; return (u8*)p->a[i].aBlob; @@ -724,8 +738,8 @@ static void jsonReturnString( }else if( jsonForceRCStr(p) ){ sqlite3RCStrRef(p->zBuf); if( pParse ){ - int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, - (char*)pParse->aBlob, pParse->nBlob); + int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, 1, + pParse->aBlob, pParse->nBlob); if( rc==SQLITE_NOMEM ){ sqlite3RCStrUnref(p->zBuf); sqlite3_result_error_nomem(ctx); @@ -2746,6 +2760,12 @@ static JsonParse *jsonParseFuncArg( return 0; } } + if( ctx ){ + int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); + int rc = jsonCacheInsert(ctx, p->zJson, p->nJson, isRCStr, + p->aBlob, p->nBlob); + if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + } return p; json_pfa_malformed: From 063d0d4c3a453939237e64c8f9b3b6d96cad0a65 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Dec 2023 18:46:14 +0000 Subject: [PATCH 131/191] Fix up the JSON cache to work better. FossilOrigin-Name: 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 --- manifest | 15 ++-- manifest.uuid | 2 +- src/json.c | 201 +++++++++++++++++++++++++------------------------- 3 files changed, 109 insertions(+), 109 deletions(-) diff --git a/manifest b/manifest index 8aa6ceaf3b..9b30ebcd15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cache\sis\sworking\sbetter,\sbut\sdoes\snot\spreserve\sthe\shasJson5\sflag. -D 2023-12-01T13:28:13.897 +C Fix\sup\sthe\sJSON\scache\sto\swork\sbetter. +D 2023-12-01T18:46:14.485 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 32e3741e5ff9f8380ebf84dcd6d3c28ddcd7f6a2923b72a05165ad108ca0c278 +F src/json.c 3556e879386b0af1c542ed13ebb2f58c48d3b8b3ab13a96ab77ad2a9493cc715 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,11 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c -R b145e990ca1c690b2dc7d50ed4bd4abc -T *branch * jsonb-cache -T *sym-jsonb-cache * -T -sym-jsonb * +P a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a +R bed96169a434a08ba8168cff3e09a739 U drh -Z 663fd17309f10179421e079add21165c +Z 599839251e392336311bcdbf8af02666 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 697f12705c..c67da5411d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a12add7ab9f5aee5bb2ede0c4d22e599dd28f7a107dce72b2ea48ef92d233e8a \ No newline at end of file +1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 5f02a1c8ed..08e0f4fb63 100644 --- a/src/json.c +++ b/src/json.c @@ -201,7 +201,6 @@ static const char jsonIsOk[256] = { /* Objects */ typedef struct JsonCache JsonCache; -typedef struct JsonCacheLine JsonCacheLine; typedef struct JsonString JsonString; typedef struct JsonParse JsonParse; @@ -217,16 +216,10 @@ typedef struct JsonParse JsonParse; ** All content, both JSON text and the JSONB blobs, is stored as RCStr ** objects. */ -struct JsonCacheLine { - u32 nJson; /* Size of the JSON text, in bytes */ - u32 nBlob; /* Size of the corresponding JSONB, in bytes */ - char *zJson; /* RCStr holding the JSON text */ - char *aBlob; /* RCStr holding the corresponding JSONB */ -}; struct JsonCache { - sqlite3 *db; /* Database connection */ - int nUsed; /* Number of active entries in the cache */ - JsonCacheLine a[JSON_CACHE_SIZE]; /* One line for each cache entry */ + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */ }; /* An instance of this object represents a JSON string @@ -285,13 +278,14 @@ struct JsonParse { u32 nBlob; /* Bytes of aBlob[] actually used */ u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ char *zJson; /* Json text used for parsing */ + int nJson; /* Length of the zJson string in bytes */ u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ - u8 bBlobIsRCStr; /* True if aBlob is an RCStr */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 bReadOnly; /* Do not modify. */ u32 nJPRef; /* Number of references to this object */ - int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ /* Search and edit information. See jsonLookupBlobStep() */ u8 eEdit; /* Edit operation to apply */ @@ -330,6 +324,9 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +static void jsonParseFree(JsonParse*); + + /************************************************************************** ** Utility routines for dealing with JsonCache objects **************************************************************************/ @@ -340,8 +337,7 @@ static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonCacheDelete(JsonCache *p){ int i; for(i=0; inUsed; i++){ - sqlite3RCStrUnref(p->a[i].zJson); - sqlite3RCStrUnref(p->a[i].aBlob); + jsonParseFree(p->a[i]); } sqlite3DbFree(p->db, p); } @@ -358,16 +354,12 @@ static void jsonCacheDeleteGeneric(void *p){ */ static int jsonCacheInsert( sqlite3_context *ctx, /* The SQL statement context holding the cache */ - char *zJson, /* The key. */ - u32 nJson, /* Number of bytes in zJson */ - int bJsonIsRCStr, /* True if zJson is already an RCStr */ - u8 *aBlob, /* The value. */ - u32 nBlob /* Number of bytes in aBlob */ + JsonParse *pParse /* The parse object to be added to the cache */ ){ JsonCache *p; - char *aRCBlob; - char *zRCJson; + assert( pParse->zJson!=0 ); + assert( pParse->bJsonIsRCStr ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ sqlite3 *db = sqlite3_context_db_handle(ctx); @@ -378,76 +370,61 @@ static int jsonCacheInsert( p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ) return SQLITE_NOMEM; } - aRCBlob = sqlite3RCStrNew( nBlob ); - if( aRCBlob==0 ) return SQLITE_NOMEM; - memcpy(aRCBlob, aBlob, nBlob); - if( bJsonIsRCStr ){ - zRCJson = sqlite3RCStrRef(zJson); - }else{ - zRCJson = sqlite3RCStrNew( nJson ); - if( zRCJson==0 ){ - sqlite3RCStrUnref(aRCBlob); - return SQLITE_NOMEM; - } - memcpy(zRCJson, zJson, nJson); - zRCJson[nJson] = 0; - } if( p->nUsed >= JSON_CACHE_SIZE ){ - sqlite3RCStrUnref(p->a[0].zJson); - sqlite3RCStrUnref(p->a[0].aBlob); + jsonParseFree(p->a[0]); memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); p->nUsed = JSON_CACHE_SIZE-1; } - p->a[p->nUsed].nJson = nJson; - p->a[p->nUsed].nBlob = nBlob; - p->a[p->nUsed].zJson = zRCJson; - p->a[p->nUsed].aBlob = aRCBlob; + pParse->eEdit = 0; + pParse->nJPRef++; + pParse->bReadOnly = 1; + p->a[p->nUsed] = pParse; p->nUsed++; return SQLITE_OK; } /* -** Search for a cached translation of zJson (size: nJson bytes) into -** JSONB. Return it if found. -** -** The returned value is an RCStr object if it is not NULL. -** The caller is responsible for incrementing the reference count. +** Search for a cached translation the json text supplied by pArg. Return +** the JsonParse object if found. */ -static u8 *jsonCacheSearch( +static JsonParse *jsonCacheSearch( sqlite3_context *ctx, /* The SQL statement context holding the cache */ - char *zJson, /* The key. Might or might not be an RCStr */ - u32 nJson, /* Size of the key in bytes */ - u32 *pnBlob /* OUT: Size of the result in bytes */ + sqlite3_value *pArg /* Function argument containing SQL text */ ){ JsonCache *p; int i; + const char *zJson; + int nJson; + + if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){ + return 0; + } + zJson = (const char*)sqlite3_value_text(pArg); + if( zJson==0 ) return 0; + nJson = sqlite3_value_bytes(pArg); - assert( pnBlob!=0 ); p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); if( p==0 ){ - *pnBlob = 0; return 0; } for(i=0; inUsed; i++){ - if( p->a[i].zJson==zJson ) break; + if( p->a[i]->zJson==zJson ) break; } if( i>=p->nUsed ){ for(i=0; inUsed; i++){ - if( p->a[i].nJson!=nJson ) continue; - if( memcmp(p->a[i].zJson, zJson, nJson)==0 ) break; + if( p->a[i]->nJson!=nJson ) continue; + if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break; } } if( inUsed ){ if( inUsed-1 ){ - JsonCacheLine tmp = p->a[i]; + JsonParse *tmp = p->a[i]; memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); p->a[p->nUsed-1] = tmp; i = p->nUsed - 1; } - *pnBlob = p->a[i].nBlob; - return (u8*)p->a[i].aBlob; + return p->a[i]; }else{ - *pnBlob = 0; return 0; } } @@ -717,9 +694,9 @@ static void jsonAppendSqlValue( ** ** The JsonString is reset. ** -** If pParse and ctx are both non-NULL and if pParse->aBlob is valid -** then an attempt is made to cache the translation from JSON text into -** the blob. +** If pParse and ctx are both non-NULL, then the SQL string in p is +** loaded into the zJson field of the pParse object as a RCStr and the +** pParse is added to the cache. */ static void jsonReturnString( JsonString *p, /* String to return */ @@ -736,18 +713,19 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ - sqlite3RCStrRef(p->zBuf); - if( pParse ){ - int rc = jsonCacheInsert(ctx, p->zBuf, p->nUsed, 1, - pParse->aBlob, pParse->nBlob); + if( pParse && pParse->bJsonIsRCStr==0 ){ + int rc; + pParse->zJson = sqlite3RCStrRef(p->zBuf); + pParse->nJson = p->nUsed; + pParse->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, pParse); if( rc==SQLITE_NOMEM ){ - sqlite3RCStrUnref(p->zBuf); sqlite3_result_error_nomem(ctx); jsonStringReset(p); return; } } - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); }else{ @@ -771,12 +749,11 @@ static void jsonReturnString( */ static void jsonParseReset(JsonParse *pParse){ assert( pParse->nJPRef<=1 ); - if( pParse->bBlobIsRCStr ){ - assert( pParse->nBlobAlloc==0 ); - sqlite3RCStrUnref((char*)pParse->aBlob); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->bBlobIsRCStr = 0; + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->nJson = 0; + pParse->bJsonIsRCStr = 0; } if( pParse->nBlobAlloc ){ sqlite3_free(pParse->aBlob); @@ -1048,6 +1025,7 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ u8 *aOld; u32 nSize; + assert( !pParse->bReadOnly ); if( pParse->nBlobAlloc>0 ) return 1; aOld = pParse->aBlob; nSize = pParse->nBlob + nExtra; @@ -2671,6 +2649,11 @@ jsonInsertIntoBlob_patherror: return; } +/* +** Make a copy of a JsonParse object. The copy will be editable. +*/ + + /* ** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, ** from the SQL function argument pArg. Return a pointer to the new @@ -2692,17 +2675,40 @@ static JsonParse *jsonParseFuncArg( sqlite3_value *pArg, u32 flgs ){ - int eType; /* Datatype of pArg */ - JsonParse *p = 0; /* Value to be returned */ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + JsonParse *pFromCache = 0; /* Value taken from cache */ assert( ctx!=0 ); eType = sqlite3_value_type(pArg); if( eType==SQLITE_NULL ){ return 0; } + pFromCache = jsonCacheSearch(ctx, pArg); + if( pFromCache ){ + pFromCache->nJPRef++; + if( (flgs & JSON_EDITABLE)==0 ){ + return pFromCache; + } + } +rebuild_from_cache: p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) goto json_pfa_oom; memset(p, 0, sizeof(*p)); + p->nJPRef = 1; + if( pFromCache!=0 ){ + u32 nBlob = pFromCache->nBlob; + p->aBlob = sqlite3_malloc64( nBlob ); + if( p->aBlob==0 ) goto json_pfa_oom; + memcpy(p->aBlob, pFromCache->aBlob, nBlob); + p->nBlobAlloc = p->nBlob = nBlob; + p->hasNonstd = pFromCache->hasNonstd; + jsonParseFree(pFromCache); + return p; + }else{ + jsonParseFree(pFromCache); + pFromCache = 0; + } if( eType==SQLITE_BLOB ){ u32 n, sz = 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); @@ -2729,29 +2735,11 @@ static JsonParse *jsonParseFuncArg( } return p; } - /* TODO: Check in the cache */ p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; if( p->zJson==0 ) goto json_pfa_oom; - - p->aBlob = jsonCacheSearch(ctx, p->zJson, p->nJson, &p->nBlob); - if( p->aBlob ){ - if( flgs & JSON_EDITABLE ){ - u8 *pNew = sqlite3_malloc64( p->nBlob ); - if( pNew==0 ) goto json_pfa_oom; - memcpy(pNew, p->aBlob, p->nBlob); - p->aBlob = pNew; - p->nBlobAlloc = p->nBlob; - }else{ - sqlite3RCStrRef((char*)p->aBlob); - p->bBlobIsRCStr = 1; - } - return p; - } - - if( flgs & JSON_KEEPERROR ) ctx = 0; - if( jsonConvertTextToBlob(p, ctx) ){ + if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; return p; @@ -2759,12 +2747,26 @@ static JsonParse *jsonParseFuncArg( jsonParseFree(p); return 0; } - } - if( ctx ){ + }else{ int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); - int rc = jsonCacheInsert(ctx, p->zJson, p->nJson, isRCStr, - p->aBlob, p->nBlob); + int rc; + if( !isRCStr ){ + char *zNew = sqlite3RCStrNew( p->nJson ); + if( zNew==0 ) goto json_pfa_oom; + memcpy(zNew, p->zJson, p->nJson); + p->zJson = zNew; + p->zJson[p->nJson] = 0; + }else{ + sqlite3RCStrRef(p->zJson); + } + p->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, p); if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + if( flgs & JSON_EDITABLE ){ + pFromCache = p; + p = 0; + goto rebuild_from_cache; + } } return p; @@ -2779,6 +2781,7 @@ json_pfa_malformed: } json_pfa_oom: + jsonParseFree(pFromCache); jsonParseFree(p); sqlite3_result_error_nomem(ctx); return 0; From b5effc0605abcf3f67a4c62ef42d0ded03a214fd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Dec 2023 20:09:59 +0000 Subject: [PATCH 132/191] Different approach to querying a tokendata=1 table. Saves cpu and memory. FossilOrigin-Name: c523f40895866e6fc979a26483dbea8206126b4bbdf4b73b77263c09e13c855e --- ext/fts5/fts5Int.h | 19 +- ext/fts5/fts5_expr.c | 6 +- ext/fts5/fts5_index.c | 553 ++++++++++++++++++++++++++--- ext/fts5/fts5_main.c | 7 +- ext/fts5/test/fts5origintext.test | 10 +- ext/fts5/test/fts5origintext2.test | 1 + ext/fts5/test/fts5origintext3.test | 9 + manifest | 24 +- manifest.uuid | 2 +- 9 files changed, 541 insertions(+), 90 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 9d2622448d..cee9528858 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -385,17 +385,18 @@ struct Fts5IndexIter { /* ** Values used as part of the flags argument passed to IndexQuery(). */ -#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ -#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ -#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ -#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ -#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 -#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 -#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 /* ** Create/destroy an Fts5Index object. @@ -467,7 +468,7 @@ int sqlite3Fts5StructureTest(Fts5Index*, void*); /* ** Used by xInstToken() and xPhraseToken(). */ -int sqlite3Fts5IterToken(Fts5IndexIter*, int, int, const char**, int*); +int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); /* ** Insert or remove data to or from the index. Each time a document is @@ -548,7 +549,7 @@ int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); /* Used to populate hash tables for xInstToken in detail=none/column mode. */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); int sqlite3Fts5IndexIterWriteTokendata( - Fts5IndexIter*, const char*, int, int iCol, int iOff + Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff ); /* diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 6589d1b3e6..89e7cbf364 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3003,6 +3003,7 @@ static int fts5ExprPopulatePoslistsCb( Fts5Expr *pExpr = p->pExpr; int i; int nQuery = nToken; + i64 iRowid = pExpr->pRoot->iRowid; UNUSED_PARAM2(iUnused1, iUnused2); @@ -3025,7 +3026,7 @@ static int fts5ExprPopulatePoslistsCb( int iCol = p->iOff>>32; int iTokOff = p->iOff & 0x7FFFFFFF; rc = sqlite3Fts5IndexIterWriteTokendata( - pT->pIter, pToken, nToken, iCol, iTokOff + pT->pIter, pToken, nToken, iRowid, iCol, iTokOff ); } if( rc ) return rc; @@ -3210,6 +3211,7 @@ int sqlite3Fts5ExprInstToken( ){ Fts5ExprPhrase *pPhrase = 0; Fts5IndexIter *pIter = 0; + i64 iRowid = pExpr->pRoot->iRowid; if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ return SQLITE_RANGE; @@ -3220,6 +3222,6 @@ int sqlite3Fts5ExprInstToken( } pIter = pPhrase->aTerm[iToken].pIter; - return sqlite3Fts5IterToken(pIter, iCol, iOff+iToken, ppOut, pnOut); + return sqlite3Fts5IterToken(pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut); } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 4b7c8d3335..5f39a1e764 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -327,6 +327,8 @@ typedef struct Fts5TokenMapEntry Fts5TokenMapEntry; typedef struct Fts5TokenMapToken Fts5TokenMapToken; typedef struct Fts5TokenMap Fts5TokenMap; +typedef struct Fts5TokenDataIter Fts5TokenDataIter; + struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ int nn; /* Size of record in bytes */ @@ -594,9 +596,16 @@ struct Fts5SegIter { ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. +** +** pColset: +** If not NULL, points to an object containing a set of column indices. +** Only matches that occur in one of these columns will be returned. +** The Fts5Iter does not own the Fts5Colset object, and so it is not +** freed when the iterator is closed - it is owned by the upper layer. */ struct Fts5Iter { Fts5IndexIter base; /* Base class containing output vars */ + Fts5TokenDataIter *pTokenDataIter; Fts5Index *pIndex; /* Index that owns this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ @@ -3780,6 +3789,28 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ } } +static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){ + int iIter; + for(iIter=pIter->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pIter, iEq, iIter); + } + } + fts5MultiIterSetEof(pIter); + fts5AssertMultiIterSetup(p, pIter); + + if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter)) + || fts5MultiIterIsDeleted(pIter) + ){ + fts5MultiIterNext(p, pIter, 0, 0); + }else if( pIter->base.bEof==0 ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + pIter->xSetOutputs(pIter, pSeg); + } +} /* ** Allocate a new Fts5Iter object. @@ -3866,26 +3897,7 @@ static void fts5MultiIterNew( ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); - } - } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew)) - || fts5MultiIterIsDeleted(pNew) - ){ - fts5MultiIterNext(p, pNew, 0, 0); - }else if( pNew->base.bEof==0 ){ - Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; - pNew->xSetOutputs(pNew, pSeg); - } - + fts5MultiIterFinishSetup(p, pNew); }else{ fts5MultiIterFree(pNew); *ppOut = 0; @@ -6718,6 +6730,431 @@ int sqlite3Fts5IndexWrite( return rc; } +/* +** pToken points to a buffer of size nToken bytes containing a search +** term, including the index number at the start, used on a tokendata=1 +** table. This function returns true if the term in buffer pBuf matches +** token pToken/nToken. +*/ +static int fts5IsTokendataPrefix( + Fts5Buffer *pBuf, + const u8 *pToken, + int nToken +){ + return ( + pBuf->n>=nToken + && 0==memcmp(pBuf->p, pToken, nToken) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + ); +} + +/* +** Ensure the segment-iterator passed as the only argument points to EOF. +*/ +static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; +} + +typedef struct Fts5TokenDataMap Fts5TokenDataMap; +struct Fts5TokenDataMap { + i64 iRowid; + i64 iPos; + int iIter; +}; + +struct Fts5TokenDataIter { + int nIter; + int nIterAlloc; + + int nMap; + int nMapAlloc; + Fts5TokenDataMap *aMap; + + Fts5PoslistReader *aPoslistReader; + Fts5Iter *apIter[1]; +}; + +static Fts5TokenDataIter *fts5AppendTokendataIter( + Fts5Index *p, + Fts5TokenDataIter *pIn, + Fts5Iter *pAppend +){ + Fts5TokenDataIter *pRet = pIn; + + if( p->rc==SQLITE_OK ){ + if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = nAlloc * sizeof(Fts5Iter*); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + if( pIn==0 ) memset(pNew, 0, nByte); + pRet = pNew; + pNew->nIterAlloc = nAlloc; + } + } + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pAppend); + }else{ + pRet->apIter[pRet->nIter++] = pAppend; + } + + return pRet; +} + +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + +static int fts5TokendataIterToken( + Fts5Iter *pIter, + i64 iRowid, + int iCol, int iOff, + const char **ppOut, int *pnOut +){ + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5TokenDataMap *aMap = pT->aMap; + i64 iPos = (((i64)iCol)<<32) + iOff; + + int i1 = 0; + int i2 = pT->nMap; + int iTest = 0; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + } + + return SQLITE_OK; +} + +static void fts5TokendataIterAppendMap( + Fts5Index *p, + Fts5TokenDataIter *pT, + int iIter, + i64 iRowid, + i64 iPos +){ + if( p->rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nByte = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->nMap++; + } +} + +static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ + int ii; + int nHit = 0; + i64 iRowid = SMALLEST_INT64; + int iMin = 0; + + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + pIter->base.nData = 0; + pIter->base.pData = 0; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 ){ + if( nHit==0 || p->base.iRowidbase.iRowid; + nHit = 1; + pIter->base.pData = p->base.pData; + pIter->base.nData = p->base.nData; + iMin = ii; + }else if( p->base.iRowid==iRowid ){ + nHit++; + } + } + } + + if( nHit==0 ){ + pIter->base.bEof = 1; + }else{ + int eDetail = pIter->pIndex->pConfig->eDetail; + pIter->base.bEof = 0; + pIter->base.iRowid = iRowid; + + if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1); + }else + if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ + int nReader = 0; + int nByte = 0; + i64 iPrev = 0; + + /* Allocate array of iterators if they are not already allocated. */ + if( pT->aPoslistReader==0 ){ + pT->aPoslistReader = sqlite3Fts5MallocZero( + &pIter->pIndex->rc, sizeof(Fts5PoslistReader) * pT->nIter + ); + if( pT->aPoslistReader==0 ) return; + } + + /* Populate an iterator for each poslist that will be merged */ + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( iRowid==p->base.iRowid ){ + sqlite3Fts5PoslistReaderInit( + p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] + ); + nByte += p->base.nData; + } + } + + /* Ensure the output buffer is large enough */ + if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10) ){ + return; + } + + /* Ensure the token-mapping is large enough */ + if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + pT->aMap, nNew*sizeof(Fts5TokenDataMap) + ); + if( aNew==0 ){ + pIter->pIndex->rc = SQLITE_NOMEM; + return; + } + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pIter->poslist.n = 0; + + while( 1 ){ + i64 iMinPos = LARGEST_INT64; + + /* Find smallest position */ + iMin = 0; + for(ii=0; iiaPoslistReader[ii]; + if( pReader->bEof==0 ){ + if( pReader->iPosiPos; + iMin = ii; + } + } + } + + /* If all readers were at EOF, break out of the loop. */ + if( iMinPos==LARGEST_INT64 ) break; + + sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos); + sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]); + + if( eDetail==FTS5_DETAIL_FULL ){ + pT->aMap[pT->nMap].iPos = iMinPos; + pT->aMap[pT->nMap].iIter = iMin; + pT->aMap[pT->nMap].iRowid = iRowid; + pT->nMap++; + } + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + } +} + +static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ + int ii; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 + && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidpIndex, p, bFrom, iFrom); + while( bFrom && p->base.bEof==0 + && p->base.iRowidpIndex->rc==SQLITE_OK + ){ + fts5MultiIterNext(p->pIndex, p, 0, 0); + } + } + } + + fts5IterSetOutputsTokendata(pIter); +} + +static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ + if( pIter && pIter->aSeg[0].pLeaf==0 ){ + fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); + } +} + +static Fts5Iter *fts5SetupTokendataIter( + Fts5Index *p, /* FTS index to query */ + int bDesc, /* True for "ORDER BY rowid DESC" */ + const u8 *pToken, /* Buffer containing query term */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset /* Colset to filter on */ +){ + Fts5Iter *pRet = 0; + Fts5TokenDataIter *pSet = 0; + Fts5Structure *pStruct = 0; + const int flags = FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_SCAN; + + assert( bDesc==0 ); + + Fts5Buffer bSeek = {0, 0, 0}; + Fts5Buffer *pSmall = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + + while( 1 ){ + Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; + Fts5Iter *pNew = 0; + Fts5SegIter *pNewIter = 0; + Fts5SegIter *pPrevIter = 0; + + int iLvl, iSeg, ii; + + pNew = fts5MultiIterAlloc(p, pStruct->nSegment); + if( pNew==0 ) break; + + if( pSmall ){ + fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); + fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); + }else{ + fts5BufferSet(&p->rc, &bSeek, nToken, pToken); + } + + pNewIter = &pNew->aSeg[0]; + pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + + pNewIter++; + if( pPrevIter ){ + if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ + fts5SegIterSetEOF(pPrevIter); + } + pPrevIter++; + } + } + } + fts5TokendataSetTermIfEof(pPrev, pSmall); + + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pColset = pColset; + fts5IterSetOutputCb(&p->rc, pNew); + + /* Loop through all segments in the new iterator. Find the smallest + ** term that any segment-iterator points to. Iterator pNew will be + ** used for this term. Also, set any iterator that points to a term that + ** does not match pToken/nToken to point to EOF */ + pSmall = 0; + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pII = &pNew->aSeg[ii]; + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + fts5SegIterSetEOF(pII); + } + if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ + pSmall = &pII->term; + } + } + + /* If pSmall is still NULL at this point, then the new iterator does + ** not point to any terms that match the query. So delete it and break + ** out of the loop - all required iterators have been collected. */ + if( pSmall==0 ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + /* Append this iterator to the set and continue. */ + pSet = fts5AppendTokendataIter(p, pSet, pNew); + } + + if( p->rc==SQLITE_OK && pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Iter *pIter = pSet->apIter[ii]; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM; + } + fts5MultiIterFinishSetup(p, pIter); + } + } + + if( p->rc==SQLITE_OK ){ + pRet = fts5MultiIterAlloc(p, 0); + } + if( pRet ){ + pRet->pTokenDataIter = pSet; + if( pSet ){ + fts5IterSetOutputsTokendata(pRet); + }else{ + pRet->base.bEof = 1; + } + }else{ + fts5TokendataIterDelete(pSet); + } + + fts5StructureRelease(pStruct); + fts5BufferFree(&bSeek); + return pRet; +} + + /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -6739,6 +7176,7 @@ int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ + int bTokendata = (flags&FTS5INDEX_QUERY_NOTOKENDATA)?0:pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this @@ -6766,7 +7204,7 @@ int sqlite3Fts5IndexQuery( } } - if( iIdx<=pConfig->nPrefix && (pConfig->bTokendata==0 || iIdx!=0) ){ + if( iIdx<=pConfig->nPrefix && (bTokendata==0 || iIdx!=0) ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); @@ -6776,6 +7214,10 @@ int sqlite3Fts5IndexQuery( ); fts5StructureRelease(pStruct); } + }else if( bTokendata && iIdx==0 ){ + int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; + buf.p[0] = '0'; + pRet = fts5SetupTokendataIter(p, bDesc, buf.p, nToken+1, pColset); }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; @@ -6816,7 +7258,11 @@ int sqlite3Fts5IndexQuery( int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 0, 0); + }else{ + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + } return fts5IndexReturn(pIter->pIndex); } @@ -6849,7 +7295,11 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 1, iMatch); + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + } return fts5IndexReturn(pIter->pIndex); } @@ -6869,12 +7319,18 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ */ int sqlite3Fts5IterToken( Fts5IndexIter *pIndexIter, + i64 iRowid, int iCol, int iOff, const char **ppOut, int *pnOut ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenMap *pMap = pIter->pTokenMap; + + if( pIter->pTokenDataIter ){ + return fts5TokendataIterToken(pIter, iRowid, iCol, iOff, ppOut, pnOut); + } + if( pMap ){ if( pMap->bHashed==0 ){ Fts5Index *p = pIter->pIndex; @@ -6899,53 +7355,29 @@ int sqlite3Fts5IterToken( void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL ); - if( pIter->pTokenMap ){ - pIter->pTokenMap->nEntry = 0; + if( pIter->pTokenDataIter ){ + pIter->pTokenDataIter->nMap = 0; } } int sqlite3Fts5IndexIterWriteTokendata( Fts5IndexIter *pIndexIter, const char *pToken, int nToken, - int iCol, int iOff + i64 iRowid, int iCol, int iOff ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; Fts5Index *p = pIter->pIndex; + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); - if( pIter->pTokenMap==0 ){ - pIter->pTokenMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); - } - if( p->rc==SQLITE_OK ){ - Fts5TokenMap *pMap = pIter->pTokenMap; + if( pT ){ int ii; - for(ii=0; iinToken; ii++){ - if( nToken==pMap->aToken[ii].nTerm - && 0==memcmp(pMap->aToken[ii].pTerm, pToken, nToken) - ){ - break; - } + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; } - if( ii==pMap->nToken ){ - fts5TokenMapTerm(p, pMap, (const u8*)pToken, nToken); - } - if( pMap->nEntry>=pMap->nEntryAlloc ){ - int nNew = pMap->nEntryAlloc ? pMap->nEntryAlloc*2 : 32; - Fts5TokenMapEntry *aNew = (Fts5TokenMapEntry*)sqlite3_realloc( - pMap->aEntry, nNew * sizeof(Fts5TokenMapEntry) - ); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - pMap->aEntry = aNew; - pMap->nEntryAlloc = nNew; - } - } - if( p->rc==SQLITE_OK ){ - Fts5TokenMapEntry *pEntry = &pMap->aEntry[pMap->nEntry++]; - pEntry->iRowid = pIndexIter->iRowid; - pEntry->iCol = iCol; - pEntry->iOff = iOff; - pEntry->iTok = ii+1; + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); } } return fts5IndexReturn(p); @@ -6958,6 +7390,7 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); fts5MultiIterFree(pIter); sqlite3Fts5IndexCloseReader(pIndex); } @@ -7465,7 +7898,9 @@ static int fts5QueryCksum( int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIter = 0; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + int rc = sqlite3Fts5IndexQuery( + p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA), 0, &pIter + ); while( rc==SQLITE_OK && ALWAYS(pIter!=0) && 0==sqlite3Fts5IterEof(pIter) ){ i64 rowid = pIter->iRowid; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 9f19b24b88..34050474f8 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -656,12 +656,15 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } idxStr[iIdxStr] = '\0'; - /* Set idxFlags flags for the ORDER BY clause */ + /* Set idxFlags flags for the ORDER BY clause + ** + ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC". + */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; if( iSort==(pConfig->nCol+1) && bSeenMatch ){ idxFlags |= FTS5_BI_ORDER_RANK; - }else if( iSort==-1 ){ + }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; } if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 845e8145db..8273b3ca4d 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -141,7 +141,7 @@ do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1 do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2 do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3 -do_execsql_test 3.0 { +do_execsql_test 3.2 { CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize="origintext unicode61", tokendata=1, @@ -160,11 +160,11 @@ do_execsql_test 3.0 { #db func b b #execsql_pp { SELECT b(term) FROM vocab } -do_execsql_test 3.1.1 { SELECT rowid FROM ft2('hello') } {1 2 3} -do_execsql_test 3.1.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} -do_execsql_test 3.1.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} +do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3} +do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} +do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} -do_execsql_test 3.1.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} +do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} #------------------------------------------------------------------------- # diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test index 26f9864098..948db1c519 100644 --- a/ext/fts5/test/fts5origintext2.test +++ b/ext/fts5/test/fts5origintext2.test @@ -102,6 +102,7 @@ breakpoint do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3} do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} +do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3} finish_test diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index 57b5984f4d..2b1e5c6387 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -46,6 +46,7 @@ foreach_detail_mode $testprefix { SELECT fts5_test_poslist(ft) FROM ft('hello'); } {{0.0.0 0.0.2 0.0.4}} +breakpoint do_execsql_test 1.3 { SELECT insttoken(ft, 0, 0), @@ -54,6 +55,14 @@ foreach_detail_mode $testprefix { FROM ft('hello'); } {hello.Hello hello.HELLO hello} + do_execsql_test 1.4 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hello') ORDER BY rank; + } {hello.Hello hello.HELLO hello} + } finish_test diff --git a/manifest b/manifest index 13a7da6a20..04745fb5d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssigned\sinteger\soverflow\sin\sfts5. -D 2023-11-29T16:22:39.960 +C Different\sapproach\sto\squerying\sa\stokendata=1\stable.\sSaves\scpu\sand\smemory. +D 2023-12-01T20:09:59.031 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -90,14 +90,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 5e5630fc81e212f658afaa5b2650dac939d2729d0723aef1eeaff908f1725648 -F ext/fts5/fts5Int.h 782151060d176be22861f57bf38e087a82cfb0dfc4b2fa6f9ccbc2641b6d01e3 +F ext/fts5/fts5Int.h 2dc73393460e5c5cab67adc7e32e1387cc225b57e05f629d490e65cddea1a8c5 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e +F ext/fts5/fts5_expr.c aac8026aedf56c9a6e32b31c89f9dd7e5548378457085093307d06be14f1a176 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c bafdef8be40a20bb86a131af4f09b56919f416fa13d1a86af1bf92bad9a2870d -F ext/fts5/fts5_main.c 55b53085dbd1693b5735463198a8d124dfbc27f08311c839637b44b8254ef7cb +F ext/fts5/fts5_index.c b6012920df8963245226bb536db7ddea62dfd7a860d6112887b175f7aaf55b82 +F ext/fts5/fts5_main.c 20596de592af135f68b9be875f0a28715f6562bbdedd215e1c89eac1b42e97f9 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -190,9 +190,9 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 7caef7634889bab8b44d145141c0d9325299398fb89b116bccd6262fde5659db -F ext/fts5/test/fts5origintext2.test 26482f4af1f2785cb01d06af9aae202289b6e8cf7b708d18aea305b459c2f302 -F ext/fts5/test/fts5origintext3.test 87a212b8235794348c56cb70f21e122d182a5af688c56057b90b7c151d0aa347 +F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0703aff5981625b527e62 +F ext/fts5/test/fts5origintext2.test 3259b331073fec918e02fd4d14d50586f9a3531da047a2a8f4624983eb654229 +F ext/fts5/test/fts5origintext3.test cb0f5835f8dff5954ee20570b68ee520cf04a08f6f9ca967b9d01d27e532da37 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2146,8 +2146,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 554fc13f2ca5f2ebd9ad0206034c25b556ff40db3106051c5e539f2e142e88ea -R 5c7c1632cd09d2b131adef895eab90a8 +P 60e46c7ec68fd8caaed960ca06d98fb06855b2d0bb860dd2fb7b5e89a5e9c7b4 +R e2c2c11ec4bab5e354a94b25a41ae3cd U dan -Z 009b023eba91b6acc35beb7d051a677a +Z f7d96142774ad25f02aad90982aa0fc8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df2c2789ad..e669309199 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60e46c7ec68fd8caaed960ca06d98fb06855b2d0bb860dd2fb7b5e89a5e9c7b4 \ No newline at end of file +c523f40895866e6fc979a26483dbea8206126b4bbdf4b73b77263c09e13c855e \ No newline at end of file From f4c2962558e19b1b86734919a74c50856121d3d6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Dec 2023 20:37:11 +0000 Subject: [PATCH 133/191] Remove old code for tokendata=1 queries. FossilOrigin-Name: b0a489e8e1bf0290c2117ab32d78b1cc7d67bcb226b55ec044c8367ebde3815b --- ext/fts5/fts5_index.c | 423 +++++---------------------------- ext/fts5/test/fts5simple2.test | 4 +- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 67 insertions(+), 376 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 5f39a1e764..05f4c5e6aa 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -323,10 +323,6 @@ typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; -typedef struct Fts5TokenMapEntry Fts5TokenMapEntry; -typedef struct Fts5TokenMapToken Fts5TokenMapToken; -typedef struct Fts5TokenMap Fts5TokenMap; - typedef struct Fts5TokenDataIter Fts5TokenDataIter; struct Fts5Data { @@ -370,7 +366,6 @@ struct Fts5Index { sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; - sqlite3_stmt *pIdxProbe; int nRead; /* Total number of blocks read */ sqlite3_stmt *pDeleteFromIdx; @@ -618,37 +613,11 @@ struct Fts5Iter { int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ - Fts5TokenMap *pTokenMap; i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; -struct Fts5TokenMapEntry { - i64 iRowid; - u16 iCol; - int iOff; - int iTok; /* Offset into aToken[] + 1 */ -}; - -struct Fts5TokenMapToken { - u8 *pTerm; - int nTerm; -}; - -struct Fts5TokenMap { - int bHashed; /* True once hashed */ - - int nEntryAlloc; - int nEntry; - Fts5TokenMapEntry *aEntry; - - int nTokenAlloc; - int nToken; - Fts5TokenMapToken *aToken; -}; - - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -2641,18 +2610,6 @@ static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ return p->pIdxSelect; } -static sqlite3_stmt *fts5IdxProbeStmt(Fts5Index *p){ - if( p->pIdxProbe==0 ){ - Fts5Config *pConfig = p->pConfig; - fts5IndexPrepareStmt(p, &p->pIdxProbe, sqlite3_mprintf( - "SELECT 1 FROM '%q'.'%q_idx' WHERE " - "segid=? AND term>? AND termzDb, pConfig->zName - )); - } - return p->pIdxProbe; -} - /* ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. @@ -3073,21 +3030,6 @@ static void fts5SegIterNextFrom( }while( p->rc==SQLITE_OK ); } -/* -** Free the Fts5TokenMap object passed as the only argument. -*/ -static void fts5TokenMapFree(Fts5TokenMap *pMap){ - if( pMap ){ - int ii; - for(ii=0; iinToken; ii++){ - sqlite3_free(pMap->aToken[ii].pTerm); - } - sqlite3_free(pMap->aToken); - sqlite3_free(pMap->aEntry); - sqlite3_free(pMap); - } -} - /* ** Free the iterator object passed as the second argument. */ @@ -3098,7 +3040,6 @@ static void fts5MultiIterFree(Fts5Iter *pIter){ fts5SegIterClear(&pIter->aSeg[i]); } fts5BufferFree(&pIter->poslist); - fts5TokenMapFree(pIter->pTokenMap); sqlite3_free(pIter); } } @@ -3915,7 +3856,6 @@ fts5MultiIterNew_post_check: static void fts5MultiIterNew2( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Data *pData, /* Doclist to iterate through */ - Fts5TokenMap *pMap, /* Token-map, if any */ int bDesc, /* True for descending rowid order */ Fts5Iter **ppOut /* New object */ ){ @@ -3923,8 +3863,6 @@ static void fts5MultiIterNew2( pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - pNew->pTokenMap = pMap; - pMap = 0; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -3947,7 +3885,6 @@ static void fts5MultiIterNew2( *ppOut = pNew; } - fts5TokenMapFree(pMap); fts5DataRelease(pData); } @@ -6141,210 +6078,9 @@ static void fts5MergePrefixLists( *p1 = out; } -static u8 *fts5IdxBufferDup(Fts5Index *p, const u8 *pDup, int nDup){ - u8 *pRet = fts5IdxMalloc(p, nDup+1); - if( pRet ){ - memcpy(pRet, pDup, nDup); - } - return pRet; -} - -static void fts5TokenMapTerm( - Fts5Index *p, - Fts5TokenMap *pMap, - const u8 *pTerm, - int nTerm -){ - if( p->rc==SQLITE_OK ){ - Fts5TokenMapToken *pToken = 0; - if( pMap->nToken==pMap->nTokenAlloc ){ - i64 nNew = (pMap->nTokenAlloc ? pMap->nTokenAlloc * 2 : 32); - Fts5TokenMapToken *aNew = sqlite3_realloc64( - pMap->aToken, nNew*sizeof(Fts5TokenMapToken) - ); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - pMap->nTokenAlloc = nNew; - pMap->aToken = aNew; - } - pToken = &pMap->aToken[pMap->nToken++]; - pToken->nTerm = nTerm; - pToken->pTerm = fts5IdxBufferDup(p, pTerm, nTerm); - } -} - - -static void fts5TokenMapPoslist( - Fts5Index *p, - Fts5TokenMap *pMap, - Fts5Iter *p1 -){ - if( p->rc==SQLITE_OK ){ - const u8 *a = p1->base.pData; - i64 iPos = 0; - int iOff = 0; - - while( 0==sqlite3Fts5PoslistNext64(a, p1->base.nData, &iOff, &iPos) ){ - Fts5TokenMapEntry *pEntry = 0; - int iCol = FTS5_POS2COLUMN(iPos); - int iTokOff = FTS5_POS2OFFSET(iPos); - - if( pMap->nEntry==pMap->nEntryAlloc ){ - i64 nNew = (pMap->nEntryAlloc ? pMap->nEntryAlloc * 2 : 32); - Fts5TokenMapEntry *aNew = sqlite3_realloc64( - pMap->aEntry, nNew*sizeof(Fts5TokenMapEntry) - ); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - pMap->nEntryAlloc = nNew; - pMap->aEntry = aNew; - } - pEntry = &pMap->aEntry[pMap->nEntry++]; - pEntry->iRowid = p1->base.iRowid; - pEntry->iCol = iCol; - pEntry->iOff = iTokOff; - pEntry->iTok = pMap->nToken; - } - } -} - -static int fts5TokenMapHash(i64 iRowid, int iCol, int iOff){ - return (iRowid + (iRowid << 3) + (iCol << 6) + (iOff << 9)) & 0x7FFFFFFF; -} - -static void fts5TokenMapHashify(Fts5Index *p, Fts5TokenMap *pMap){ - int nHash = pMap->nEntry*2; - Fts5TokenMapEntry *aHash = 0; - - aHash = (Fts5TokenMapEntry*)fts5IdxMalloc(p, nHash*sizeof(Fts5TokenMapEntry)); - if( aHash ){ - int ii; - for(ii=0; iinEntry; ii++){ - Fts5TokenMapEntry *pEntry = &pMap->aEntry[ii]; - Fts5TokenMapEntry *pCopy = 0; - int iHash = fts5TokenMapHash(pEntry->iRowid, pEntry->iCol, pEntry->iOff); - - while( aHash[iHash % nHash].iTok ){ - iHash++; - } - pCopy = &aHash[iHash % nHash]; - memcpy(pCopy, pEntry, sizeof(Fts5TokenMapEntry)); - } - - sqlite3_free(pMap->aEntry); - pMap->aEntry = aHash; - pMap->nEntry = pMap->nEntryAlloc = nHash; - } -} - -static const u8 *fts5TokenMapLookup( - Fts5TokenMap *pMap, - i64 iRowid, - int iCol, - int iOff, - int *pnOut -){ - int iHash = fts5TokenMapHash(iRowid, iCol, iOff) % pMap->nEntry; - - for(; pMap->aEntry[iHash].iTok!=0; iHash = (iHash+1)%pMap->nEntry){ - Fts5TokenMapEntry *pEntry = &pMap->aEntry[iHash]; - if( pEntry->iRowid==iRowid && pEntry->iCol==iCol && pEntry->iOff==iOff ){ - *pnOut = pMap->aToken[pEntry->iTok-1].nTerm; - return pMap->aToken[pEntry->iTok-1].pTerm; - } - } - - *pnOut = 0; - return 0; -} - -/* -** The iterator passed as the second argument has been opened to scan and -** merge doclists for a series of tokens in tokendata=1 mode. This function -** tests whether or not, instead of using the cursor to read doclists to -** merge, it can be used directly by the upper layer. This is the case -** if the cursor currently points to the only token that corresponds to -** the queried term. i.e. if the next token that will be visited by the -** iterator does not match the query. -*/ -int fts5TokendataIterIsOk( - Fts5Index *p, - Fts5Iter *pIter, - const u8 *pToken, - int nToken -){ - int ii; - Fts5Buffer buf = {0, 0, 0}; - int bRet = 1; - Fts5Buffer *pTerm = 0; - - /* Iterator is not usable if it uses the hash table */ - if( pIter->aSeg[0].pSeg==0 ) return 0; - - for(ii=0; bRet && iinSeg; ii++){ - Fts5SegIter *pSeg = &pIter->aSeg[ii]; - Fts5Data *pLeaf = pSeg->pLeaf; - if( pLeaf ){ - - if( pTerm==0 ){ - pTerm = &pSeg->term; - }else{ - if( pSeg->term.n!=pTerm->n - || memcmp(pSeg->term.p, pTerm->p, pTerm->n) - ){ - bRet = 0; - break; - } - } - - if( pSeg->iEndofDoclistszLeaf ){ - /* Next term is on this node. Check it directly. */ - int nPrefix = 0; - fts5GetVarint32(&pLeaf->p[pSeg->iEndofDoclist], nPrefix); - if( nPrefix>=nToken ) bRet = 0; - }else{ - /* Next term is on a subsequent page. In this case query the %_idx - ** table to discover exactly what that next term is. */ - sqlite3_stmt *pProbe = fts5IdxProbeStmt(p); - if( pProbe ){ - int rc = SQLITE_OK; - if( buf.n==0 ){ - sqlite3Fts5BufferAppendBlob(&p->rc, &buf, nToken, pToken); - sqlite3Fts5BufferAppendBlob(&p->rc, &buf, 1, (const u8*)"\1"); - } - sqlite3_bind_int(pProbe, 1, pSeg->pSeg->iSegid); - sqlite3_bind_blob(pProbe,2, pSeg->term.p,pSeg->term.n, SQLITE_STATIC); - sqlite3_bind_blob(pProbe,3, buf.p, buf.n, SQLITE_STATIC); - - if( sqlite3_step(pProbe)==SQLITE_ROW ){ - bRet = 0; - } - rc = sqlite3_reset(pProbe); - if( p->rc==SQLITE_OK ) p->rc = rc; - } - } - } - } - - if( bRet ){ - for(ii=0; iinSeg; ii++){ - Fts5SegIter *pSeg = &pIter->aSeg[ii]; - pSeg->flags |= FTS5_SEGITER_ONETERM; - } - } - - fts5BufferFree(&buf); - return bRet; -} - static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ - int bTokenscan, int iIdx, /* Index to scan for data */ u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ @@ -6355,7 +6091,6 @@ static void fts5SetupPrefixIter( Fts5Buffer *aBuf; int nBuf = 32; int nMerge = 1; - Fts5TokenMap *pMap = 0; void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); @@ -6369,8 +6104,6 @@ static void fts5SetupPrefixIter( xAppend = fts5AppendPoslist; } - assert( bTokenscan==0 || iIdx==0 ); - aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); @@ -6417,90 +6150,69 @@ static void fts5SetupPrefixIter( fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); - if( bDesc==0 && bTokenscan && fts5TokendataIterIsOk(p, p1, pToken,nToken) ){ - /* In this case iterator p1 may be used as is. */ - *ppIter = p1; - }else{ + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); - if( iIdx==0 && p->pConfig->eDetail==FTS5_DETAIL_FULL && bTokenscan ){ - pMap = (Fts5TokenMap*)fts5IdxMalloc(p, sizeof(Fts5TokenMap)); + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); + if( bNewTerm ){ + if( nTermrc!=SQLITE_OK || (aBuf && pStruct) ); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - p1->xSetOutputs(p1, pSeg); - - assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( bNewTerm ){ - if( nTermnToken && pTerm[nToken]!=0x00 ) break; - } - - if( pMap ){ - if( bNewTerm ){ - fts5TokenMapTerm(p, pMap, &pTerm[1], nTerm-1); + if( p1->base.nData==0 ) continue; + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ + for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=nBuf ); + for(iStore=i1; iStorebase.nData==0 ) continue; - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ - for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - int i1 = i*nMerge; - int iStore; - assert( i1+nMerge<=nBuf ); + if( iStore==i1+nMerge ){ + xMerge(p, &doclist, nMerge, &aBuf[i1]); for(iStore=i1; iStorebase.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; + iLastRowid = 0; } - - assert( (nBuf%nMerge)==0 ); - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, nMerge, &aBuf[i]); - } - for(iFree=i; iFreep = (u8*)&pData[1]; - pData->nn = pData->szLeaf = doclist.n; - if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); - fts5MultiIterNew2(p, pData, pMap, bDesc, ppIter); - pMap = 0; - } - fts5BufferFree(&doclist); + + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; } + + assert( (nBuf%nMerge)==0 ); + for(i=0; irc==SQLITE_OK ){ + xMerge(p, &doclist, nMerge, &aBuf[i]); + } + for(iFree=i; iFreep = (u8*)&pData[1]; + pData->nn = pData->szLeaf = doclist.n; + if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + fts5MultiIterNew2(p, pData, bDesc, ppIter); + } + fts5BufferFree(&doclist); } - fts5TokenMapFree(pMap); fts5StructureRelease(pStruct); sqlite3_free(aBuf); } @@ -6634,7 +6346,6 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); - sqlite3_finalize(p->pIdxProbe); sqlite3_finalize(p->pDataVersion); sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); @@ -7038,7 +6749,6 @@ static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ static Fts5Iter *fts5SetupTokendataIter( Fts5Index *p, /* FTS index to query */ - int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing query term */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset /* Colset to filter on */ @@ -7048,8 +6758,6 @@ static Fts5Iter *fts5SetupTokendataIter( Fts5Structure *pStruct = 0; const int flags = FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_SCAN; - assert( bDesc==0 ); - Fts5Buffer bSeek = {0, 0, 0}; Fts5Buffer *pSmall = 0; @@ -7204,7 +6912,10 @@ int sqlite3Fts5IndexQuery( } } - if( iIdx<=pConfig->nPrefix && (bTokendata==0 || iIdx!=0) ){ + if( bTokendata && iIdx==0 ){ + buf.p[0] = '0'; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); @@ -7214,17 +6925,10 @@ int sqlite3Fts5IndexQuery( ); fts5StructureRelease(pStruct); } - }else if( bTokendata && iIdx==0 ){ - int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; - buf.p[0] = '0'; - pRet = fts5SetupTokendataIter(p, bDesc, buf.p, nToken+1, pColset); }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; - int bTokenscan = (iIdx==0); - fts5SetupPrefixIter( - p, bDesc, bTokenscan, iPrefixIdx, buf.p, nToken+1, pColset, &pRet - ); + fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); if( pRet==0 ){ assert( p->rc!=SQLITE_OK ); }else{ @@ -7325,26 +7029,11 @@ int sqlite3Fts5IterToken( const char **ppOut, int *pnOut ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - Fts5TokenMap *pMap = pIter->pTokenMap; if( pIter->pTokenDataIter ){ return fts5TokendataIterToken(pIter, iRowid, iCol, iOff, ppOut, pnOut); } - if( pMap ){ - if( pMap->bHashed==0 ){ - Fts5Index *p = pIter->pIndex; - fts5TokenMapHashify(p, pMap); - if( p->rc ){ - return fts5IndexReturn(p); - } - } - *ppOut = (const char*)fts5TokenMapLookup( - pIter->pTokenMap, pIndexIter->iRowid, iCol, iOff, pnOut - ); - }else{ - *ppOut = sqlite3Fts5IterTerm(pIndexIter, pnOut); - } return SQLITE_OK; } diff --git a/ext/fts5/test/fts5simple2.test b/ext/fts5/test/fts5simple2.test index e57cea70fa..6c0e0e1662 100644 --- a/ext/fts5/test/fts5simple2.test +++ b/ext/fts5/test/fts5simple2.test @@ -343,7 +343,9 @@ do_execsql_test 17.0 { INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); COMMIT; } -do_execsql_test 17.1 { SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 } +do_execsql_test 17.1 { + SELECT * FROM t2('y:a*') WHERE rowid BETWEEN 10 AND 20 +} do_execsql_test 17.2 { BEGIN; INSERT INTO t2 VALUES('a aa aaa', 'b bb bbb'); diff --git a/manifest b/manifest index 07814168f5..92cc5ca8d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\swith\sthis\sbranch. -D 2023-12-01T20:10:20.295 +C Remove\sold\scode\sfor\stokendata=1\squeries. +D 2023-12-01T20:37:11.688 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c aac8026aedf56c9a6e32b31c89f9dd7e5548378457085093307d06be14f1a176 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c b6012920df8963245226bb536db7ddea62dfd7a860d6112887b175f7aaf55b82 +F ext/fts5/fts5_index.c 2296bcd6736eaf093212892474619dfdb7ac1e262732e2b72cb528172c0b13d6 F ext/fts5/fts5_main.c 20596de592af135f68b9be875f0a28715f6562bbdedd215e1c89eac1b42e97f9 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -215,7 +215,7 @@ F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114 F ext/fts5/test/fts5secure8.test eb3579e9d58b0acad97e8082dee1f99b2d393198f03500b453c2b25761c0c298 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b -F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 +F ext/fts5/test/fts5simple2.test 8dd2389ee75e21a1429fe87e5f8c7d9a97ad1470304a8a2d3ba4b8c3c345fecd F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f F ext/fts5/test/fts5synonym.test 1651815b8008de170e8e600dcacc17521d765482ea8f074ae82cfa870d8bb7fb F ext/fts5/test/fts5synonym2.test 8f891fc49cc1e8daed727051e77e1f42849c784a6a54bef82564761b2cb3e016 @@ -2148,8 +2148,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c523f40895866e6fc979a26483dbea8206126b4bbdf4b73b77263c09e13c855e 883990e7938c1f63906300a6113f0fadce143913b7c384e8aeb5f886f0be7c62 -R 070accca4f04b7da788d8a587600de24 +P 8258967411d3ff212424b25fec79ded0d8ae83e773cd35a0bbf300c94923f25b +R d3ee6e0cbe8554b06da39b239b5142bc U dan -Z 0994bbca14fc53076151cd58ec1e7d74 +Z cbe01276ff2d44c618e4cc899051c453 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1398e08598..359f2cca45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8258967411d3ff212424b25fec79ded0d8ae83e773cd35a0bbf300c94923f25b \ No newline at end of file +b0a489e8e1bf0290c2117ab32d78b1cc7d67bcb226b55ec044c8367ebde3815b \ No newline at end of file From 3af20cf3a0523ba8a8c7e966abc56eed218f4dd4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 1 Dec 2023 22:01:26 +0000 Subject: [PATCH 134/191] Performance optimization in the JSON parser. FossilOrigin-Name: 68d191f40e708962ec88e0c245b4496bc4a671300484b1cc0f3fc7e6d199a6e6 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/json.c | 6 ------ 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 05817d816e..6df01a3cfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JSON\scache\sis\snow\smore\seffective. -D 2023-12-01T18:49:02.694 +C Performance\soptimization\sin\sthe\sJSON\sparser. +D 2023-12-01T22:01:26.348 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3556e879386b0af1c542ed13ebb2f58c48d3b8b3ab13a96ab77ad2a9493cc715 +F src/json.c d524f2f13338f84327641edd5b422252d7c93a4f7fc57f083a0d41c767561554 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,9 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25ed295f300fea6185104a73721076bccd2b2a6e411c78564266fa6dca4ff70c 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 -R bed96169a434a08ba8168cff3e09a739 -T +closed 1fdbc39521f63aedc6f08ecaafa54ea467b8c6316a692a18ad01eecbf22a0977 +P 443a3f3a8e64d81cad8300a30e2cc57c4e39f69b5669ac8b550c590ae9f1134a +R a25493165248fa490fe7a3cb972ea0d2 U drh -Z 38a6916a5a09665f20ec73466c161c5c +Z 51cb7fe7cb06e4de05d17231c2159ec6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index caad112349..0d822e57d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -443a3f3a8e64d81cad8300a30e2cc57c4e39f69b5669ac8b550c590ae9f1134a \ No newline at end of file +68d191f40e708962ec88e0c245b4496bc4a671300484b1cc0f3fc7e6d199a6e6 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 08e0f4fb63..3ae88be9d4 100644 --- a/src/json.c +++ b/src/json.c @@ -1366,7 +1366,6 @@ json_parse_restart: case '\'': { u8 opcode; char cDelim; - int nn; pParse->hasNonstd = 1; opcode = JSONB_TEXT; goto parse_string; @@ -1375,12 +1374,7 @@ json_parse_restart: opcode = JSONB_TEXT; parse_string: cDelim = z[i]; - nn = pParse->nJson; for(j=i+1; 1; j++){ - if( j>=nn ){ - pParse->iErr = j; - return -1; - } if( jsonIsOk[(unsigned char)z[j]] ) continue; c = z[j]; if( c==cDelim ){ From 5ec9c916ad97600b14871534137b0bc3bfd5bf49 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 01:06:33 +0000 Subject: [PATCH 135/191] Fix harmless compiler warnings and enhance performance the parser. FossilOrigin-Name: 285633da6d188547e52f07779e209c9e5f3dc33ce0668e14858f3337889ef4b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 41 +++++++++++++++++++++++++++++------------ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 6df01a3cfa..776e83a5e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\sJSON\sparser. -D 2023-12-01T22:01:26.348 +C Fix\sharmless\scompiler\swarnings\sand\senhance\sperformance\sthe\sparser. +D 2023-12-02T01:06:33.225 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d524f2f13338f84327641edd5b422252d7c93a4f7fc57f083a0d41c767561554 +F src/json.c 58c3ded3bdc94125235a805214016147ecd63f8173cff19a7084c9fb0ea0ff9c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 443a3f3a8e64d81cad8300a30e2cc57c4e39f69b5669ac8b550c590ae9f1134a -R a25493165248fa490fe7a3cb972ea0d2 +P 68d191f40e708962ec88e0c245b4496bc4a671300484b1cc0f3fc7e6d199a6e6 +R 398de78fe5dea0e1b0bc6535ce40a9fc U drh -Z 51cb7fe7cb06e4de05d17231c2159ec6 +Z c1eeb98e2ab1260bdea70d2e8aff46bd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d822e57d7..b7c120681c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68d191f40e708962ec88e0c245b4496bc4a671300484b1cc0f3fc7e6d199a6e6 \ No newline at end of file +285633da6d188547e52f07779e209c9e5f3dc33ce0668e14858f3337889ef4b8 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3ae88be9d4..6c86d98f2f 100644 --- a/src/json.c +++ b/src/json.c @@ -1075,6 +1075,17 @@ static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ return 0; } +static void jsonBlobAppendNodeType(JsonParse*,u8,u32); +static SQLITE_NOINLINE void jsonBlobExpandAndAppendNodeType( + JsonParse *pParse, + u8 eType, + u32 szPayload +){ + if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; + jsonBlobAppendNodeType(pParse, eType, szPayload); +} + + /* Append an node type byte together with the payload size. */ static void jsonBlobAppendNodeType( @@ -1082,25 +1093,31 @@ static void jsonBlobAppendNodeType( u8 eType, u32 szPayload ){ - u8 a[5]; + u8 *a; + if( pParse->nBlob+szPayload+9 > pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendNodeType(pParse,eType,szPayload); + return; + } + a = &pParse->aBlob[pParse->nBlob]; if( szPayload<=11 ){ - jsonBlobAppendOneByte(pParse, eType | (szPayload<<4)); + a[0] = eType | (szPayload<<4); + pParse->nBlob += 1; }else if( szPayload<=0xff ){ a[0] = eType | 0xc0; a[1] = szPayload & 0xff; - jsonBlobAppendNBytes(pParse, a, 2); + pParse->nBlob += 2; }else if( szPayload<=0xffff ){ a[0] = eType | 0xd0; a[1] = (szPayload >> 8) & 0xff; a[2] = szPayload & 0xff; - jsonBlobAppendNBytes(pParse, a, 3); + pParse->nBlob += 3; }else{ a[0] = eType | 0xe0; a[1] = (szPayload >> 24) & 0xff; a[2] = (szPayload >> 16) & 0xff; a[3] = (szPayload >> 8) & 0xff; a[4] = szPayload & 0xff; - jsonBlobAppendNBytes(pParse, a, 5); + pParse->nBlob += 5; } } @@ -2169,12 +2186,9 @@ static u32 jsonLookupBlobStep( u32 nIns; /* Total bytes to insert (label+value) */ JsonParse v; /* BLOB encoding of the value to be inserted */ JsonParse ix; /* Header of the label to be inserted */ - u8 aLabel[16]; /* Buffer space for ix */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); - ix.aBlob = aLabel; - ix.nBlobAlloc = sizeof(aLabel); jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); memset(&v, 0, sizeof(v)); if( zPath[i]==0 ){ @@ -2190,9 +2204,11 @@ static u32 jsonLookupBlobStep( if( JSON_BLOB_ISERROR(rc) || v.oom ){ pParse->oom |= v.oom; jsonParseReset(&v); + jsonParseReset(&ix); return rc; } } + pParse->oom |= ix.oom; if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) ){ nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); @@ -2204,6 +2220,7 @@ static u32 jsonLookupBlobStep( if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); } jsonParseReset(&v); + jsonParseReset(&ix); return j; } }else if( zPath[0]=='[' ){ @@ -3323,11 +3340,11 @@ static int jsonMergePatchBlob( u32 iTEndBE; /* Original first byte past end of target, before edit */ u32 iTEnd; /* Current first byte past end of target */ u8 eTLabel; /* Node type of the target label */ - u32 iTLabel; /* Index of the label */ - u32 nTLabel; /* Header size in bytes for the target label */ + u32 iTLabel = 0; /* Index of the label */ + u32 nTLabel = 0; /* Header size in bytes for the target label */ u32 szTLabel; /* Size of the target label payload */ - u32 iTValue; /* Index of the target value */ - u32 nTValue; /* Header size of the target value */ + u32 iTValue = 0; /* Index of the target value */ + u32 nTValue = 0; /* Header size of the target value */ u32 szTValue; /* Payload size for the target value */ u32 iPCursor; /* Cursor position while scanning the patch */ From 6df61985d4930d11d6f6fe65997b54f512e9aff5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 01:38:53 +0000 Subject: [PATCH 136/191] Unroll a loop in the parser for a performance increase. FossilOrigin-Name: a6dc29e4d5e13949e0fcd9d5dde575c2670eb10a230ab9df3806fc8c3016c540 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 13 +++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 776e83a5e6..54838edff6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sand\senhance\sperformance\sthe\sparser. -D 2023-12-02T01:06:33.225 +C Unroll\sa\sloop\sin\sthe\sparser\sfor\sa\sperformance\sincrease. +D 2023-12-02T01:38:53.624 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 58c3ded3bdc94125235a805214016147ecd63f8173cff19a7084c9fb0ea0ff9c +F src/json.c 5363d3a9e6d4f8a77b735f279aedc0139f5ba4a1c6dff5052cc8b8f31012d9ef F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68d191f40e708962ec88e0c245b4496bc4a671300484b1cc0f3fc7e6d199a6e6 -R 398de78fe5dea0e1b0bc6535ce40a9fc +P 285633da6d188547e52f07779e209c9e5f3dc33ce0668e14858f3337889ef4b8 +R 5a9d6175e86127f56f06090dd3d036f6 U drh -Z c1eeb98e2ab1260bdea70d2e8aff46bd +Z 9e063f5d4d8d31b6e06beb4f188d0a08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7c120681c..beaced137d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -285633da6d188547e52f07779e209c9e5f3dc33ce0668e14858f3337889ef4b8 \ No newline at end of file +a6dc29e4d5e13949e0fcd9d5dde575c2670eb10a230ab9df3806fc8c3016c540 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6c86d98f2f..62e3d652fd 100644 --- a/src/json.c +++ b/src/json.c @@ -1391,8 +1391,16 @@ json_parse_restart: opcode = JSONB_TEXT; parse_string: cDelim = z[i]; - for(j=i+1; 1; j++){ - if( jsonIsOk[(unsigned char)z[j]] ) continue; + j = i+1; + while( 1 /*exit-by-break*/ ){ + if( jsonIsOk[(u8)z[j]] ){ + if( jsonIsOk[(u8)z[j+1]] ){ + j += 2; + continue; + }else{ + j += 1; + } + } c = z[j]; if( c==cDelim ){ break; @@ -1421,6 +1429,7 @@ json_parse_restart: pParse->iErr = j; return -1; } + j++; } jsonBlobAppendNodeType(pParse, opcode, j-1-i); jsonBlobAppendNBytes(pParse, (const u8*)&z[i+1], j-1-i); From 768b6e32f699a2e76b21f1ef512d7c1811899bac Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 12:23:34 +0000 Subject: [PATCH 137/191] Remove a NEVER that can be true if a virtual table column is declared to have a DEFAULT. See [forum:/forumpost/3d4de8917627d058|forum post 3d4de8917627d058]. FossilOrigin-Name: 8abc2ccaf8106f20243568cd7fa74174386eb85d7ea381201e97e2fd527033e0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d6a02fa193..e9c843fd00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sbasic\sbatch-mode\sSQL\srunner\sfor\sthe\sSAH\sPool\sVFS,\sfor\suse\sin\scomparing\sit\sagainst\sWebSQL.\sBring\sthe\sWebSQL\sbatch\srunner\sup\sto\sdate,\snoting\sthat\sit\scannot\srun\swithout\saddition\sof\san\s"origin\strial"\sactivation\skey\sfrom\sGoogle\sbecause\sthat's\snow\sthe\sonly\sway\sto\senable\sWebSQL\sin\sChrome\s(that\spart\sis\snot\schecked\sin\sbecause\sthat\skey\sis\sprivate).\sMinor\scode-adjacent\scleanups. -D 2023-11-30T20:34:24.440 +C Remove\sa\sNEVER\sthat\scan\sbe\strue\sif\sa\svirtual\stable\scolumn\sis\sdeclared\sto\shave\na\sDEFAULT.\s\sSee\n[forum:/forumpost/3d4de8917627d058|forum\spost\s3d4de8917627d058]. +D 2023-12-02T12:23:34.118 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -672,7 +672,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 -F src/build.c 189e4517d67f09f0a3e0d8e1faa6e2ef0c2e95f6ac82e33c912cb7efa2a359cc +F src/build.c d0bb02989e9e652ee8a012c7780e32087c0dd99643d0cf609971029f78cb738a F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6802b6459d0d16c961ff41d240a6c88287f197d8f609090f79308707490a49c2 -R 0685d189d0b0c5921f2e4eda3787211e -U stephan -Z 5692f67161061876d9b3bcabcb3928cc +P 883990e7938c1f63906300a6113f0fadce143913b7c384e8aeb5f886f0be7c62 +R 9588fe6fe9170e1bad62d9178cc8ac2d +U drh +Z fe60aea9a6abab8faa2bce1247eb25bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 98d7aac12d..494cc91006 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -883990e7938c1f63906300a6113f0fadce143913b7c384e8aeb5f886f0be7c62 \ No newline at end of file +8abc2ccaf8106f20243568cd7fa74174386eb85d7ea381201e97e2fd527033e0 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 3d06792358..14639f096e 100644 --- a/src/build.c +++ b/src/build.c @@ -720,7 +720,7 @@ void sqlite3ColumnSetExpr( */ Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ if( pCol->iDflt==0 ) return 0; - if( NEVER(!IsOrdinaryTable(pTab)) ) return 0; + if( !IsOrdinaryTable(pTab) ) return 0; if( NEVER(pTab->u.tab.pDfltList==0) ) return 0; if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; From 679c90850c7df5b81aa43e5af30737748ea06954 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 13:36:52 +0000 Subject: [PATCH 138/191] Simplification and optimization of the JSON parser. FossilOrigin-Name: f5ec9485119a2a6cb33eb864c7ca9b41d4a2ed08ab6ad9a6b0dd9358ab253576 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/json.c | 75 +++++++++++++++++++++++++-------------------------- 3 files changed, 43 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 54838edff6..07d971dc38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Unroll\sa\sloop\sin\sthe\sparser\sfor\sa\sperformance\sincrease. -D 2023-12-02T01:38:53.624 +C Simplification\sand\soptimization\sof\sthe\sJSON\sparser. +D 2023-12-02T13:36:52.119 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 5363d3a9e6d4f8a77b735f279aedc0139f5ba4a1c6dff5052cc8b8f31012d9ef +F src/json.c 73558b9dca8f326045c42350bd8274c9a9a6489cdb145b2208a8fc7e50a04247 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 285633da6d188547e52f07779e209c9e5f3dc33ce0668e14858f3337889ef4b8 -R 5a9d6175e86127f56f06090dd3d036f6 +P a6dc29e4d5e13949e0fcd9d5dde575c2670eb10a230ab9df3806fc8c3016c540 +R d65ba1fc2a2f76e7b171a7dc72bb1070 U drh -Z 9e063f5d4d8d31b6e06beb4f188d0a08 +Z b0de3fe3c23565cae5077bfe62fcd8b8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index beaced137d..9855953161 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6dc29e4d5e13949e0fcd9d5dde575c2670eb10a230ab9df3806fc8c3016c540 \ No newline at end of file +f5ec9485119a2a6cb33eb864c7ca9b41d4a2ed08ab6ad9a6b0dd9358ab253576 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 62e3d652fd..af8f461d46 100644 --- a/src/json.c +++ b/src/json.c @@ -1064,38 +1064,39 @@ static int jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ return 0; } -/* Append bytes. Return 1 if an error occurs. +/* Slow version of jsonBlobAppendNode() that first resizes the +** pParse->aBlob structure. */ -static int jsonBlobAppendNBytes(JsonParse *pParse, const u8 *aData, u32 N){ - if( pParse->nBlob+N > pParse->nBlobAlloc ){ - return jsonBlobExpandAndAppend(pParse, aData, N); - } - memcpy(&pParse->aBlob[pParse->nBlob], aData, N); - pParse->nBlob += N; - return 0; -} - -static void jsonBlobAppendNodeType(JsonParse*,u8,u32); -static SQLITE_NOINLINE void jsonBlobExpandAndAppendNodeType( +static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( JsonParse *pParse, u8 eType, - u32 szPayload + u32 szPayload, + const void *aPayload ){ if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; - jsonBlobAppendNodeType(pParse, eType, szPayload); + jsonBlobAppendNode(pParse, eType, szPayload, aPayload); } -/* Append an node type byte together with the payload size. +/* Append an node type byte together with the payload size and +** possibly also the payload. +** +** If aPayload is not NULL, then it is a pointer to the payload which +** is also appended. If aPayload is NULL, the pParse->aBlob[] array +** is resized (if necessary) so that it is big enough to hold the +** payload, but the payload is not appended and pParse->nBlob is left +** pointing to where the first byte of payload will eventually be. */ -static void jsonBlobAppendNodeType( - JsonParse *pParse, - u8 eType, - u32 szPayload +static void jsonBlobAppendNode( + JsonParse *pParse, /* The JsonParse object under construction */ + u8 eType, /* Node type. One of JSONB_* */ + u32 szPayload, /* Number of bytes of payload */ + const void *aPayload /* The payload. Might be NULL */ ){ u8 *a; if( pParse->nBlob+szPayload+9 > pParse->nBlobAlloc ){ - jsonBlobExpandAndAppendNodeType(pParse,eType,szPayload); + jsonBlobExpandAndAppendNode(pParse,eType,szPayload,aPayload); return; } a = &pParse->aBlob[pParse->nBlob]; @@ -1119,6 +1120,10 @@ static void jsonBlobAppendNodeType( a[4] = szPayload & 0xff; pParse->nBlob += 5; } + if( aPayload ){ + pParse->nBlob += szPayload; + memcpy(&pParse->aBlob[pParse->nBlob-szPayload], aPayload, szPayload); + } } /* Change the payload size for the node at index i to be szPayload. @@ -1230,7 +1235,7 @@ json_parse_restart: case '{': { /* Parse object */ iThis = pParse->nBlob; - jsonBlobAppendNodeType(pParse, JSONB_OBJECT, (pParse->nJson-i)*2); + jsonBlobAppendNode(pParse, JSONB_OBJECT, (pParse->nJson-i)*2, 0); if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; return -1; @@ -1258,8 +1263,7 @@ json_parse_restart: k++; } assert( iBlob==pParse->nBlob ); - jsonBlobAppendNodeType(pParse, op, k-j); - jsonBlobAppendNBytes(pParse, (const u8*)&z[j], k-j); + jsonBlobAppendNode(pParse, op, k-j, &z[j]); pParse->hasNonstd = 1; x = k; }else{ @@ -1331,7 +1335,7 @@ json_parse_restart: case '[': { /* Parse array */ iThis = pParse->nBlob; - jsonBlobAppendNodeType(pParse, JSONB_ARRAY, pParse->nJson - i); + jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); iStart = pParse->nBlob; if( pParse->oom ) return -1; if( ++pParse->iDepth > JSON_MAX_DEPTH ){ @@ -1431,8 +1435,7 @@ json_parse_restart: } j++; } - jsonBlobAppendNodeType(pParse, opcode, j-1-i); - jsonBlobAppendNBytes(pParse, (const u8*)&z[i+1], j-1-i); + jsonBlobAppendNode(pParse, opcode, j-1-i, &z[i+1]); return j+1; } case 't': { @@ -1507,11 +1510,9 @@ json_parse_restart: ){ pParse->hasNonstd = 1; if( z[i]=='-' ){ - jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 6); - jsonBlobAppendNBytes(pParse, (const u8*)"-9e999", 6); + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); }else{ - jsonBlobAppendNodeType(pParse, JSONB_FLOAT, 5); - jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); } return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); } @@ -1592,8 +1593,7 @@ json_parse_restart: assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); assert( JSONB_INT+0x02==JSONB_FLOAT ); if( z[i]=='+' ) i++; - jsonBlobAppendNodeType(pParse, JSONB_INT+t, j-i); - jsonBlobAppendNBytes(pParse, (const u8*)&z[i], j-i); + jsonBlobAppendNode(pParse, JSONB_INT+t, j-i, &z[i]); return j; } case '}': { @@ -1660,8 +1660,7 @@ json_parse_restart: } if( sqlite3Isalnum(z[i+nn]) ) continue; if( aNanInfName[k].eType==JSONB_FLOAT ){ - jsonBlobAppendOneByte(pParse, JSONB_FLOAT | 0x50); - jsonBlobAppendNBytes(pParse, (const u8*)"9e999", 5); + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); }else{ jsonBlobAppendOneByte(pParse, JSONB_NULL); } @@ -2198,7 +2197,7 @@ static u32 jsonLookupBlobStep( testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); - jsonBlobAppendNodeType(&ix,JSONB_TEXTRAW, nKey); + jsonBlobAppendNode(&ix,JSONB_TEXTRAW, nKey, 0); memset(&v, 0, sizeof(v)); if( zPath[i]==0 ){ v.nBlob = pParse->nIns; @@ -2563,8 +2562,7 @@ static int jsonFunctionArgToBlob( return 1; } }else{ - jsonBlobAppendNodeType(pParse, JSONB_TEXTRAW, nJson); - jsonBlobAppendNBytes(pParse, (const u8*)zJson, nJson); + jsonBlobAppendNode(pParse, JSONB_TEXTRAW, nJson, zJson); } break; } @@ -2574,8 +2572,7 @@ static int jsonFunctionArgToBlob( const char *z = (const char*)sqlite3_value_text(pArg); int e = eType==SQLITE_INTEGER ? JSONB_INT : JSONB_FLOAT; if( z==0 ) return 1; - jsonBlobAppendNodeType(pParse, e, n); - jsonBlobAppendNBytes(pParse, (const u8*)z, n); + jsonBlobAppendNode(pParse, e, n, z); break; } } From f0b8b16317a0449d00d45246a2d78598e8322448 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 14:16:47 +0000 Subject: [PATCH 139/191] Performance optimization in jsonAppendString(). FossilOrigin-Name: fdf00e96239c73fb67e2acecc5b95f55a1fc51c3deed4512613c0d6070ce5805 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 36 +++++++++++++++++++++++++++--------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 07d971dc38..71a33c080b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sand\soptimization\sof\sthe\sJSON\sparser. -D 2023-12-02T13:36:52.119 +C Performance\soptimization\sin\sjsonAppendString(). +D 2023-12-02T14:16:47.613 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 73558b9dca8f326045c42350bd8274c9a9a6489cdb145b2208a8fc7e50a04247 +F src/json.c a3eabd185cfd13250503ac8a17ac3ec61c9bb88730f99134de9df6c9a3446d46 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6dc29e4d5e13949e0fcd9d5dde575c2670eb10a230ab9df3806fc8c3016c540 -R d65ba1fc2a2f76e7b171a7dc72bb1070 +P f5ec9485119a2a6cb33eb864c7ca9b41d4a2ed08ab6ad9a6b0dd9358ab253576 +R a5df9ac2834f12883789ae2220af8e65 U drh -Z b0de3fe3c23565cae5077bfe62fcd8b8 +Z f68f81753669c00bcaaaadfbdd6d3188 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9855953161..eb14e17380 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5ec9485119a2a6cb33eb864c7ca9b41d4a2ed08ab6ad9a6b0dd9358ab253576 \ No newline at end of file +fdf00e96239c73fb67e2acecc5b95f55a1fc51c3deed4512613c0d6070ce5805 \ No newline at end of file diff --git a/src/json.c b/src/json.c index af8f461d46..9d6554f415 100644 --- a/src/json.c +++ b/src/json.c @@ -595,17 +595,33 @@ static void jsonAppendSeparator(JsonString *p){ ** string. */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( zIn==0 ) return; + u32 k; + u8 c; + const u8 *z = (const u8*)zIn; + if( z==0 ) return; if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; - for(i=0; izBuf[p->nUsed++] = c; - }else if( c=='"' || c=='\\' ){ + while( 1 /*exit-by-break*/ ){ + k = 0; + while( k=N ){ + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + } + break; + } + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + z += k; + N -= k; + } + c = z[0]; + if( c=='"' || c=='\\' ){ json_simple_escape: - if( (p->nUsed+N+3-i > p->nAlloc) && jsonStringGrow(p,N+3-i)!=0 ) return; + if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = c; }else if( c=='\'' ){ @@ -626,7 +642,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ c = aSpecial[c]; goto json_simple_escape; } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonStringGrow(p,N+7-i)!=0 ) return; + if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; p->zBuf[p->nUsed++] = 'u'; p->zBuf[p->nUsed++] = '0'; @@ -634,6 +650,8 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; } + z++; + N--; } p->zBuf[p->nUsed++] = '"'; assert( p->nUsednAlloc ); From 82136d90f868f01c4573b52802f6e2b00bb20f45 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 14:55:46 +0000 Subject: [PATCH 140/191] Minor fix to the header comment on jsonXlateTextToBlob(). FossilOrigin-Name: c3677ba410208c07b711f5f526eb5cf039a8eee49f632c7ae04fa55cdfbb9058 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 71a33c080b..ab5b5ba123 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sjsonAppendString(). -D 2023-12-02T14:16:47.613 +C Minor\sfix\sto\sthe\sheader\scomment\son\sjsonXlateTextToBlob(). +D 2023-12-02T14:55:46.474 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a3eabd185cfd13250503ac8a17ac3ec61c9bb88730f99134de9df6c9a3446d46 +F src/json.c 9ff2d40ed0d0199d88fa414f519a977bee38df27907702663b1cc5bc85274b05 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f5ec9485119a2a6cb33eb864c7ca9b41d4a2ed08ab6ad9a6b0dd9358ab253576 -R a5df9ac2834f12883789ae2220af8e65 +P fdf00e96239c73fb67e2acecc5b95f55a1fc51c3deed4512613c0d6070ce5805 +R dbfc8fdcf920d7c2aea651d65adda878 U drh -Z f68f81753669c00bcaaaadfbdd6d3188 +Z ecc9fe8f5fc6f3c09b592aceda67d13d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb14e17380..f505eb9a85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdf00e96239c73fb67e2acecc5b95f55a1fc51c3deed4512613c0d6070ce5805 \ No newline at end of file +c3677ba410208c07b711f5f526eb5cf039a8eee49f632c7ae04fa55cdfbb9058 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9d6554f415..095ae870bf 100644 --- a/src/json.c +++ b/src/json.c @@ -1235,7 +1235,7 @@ static int jsonIs4HexB(const char *z, int *pOp){ ** or one of the following special result codes: ** ** 0 End of input -** -1 Syntax error +** -1 Syntax error or OOM ** -2 '}' seen \ ** -3 ']' seen \___ For these returns, pParse->iErr is set to ** -4 ',' seen / the index in zJson[] of the seen character From 4cd397c0d9b3ded36f9838ca04b88a74f40931ce Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 15:06:43 +0000 Subject: [PATCH 141/191] Fix potential unsigned integer underflow in jsonAppendString(). FossilOrigin-Name: d2fba2cbdc3870d34228c1a9446eced884325acc183900d7dd0b96132570fb4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab5b5ba123..38a7efdf50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfix\sto\sthe\sheader\scomment\son\sjsonXlateTextToBlob(). -D 2023-12-02T14:55:46.474 +C Fix\spotential\sunsigned\sinteger\sunderflow\sin\sjsonAppendString(). +D 2023-12-02T15:06:43.574 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9ff2d40ed0d0199d88fa414f519a977bee38df27907702663b1cc5bc85274b05 +F src/json.c c3167c9737da0f3f45538c7599bbf3f090d1ba1e444e4e743935496f37cbddcc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fdf00e96239c73fb67e2acecc5b95f55a1fc51c3deed4512613c0d6070ce5805 -R dbfc8fdcf920d7c2aea651d65adda878 +P c3677ba410208c07b711f5f526eb5cf039a8eee49f632c7ae04fa55cdfbb9058 +R 13d01dca5d473a81616de1c5ea4e5269 U drh -Z ecc9fe8f5fc6f3c09b592aceda67d13d +Z bee957be31c329bcce231111ed4f35d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f505eb9a85..0f4a0c9d1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3677ba410208c07b711f5f526eb5cf039a8eee49f632c7ae04fa55cdfbb9058 \ No newline at end of file +d2fba2cbdc3870d34228c1a9446eced884325acc183900d7dd0b96132570fb4a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 095ae870bf..83bf3e7337 100644 --- a/src/json.c +++ b/src/json.c @@ -603,7 +603,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; while( 1 /*exit-by-break*/ ){ k = 0; - while( k=N ){ if( k>0 ){ From 2c26bde4ff7e022c4d774ce9a4c27ca56ce65279 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 15:59:48 +0000 Subject: [PATCH 142/191] Do not allow a JsonParse object to be considered "editable" after an OOM. FossilOrigin-Name: c6bacf57bd6fe0fee00c9d41163a270b60997c20659949971bbf5c6c62622bfe --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 38a7efdf50..1affea972f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\spotential\sunsigned\sinteger\sunderflow\sin\sjsonAppendString(). -D 2023-12-02T15:06:43.574 +C Do\snot\sallow\sa\sJsonParse\sobject\sto\sbe\sconsidered\s"editable"\safter\san\sOOM. +D 2023-12-02T15:59:48.114 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c c3167c9737da0f3f45538c7599bbf3f090d1ba1e444e4e743935496f37cbddcc +F src/json.c 837599bcccc7b1533273d17697b33acb747f6a742e0789a716a1d991e83b76ea F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3677ba410208c07b711f5f526eb5cf039a8eee49f632c7ae04fa55cdfbb9058 -R 13d01dca5d473a81616de1c5ea4e5269 +P d2fba2cbdc3870d34228c1a9446eced884325acc183900d7dd0b96132570fb4a +R 4ccb83954df0ebdab5ce51302095cafb U drh -Z bee957be31c329bcce231111ed4f35d1 +Z 5295c6ae0590ca2382d336c94f2f32b0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0f4a0c9d1d..ba27b52b90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2fba2cbdc3870d34228c1a9446eced884325acc183900d7dd0b96132570fb4a \ No newline at end of file +c6bacf57bd6fe0fee00c9d41163a270b60997c20659949971bbf5c6c62622bfe \ No newline at end of file diff --git a/src/json.c b/src/json.c index 83bf3e7337..334829f9cf 100644 --- a/src/json.c +++ b/src/json.c @@ -1044,6 +1044,7 @@ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ u8 *aOld; u32 nSize; assert( !pParse->bReadOnly ); + if( pParse->oom ) return 0; if( pParse->nBlobAlloc>0 ) return 1; aOld = pParse->aBlob; nSize = pParse->nBlob + nExtra; From 05db513435d21909be60b7dcf48574fdc74b8308 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 16:11:22 +0000 Subject: [PATCH 143/191] Protect a memcpy() against OOM conditions. FossilOrigin-Name: 26144d1c25ae0435db568009ba05e485d23d146f2b1f29f3a426c87860316aed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1affea972f..4e9b31f235 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sa\sJsonParse\sobject\sto\sbe\sconsidered\s"editable"\safter\san\sOOM. -D 2023-12-02T15:59:48.114 +C Protect\sa\smemcpy()\sagainst\sOOM\sconditions. +D 2023-12-02T16:11:22.802 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 837599bcccc7b1533273d17697b33acb747f6a742e0789a716a1d991e83b76ea +F src/json.c 21ffece5a6e846480e6f108203d5ee3838261c0d8af773986895b1fd33fea593 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2fba2cbdc3870d34228c1a9446eced884325acc183900d7dd0b96132570fb4a -R 4ccb83954df0ebdab5ce51302095cafb +P c6bacf57bd6fe0fee00c9d41163a270b60997c20659949971bbf5c6c62622bfe +R 0ee4315c57b01c35ef946afcac5153f9 U drh -Z 5295c6ae0590ca2382d336c94f2f32b0 +Z 15254a8c6732e3004ca75d702e3d0bea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ba27b52b90..115ae173de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6bacf57bd6fe0fee00c9d41163a270b60997c20659949971bbf5c6c62622bfe \ No newline at end of file +26144d1c25ae0435db568009ba05e485d23d146f2b1f29f3a426c87860316aed \ No newline at end of file diff --git a/src/json.c b/src/json.c index 334829f9cf..cdc0d60c71 100644 --- a/src/json.c +++ b/src/json.c @@ -2239,12 +2239,14 @@ static u32 jsonLookupBlobStep( if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) ){ nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); - memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); - k = j + ix.nBlob; - memcpy(&pParse->aBlob[k], zKey, nKey); - k += nKey; - memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); - if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + if( !pParse->oom ){ + memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); + k = j + ix.nBlob; + memcpy(&pParse->aBlob[k], zKey, nKey); + k += nKey; + memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + } } jsonParseReset(&v); jsonParseReset(&ix); From c44041e03bc4d7ad0a5edbe8277a325eaaf5f5e6 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Dec 2023 17:32:16 +0000 Subject: [PATCH 144/191] Ensure that tokendata=1 queries avoid loading large doclists for queries like "common AND uncommon", just as tokendata=0 queries do. FossilOrigin-Name: 7bda09ab404a110d57449e149a3281fca8dc4cacf7bd9832ea2a1356ad20fe8e --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_expr.c | 25 ++++--- ext/fts5/fts5_index.c | 109 ++++++++++++++++++++++++++--- ext/fts5/fts5_main.c | 10 +++ ext/fts5/test/fts5origintext2.test | 1 - ext/fts5/test/fts5origintext3.test | 13 +++- ext/fts5/test/fts5origintext4.test | 66 +++++++++++++++++ manifest | 23 +++--- manifest.uuid | 2 +- 9 files changed, 217 insertions(+), 34 deletions(-) create mode 100644 ext/fts5/test/fts5origintext4.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index cee9528858..911f547d17 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -397,6 +397,7 @@ struct Fts5IndexIter { #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 #define FTS5INDEX_QUERY_SKIPHASH 0x0040 #define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 +#define FTS5INDEX_QUERY_SCANONETERM 0x0100 /* ** Create/destroy an Fts5Index object. @@ -786,6 +787,7 @@ int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); int sqlite3Fts5ExprInstToken(Fts5Expr*, int, int, int, int, const char**, int*); +void sqlite3Fts5ExprClearTokens(Fts5Expr*); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 89e7cbf364..95d102062d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3050,17 +3050,6 @@ int sqlite3Fts5ExprPopulatePoslists( sCtx.aPopulator = aPopulator; sCtx.iOff = (((i64)iCol) << 32) - 1; - /* If this is a tokendata=1 table, clear out the hash tables of - ** full-terms. */ - if( pConfig->bTokendata ){ - for(i=0; inPhrase; i++){ - Fts5ExprTerm *pT; - for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ - sqlite3Fts5IndexIterClearTokendata(pT->pIter); - } - } - } - for(i=0; inPhrase; i++){ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; Fts5Colset *pColset = pNode->pNear->pColset; @@ -3225,3 +3214,17 @@ int sqlite3Fts5ExprInstToken( return sqlite3Fts5IterToken(pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut); } +/* +** Clear the token mappings for all Fts5IndexIter objects mannaged by +** the expression passed as the only argument. +*/ +void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ + int ii; + for(ii=0; iinPhrase; ii++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } +} + diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 05f4c5e6aa..94b4767677 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -366,6 +366,7 @@ struct Fts5Index { sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxNextSelect; int nRead; /* Total number of blocks read */ sqlite3_stmt *pDeleteFromIdx; @@ -2660,7 +2661,7 @@ static void fts5SegIterSeekInit( fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); } - if( p->rc==SQLITE_OK && bGe==0 ){ + if( p->rc==SQLITE_OK && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM)) ){ pIter->flags |= FTS5_SEGITER_ONETERM; if( pIter->pLeaf ){ if( flags & FTS5INDEX_QUERY_DESC ){ @@ -2693,6 +2694,79 @@ static void fts5SegIterSeekInit( ); } + +/* +** SQL used by fts5SegIterNextInit() to find the page to open. +*/ +static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){ + if( p->pIdxNextSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? ORDER BY term ASC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + + } + return p->pIdxNextSelect; +} + +/* +** This is similar to fts5SegIterSeekInit(), except that it initializes +** the segment iterator to point to the first term following the page +** with pToken/nToken on it. +*/ +static void fts5SegIterNextInit( + Fts5Index *p, + const char *pTerm, int nTerm, + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = -1; /* Page of segment to open */ + int bDlidx = 0; + sqlite3_stmt *pSel = 0; /* SELECT to find iPg */ + + pSel = fts5IdxNextStmt(p); + if( pSel ){ + assert( p->rc==SQLITE_OK ); + sqlite3_bind_int(pSel, 1, pSeg->iSegid); + sqlite3_bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC); + + if( sqlite3_step(pSel)==SQLITE_ROW ){ + i64 val = sqlite3_column_int64(pSel, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pSel); + if( p->rc ) return; + } + + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + pIter->flags |= FTS5_SEGITER_ONETERM; + if( iPg>=0 ){ + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + fts5SegIterSetNext(p, pIter); + fts5SegIterAllocTombstone(p, pIter); + } + if( pIter->pLeaf ){ + const u8 *a = pIter->pLeaf->p; + int iTermOff = 0; + + pIter->iPgidxOff = pIter->pLeaf->szLeaf; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff); + pIter->iLeafOffset = iTermOff; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); + + assert( p->rc!=SQLITE_OK || + fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0 + ); + } +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within the ** in-memory hash table. If there is no such term in the hash-table, the @@ -6346,6 +6420,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxNextSelect); sqlite3_finalize(p->pDataVersion); sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); @@ -6496,7 +6571,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = nAlloc * sizeof(Fts5Iter*); + int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); if( pNew==0 ){ @@ -6513,6 +6588,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( }else{ pRet->apIter[pRet->nIter++] = pAppend; } + assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc ); return pRet; } @@ -6747,6 +6823,10 @@ static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ } } +/* +** This function sets up an iterator to use for a non-prefix query on a +** tokendata=1 table. +*/ static Fts5Iter *fts5SetupTokendataIter( Fts5Index *p, /* FTS index to query */ const u8 *pToken, /* Buffer containing query term */ @@ -6756,7 +6836,7 @@ static Fts5Iter *fts5SetupTokendataIter( Fts5Iter *pRet = 0; Fts5TokenDataIter *pSet = 0; Fts5Structure *pStruct = 0; - const int flags = FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_SCAN; + const int flags = FTS5INDEX_QUERY_SCANONETERM | FTS5INDEX_QUERY_SCAN; Fts5Buffer bSeek = {0, 0, 0}; Fts5Buffer *pSmall = 0; @@ -6787,20 +6867,32 @@ static Fts5Iter *fts5SetupTokendataIter( for(iLvl=0; iLvlnLevel; iLvl++){ for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; - fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + int bDone = 0; - pNewIter++; if( pPrevIter ){ if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ - fts5SegIterSetEOF(pPrevIter); + memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); + memset(pPrevIter, 0, sizeof(Fts5SegIter)); + bDone = 1; + }else if( pPrevIter->pLeaf + && pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf + ){ + fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); + bDone = 1; } - pPrevIter++; } + + if( bDone==0 ){ + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + } + + pNewIter++; + if( pPrevIter ) pPrevIter++; } } fts5TokendataSetTermIfEof(pPrev, pSmall); - pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->bSkipEmpty = 1; pNew->pColset = pColset; fts5IterSetOutputCb(&p->rc, pNew); @@ -7043,7 +7135,6 @@ int sqlite3Fts5IterToken( */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL ); if( pIter->pTokenDataIter ){ pIter->pTokenDataIter->nMap = 0; } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 34050474f8..e911b0c0f9 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -916,6 +916,16 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table, + ** clear any token mappings accumulated at the fts5_index.c level. In + ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH, + ** we need to retain the mappings for the entire query. */ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata + ){ + sqlite3Fts5ExprClearTokens(pCsr->pExpr); + } + if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test index 948db1c519..a27309fe0c 100644 --- a/ext/fts5/test/fts5origintext2.test +++ b/ext/fts5/test/fts5origintext2.test @@ -98,7 +98,6 @@ do_execsql_test 1.10 { INSERT INTO ft VALUES('WORLD'); } -breakpoint do_execsql_test 1.11 { SELECT rowid FROM ft('hello'); } {1 2 3} do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index 2b1e5c6387..ac00bfabc0 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -46,7 +46,6 @@ foreach_detail_mode $testprefix { SELECT fts5_test_poslist(ft) FROM ft('hello'); } {{0.0.0 0.0.2 0.0.4}} -breakpoint do_execsql_test 1.3 { SELECT insttoken(ft, 0, 0), @@ -63,6 +62,18 @@ breakpoint FROM ft('hello') ORDER BY rank; } {hello.Hello hello.HELLO hello} + do_execsql_test 1.5 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + INSERT INTO ft2(rowid, x) VALUES(1, 'ONE one two three ONE'); + INSERT INTO ft2(rowid, x) VALUES(2, 'TWO one two three TWO'); + INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE'); + } + + do_execsql_test 1.6 { + SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank; + } {three.THREE 3 three 1 three 2} } finish_test diff --git a/ext/fts5/test/fts5origintext4.test b/ext/fts5/test/fts5origintext4.test new file mode 100644 index 0000000000..8973a24b05 --- /dev/null +++ b/ext/fts5/test/fts5origintext4.test @@ -0,0 +1,66 @@ +# 2023 November 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext4 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft SELECT 'the first thing'; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<90000 + ) + INSERT INTO ft SELECT 'The second thing' FROM s; + + INSERT INTO ft SELECT 'the first thing'; + COMMIT; + INSERT INTO ft(ft) VALUES('optimize'); +} + +foreach {tn sql expr} { + 1 { SELECT rowid FROM ft('the') } {$mem > 250000} + 2 { SELECT rowid FROM ft('first') } {$mem < 50000} + 3 { SELECT rowid FROM ft('the first') } {$mem < 50000} +} { + db close + sqlite3 db test.db + sqlite3_fts5_register_origintext db + + execsql $sql + do_test 1.2.$tn { + set mem [lindex [sqlite3_db_status db CACHE_USED 0] 1] + expr $expr + } 1 +} + +proc b {x} { string map [list "\0" "."] $x } +db func b b +# execsql_pp { SELECT segid, b(term), pgno from ft_idx } + +finish_test + diff --git a/manifest b/manifest index 92cc5ca8d2..60fd22f769 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sold\scode\sfor\stokendata=1\squeries. -D 2023-12-01T20:37:11.688 +C Ensure\sthat\stokendata=1\squeries\savoid\sloading\slarge\sdoclists\sfor\squeries\slike\s"common\sAND\suncommon",\sjust\sas\stokendata=0\squeries\sdo. +D 2023-12-02T17:32:16.568 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -90,14 +90,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 5e5630fc81e212f658afaa5b2650dac939d2729d0723aef1eeaff908f1725648 -F ext/fts5/fts5Int.h 2dc73393460e5c5cab67adc7e32e1387cc225b57e05f629d490e65cddea1a8c5 +F ext/fts5/fts5Int.h 285118aa6dfccb382e84eaeb9f7bec334e4f7104efa9303240605447003445c9 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c aac8026aedf56c9a6e32b31c89f9dd7e5548378457085093307d06be14f1a176 +F ext/fts5/fts5_expr.c f83259b52b7b3e76768b835fe155cb7e345affdfafb96574372b4127d5f5496a F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 2296bcd6736eaf093212892474619dfdb7ac1e262732e2b72cb528172c0b13d6 -F ext/fts5/fts5_main.c 20596de592af135f68b9be875f0a28715f6562bbdedd215e1c89eac1b42e97f9 +F ext/fts5/fts5_index.c a02b6ff2d391dd9c2119f437eba1e8af5ac4b2f1798c7c39a93d73de95ad2337 +F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -191,8 +191,9 @@ F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0703aff5981625b527e62 -F ext/fts5/test/fts5origintext2.test 3259b331073fec918e02fd4d14d50586f9a3531da047a2a8f4624983eb654229 -F ext/fts5/test/fts5origintext3.test cb0f5835f8dff5954ee20570b68ee520cf04a08f6f9ca967b9d01d27e532da37 +F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 +F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a +F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2148,8 +2149,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8258967411d3ff212424b25fec79ded0d8ae83e773cd35a0bbf300c94923f25b -R d3ee6e0cbe8554b06da39b239b5142bc +P b0a489e8e1bf0290c2117ab32d78b1cc7d67bcb226b55ec044c8367ebde3815b +R aa740197e8da931b3ffaf4ddf853a1ed U dan -Z cbe01276ff2d44c618e4cc899051c453 +Z 62655ccf950056e7477fcaff4611778d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 359f2cca45..7fc4017fab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0a489e8e1bf0290c2117ab32d78b1cc7d67bcb226b55ec044c8367ebde3815b \ No newline at end of file +7bda09ab404a110d57449e149a3281fca8dc4cacf7bd9832ea2a1356ad20fe8e \ No newline at end of file From a11aaff05ae0f9b43f4b717e0b3158c1e36e1c10 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 18:04:27 +0000 Subject: [PATCH 145/191] Take extra care to ensure that JSONB values that are in cache are actually owned by the JSON subsystem, and that ownership of such values is not handed back to the bytecode engine. FossilOrigin-Name: 1304534001e9ef66c6b12752b69d790bfa3427cc803f87cc48ca22ae12df0fdf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4e9b31f235..87767f5859 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Protect\sa\smemcpy()\sagainst\sOOM\sconditions. -D 2023-12-02T16:11:22.802 +C Take\sextra\scare\sto\sensure\sthat\sJSONB\svalues\sthat\sare\sin\scache\sare\sactually\nowned\sby\sthe\sJSON\ssubsystem,\sand\sthat\sownership\sof\ssuch\svalues\sis\snot\shanded\nback\sto\sthe\sbytecode\sengine. +D 2023-12-02T18:04:27.395 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 21ffece5a6e846480e6f108203d5ee3838261c0d8af773986895b1fd33fea593 +F src/json.c 4c6b5c0c731fe7a2b2d28467af747c4744370bd47b5f9d6b7531efb8617eda37 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c6bacf57bd6fe0fee00c9d41163a270b60997c20659949971bbf5c6c62622bfe -R 0ee4315c57b01c35ef946afcac5153f9 +P 26144d1c25ae0435db568009ba05e485d23d146f2b1f29f3a426c87860316aed +R 4e2a070d847085a145ada5a2fea29659 U drh -Z 15254a8c6732e3004ca75d702e3d0bea +Z 2cced16db85959c5b0bfa29c3e3e2f71 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 115ae173de..18c274186a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -26144d1c25ae0435db568009ba05e485d23d146f2b1f29f3a426c87860316aed \ No newline at end of file +1304534001e9ef66c6b12752b69d790bfa3427cc803f87cc48ca22ae12df0fdf \ No newline at end of file diff --git a/src/json.c b/src/json.c index cdc0d60c71..586371d9d4 100644 --- a/src/json.c +++ b/src/json.c @@ -375,6 +375,7 @@ static int jsonCacheInsert( memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); p->nUsed = JSON_CACHE_SIZE-1; } + assert( pParse->nBlobAlloc>0 ); pParse->eEdit = 0; pParse->nJPRef++; pParse->bReadOnly = 1; @@ -731,7 +732,7 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); }else if( jsonForceRCStr(p) ){ - if( pParse && pParse->bJsonIsRCStr==0 ){ + if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ int rc; pParse->zJson = sqlite3RCStrRef(p->zBuf); pParse->nJson = p->nUsed; @@ -1751,6 +1752,8 @@ static void jsonReturnStringAsBlob(JsonString *pStr){ sqlite3_free(px.aBlob); sqlite3_result_error_nomem(pStr->pCtx); }else{ + assert( px.nBlobAlloc>0 ); + assert( !px.bReadOnly ); sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, sqlite3_free); } } @@ -2841,9 +2844,12 @@ static void jsonReturnParse( } flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( flgs & JSON_BLOB ){ - sqlite3_result_blob(ctx, p->aBlob, p->nBlob, - p->nBlobAlloc>0 ? SQLITE_DYNAMIC : SQLITE_TRANSIENT); - p->nBlobAlloc = 0; + if( p->nBlobAlloc>0 && !p->bReadOnly ){ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_DYNAMIC); + p->nBlobAlloc = 0; + }else{ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_TRANSIENT); + } }else{ JsonString s; jsonStringInit(&s, ctx); @@ -3063,6 +3069,8 @@ static void jsonbFunc( if( jsonConvertTextToBlob(pParse, ctx) ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ + assert( pParse->nBlobAlloc>0 ); + assert( !pParse->bReadOnly ); sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); pParse->aBlob = 0; pParse->nBlob = 0; From 94c521295aa898eb07dcfc4cf4ccdeb04ff7d735 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Dec 2023 18:14:07 +0000 Subject: [PATCH 146/191] When tokendata=1 queries require multiple segment-cursors, allow those cursors to share a single array of in-memory tombstone pages. FossilOrigin-Name: e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f --- ext/fts5/fts5_index.c | 65 ++++++++++++++++++++++++++++++++----------- manifest | 12 ++++---- manifest.uuid | 2 +- 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 94b4767677..9edf184c71 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -324,6 +324,7 @@ typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; typedef struct Fts5TokenDataIter Fts5TokenDataIter; +typedef struct Fts5TombstoneArray Fts5TombstoneArray; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ @@ -520,8 +521,7 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ i64 iLeafOffset; /* Byte offset within current leaf */ - Fts5Data **apTombstone; /* Array of tombstone pages */ - int nTombstone; + Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); @@ -548,6 +548,12 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +struct Fts5TombstoneArray { + int nRef; /* Number of pointers to this object */ + int nTombstone; + Fts5Data *apTombstone[1]; /* Array of tombstone pages */ +}; + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -1924,11 +1930,13 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ const int nTomb = pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - Fts5Data **apTomb = 0; - apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb); - if( apTomb ){ - pIter->apTombstone = apTomb; - pIter->nTombstone = nTomb; + int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + Fts5TombstoneArray *pNew; + pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ + pNew->nTombstone = nTomb; + pNew->nRef = 1; + pIter->pTombArray = pNew; } } } @@ -2677,7 +2685,9 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); - fts5SegIterAllocTombstone(p, pIter); + if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM) ){ + fts5SegIterAllocTombstone(p, pIter); + } /* Either: ** @@ -2852,6 +2862,19 @@ static void fts5IndexFreeArray(Fts5Data **ap, int n){ } } +static void fts5TombstoneArrayDelete(Fts5TombstoneArray *p){ + if( p ){ + p->nRef--; + if( p->nRef<=0 ){ + int ii; + for(ii=0; iinTombstone; ii++){ + fts5DataRelease(p->apTombstone[ii]); + } + sqlite3_free(p); + } + } +} + /* ** Zero the iterator passed as the only argument. */ @@ -2859,7 +2882,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); - fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone); + fts5TombstoneArrayDelete(pIter->pTombArray); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -3248,24 +3271,25 @@ static int fts5IndexTombstoneQuery( static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + Fts5TombstoneArray *pArray = pSeg->pTombArray; - if( pSeg->pLeaf && pSeg->nTombstone ){ + if( pSeg->pLeaf && pArray ){ /* Figure out which page the rowid might be present on. */ - int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone; + int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone; assert( iPg>=0 ); /* If tombstone hash page iPg has not yet been loaded from the ** database, load it now. */ - if( pSeg->apTombstone[iPg]==0 ){ - pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + if( pArray->apTombstone[iPg]==0 ){ + pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex, FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) ); - if( pSeg->apTombstone[iPg]==0 ) return 0; + if( pArray->apTombstone[iPg]==0 ) return 0; } return fts5IndexTombstoneQuery( - pSeg->apTombstone[iPg], - pSeg->nTombstone, + pArray->apTombstone[iPg], + pArray->nTombstone, pSeg->iRowid ); } @@ -6886,6 +6910,15 @@ static Fts5Iter *fts5SetupTokendataIter( fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); } + if( pPrevIter ){ + if( pPrevIter->pTombArray ){ + pNewIter->pTombArray = pPrevIter->pTombArray; + pNewIter->pTombArray->nRef++; + } + }else{ + fts5SegIterAllocTombstone(p, pNewIter); + } + pNewIter++; if( pPrevIter ) pPrevIter++; } diff --git a/manifest b/manifest index 60fd22f769..699d024ca1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\stokendata=1\squeries\savoid\sloading\slarge\sdoclists\sfor\squeries\slike\s"common\sAND\suncommon",\sjust\sas\stokendata=0\squeries\sdo. -D 2023-12-02T17:32:16.568 +C When\stokendata=1\squeries\srequire\smultiple\ssegment-cursors,\sallow\sthose\scursors\sto\sshare\sa\ssingle\sarray\sof\sin-memory\stombstone\spages. +D 2023-12-02T18:14:07.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c f83259b52b7b3e76768b835fe155cb7e345affdfafb96574372b4127d5f5496a F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c a02b6ff2d391dd9c2119f437eba1e8af5ac4b2f1798c7c39a93d73de95ad2337 +F ext/fts5/fts5_index.c 21f8f449666ac44c12d5051e153ad84a886a729cb2f5d6ad02a113095c3f8ec6 F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2149,8 +2149,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b0a489e8e1bf0290c2117ab32d78b1cc7d67bcb226b55ec044c8367ebde3815b -R aa740197e8da931b3ffaf4ddf853a1ed +P 7bda09ab404a110d57449e149a3281fca8dc4cacf7bd9832ea2a1356ad20fe8e +R 1ce6343b4aa590c869e9b9aa51095415 U dan -Z 62655ccf950056e7477fcaff4611778d +Z 8664660cde606ab1a6fdc20b18622c4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7fc4017fab..0cbd084480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bda09ab404a110d57449e149a3281fca8dc4cacf7bd9832ea2a1356ad20fe8e \ No newline at end of file +e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f \ No newline at end of file From 53c2160db04921c76e62c6eef3329cbe637b8984 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 18:17:38 +0000 Subject: [PATCH 147/191] Fix harmless compiler warnings. Refactor some identifier names for clearer presentation. FossilOrigin-Name: 7e3941502789c5afaf19b08112f464abf5e3cba7f92fc9290af2a0f96127ad9a --- manifest | 12 ++--- manifest.uuid | 2 +- src/json.c | 124 ++++++++++++++++++++++++++------------------------ 3 files changed, 71 insertions(+), 67 deletions(-) diff --git a/manifest b/manifest index 87767f5859..6f7377cfb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sextra\scare\sto\sensure\sthat\sJSONB\svalues\sthat\sare\sin\scache\sare\sactually\nowned\sby\sthe\sJSON\ssubsystem,\sand\sthat\sownership\sof\ssuch\svalues\sis\snot\shanded\nback\sto\sthe\sbytecode\sengine. -D 2023-12-02T18:04:27.395 +C Fix\sharmless\scompiler\swarnings.\s\sRefactor\ssome\sidentifier\snames\sfor\nclearer\spresentation. +D 2023-12-02T18:17:38.516 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4c6b5c0c731fe7a2b2d28467af747c4744370bd47b5f9d6b7531efb8617eda37 +F src/json.c b67328611c8ebc5aca49e2d73b1bad9962948aa5c952f3d96c44eb74ee3d0010 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 26144d1c25ae0435db568009ba05e485d23d146f2b1f29f3a426c87860316aed -R 4e2a070d847085a145ada5a2fea29659 +P 1304534001e9ef66c6b12752b69d790bfa3427cc803f87cc48ca22ae12df0fdf +R 1b492fad564bbccdff370d30a804d641 U drh -Z 2cced16db85959c5b0bfa29c3e3e2f71 +Z f1274565e5b6e1f5da018a4ab14dfe40 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 18c274186a..05b1875e6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1304534001e9ef66c6b12752b69d790bfa3427cc803f87cc48ca22ae12df0fdf \ No newline at end of file +7e3941502789c5afaf19b08112f464abf5e3cba7f92fc9290af2a0f96127ad9a \ No newline at end of file diff --git a/src/json.c b/src/json.c index 586371d9d4..064d9457ce 100644 --- a/src/json.c +++ b/src/json.c @@ -287,7 +287,7 @@ struct JsonParse { u8 bReadOnly; /* Do not modify. */ u32 nJPRef; /* Number of references to this object */ u32 iErr; /* Error location in zJson[] */ - /* Search and edit information. See jsonLookupBlobStep() */ + /* Search and edit information. See jsonLookupStep() */ u8 eEdit; /* Edit operation to apply */ int delta; /* Size change due to the edit */ u32 nIns; /* Number of bytes to insert */ @@ -2119,20 +2119,24 @@ static void jsonBlobEdit( } /* -** Error returns from jsonLookupBlobStep() +** Error returns from jsonLookupStep() */ -#define JSON_BLOB_ERROR 0xffffffff -#define JSON_BLOB_NOTFOUND 0xfffffffe -#define JSON_BLOB_PATHERROR 0xfffffffd -#define JSON_BLOB_ISERROR(x) ((x)>=JSON_BLOB_PATHERROR) +#define JSON_LOOKUP_ERROR 0xffffffff +#define JSON_LOOKUP_NOTFOUND 0xfffffffe +#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* ** Search along zPath to find the Json element specified. Return an ** index into pParse->aBlob[] for the start of that element's value. ** -** Return JSON_BLOB_NOTFOUND if no such element exists. +** If the value found by this routine is the value half of label/value pair +** within an object, then set pPath->iLabel to the start of the corresponding +** label, before returning. +** +** Return one of the JSON_LOOKUP error codes if problems are seen. */ -static u32 jsonLookupBlobStep( +static u32 jsonLookupStep( JsonParse *pParse, /* The JSON to search */ u32 iRoot, /* Begin the search at this element of aBlob[] */ const char *zPath, /* The path to search */ @@ -2173,7 +2177,7 @@ static u32 jsonLookupBlobStep( if( zPath[i] ){ i++; }else{ - return JSON_BLOB_PATHERROR; + return JSON_LOOKUP_PATHERROR; } testcase( nKey==0 ); }else{ @@ -2181,37 +2185,37 @@ static u32 jsonLookupBlobStep( for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} nKey = i; if( nKey==0 ){ - return JSON_BLOB_PATHERROR; + return JSON_LOOKUP_PATHERROR; } } - if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_BLOB_NOTFOUND; + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); j = iRoot + n; /* j is the index of a label */ iEnd = j+sz; while( jaBlob[j] & 0x0f; - if( xJSONB_TEXTRAW ) return JSON_BLOB_ERROR; + if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; n = jsonbPayloadSize(pParse, j, &sz); - if( n==0 ) return JSON_BLOB_ERROR; + if( n==0 ) return JSON_LOOKUP_ERROR; k = j+n; /* k is the index of the label text */ - if( k+sz>=iEnd ) return JSON_BLOB_ERROR; + if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ u32 v = k+sz; /* v is the index of the value */ - if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; n = jsonbPayloadSize(pParse, v, &sz); - if( n==0 || v+n+sz>iEnd ) return JSON_BLOB_ERROR; + if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; assert( j>0 ); - rc = jsonLookupBlobStep(pParse, v, &zPath[i], j); + rc = jsonLookupStep(pParse, v, &zPath[i], j); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } j = k+sz; - if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_BLOB_ERROR; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; n = jsonbPayloadSize(pParse, j, &sz); - if( n==0 ) return JSON_BLOB_ERROR; + if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } - if( j>iEnd ) return JSON_BLOB_ERROR; + if( j>iEnd ) return JSON_LOOKUP_ERROR; if( pParse->eEdit>=JEDIT_INS ){ u32 nIns; /* Total bytes to insert (label+value) */ JsonParse v; /* BLOB encoding of the value to be inserted */ @@ -2230,8 +2234,8 @@ static u32 jsonLookupBlobStep( v.eEdit = pParse->eEdit; v.nIns = pParse->nIns; v.aIns = pParse->aIns; - rc = jsonLookupBlobStep(&v, 0, &zPath[i], 0); - if( JSON_BLOB_ISERROR(rc) || v.oom ){ + rc = jsonLookupStep(&v, 0, &zPath[i], 0); + if( JSON_LOOKUP_ISERROR(rc) || v.oom ){ pParse->oom |= v.oom; jsonParseReset(&v); jsonParseReset(&ix); @@ -2257,7 +2261,7 @@ static u32 jsonLookupBlobStep( } }else if( zPath[0]=='[' ){ x = pParse->aBlob[iRoot] & 0x0f; - if( x!=JSONB_ARRAY ) return JSON_BLOB_NOTFOUND; + if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); k = 0; i = 1; @@ -2276,31 +2280,31 @@ static u32 jsonLookupBlobStep( nn = nn*10 + zPath[i] - '0'; i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>k ) return JSON_BLOB_NOTFOUND; + if( nn>k ) return JSON_LOOKUP_NOTFOUND; k -= nn; } if( zPath[i]!=']' ){ - return JSON_BLOB_PATHERROR; + return JSON_LOOKUP_PATHERROR; } }else{ - return JSON_BLOB_PATHERROR; + return JSON_LOOKUP_PATHERROR; } } j = iRoot+n; iEnd = j+sz; while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } k--; n = jsonbPayloadSize(pParse, j, &sz); - if( n==0 ) return JSON_BLOB_ERROR; + if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } - if( j>iEnd ) return JSON_BLOB_ERROR; - if( k>0 ) return JSON_BLOB_NOTFOUND; + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( k>0 ) return JSON_LOOKUP_NOTFOUND; if( pParse->eEdit>=JEDIT_INS ){ JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); @@ -2315,8 +2319,8 @@ static u32 jsonLookupBlobStep( v.eEdit = pParse->eEdit; v.nIns = pParse->nIns; v.aIns = pParse->aIns; - rc = jsonLookupBlobStep(&v, 0, &zPath[i+1], 0); - if( JSON_BLOB_ISERROR(rc) || v.oom ){ + rc = jsonLookupStep(&v, 0, &zPath[i+1], 0); + if( JSON_LOOKUP_ISERROR(rc) || v.oom ){ pParse->oom |= v.oom; jsonParseReset(&v); return rc; @@ -2330,9 +2334,9 @@ static u32 jsonLookupBlobStep( return j; } }else{ - return JSON_BLOB_PATHERROR; + return JSON_LOOKUP_PATHERROR; } - return JSON_BLOB_NOTFOUND; + return JSON_LOOKUP_NOTFOUND; } /* @@ -2670,11 +2674,11 @@ static void jsonInsertIntoBlob( p->nIns = ax.nBlob; p->aIns = ax.aBlob; p->delta = 0; - rc = jsonLookupBlobStep(p, 0, zPath+1, 0); + rc = jsonLookupStep(p, 0, zPath+1, 0); } jsonParseReset(&ax); - if( rc==JSON_BLOB_NOTFOUND ) continue; - if( JSON_BLOB_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; + if( rc==JSON_LOOKUP_NOTFOUND ) continue; + if( JSON_LOOKUP_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; } jsonReturnParse(ctx, p); jsonParseFree(p); @@ -2682,7 +2686,7 @@ static void jsonInsertIntoBlob( jsonInsertIntoBlob_patherror: jsonParseFree(p); - if( rc==JSON_BLOB_ERROR ){ + if( rc==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ jsonBadPathError(ctx, zPath); @@ -3149,11 +3153,11 @@ static void jsonArrayLengthFunc( jsonParseFree(p); return; } - i = jsonLookupBlobStep(p, 0, zPath[0]=='$' ? zPath+1 : "@", 0); - if( JSON_BLOB_ISERROR(i) ){ - if( i==JSON_BLOB_NOTFOUND ){ + i = jsonLookupStep(p, 0, zPath[0]=='$' ? zPath+1 : "@", 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_BLOB_PATHERROR ){ + }else if( i==JSON_LOOKUP_PATHERROR ){ jsonBadPathError(ctx, zPath); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); @@ -3230,7 +3234,7 @@ static void jsonExtractFunc( u32 j; if( zPath==0 ) goto json_extract_error; if( zPath[0]=='$' ){ - j = jsonLookupBlobStep(p, 0, zPath+1, 0); + j = jsonLookupStep(p, 0, zPath+1, 0); }else if( (flags & JSON_ABPATH) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for @@ -3253,7 +3257,7 @@ static void jsonExtractFunc( jsonAppendRaw(&jx, zPath, nPath); } jsonStringTerminate(&jx); - j = jsonLookupBlobStep(p, 0, jx.zBuf, 0); + j = jsonLookupStep(p, 0, jx.zBuf, 0); jsonStringReset(&jx); }else{ jsonBadPathError(ctx, zPath); @@ -3280,14 +3284,14 @@ static void jsonExtractFunc( jsonAppendSeparator(&jx); jsonXlateBlobToText(p, j, &jx); } - }else if( j==JSON_BLOB_NOTFOUND ){ + }else if( j==JSON_LOOKUP_NOTFOUND ){ if( argc==2 ){ goto json_extract_error; /* Return NULL if not found */ }else{ jsonAppendSeparator(&jx); jsonAppendRawNZ(&jx, "null", 4); } - }else if( j==JSON_BLOB_ERROR ){ + }else if( j==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); goto json_extract_error; }else{ @@ -3613,8 +3617,8 @@ static void jsonRemoveFunc( ){ JsonParse *p; /* The parse */ const char *zPath = 0; /* Path of element to be removed */ - u32 i; /* Loop counter */ - int rc; /* Subroutine return code */ + int i; /* Loop counter */ + u32 rc; /* Subroutine return code */ if( argc<1 ) return; p = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); @@ -3636,9 +3640,9 @@ static void jsonRemoveFunc( } p->eEdit = JEDIT_DEL; p->delta = 0; - rc = jsonLookupBlobStep(p, 0, zPath+1, 0); - if( rc==JSON_BLOB_NOTFOUND ) continue; - if( JSON_BLOB_ISERROR(rc) ) goto json_remove_patherror; + rc = jsonLookupStep(p, 0, zPath+1, 0); + if( rc==JSON_LOOKUP_NOTFOUND ) continue; + if( JSON_LOOKUP_ISERROR(rc) ) goto json_remove_patherror; } jsonReturnParse(ctx, p); jsonParseFree(p); @@ -3738,11 +3742,11 @@ static void jsonTypeFunc( jsonBadPathError(ctx, zPath); goto json_type_done; } - i = jsonLookupBlobStep(p, 0, zPath+1, 0); - if( JSON_BLOB_ISERROR(i) ){ - if( i==JSON_BLOB_NOTFOUND ){ + i = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_BLOB_PATHERROR ){ + }else if( i==JSON_LOOKUP_PATHERROR ){ jsonBadPathError(ctx, zPath); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); @@ -4361,8 +4365,8 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ if( levelChange ){ if( p->nParent>0 ){ JsonParent *pParent = &p->aParent[p->nParent-1]; - u32 i = pParent->iValue; - p->eType = p->sParse.aBlob[i] & 0x0f; + u32 iVal = pParent->iValue; + p->eType = p->sParse.aBlob[iVal] & 0x0f; }else{ p->eType = 0; } @@ -4628,9 +4632,9 @@ static int jsonEachFilter( i = p->i = 0; p->eType = 0; }else{ - i = jsonLookupBlobStep(&p->sParse, 0, zRoot+1, 0); - if( JSON_BLOB_ISERROR(i) ){ - if( i==JSON_BLOB_NOTFOUND ){ + i = jsonLookupStep(&p->sParse, 0, zRoot+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ p->i = 0; p->eType = 0; p->iEnd = 0; From c1e85742da3633cb9a154043b22ffb3c64c31bdc Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 20:25:36 +0000 Subject: [PATCH 148/191] Code and comment cleanup. Everything should work the same. FossilOrigin-Name: c640754df0d3ffdad994745f0d0e10c8f19f424b87f6a6e6e269491a0350b950 --- manifest | 12 +-- manifest.uuid | 2 +- src/json.c | 201 +++++++++++++++++++++++++++++--------------------- 3 files changed, 125 insertions(+), 90 deletions(-) diff --git a/manifest b/manifest index 6f7377cfb8..6e1a782d31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings.\s\sRefactor\ssome\sidentifier\snames\sfor\nclearer\spresentation. -D 2023-12-02T18:17:38.516 +C Code\sand\scomment\scleanup.\s\sEverything\sshould\swork\sthe\ssame. +D 2023-12-02T20:25:36.578 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c b67328611c8ebc5aca49e2d73b1bad9962948aa5c952f3d96c44eb74ee3d0010 +F src/json.c 5ad1a1be6199359b09bb4eca690789b77375e00f17c55697b6288f1b0fbbe8b0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1304534001e9ef66c6b12752b69d790bfa3427cc803f87cc48ca22ae12df0fdf -R 1b492fad564bbccdff370d30a804d641 +P 7e3941502789c5afaf19b08112f464abf5e3cba7f92fc9290af2a0f96127ad9a +R 9f984966a856d28c36c8f0d6d69732a4 U drh -Z f1274565e5b6e1f5da018a4ab14dfe40 +Z ea3e13091dceb571341f1dccf0e97828 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 05b1875e6c..9a68b3807a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e3941502789c5afaf19b08112f464abf5e3cba7f92fc9290af2a0f96127ad9a \ No newline at end of file +c640754df0d3ffdad994745f0d0e10c8f19f424b87f6a6e6e269491a0350b950 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 064d9457ce..9356bfa8d5 100644 --- a/src/json.c +++ b/src/json.c @@ -133,12 +133,13 @@ #define JSONB_ARRAY 11 /* An array */ #define JSONB_OBJECT 12 /* An object */ -/* Human-readable names for the JSONB values: +/* Human-readable names for the JSONB values. The index for each +** string must correspond to the JSONB_* integer above. */ static const char * const jsonbType[] = { "null", "true", "false", "integer", "integer", "real", "real", "text", "text", "text", - "text", "array", "object" + "text", "array", "object", "", "", "", "" }; /* @@ -168,7 +169,7 @@ static const char jsonIsSpace[] = { #define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) /* -** Characters that are special to JSON. Control charaters, +** Characters that are special to JSON. Control characters, ** '"' and '\\'. */ static const char jsonIsOk[256] = { @@ -204,7 +205,6 @@ typedef struct JsonCache JsonCache; typedef struct JsonString JsonString; typedef struct JsonParse JsonParse; - /* ** Magic number used for the JSON parse cache in sqlite3_get_auxdata() */ @@ -213,8 +213,14 @@ typedef struct JsonParse JsonParse; /* A cache mapping JSON text into JSONB blobs. ** -** All content, both JSON text and the JSONB blobs, is stored as RCStr -** objects. +** Each cache entry is a JsonParse object with the following restrictions: +** +** * The bReadOnly flag must be set +** +** * The aBlob[] array must be owned by the JsonParse object. In other +** words, nBlobAlloc must be non-zero. +** +** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. */ struct JsonCache { sqlite3 *db; /* Database connection */ @@ -225,6 +231,10 @@ struct JsonCache { /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. +** +** If the generated string is longer than will fit into the zSpace[] buffer, +** then it will be an RCStr string. This aids with caching of large +** JSON strings. */ struct JsonString { sqlite3_context *pCtx; /* Function context - put error messages here */ @@ -260,18 +270,17 @@ struct JsonString { /* A parsed JSON value. Lifecycle: ** ** 1. JSON comes in and is parsed into a JSONB value in aBlob. The -** original text is stored in zJson. +** original text is stored in zJson. This step is skipped if the +** input is JSONB instead of text JSON. ** -** 2. The aBlob is searched using the JSON path notation, if needed. +** 2. The aBlob[] array is searched using the JSON path notation, if needed. ** -** 3. Zero or more changes are made to aBlob (via json_remove() or -** json_replace() or similar). +** 3. Zero or more changes are made to aBlob[] (via json_remove() or +** json_replace() or json_patch() or similar). ** -** 4. New JSON text is generated from the aBlob for output. -** -** Step 1 is omitted if the input is a BLOB in the JSONB format. Step 4 -** is omitted if the output is JSONB or some other value that is not -** JSON text. +** 4. New JSON text is generated from the aBlob[] for output. This step +** is skipped the function is one of the jsonb_* functions that returns +** JSONB instead of text JSON. */ struct JsonParse { u8 *aBlob; /* JSONB representation of JSON value */ @@ -326,7 +335,6 @@ static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); - /************************************************************************** ** Utility routines for dealing with JsonCache objects **************************************************************************/ @@ -346,11 +354,11 @@ static void jsonCacheDeleteGeneric(void *p){ } /* -** Insert a new entry into the cache. If the cache is full, expell +** Insert a new entry into the cache. If the cache is full, expel ** the least recently used entry. Return SQLITE_OK on success or a ** result code otherwise. ** -** Both the input JSON and JSONB must be RCStr objects. +** Cache entries are stored in age order, oldest first. */ static int jsonCacheInsert( sqlite3_context *ctx, /* The SQL statement context holding the cache */ @@ -386,7 +394,14 @@ static int jsonCacheInsert( /* ** Search for a cached translation the json text supplied by pArg. Return -** the JsonParse object if found. +** the JsonParse object if found. Return NULL if not found. +** +** When a match if found, the matching entry is moved to become the +** most-recently used entry if it isn't so already. +** +** The JsonParse object returned still belongs to the Cache and might +** be deleted at any moment. If the caller whants the JsonParse to +** linger, it needs to increment the nPJRef reference counter. */ static JsonParse *jsonCacheSearch( sqlite3_context *ctx, /* The SQL statement context holding the cache */ @@ -419,6 +434,7 @@ static JsonParse *jsonCacheSearch( } if( inUsed ){ if( inUsed-1 ){ + /* Make the matching entry the most recently used entry */ JsonParse *tmp = p->a[i]; memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); p->a[p->nUsed-1] = tmp; @@ -561,7 +577,9 @@ static void jsonStringTerminate(JsonString *p){ } } -/* Try to force the string to be a zero-terminated RCStr string. +/* Try to force the string to be a zero-terminated RCStr string. In other +** words, make sure it is not still using the internal zSpace[] static +** buffer. ** ** Return true on success. Return false if an OOM prevents this ** from happening. @@ -591,9 +609,12 @@ static void jsonAppendSeparator(JsonString *p){ } /* Append the N-byte string in zIn to the end of the JsonString string -** under construction. Enclose the string in "..." and escape -** any double-quotes or backslash characters contained within the +** under construction. Enclose the string in double-quotes ("...") and +** escape any double-quotes or backslash characters contained within the ** string. +** +** This routine is a high-runner. There is a measurable performance +** increase associated with unwinding the jsonIsOk[] loop. */ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 k; @@ -604,8 +625,8 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; while( 1 /*exit-by-break*/ ){ k = 0; - while( k+1=N ){ if( k>0 ){ memcpy(&p->zBuf[p->nUsed], z, k); @@ -783,12 +804,8 @@ static void jsonParseReset(JsonParse *pParse){ } /* -** Free a JsonParse object that was obtained from sqlite3_malloc(). -** -** Note that destroying JsonParse might call sqlite3RCStrUnref() to -** destroy the zJson value. The RCStr object might recursively invoke -** JsonParse to destroy this pParse object again. Take care to ensure -** that this recursive destructor sequence terminates harmlessly. +** Decrement the reference count on the JsonParse object. When the +** count reaches zero, free the object. */ static void jsonParseFree(JsonParse *pParse){ if( pParse ){ @@ -1010,7 +1027,6 @@ static void jsonWrongNumArgs( ** Utility routines for dealing with the binary BLOB representation of JSON ****************************************************************************/ - /* ** Expand pParse->aBlob so that it holds at least N bytes. ** @@ -2068,7 +2084,8 @@ static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ } /* -** Edit the size of the element at iRoot by the amount in pParse->delta. +** Edit the payload size of the element at iRoot by the amount in +** pParse->delta. */ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ u32 sz = 0; @@ -2126,6 +2143,55 @@ static void jsonBlobEdit( #define JSON_LOOKUP_PATHERROR 0xfffffffd #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) +/* Forward declaration */ +static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); + + +/* This helper routine for jsonLookupStep() populates pIns with +** binary data that is to be inserted into pParse. +** +** In the common case, pIns just points to pParse->aIns and pParse->nIns. +** But if the zPath of the original edit operation includes path elements +** that go deeper, additional substructure must be created. +** +** For example: +** +** json_insert('{}', '$.a.b.c', 123); +** +** The search stops at '$.a' But additional substructure must be +** created for the ".b.c" part of the patch so that the final result +** is: {"a":{"b":{"c"::123}}}. This routine populates pIns with +** the binary equivalent of {"b":{"c":123}} so that it can be inserted. +** +** The caller is responsible for resetting pIns when it has finished +** using the substructure. +*/ +static u32 jsonCreateEditSubstructure( + JsonParse *pParse, /* The original JSONB that is being edited */ + JsonParse *pIns, /* Populate this with the blob data to insert */ + const char *zTail /* Tail of the path that determins substructure */ +){ + static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; + int rc; + memset(pIns, 0, sizeof(*pIns)); + if( zTail[0]==0 ){ + /* No substructure. Just insert what is given in pParse. */ + pIns->aBlob = pParse->aIns; + pIns->nBlob = pParse->nIns; + rc = 0; + }else{ + /* Construct the binary substructure */ + pIns->nBlob = 1; + pIns->aBlob = (u8*)&emptyObject[zTail[0]=='.']; + pIns->eEdit = pParse->eEdit; + pIns->nIns = pParse->nIns; + pIns->aIns = pParse->aIns; + rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->oom |= pIns->oom; + } + return rc; /* Error code only */ +} + /* ** Search along zPath to find the Json element specified. Return an ** index into pParse->aBlob[] for the start of that element's value. @@ -2135,6 +2201,13 @@ static void jsonBlobEdit( ** label, before returning. ** ** Return one of the JSON_LOOKUP error codes if problems are seen. +** +** This routine will also modify the blob. If pParse->eEdit is one of +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be +** made to the selected value. If an edit is performed, then the return +** value does not necessarily point to the select element. If an edit +** is performed, the return value is only useful for detecting error +** conditions. */ static u32 jsonLookupStep( JsonParse *pParse, /* The JSON to search */ @@ -2145,7 +2218,6 @@ static u32 jsonLookupStep( u32 i, j, k, nKey, sz, n, iEnd, rc; const char *zKey; u8 x; - static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; if( zPath[0]==0 ){ if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ @@ -2224,26 +2296,12 @@ static u32 jsonLookupStep( testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); jsonBlobAppendNode(&ix,JSONB_TEXTRAW, nKey, 0); - memset(&v, 0, sizeof(v)); - if( zPath[i]==0 ){ - v.nBlob = pParse->nIns; - v.aBlob = pParse->aIns; - }else{ - v.nBlob = 1; - v.aBlob = (u8*)&emptyObject[zPath[i]=='.']; - v.eEdit = pParse->eEdit; - v.nIns = pParse->nIns; - v.aIns = pParse->aIns; - rc = jsonLookupStep(&v, 0, &zPath[i], 0); - if( JSON_LOOKUP_ISERROR(rc) || v.oom ){ - pParse->oom |= v.oom; - jsonParseReset(&v); - jsonParseReset(&ix); - return rc; - } - } pParse->oom |= ix.oom; - if( jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) ){ + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) + ){ + assert( !pParse->oom ); nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); if( !pParse->oom ){ @@ -2257,7 +2315,7 @@ static u32 jsonLookupStep( } jsonParseReset(&v); jsonParseReset(&ix); - return j; + return rc; } }else if( zPath[0]=='[' ){ x = pParse->aBlob[iRoot] & 0x0f; @@ -2309,29 +2367,16 @@ static u32 jsonLookupStep( JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); - memset(&v, 0, sizeof(v)); - if( zPath[i+1]==0 ){ - v.aBlob = pParse->aIns; - v.nBlob = pParse->nIns; - }else{ - v.nBlob = 1; - v.aBlob = (u8*)&emptyObject[zPath[i+1]=='.']; - v.eEdit = pParse->eEdit; - v.nIns = pParse->nIns; - v.aIns = pParse->aIns; - rc = jsonLookupStep(&v, 0, &zPath[i+1], 0); - if( JSON_LOOKUP_ISERROR(rc) || v.oom ){ - pParse->oom |= v.oom; - jsonParseReset(&v); - return rc; - } - } - if( jsonBlobMakeEditable(pParse, v.nBlob) ){ + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, v.nBlob) + ){ + assert( !pParse->oom ); jsonBlobEdit(pParse, j, 0, v.aBlob, v.nBlob); } jsonParseReset(&v); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); - return j; + return rc; } }else{ return JSON_LOOKUP_PATHERROR; @@ -3169,17 +3214,7 @@ static void jsonArrayLengthFunc( i = 0; } if( (p->aBlob[i] & 0x0f)==JSONB_ARRAY ){ - u32 n, sz = 0, iEnd; - n = jsonbPayloadSize(p, i, &sz); - if( n==0 ) eErr = 2; - iEnd = i+n+sz; - i += n; - while( eErr==0 && i Date: Sat, 2 Dec 2023 20:35:04 +0000 Subject: [PATCH 149/191] Fix various compiler warnings and other problems with the new code on this branch. FossilOrigin-Name: 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0 --- ext/fts5/fts5.h | 2 +- ext/fts5/fts5Int.h | 5 ++- ext/fts5/fts5_expr.c | 8 ++++ ext/fts5/fts5_index.c | 92 +++++++++++++++++++++++++++++++++++++------ manifest | 18 ++++----- manifest.uuid | 2 +- 6 files changed, 102 insertions(+), 25 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 9feedbba19..63c9765eb6 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -281,7 +281,7 @@ struct Fts5PhraseIter { ** includes any embedded 0x00 and trailing data. ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. +** "detail=none" or "detail=column" option. */ struct Fts5ExtensionApi { int iVersion; /* Currently always set to 3 */ diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 911f547d17..4e4385ce2b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -467,7 +467,7 @@ void sqlite3Fts5StructureRelease(void*); int sqlite3Fts5StructureTest(Fts5Index*, void*); /* -** Used by xInstToken() and xPhraseToken(). +** Used by xInstToken(): */ int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); @@ -547,8 +547,9 @@ int sqlite3Fts5IndexLoadConfig(Fts5Index *p); int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); -/* Used to populate hash tables for xInstToken in detail=none/column mode. */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); + +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ int sqlite3Fts5IndexIterWriteTokendata( Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff ); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 95d102062d..6f58cf8735 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2985,6 +2985,11 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +/* +** pToken is a buffer nToken bytes in size that may or may not contain +** an embedded 0x00 byte. If it does, return the number of bytes in +** the buffer before the 0x00. If it does not, return nToken. +*/ static int fts5QueryTerm(const char *pToken, int nToken){ int ii; for(ii=0; iiapTombstone) for the -** iterator passed as the second argument. If an OOM error occurs, leave -** an error in the Fts5Index object. +** Allocate a tombstone hash page array object (pIter->pTombArray) for +** the iterator passed as the second argument. If an OOM error occurs, +** leave an error in the Fts5Index object. */ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ const int nTomb = pIter->pSeg->nPgTombstone; @@ -2748,6 +2752,7 @@ static void fts5SegIterNextInit( bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(pSel); + sqlite3_bind_null(pSel, 2); if( p->rc ) return; } @@ -2772,7 +2777,7 @@ static void fts5SegIterNextInit( if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); assert( p->rc!=SQLITE_OK || - fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0 + fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0 ); } } @@ -2862,6 +2867,10 @@ static void fts5IndexFreeArray(Fts5Data **ap, int n){ } } +/* +** Decrement the ref-count of the object passed as the only argument. If it +** reaches 0, free it and its contents. +*/ static void fts5TombstoneArrayDelete(Fts5TombstoneArray *p){ if( p ){ p->nRef--; @@ -3828,6 +3837,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ } } +/* +** All the component segment-iterators of pIter have been set up. This +** functions finishes setup for iterator pIter itself. +*/ static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){ int iIter; for(iIter=pIter->nSeg-1; iIter>0; iIter--){ @@ -6566,13 +6579,25 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ pSeg->pLeaf = 0; } -typedef struct Fts5TokenDataMap Fts5TokenDataMap; +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits. Or, for an iterator used by an +** "ORDER BY rank" query, it accumulates an array of these for the entire +** query. +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +*/ struct Fts5TokenDataMap { - i64 iRowid; - i64 iPos; - int iIter; + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ }; +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +*/ struct Fts5TokenDataIter { int nIter; int nIterAlloc; @@ -6585,10 +6610,14 @@ struct Fts5TokenDataIter { Fts5Iter *apIter[1]; }; +/* +** This function appends iterator pAppend to Fts5TokenDataIter pIn and +** returns the result. +*/ static Fts5TokenDataIter *fts5AppendTokendataIter( - Fts5Index *p, - Fts5TokenDataIter *pIn, - Fts5Iter *pAppend + Fts5Index *p, /* Index object (for error code) */ + Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */ + Fts5Iter *pAppend /* Append this iterator */ ){ Fts5TokenDataIter *pRet = pIn; @@ -6617,6 +6646,9 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( return pRet; } +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ if( pSet ){ int ii; @@ -6629,6 +6661,13 @@ static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ } } +/* +** The iterator passed as the first argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function is used to access the token +** instance located at offset iOff of column iCol of row iRowid. It is +** returned via output pointers *ppOut and *pnOut. This is used by the +** xInstToken() API. +*/ static int fts5TokendataIterToken( Fts5Iter *pIter, i64 iRowid, @@ -6673,6 +6712,9 @@ static int fts5TokendataIterToken( return SQLITE_OK; } +/* +** Append a mapping to the token-map belonging to object pT. +*/ static void fts5TokendataIterAppendMap( Fts5Index *p, Fts5TokenDataIter *pT, @@ -6703,6 +6745,12 @@ static void fts5TokendataIterAppendMap( } } +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function sets the iterator output +** variables (pIter->base.*) according to the contents of the current +** row. +*/ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ int ii; int nHit = 0; @@ -6819,6 +6867,13 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ } } +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function advances the iterator. If +** argument bFrom is false, then the iterator is advanced to the next +** entry. Or, if bFrom is true, it is advanced to the first entry with +** a rowid of iFrom or greater. +*/ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ int ii; Fts5TokenDataIter *pT = pIter->pTokenDataIter; @@ -6841,6 +6896,10 @@ static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ fts5IterSetOutputsTokendata(pIter); } +/* +** If the segment-iterator passed as the first argument is at EOF, then +** set pIter->term to a copy of buffer pTerm. +*/ static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ if( pIter && pIter->aSeg[0].pLeaf==0 ){ fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); @@ -7144,7 +7203,9 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ } /* -** +** This is used by xInstToken() to access the token at offset iOff, column +** iCol of row iRowid. The token is returned via output variables *ppOut +** and *pnOut. */ int sqlite3Fts5IterToken( Fts5IndexIter *pIndexIter, @@ -7173,6 +7234,13 @@ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ } } +/* +** Set a token-mapping for the iterator passed as the first argument. This +** is used in detail=column or detail=none mode when a token is requested +** using the xInstToken() API. In this case the caller tokenizers the +** current row and configures the token-mapping via multiple calls to this +** function. +*/ int sqlite3Fts5IndexIterWriteTokendata( Fts5IndexIter *pIndexIter, const char *pToken, int nToken, diff --git a/manifest b/manifest index 699d024ca1..2353c55e4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\stokendata=1\squeries\srequire\smultiple\ssegment-cursors,\sallow\sthose\scursors\sto\sshare\sa\ssingle\sarray\sof\sin-memory\stombstone\spages. -D 2023-12-02T18:14:07.393 +C Fix\svarious\scompiler\swarnings\sand\sother\sproblems\swith\sthe\snew\scode\son\sthis\sbranch. +D 2023-12-02T20:35:04.768 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -89,14 +89,14 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 5e5630fc81e212f658afaa5b2650dac939d2729d0723aef1eeaff908f1725648 -F ext/fts5/fts5Int.h 285118aa6dfccb382e84eaeb9f7bec334e4f7104efa9303240605447003445c9 +F ext/fts5/fts5.h ff90acaa97f8e865b66d1177d1b56b8c110fd5548ab5863bab43f055a1d745fe +F ext/fts5/fts5Int.h 1fdbf3d16bdd481fe2ee99927919e4c3db835efae00f8efd7efb5e6a93277459 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c f83259b52b7b3e76768b835fe155cb7e345affdfafb96574372b4127d5f5496a +F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 21f8f449666ac44c12d5051e153ad84a886a729cb2f5d6ad02a113095c3f8ec6 +F ext/fts5/fts5_index.c b31bf4f0fb51a15cc1aa54c2f337197740f4f8898347266781ca6970ca751302 F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2149,8 +2149,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7bda09ab404a110d57449e149a3281fca8dc4cacf7bd9832ea2a1356ad20fe8e -R 1ce6343b4aa590c869e9b9aa51095415 +P e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f +R a72f98879c56fe0b2489dc56b6289649 U dan -Z 8664660cde606ab1a6fdc20b18622c4a +Z 6a8664529c8f348c40cce12fb229aa10 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0cbd084480..b4349493de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f \ No newline at end of file +3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0 \ No newline at end of file From 8f8d481485b6409b739a2a5b4693a35b4aab5c77 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 20:37:45 +0000 Subject: [PATCH 150/191] Fix harmless compiler warnings reported by MSVC. FossilOrigin-Name: 419652c0c82980bd043584dcd2976f91dfff7b926b216d597698299850b855c0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6e1a782d31..cc84f09d52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Code\sand\scomment\scleanup.\s\sEverything\sshould\swork\sthe\ssame. -D 2023-12-02T20:25:36.578 +C Fix\sharmless\scompiler\swarnings\sreported\sby\sMSVC. +D 2023-12-02T20:37:45.416 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 5ad1a1be6199359b09bb4eca690789b77375e00f17c55697b6288f1b0fbbe8b0 +F src/json.c fc67289aebe1ab7d3ccc31ec61b0fcea1d724f23026d072c05242dcc57049edf F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e3941502789c5afaf19b08112f464abf5e3cba7f92fc9290af2a0f96127ad9a -R 9f984966a856d28c36c8f0d6d69732a4 +P c640754df0d3ffdad994745f0d0e10c8f19f424b87f6a6e6e269491a0350b950 +R cbbd74dd3f256fc8164ba32c868936ce U drh -Z ea3e13091dceb571341f1dccf0e97828 +Z de70fb53fc1b64715f23def652af30f4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9a68b3807a..304fd5742b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c640754df0d3ffdad994745f0d0e10c8f19f424b87f6a6e6e269491a0350b950 \ No newline at end of file +419652c0c82980bd043584dcd2976f91dfff7b926b216d597698299850b855c0 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9356bfa8d5..067bdb4739 100644 --- a/src/json.c +++ b/src/json.c @@ -3416,10 +3416,10 @@ static int jsonMergePatchBlob( u8 eTLabel; /* Node type of the target label */ u32 iTLabel = 0; /* Index of the label */ u32 nTLabel = 0; /* Header size in bytes for the target label */ - u32 szTLabel; /* Size of the target label payload */ + u32 szTLabel = 0; /* Size of the target label payload */ u32 iTValue = 0; /* Index of the target value */ u32 nTValue = 0; /* Header size of the target value */ - u32 szTValue; /* Payload size for the target value */ + u32 szTValue = 0; /* Payload size for the target value */ u32 iPCursor; /* Cursor position while scanning the patch */ u32 iPEnd; /* First byte past the end of the patch */ From c78c3c91ae722722b0f9ba59dd850885b5ffe5a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 2 Dec 2023 21:39:34 +0000 Subject: [PATCH 151/191] Implement strict JSONB checking in the json_valid() function. FossilOrigin-Name: 0f26d38880fcbc207abcc94dbc170a7428bab1b4f0b7731aaf5bee0224000994 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 35 +++++++++++++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cc84f09d52..95a05c007f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sreported\sby\sMSVC. -D 2023-12-02T20:37:45.416 +C Implement\sstrict\sJSONB\schecking\sin\sthe\sjson_valid()\sfunction. +D 2023-12-02T21:39:34.516 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fc67289aebe1ab7d3ccc31ec61b0fcea1d724f23026d072c05242dcc57049edf +F src/json.c fd321d1ab2d6f42354bd2ff7bc4974ab8dd9eee977e4e8e137cf1360873ef052 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c640754df0d3ffdad994745f0d0e10c8f19f424b87f6a6e6e269491a0350b950 -R cbbd74dd3f256fc8164ba32c868936ce +P 419652c0c82980bd043584dcd2976f91dfff7b926b216d597698299850b855c0 +R ed8c7ad5ad50d0c7dab9c9a4109db3da U drh -Z de70fb53fc1b64715f23def652af30f4 +Z 50662e1c3dbad4c9b4aaea32a805ad55 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 304fd5742b..c4f20a42ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -419652c0c82980bd043584dcd2976f91dfff7b926b216d597698299850b855c0 \ No newline at end of file +0f26d38880fcbc207abcc94dbc170a7428bab1b4f0b7731aaf5bee0224000994 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 067bdb4739..ae6458cd59 100644 --- a/src/json.c +++ b/src/json.c @@ -3880,8 +3880,39 @@ static void jsonValidFunc( } case SQLITE_BLOB: { if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ - /* TO-DO: strict checking if flags & 0x08 */ - res = 1; + if( flags & 0x04 ){ + /* Superficial checking only - accomplisehd by the + ** jsonFuncArgMightBeBinary() call above. */ + res = 1; + }else{ + /* Strict checking. Check by translating BLOB->TEXT->BLOB. If + ** no errors occur, call that a "strict check". */ + JsonParse px; + JsonString sx; + u8 oom = 0; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + px.nBlob = sqlite3_value_bytes(argv[0]); + jsonStringInit(&sx, 0); + jsonXlateBlobToText(&px, 0, &sx); + jsonParseReset(&px); + if( sx.eErr & JSTRING_OOM ) oom = 1; + if( sx.eErr==0 ){ + memset(&px, 0, sizeof(px)); + px.zJson = sx.zBuf; + px.nJson = sx.nUsed; + if( jsonXlateTextToBlob(&px, 0)==px.nJson ){ + res = 1; + } + oom |= px.oom; + jsonParseReset(&px); + } + jsonStringReset(&sx); + if( oom ){ + sqlite3_result_error_nomem(ctx); + return; + } + } } break; } From eb18ae3089ac409ed7c0bd943a993ea8b357c2f5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 00:51:30 +0000 Subject: [PATCH 152/191] Minor code changes for consistency and to simplify testing. FossilOrigin-Name: df272bd837910ad9e03e222716a1201a601399664365f1dcf73d5932372518ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 44 ++++++++++++++------------------------------ 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 95a05c007f..0746333f7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\sstrict\sJSONB\schecking\sin\sthe\sjson_valid()\sfunction. -D 2023-12-02T21:39:34.516 +C Minor\scode\schanges\sfor\sconsistency\sand\sto\ssimplify\stesting. +D 2023-12-03T00:51:30.732 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fd321d1ab2d6f42354bd2ff7bc4974ab8dd9eee977e4e8e137cf1360873ef052 +F src/json.c 9a79d32ad7df0f37aaca77a78278af177b7c898f719f7f3c1beb6626cf0cd88e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 419652c0c82980bd043584dcd2976f91dfff7b926b216d597698299850b855c0 -R ed8c7ad5ad50d0c7dab9c9a4109db3da +P 0f26d38880fcbc207abcc94dbc170a7428bab1b4f0b7731aaf5bee0224000994 +R 66de56e0a2d95958c6880cf67a24bb7b U drh -Z 50662e1c3dbad4c9b4aaea32a805ad55 +Z 5f7e6173dda49cfc3454a89f079d029d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c4f20a42ae..8eca6d0777 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f26d38880fcbc207abcc94dbc170a7428bab1b4f0b7731aaf5bee0224000994 \ No newline at end of file +df272bd837910ad9e03e222716a1201a601399664365f1dcf73d5932372518ed \ No newline at end of file diff --git a/src/json.c b/src/json.c index ae6458cd59..a35306b62e 100644 --- a/src/json.c +++ b/src/json.c @@ -534,7 +534,7 @@ static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ } } static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ - if( N==0 ) return; + assert( N>0 ); if( N+p->nUsed >= p->nAlloc ){ jsonStringExpandAndAppend(p,zIn,N); }else{ @@ -572,9 +572,8 @@ static void jsonAppendChar(JsonString *p, char c){ /* Make sure there is a zero terminator on p->zBuf[] */ static void jsonStringTerminate(JsonString *p){ - if( p->nUsednAlloc || jsonStringGrow(p,1) ){ - p->zBuf[p->nUsed] = 0; - } + jsonAppendChar(p, 0); + p->nUsed--; } /* Try to force the string to be a zero-terminated RCStr string. In other @@ -991,24 +990,6 @@ static const struct NanInfName { }; -/* -** Compute the text of an error in JSON path syntax. -** -** If ctx is not NULL then push the error message into ctx and return NULL. -* If ctx is NULL, then return the text of the error message. -*/ -static char *jsonPathSyntaxError(const char *zErr, sqlite3_context *ctx){ - char *zMsg = sqlite3_mprintf("JSON path error near '%q'", zErr); - if( ctx==0 ) return zMsg; - if( zMsg==0 ){ - sqlite3_result_error_nomem(ctx); - }else{ - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - } - return 0; -} - /* ** Report the wrong number of arguments for json_insert(), json_replace() ** or json_set(). @@ -2659,15 +2640,19 @@ static int jsonFunctionArgToBlob( /* ** Generate a bad path error. +** +** If ctx is not NULL then push the error message into ctx and return NULL. +** If ctx is NULL, then return the text of the error message. */ -static void jsonBadPathError( +static char *jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ const char *zPath /* The path with the problem */ ){ - sqlite3 *db = sqlite3_context_db_handle(ctx); - char *zMsg = sqlite3MPrintf(db, "bad JSON path: %Q", zPath); + char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + if( ctx==0 ) return zMsg; sqlite3_result_error(ctx, zMsg, -1); - sqlite3DbFree(db, zMsg); + sqlite3_free(zMsg); + return 0; } /* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent @@ -3287,7 +3272,6 @@ static void jsonExtractFunc( }else if( zPath[0]!='[' ){ jsonAppendRawNZ(&jx, ".", 1); jsonAppendRaw(&jx, zPath, nPath); - jsonAppendChar(&jx, 0); }else{ jsonAppendRaw(&jx, zPath, nPath); } @@ -3685,7 +3669,7 @@ static void jsonRemoveFunc( json_remove_patherror: jsonParseFree(p); - jsonPathSyntaxError(zPath, ctx); + jsonBadPathError(ctx, zPath); return; json_remove_return_null: @@ -4689,7 +4673,7 @@ static int jsonEachFilter( if( zRoot==0 ) return SQLITE_OK; if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -4707,7 +4691,7 @@ static int jsonEachFilter( return SQLITE_OK; } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot, 0); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } From 78fa0186b822e137d6d6e157a28cbcad67275cdb Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 11:54:39 +0000 Subject: [PATCH 153/191] Do not let bad hexadecimal digits in malformed JSONB cause an assertion fault. FossilOrigin-Name: 8dec1ba1e5076ff596756e00c1e2ada0245f168a503dd1cadadf848331acfac3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 26 +++++++++++++++++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0746333f7a..75a751773f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scode\schanges\sfor\sconsistency\sand\sto\ssimplify\stesting. -D 2023-12-03T00:51:30.732 +C Do\snot\slet\sbad\shexadecimal\sdigits\sin\smalformed\sJSONB\scause\san\sassertion\sfault. +D 2023-12-03T11:54:39.886 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9a79d32ad7df0f37aaca77a78278af177b7c898f719f7f3c1beb6626cf0cd88e +F src/json.c 0bbd116e9054eae2aa8088071f163c47050388ef83ab871c6c85b57a52bef984 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f26d38880fcbc207abcc94dbc170a7428bab1b4f0b7731aaf5bee0224000994 -R 66de56e0a2d95958c6880cf67a24bb7b +P df272bd837910ad9e03e222716a1201a601399664365f1dcf73d5932372518ed +R 214d8647778f37447f46e39ad90ab38f U drh -Z 5f7e6173dda49cfc3454a89f079d029d +Z 2537746c5900cbcfb749d720f1ba2129 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8eca6d0777..05fe5681c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df272bd837910ad9e03e222716a1201a601399664365f1dcf73d5932372518ed \ No newline at end of file +8dec1ba1e5076ff596756e00c1e2ada0245f168a503dd1cadadf848331acfac3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a35306b62e..0668ff6daa 100644 --- a/src/json.c +++ b/src/json.c @@ -817,15 +817,31 @@ static void jsonParseFree(JsonParse *pParse){ } } +/* +** Translate a single byte of Hex into an integer. +** This routine only gives a correct answer if h really is a valid hexadecimal +** character: 0..9a..fA..F. But unlike sqlite3HexToInt(), it does not +** assert() if the digit is not hex. +*/ +static u8 jsonHexToInt(int h){ +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + /* ** Convert a 4-byte hex string into an integer */ static u32 jsonHexToInt4(const char *z){ u32 v; - v = (sqlite3HexToInt(z[0])<<12) - + (sqlite3HexToInt(z[1])<<8) - + (sqlite3HexToInt(z[2])<<4) - + sqlite3HexToInt(z[3]); + v = (jsonHexToInt(z[0])<<12) + + (jsonHexToInt(z[1])<<8) + + (jsonHexToInt(z[2])<<4) + + jsonHexToInt(z[3]); return v; } @@ -2524,7 +2540,7 @@ static void jsonReturnFromBlob( }else if( c=='0' ){ c = 0; }else if( c=='x' ){ - c = (sqlite3HexToInt(z[iIn+1])<<4) | sqlite3HexToInt(z[iIn+2]); + c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); iIn += 2; }else if( c=='\r' && z[i+1]=='\n' ){ iIn++; From 0a18a5807a43f44a43a84f446f82201d389886a2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 19:59:45 +0000 Subject: [PATCH 154/191] Enable incorrect JSONB to be rendered into text without hitting an assertion for a bad whitespace escape in a string. FossilOrigin-Name: 4d6a9a217df6792b41766b774fb0c0553b45f9104c26a0955bf4a30862d7d7bf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 75a751773f..7f9fac50f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\slet\sbad\shexadecimal\sdigits\sin\smalformed\sJSONB\scause\san\sassertion\sfault. -D 2023-12-03T11:54:39.886 +C Enable\sincorrect\sJSONB\sto\sbe\srendered\sinto\stext\swithout\shitting\san\nassertion\sfor\sa\sbad\swhitespace\sescape\sin\sa\sstring. +D 2023-12-03T19:59:45.738 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 0bbd116e9054eae2aa8088071f163c47050388ef83ab871c6c85b57a52bef984 +F src/json.c a575d568a65576657aad25fb7ada56007a86faf41d967ddbc80f2109caf00b0e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df272bd837910ad9e03e222716a1201a601399664365f1dcf73d5932372518ed -R 214d8647778f37447f46e39ad90ab38f +P 8dec1ba1e5076ff596756e00c1e2ada0245f168a503dd1cadadf848331acfac3 +R 02988a1f4c6dc45085ceececc56fddf4 U drh -Z 2537746c5900cbcfb749d720f1ba2129 +Z c32b25a78a87ced6ccf27a2c8ddc3f70 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 05fe5681c0..d4318802f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dec1ba1e5076ff596756e00c1e2ada0245f168a503dd1cadadf848331acfac3 \ No newline at end of file +4d6a9a217df6792b41766b774fb0c0553b45f9104c26a0955bf4a30862d7d7bf \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0668ff6daa..9d29685eb0 100644 --- a/src/json.c +++ b/src/json.c @@ -2545,9 +2545,10 @@ static void jsonReturnFromBlob( }else if( c=='\r' && z[i+1]=='\n' ){ iIn++; continue; - }else if( 0xe2==(u8)c ){ - assert( 0x80==(u8)z[i+1] ); - assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); + }else if( 0xe2==(u8)c + && 0x80==(u8)z[i+1] + && (0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2]) + ){ iIn += 2; continue; }else{ From a3bf077b60d1dc08dea1014d7eb84ff57b439a05 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 20:11:35 +0000 Subject: [PATCH 155/191] Ensure that OOM conditions in the generation of the "bad JSON path" error message result in an SQLITE_NOMEM error. FossilOrigin-Name: aa0e02b5c26a2ef3d6216a0ed8bc01382be43173485f898cb63f2a8c559f2e74 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7f9fac50f0..31e68565bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sincorrect\sJSONB\sto\sbe\srendered\sinto\stext\swithout\shitting\san\nassertion\sfor\sa\sbad\swhitespace\sescape\sin\sa\sstring. -D 2023-12-03T19:59:45.738 +C Ensure\sthat\sOOM\sconditions\sin\sthe\sgeneration\sof\sthe\s"bad\sJSON\spath"\serror\nmessage\sresult\sin\san\sSQLITE_NOMEM\serror. +D 2023-12-03T20:11:35.681 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a575d568a65576657aad25fb7ada56007a86faf41d967ddbc80f2109caf00b0e +F src/json.c 6d92343776fe32e9d703d6e13c1d1883d75e1fd17e081159dd257b21a859f5f9 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8dec1ba1e5076ff596756e00c1e2ada0245f168a503dd1cadadf848331acfac3 -R 02988a1f4c6dc45085ceececc56fddf4 +P 4d6a9a217df6792b41766b774fb0c0553b45f9104c26a0955bf4a30862d7d7bf +R 1defa3c713a22ae4567d6d3fb033b34d U drh -Z c32b25a78a87ced6ccf27a2c8ddc3f70 +Z e3833547911441ddc3c3cd95bc32393c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4318802f6..c9937a03fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d6a9a217df6792b41766b774fb0c0553b45f9104c26a0955bf4a30862d7d7bf \ No newline at end of file +aa0e02b5c26a2ef3d6216a0ed8bc01382be43173485f898cb63f2a8c559f2e74 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9d29685eb0..9fcfd3e10f 100644 --- a/src/json.c +++ b/src/json.c @@ -2667,8 +2667,12 @@ static char *jsonBadPathError( ){ char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); if( ctx==0 ) return zMsg; - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); + if( zMsg ){ + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(ctx); + } return 0; } From 16e8a5b2f35450711d9c8bac8326ca79718d86e0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 23:30:59 +0000 Subject: [PATCH 156/191] Avoid problems when the path argument to json_tree() contains embedded U+0000 characters. FossilOrigin-Name: 9f055091af01a5dddba1a7e9868ad030c8f206237e1569215cb161e53e54aa71 --- manifest | 12 +++---- manifest.uuid | 2 +- src/json.c | 94 ++++++++++++++------------------------------------- 3 files changed, 32 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 31e68565bc..fb3e811b9c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sOOM\sconditions\sin\sthe\sgeneration\sof\sthe\s"bad\sJSON\spath"\serror\nmessage\sresult\sin\san\sSQLITE_NOMEM\serror. -D 2023-12-03T20:11:35.681 +C Avoid\sproblems\swhen\sthe\spath\sargument\sto\sjson_tree()\scontains\sembedded\sU+0000\ncharacters. +D 2023-12-03T23:30:59.212 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6d92343776fe32e9d703d6e13c1d1883d75e1fd17e081159dd257b21a859f5f9 +F src/json.c 6ac93a6cc3cbfa8be1b4058c7aa335ede339a36b65be5fdde0593bd428ee74cc F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d6a9a217df6792b41766b774fb0c0553b45f9104c26a0955bf4a30862d7d7bf -R 1defa3c713a22ae4567d6d3fb033b34d +P aa0e02b5c26a2ef3d6216a0ed8bc01382be43173485f898cb63f2a8c559f2e74 +R 04d47333f96a5f7bc80310aa6a2b6c34 U drh -Z e3833547911441ddc3c3cd95bc32393c +Z 051e2555795818c726068d0b4b3820ce # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c9937a03fa..a0e2cc933f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa0e02b5c26a2ef3d6216a0ed8bc01382be43173485f898cb63f2a8c559f2e74 \ No newline at end of file +9f055091af01a5dddba1a7e9868ad030c8f206237e1569215cb161e53e54aa71 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9fcfd3e10f..f7e1591ad2 100644 --- a/src/json.c +++ b/src/json.c @@ -3082,59 +3082,6 @@ static void jsonbTest2( ** Scalar SQL function implementations ****************************************************************************/ -/* SQL Function: jsonb(JSON) -** -** Convert the input argument into JSONB (the SQLite binary encoding of -** JSON). -** -** If the input is TEXT, or NUMERIC, try to parse it as JSON. If the fails, -** raise an error. Otherwise, return the resulting BLOB value. -** -** If the input is a BLOB, check to see if the input is a plausible -** JSONB. If it is, return it unchanged. Raise an error if it is not. -** Note that there could be internal inconsistencies in the BLOB - this -** routine does not do a full byte-for-byte validity check of the -** JSON blob. -** -** If the input is NULL, return NULL. -*/ -static void jsonbFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse *pParse; - int nJson; - const char *zJson; - JsonParse x; - UNUSED_PARAMETER(argc); - - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - /* No-op */ - }else if( jsonFuncArgMightBeBinary(argv[0]) ){ - sqlite3_result_value(ctx, argv[0]); - }else{ - zJson = (const char*)sqlite3_value_text(argv[0]); - if( zJson==0 ) return; - nJson = sqlite3_value_bytes(argv[0]); - pParse = &x; - memset(&x, 0, sizeof(x)); - x.zJson = (char*)zJson; - x.nJson = nJson; - if( jsonConvertTextToBlob(pParse, ctx) ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - }else{ - assert( pParse->nBlobAlloc>0 ); - assert( !pParse->bReadOnly ); - sqlite3_result_blob(ctx, pParse->aBlob, pParse->nBlob, sqlite3_free); - pParse->aBlob = 0; - pParse->nBlob = 0; - pParse->nBlobAlloc = 0; - } - jsonParseReset(pParse); - } -} - /* ** Implementation of the json_quote(VALUE) function. Return a JSON value ** corresponding to the SQL value input. Mostly this means putting @@ -3271,9 +3218,10 @@ static void jsonExtractFunc( for(i=1; ipCtx = ctx; z = (const char*)sqlite3_value_text(argv[0]); - n = (u32)sqlite3_value_bytes(argv[0]); + n = z ? sqlite3Strlen30(z) : 0; jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); jsonAppendSqlValue(pStr, argv[1]); @@ -4462,15 +4410,16 @@ static int jsonEachPathLength(JsonEachCursor *p){ if( p->iRowid==0 && p->bRecursive && n>1 ){ if( p->path.zBuf[n-1]==']' ){ do{ - n--; assert( n>0 ); + n--; }while( p->path.zBuf[n]!='[' ); }else{ u32 sz = 0; jsonbPayloadSize(&p->sParse, p->i, &sz); if( p->path.zBuf[n-1]=='"' ) sz += 2; + assert( szpath.zBuf[n]!='.' ){ + while( p->path.zBuf[n]!='.' && ALWAYS(n>0) ){ n--; assert( n>0 ); } @@ -4666,11 +4615,15 @@ static int jsonEachFilter( if( idxNum==0 ) return SQLITE_OK; memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); - if( p->sParse.aBlob==0 ){ - return SQLITE_NOMEM; + if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( p->sParse.aBlob==0 ){ + return SQLITE_NOMEM; + } + }else{ + goto json_each_malformed_input; } }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); @@ -4683,10 +4636,7 @@ static int jsonEachFilter( if( p->sParse.oom ){ return SQLITE_NOMEM; } - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - jsonEachCursorReset(p); - return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + goto json_each_malformed_input; } } if( idxNum==3 ){ @@ -4698,7 +4648,7 @@ static int jsonEachFilter( jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } - p->nRoot = sqlite3_value_bytes(argv[1]); + p->nRoot = sqlite3Strlen30(zRoot); if( zRoot[1]==0 ){ i = p->i = 0; p->eType = 0; @@ -4747,6 +4697,12 @@ static int jsonEachFilter( p->aParent[0].iValue = i; } return SQLITE_OK; + +json_each_malformed_input: + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } /* The methods of the json_each virtual table */ @@ -4822,7 +4778,7 @@ void sqlite3RegisterJsonFunctions(void){ /* Number of arguments ---, | | | | ,--- Flags */ /* | | | | | | */ JFUNCTION(json, 1,1,1, 0,0,0, jsonRemoveFunc), - JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonbFunc), + JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), From 9af45dc4826dd6acb24b11759346bf43a3cf4503 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 3 Dec 2023 23:38:24 +0000 Subject: [PATCH 157/191] Remove dead code. Improved reporting of errors in JSON inputs. FossilOrigin-Name: 2eaa738e6b5c1b67b3e57c868d9c3a30eea38a0b3b8b02482f06d57a45b10921 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index fb3e811b9c..6b01ad342f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sproblems\swhen\sthe\spath\sargument\sto\sjson_tree()\scontains\sembedded\sU+0000\ncharacters. -D 2023-12-03T23:30:59.212 +C Remove\sdead\scode.\s\sImproved\sreporting\sof\serrors\sin\sJSON\sinputs. +D 2023-12-03T23:38:24.696 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 6ac93a6cc3cbfa8be1b4058c7aa335ede339a36b65be5fdde0593bd428ee74cc +F src/json.c 2f824bf3cb7f6018476fef922afd13aaa5fbe063ded53cf556d9542c5e81a95c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa0e02b5c26a2ef3d6216a0ed8bc01382be43173485f898cb63f2a8c559f2e74 -R 04d47333f96a5f7bc80310aa6a2b6c34 +P 9f055091af01a5dddba1a7e9868ad030c8f206237e1569215cb161e53e54aa71 +R 37598996ac6084b4bcb164d59f053e89 U drh -Z 051e2555795818c726068d0b4b3820ce +Z 62d813bb1b1f385603ccfef947a3746b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0e2cc933f..dc5d8dab10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f055091af01a5dddba1a7e9868ad030c8f206237e1569215cb161e53e54aa71 \ No newline at end of file +2eaa738e6b5c1b67b3e57c868d9c3a30eea38a0b3b8b02482f06d57a45b10921 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f7e1591ad2..1e88b51743 100644 --- a/src/json.c +++ b/src/json.c @@ -1729,6 +1729,7 @@ static int jsonConvertTextToBlob( if( zJson[i] ){ i += json5Whitespace(&zJson[i]); if( zJson[i] ){ + if( pCtx ) sqlite3_result_error(pCtx, "malformed JSON", -1); jsonParseReset(pParse); return 1; } @@ -2801,9 +2802,6 @@ rebuild_from_cache: p->hasNonstd = pFromCache->hasNonstd; jsonParseFree(pFromCache); return p; - }else{ - jsonParseFree(pFromCache); - pFromCache = 0; } if( eType==SQLITE_BLOB ){ u32 n, sz = 0; From b7fd951be4b5ca926c8daa787d5a627c3772df3e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 00:31:58 +0000 Subject: [PATCH 158/191] Back off on the use of strlen() for situations where sqlite3_value_bytes() will work as well, for performance. FossilOrigin-Name: 79fb54fbb8b9c30f47cdbd437d24a21542716241e822749e5e28c9fbc449bfa8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6b01ad342f..a102fa7874 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sdead\scode.\s\sImproved\sreporting\sof\serrors\sin\sJSON\sinputs. -D 2023-12-03T23:38:24.696 +C Back\soff\son\sthe\suse\sof\sstrlen()\sfor\ssituations\swhere\ssqlite3_value_bytes()\nwill\swork\sas\swell,\sfor\sperformance. +D 2023-12-04T00:31:58.791 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2f824bf3cb7f6018476fef922afd13aaa5fbe063ded53cf556d9542c5e81a95c +F src/json.c 4cf9581f5d58cac15136c40dc359910eda0bac0508173ac1e2c2d988c7b9f310 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f055091af01a5dddba1a7e9868ad030c8f206237e1569215cb161e53e54aa71 -R 37598996ac6084b4bcb164d59f053e89 +P 2eaa738e6b5c1b67b3e57c868d9c3a30eea38a0b3b8b02482f06d57a45b10921 +R 38b2727d0c66c3371d75d1e0b8d86ea5 U drh -Z 62d813bb1b1f385603ccfef947a3746b +Z 2acd0d3ecb5d5f83c97261113a2445b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc5d8dab10..965abfdee1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2eaa738e6b5c1b67b3e57c868d9c3a30eea38a0b3b8b02482f06d57a45b10921 \ No newline at end of file +79fb54fbb8b9c30f47cdbd437d24a21542716241e822749e5e28c9fbc449bfa8 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1e88b51743..f5eef834d0 100644 --- a/src/json.c +++ b/src/json.c @@ -3579,7 +3579,7 @@ static void jsonObjectFunc( } jsonAppendSeparator(&jx); z = (const char*)sqlite3_value_text(argv[i]); - n = z!=0 ? sqlite3Strlen30(z) : 0; + n = sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); jsonAppendSqlValue(&jx, argv[i+1]); @@ -4099,7 +4099,7 @@ static void jsonObjectStep( } pStr->pCtx = ctx; z = (const char*)sqlite3_value_text(argv[0]); - n = z ? sqlite3Strlen30(z) : 0; + n = sqlite3Strlen30(z); jsonAppendString(pStr, z, n); jsonAppendChar(pStr, ':'); jsonAppendSqlValue(pStr, argv[1]); From 2ff73a5f3e6eef0c1cc5423f7d7b41db1f435378 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 01:14:23 +0000 Subject: [PATCH 159/191] Better pre-scan size estimations for objects in the JSON parser resulting in fewer reallocations and memmove operations. FossilOrigin-Name: 526b27f90897f5e35dfff7257daf6c4ce4798d649b09b8aecfb02df0449e3c51 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a102fa7874..f5a4a19dce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\soff\son\sthe\suse\sof\sstrlen()\sfor\ssituations\swhere\ssqlite3_value_bytes()\nwill\swork\sas\swell,\sfor\sperformance. -D 2023-12-04T00:31:58.791 +C Better\spre-scan\ssize\sestimations\sfor\sobjects\sin\sthe\sJSON\sparser\sresulting\nin\sfewer\sreallocations\sand\smemmove\soperations. +D 2023-12-04T01:14:23.183 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4cf9581f5d58cac15136c40dc359910eda0bac0508173ac1e2c2d988c7b9f310 +F src/json.c fe5c8d7494ed56126553dbcb5c970dcbbadddfd56871e08b905b8d7fb9903417 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2eaa738e6b5c1b67b3e57c868d9c3a30eea38a0b3b8b02482f06d57a45b10921 -R 38b2727d0c66c3371d75d1e0b8d86ea5 +P 79fb54fbb8b9c30f47cdbd437d24a21542716241e822749e5e28c9fbc449bfa8 +R dfc9eac13900dae64608fce7f4f645e4 U drh -Z 2acd0d3ecb5d5f83c97261113a2445b2 +Z 0536405657c5f5cc41f38cd4401ae262 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 965abfdee1..deb93d36ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79fb54fbb8b9c30f47cdbd437d24a21542716241e822749e5e28c9fbc449bfa8 \ No newline at end of file +526b27f90897f5e35dfff7257daf6c4ce4798d649b09b8aecfb02df0449e3c51 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f5eef834d0..598b290510 100644 --- a/src/json.c +++ b/src/json.c @@ -1268,7 +1268,7 @@ json_parse_restart: case '{': { /* Parse object */ iThis = pParse->nBlob; - jsonBlobAppendNode(pParse, JSONB_OBJECT, (pParse->nJson-i)*2, 0); + jsonBlobAppendNode(pParse, JSONB_OBJECT, pParse->nJson-i, 0); if( ++pParse->iDepth > JSON_MAX_DEPTH ){ pParse->iErr = i; return -1; From dc138cb186b24784128f5fc0452c22de0b3b5c9d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 13:12:45 +0000 Subject: [PATCH 160/191] Repair issues and inefficiencies found during testing. FossilOrigin-Name: ae973cb1515f9d76409c92a2ca2ffd6b71f32b0b490a4886770e7c1b90f12611 --- manifest | 12 ++--- manifest.uuid | 2 +- src/json.c | 121 +++++++++++++------------------------------------- 3 files changed, 39 insertions(+), 96 deletions(-) diff --git a/manifest b/manifest index f5a4a19dce..9ee6f8d2ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\spre-scan\ssize\sestimations\sfor\sobjects\sin\sthe\sJSON\sparser\sresulting\nin\sfewer\sreallocations\sand\smemmove\soperations. -D 2023-12-04T01:14:23.183 +C Repair\sissues\sand\sinefficiencies\sfound\sduring\stesting. +D 2023-12-04T13:12:45.155 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fe5c8d7494ed56126553dbcb5c970dcbbadddfd56871e08b905b8d7fb9903417 +F src/json.c fdab711d10347af4e70bbdc4dc3aca516a4ef0e6c4388cc70335ab9201bdf7eb F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 79fb54fbb8b9c30f47cdbd437d24a21542716241e822749e5e28c9fbc449bfa8 -R dfc9eac13900dae64608fce7f4f645e4 +P 526b27f90897f5e35dfff7257daf6c4ce4798d649b09b8aecfb02df0449e3c51 +R a8fc2b32fcf7c8e7f15da5575d63ccf0 U drh -Z 0536405657c5f5cc41f38cd4401ae262 +Z 9f46540ea8d4faeba010e6034ffe6d00 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index deb93d36ac..b7d0da89ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -526b27f90897f5e35dfff7257daf6c4ce4798d649b09b8aecfb02df0449e3c51 \ No newline at end of file +ae973cb1515f9d76409c92a2ca2ffd6b71f32b0b490a4886770e7c1b90f12611 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 598b290510..6df07b0e66 100644 --- a/src/json.c +++ b/src/json.c @@ -570,32 +570,16 @@ static void jsonAppendChar(JsonString *p, char c){ } /* Make sure there is a zero terminator on p->zBuf[] -*/ -static void jsonStringTerminate(JsonString *p){ - jsonAppendChar(p, 0); - p->nUsed--; -} - -/* Try to force the string to be a zero-terminated RCStr string. In other -** words, make sure it is not still using the internal zSpace[] static -** buffer. ** ** Return true on success. Return false if an OOM prevents this ** from happening. */ -static int jsonForceRCStr(JsonString *p){ +static int jsonStringTerminate(JsonString *p){ jsonAppendChar(p, 0); - if( p->eErr ) return 0; p->nUsed--; - if( p->bStatic==0 ) return 1; - p->nAlloc = 0; - p->nUsed++; - jsonStringGrow(p, p->nUsed); - p->nUsed--; - return p->bStatic==0; + return p->eErr==0; } - /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. */ @@ -751,7 +735,7 @@ static void jsonReturnString( }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); - }else if( jsonForceRCStr(p) ){ + }else if( jsonStringTerminate(p) ){ if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ int rc; pParse->zJson = sqlite3RCStrRef(p->zBuf); @@ -1071,30 +1055,27 @@ static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ return 1; } - -/* Expand pParse->aBlob and append N bytes. -** -** Return the number of errors. +/* Expand pParse->aBlob and append one bytes. */ -static SQLITE_NOINLINE int jsonBlobExpandAndAppend( +static SQLITE_NOINLINE void jsonBlobExpandAndAppendOneByte( JsonParse *pParse, - const u8 *aData, - u32 N + u8 c ){ - if( jsonBlobExpand(pParse, pParse->nBlob+N) ) return 1; - memcpy(&pParse->aBlob[pParse->nBlob], aData, N); - pParse->nBlob += N; - return 0; + jsonBlobExpand(pParse, pParse->nBlob+1); + if( pParse->oom==0 ){ + assert( pParse->nBlob+1<=pParse->nBlobAlloc ); + pParse->aBlob[pParse->nBlob++] = c; + } } -/* Append a single character. Return 1 if an error occurs. +/* Append a single character. */ -static int jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ +static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ if( pParse->nBlob >= pParse->nBlobAlloc ){ - return jsonBlobExpandAndAppend(pParse, &c, 1); + jsonBlobExpandAndAppendOneByte(pParse, c); + }else{ + pParse->aBlob[pParse->nBlob++] = c; } - pParse->aBlob[pParse->nBlob++] = c; - return 0; } /* Slow version of jsonBlobAppendNode() that first resizes the @@ -1937,11 +1918,12 @@ static u32 jsonXlateBlobToText( sz2--; continue; } - if( sz2<2 ){ - if( sz2>0 ) pOut->eErr |= JSTRING_MALFORMED; - if( sz2==0 ) break; - } assert( zIn[0]=='\\' ); + assert( sz2>=1 ); + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } switch( (u8)zIn[1] ){ case '\'': jsonAppendChar(pOut, '\''); @@ -1950,9 +1932,9 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, "\\u0009", 6); break; case 'x': - if( sz2<2 ){ + if( sz2<4 ){ pOut->eErr |= JSTRING_MALFORMED; - sz2 = 0; + sz2 = 2; break; } jsonAppendRawNZ(pOut, "\\u00", 4); @@ -1980,7 +1962,7 @@ static u32 jsonXlateBlobToText( || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) ){ pOut->eErr |= JSTRING_MALFORMED; - k = sz2; + sz2 = 2; break; } zIn += 2; @@ -1990,11 +1972,7 @@ static u32 jsonXlateBlobToText( jsonAppendRawNZ(pOut, zIn, 2); break; } - if( sz2<2 ){ - sz2 = 0; - pOut->eErr |= JSTRING_MALFORMED; - break; - } + assert( sz2>=2 ); zIn += 2; sz2 -= 2; } @@ -2051,11 +2029,11 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ const u8 *aBlob; int nBlob; JsonParse s; - if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + assert( sqlite3_value_type(pJson)==SQLITE_BLOB ); aBlob = sqlite3_value_blob(pJson); nBlob = sqlite3_value_bytes(pJson); if( nBlob<1 ) return 0; - if( aBlob==0 || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; memset(&s, 0, sizeof(s)); s.aBlob = (u8*)aBlob; s.nBlob = nBlob; @@ -2302,13 +2280,13 @@ static u32 jsonLookupStep( assert( !pParse->oom ); nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); - if( !pParse->oom ){ + if( ALWAYS(!pParse->oom) ){ memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); k += nKey; memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); - if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + if( ALWAYS(pParse->delta) ) jsonAfterEditSizeAdjust(pParse, iRoot); } } jsonParseReset(&v); @@ -2394,7 +2372,7 @@ static void jsonReturnTextJsonFromBlob( JsonParse x; JsonString s; - if( aBlob==0 ) return; + if( NEVER(aBlob==0) ) return; memset(&x, 0, sizeof(x)); x.aBlob = (u8*)aBlob; x.nBlob = nBlob; @@ -2423,7 +2401,7 @@ static void jsonReturnFromBlob( sqlite3 *db = sqlite3_context_db_handle(pCtx); n = jsonbPayloadSize(pParse, i, &sz); - if( n==0 ) return; + if( NEVER(n==0) ) return; switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { sqlite3_result_null(pCtx); @@ -2605,7 +2583,7 @@ static int jsonFunctionArgToBlob( static u8 aNull[] = { 0x00 }; memset(pParse, 0, sizeof(pParse[0])); switch( eType ){ - case SQLITE_NULL: { + default: { pParse->aBlob = aNull; pParse->nBlob = 1; return 0; @@ -3041,39 +3019,6 @@ static void jsonParseFunc( jsonShowParse(p); jsonParseFree(p); } - -/* -** The json_test1(JSON) function return true (1) if the input is JSON -** text generated by another json function. It returns (0) if the input -** is not known to be JSON. -*/ -static void jsonTest1Func( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - UNUSED_PARAMETER(argc); - sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); -} - -/* SQL Function: jsonb_test2(BLOB_JSON) -** -** Render BLOB_JSON back into text. -** Development testing only. -*/ -static void jsonbTest2( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - const u8 *aBlob; - int nBlob; - UNUSED_PARAMETER(argc); - - aBlob = (const u8*)sqlite3_value_blob(argv[0]); - nBlob = sqlite3_value_bytes(argv[0]); - jsonReturnTextJsonFromBlob(ctx, aBlob, nBlob); -} #endif /* SQLITE_DEBUG */ /**************************************************************************** @@ -4805,8 +4750,6 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_valid, 2,1,0, 0,0,0, jsonValidFunc), #if SQLITE_DEBUG JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), - JFUNCTION(json_test1, 1,1,0, 1,0,0, jsonTest1Func), - JFUNCTION(jsonb_test2, 1,1,0, 0,1,0, jsonbTest2), #endif WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, From 43b4864a98da1537203eab92e5112d88451884f4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 15:08:21 +0000 Subject: [PATCH 161/191] Add tests for using tokendata=1 and contentless_delete=1 together. FossilOrigin-Name: a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10 --- ext/fts5/test/fts5origintext5.test | 135 +++++++++++++++++++++++++++++ manifest | 11 +-- manifest.uuid | 2 +- 3 files changed, 142 insertions(+), 6 deletions(-) create mode 100644 ext/fts5/test/fts5origintext5.test diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test new file mode 100644 index 0000000000..a48dfbb3ab --- /dev/null +++ b/ext/fts5/test/fts5origintext5.test @@ -0,0 +1,135 @@ +# 2023 Dec 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for tables that use both tokendata=1 and contentless_delete=1. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +# Return a random integer between 0 and n-1. +# +proc random {n} { expr {abs(int(rand()*$n))} } + +# Select an element of the list passed as the only argument at random and +# return it. +# +proc select_one {list} { + set n [llength $list] + lindex $list [random $n] +} + +# Given a term that consists entirely of alphabet characters, return all +# permutations of the term using upper and lower case characters. e.g. +# +# "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba} +# +proc casify {term {lRet {{}}}} { + if {$term==""} { return $lRet } + set t [string range $term 1 end] + set f1 [string toupper [string range $term 0 0]] + set f2 [string tolower [string range $term 0 0]] + set ret [list] + foreach x $lRet { + lappend ret "$x$f1" + lappend ret "$x$f2" + } + return [casify $t $ret] +} + +proc vocab {} { + list abc def ghi jkl mno pqr stu vwx yza +} + +# Return a random 3 letter term. +# +proc term {} { + if {[info exists ::expanded_vocab]==0} { + foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] } + } + + select_one $::expanded_vocab +} + +# Return a document - between 3 and 6 terms. +# +proc document {} { + set nTerm [expr [random 3] + 3] + set doc "" + for {set ii 0} {$ii < $nTerm} {incr ii} { + lappend doc [term] + } + set doc +} +db func document document + +#------------------------------------------------------------------------- + +set NDOC 200 +set NLOOP 100 + + +sqlite3_fts5_register_origintext db +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize="origintext unicode61", contentless_delete=1, content=, + tokendata=1 + ); + + CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT); + INSERT INTO ft(ft, rank) VALUES('pgsz', 64); +} +do_test 1.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc [document] + execsql { + INSERT INTO ft(rowid, x) VALUES($ii, $doc); + INSERT INTO ctrl(id, x) VALUES($ii, $doc); + } + } +} {} + +proc do_all_vocab_test {tn} { + foreach ::v [vocab] { + set answer [execsql {SELECT id FROM ctrl WHERE x LIKE '%' || $::v || '%'}] + do_execsql_test $tn.$::v { + SELECT rowid FROM ft($::v) + } $answer + } +} + +do_all_vocab_test 1.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft WHERE rowid = $r } + execsql { DELETE FROM ctrl WHERE rowid = $r } + + set doc [document] + execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) } + execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) } + } + do_all_vocab_test 1.3.$ii +} + + + + + +finish_test + diff --git a/manifest b/manifest index 2353c55e4c..61b1c976fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\scompiler\swarnings\sand\sother\sproblems\swith\sthe\snew\scode\son\sthis\sbranch. -D 2023-12-02T20:35:04.768 +C Add\stests\sfor\susing\stokendata=1\sand\scontentless_delete=1\stogether. +D 2023-12-04T15:08:21.125 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -194,6 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 +F ext/fts5/test/fts5origintext5.test 5b9fa1b7d2f8c5f933076000c30aea5b104c00c3f1b767334b87b76d46492e59 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2149,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0175d07e4094db5ea4b0378a5ff480dafb6ba9da86a113fa767c4c89c3c866f -R a72f98879c56fe0b2489dc56b6289649 +P 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0 +R 977c39d056cdc1226d7f3117e124e785 U dan -Z 6a8664529c8f348c40cce12fb229aa10 +Z 8168409535c63203870031d6c1a58cc5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b4349493de..0187da7a6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0 \ No newline at end of file +a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10 \ No newline at end of file From 732fb64ad38beb5f79742b25d524916d8dfae2ac Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 15:22:42 +0000 Subject: [PATCH 162/191] Two new NEVER macros. FossilOrigin-Name: 52632c92cb06faf0e804654b3490fd6c199521107bd30c8fcbc3a2a5a488098f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9ee6f8d2ea..17837b4093 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Repair\sissues\sand\sinefficiencies\sfound\sduring\stesting. -D 2023-12-04T13:12:45.155 +C Two\snew\sNEVER\smacros. +D 2023-12-04T15:22:42.529 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c fdab711d10347af4e70bbdc4dc3aca516a4ef0e6c4388cc70335ab9201bdf7eb +F src/json.c f9628f4dc6cc2ddf26b5edb1c275e80db886634003cead23ec8c7c8db8b4fd04 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 526b27f90897f5e35dfff7257daf6c4ce4798d649b09b8aecfb02df0449e3c51 -R a8fc2b32fcf7c8e7f15da5575d63ccf0 +P ae973cb1515f9d76409c92a2ca2ffd6b71f32b0b490a4886770e7c1b90f12611 +R 34754ed91bc896045fb5083333f2e977 U drh -Z 9f46540ea8d4faeba010e6034ffe6d00 +Z 8fd37e932957842ac050d54bc05491f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7d0da89ae..7bf52474de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae973cb1515f9d76409c92a2ca2ffd6b71f32b0b490a4886770e7c1b90f12611 \ No newline at end of file +52632c92cb06faf0e804654b3490fd6c199521107bd30c8fcbc3a2a5a488098f \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6df07b0e66..feab5f4dfe 100644 --- a/src/json.c +++ b/src/json.c @@ -2788,7 +2788,7 @@ rebuild_from_cache: if( p->nBlob==0 ){ goto json_pfa_malformed; } - if( p->aBlob==0 ){ + if( NEVER(p->aBlob==0) ){ goto json_pfa_oom; } if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){ @@ -2810,7 +2810,7 @@ rebuild_from_cache: p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); if( p->nJson==0 ) goto json_pfa_malformed; - if( p->zJson==0 ) goto json_pfa_oom; + if( NEVER(p->zJson==0) ) goto json_pfa_oom; if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ if( flgs & JSON_KEEPERROR ){ p->nErr = 1; From 99c41692f1297f2072345f895207ce5ecc881e3b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 16:01:39 +0000 Subject: [PATCH 163/191] Remove reachable ALWAYS and NEVER macros. FossilOrigin-Name: f601de3eeabd85993c1f5ee96b62de6fdabbeae2fe8950e00d08feb48d42c498 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 17837b4093..71f63d320e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Two\snew\sNEVER\smacros. -D 2023-12-04T15:22:42.529 +C Remove\sreachable\sALWAYS\sand\sNEVER\smacros. +D 2023-12-04T16:01:39.513 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c f9628f4dc6cc2ddf26b5edb1c275e80db886634003cead23ec8c7c8db8b4fd04 +F src/json.c 486643d34e35a9d33c88ba13810119eb08edf8f7585378379b2880ab2f20884f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae973cb1515f9d76409c92a2ca2ffd6b71f32b0b490a4886770e7c1b90f12611 -R 34754ed91bc896045fb5083333f2e977 +P 52632c92cb06faf0e804654b3490fd6c199521107bd30c8fcbc3a2a5a488098f +R e34d44ceb9e1b582fec5d492940f4291 U drh -Z 8fd37e932957842ac050d54bc05491f1 +Z 6c24af6356a01e718b1f63d8aebe720b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7bf52474de..feb883af6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52632c92cb06faf0e804654b3490fd6c199521107bd30c8fcbc3a2a5a488098f \ No newline at end of file +f601de3eeabd85993c1f5ee96b62de6fdabbeae2fe8950e00d08feb48d42c498 \ No newline at end of file diff --git a/src/json.c b/src/json.c index feab5f4dfe..10470e62df 100644 --- a/src/json.c +++ b/src/json.c @@ -2280,7 +2280,7 @@ static u32 jsonLookupStep( assert( !pParse->oom ); nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); - if( ALWAYS(!pParse->oom) ){ + if( !pParse->oom ){ memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); @@ -2401,7 +2401,7 @@ static void jsonReturnFromBlob( sqlite3 *db = sqlite3_context_db_handle(pCtx); n = jsonbPayloadSize(pParse, i, &sz); - if( NEVER(n==0) ) return; + if( n==0 ) return; switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { sqlite3_result_null(pCtx); From 9d373ca1c5eee2771e97af122f23264ba2f92576 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 17:05:37 +0000 Subject: [PATCH 164/191] Fix bug in xInstToken() causing the wrong token to be returned. FossilOrigin-Name: da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f --- ext/fts5/fts5_index.c | 10 +++++-- ext/fts5/test/fts5origintext5.test | 47 ++++++++++++++++++++++++++---- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index df08ca2127..095b945838 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6607,6 +6607,7 @@ struct Fts5TokenDataIter { Fts5TokenDataMap *aMap; Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; Fts5Iter *apIter[1]; }; @@ -6794,16 +6795,19 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Allocate array of iterators if they are not already allocated. */ if( pT->aPoslistReader==0 ){ - pT->aPoslistReader = sqlite3Fts5MallocZero( - &pIter->pIndex->rc, sizeof(Fts5PoslistReader) * pT->nIter + int nByte = pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)); + pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( + &pIter->pIndex->rc, nByte ); if( pT->aPoslistReader==0 ) return; + pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; } /* Populate an iterator for each poslist that will be merged */ for(ii=0; iinIter; ii++){ Fts5Iter *p = pT->apIter[ii]; if( iRowid==p->base.iRowid ){ + pT->aPoslistToIter[nReader] = ii; sqlite3Fts5PoslistReaderInit( p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] ); @@ -6855,7 +6859,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ if( eDetail==FTS5_DETAIL_FULL ){ pT->aMap[pT->nMap].iPos = iMinPos; - pT->aMap[pT->nMap].iIter = iMin; + pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin]; pT->aMap[pT->nMap].iRowid = iRowid; pT->nMap++; } diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test index a48dfbb3ab..9421758f88 100644 --- a/ext/fts5/test/fts5origintext5.test +++ b/ext/fts5/test/fts5origintext5.test @@ -65,10 +65,10 @@ proc term {} { select_one $::expanded_vocab } -# Return a document - between 3 and 6 terms. +# Return a document - between 3 and 10 terms. # proc document {} { - set nTerm [expr [random 3] + 3] + set nTerm [expr [random 3] + 7] set doc "" for {set ii 0} {$ii < $nTerm} {incr ii} { lappend doc [term] @@ -82,11 +82,39 @@ db func document document set NDOC 200 set NLOOP 100 - sqlite3_fts5_register_origintext db + +proc tokens {cmd} { + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + if {$txt==""} break + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +proc ctrl_tokens {doc term} { + set ret [list] + set term [string tolower $term] + foreach a $doc { + if {[string tolower $a]==$term} { + if {$a==$term} { + lappend ret $a + } else { + lappend ret [string tolower $a].$a + } + } + } + set ret +} +db func ctrl_tokens ctrl_tokens + + do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( - x, tokenize="origintext unicode61", contentless_delete=1, content=, + x, tokenize="origintext unicode61", tokendata=1 ); @@ -105,13 +133,20 @@ do_test 1.1 { proc do_all_vocab_test {tn} { foreach ::v [vocab] { - set answer [execsql {SELECT id FROM ctrl WHERE x LIKE '%' || $::v || '%'}] + set answer [execsql { + SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%' + }] do_execsql_test $tn.$::v { - SELECT rowid FROM ft($::v) + SELECT rowid, tokens(ft) FROM ft($::v) } $answer } } +#execsql_pp { SELECT * FROM ctrl } +#execsql_pp { SELECT * FROM ft } +#fts5_aux_test_functions db +#execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); } + do_all_vocab_test 1.2 for {set ii 0} {$ii < $NLOOP} {incr ii} { diff --git a/manifest b/manifest index 61b1c976fd..4ecc82d692 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\susing\stokendata=1\sand\scontentless_delete=1\stogether. -D 2023-12-04T15:08:21.125 +C Fix\sbug\sin\sxInstToken()\scausing\sthe\swrong\stoken\sto\sbe\sreturned. +D 2023-12-04T17:05:37.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c b31bf4f0fb51a15cc1aa54c2f337197740f4f8898347266781ca6970ca751302 +F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10 F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 -F ext/fts5/test/fts5origintext5.test 5b9fa1b7d2f8c5f933076000c30aea5b104c00c3f1b767334b87b76d46492e59 +F ext/fts5/test/fts5origintext5.test f4377b67debb10e3731030ce51245b9843ffb31f85725615b3b8820bd5912702 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a623cfa173b4035c759cb84985d11d8727053beb383648503987d6ab15c0ef0 -R 977c39d056cdc1226d7f3117e124e785 +P a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10 +R 135aeb3f5184e1f0131c446ddc8eab37 U dan -Z 8168409535c63203870031d6c1a58cc5 +Z 33215e49f8892bbcd31004428ffced69 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0187da7a6e..60beff1eed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10 \ No newline at end of file +da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f \ No newline at end of file From 9c794b9bff393410d9efbd4d43cb5540ed534531 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 17:40:28 +0000 Subject: [PATCH 165/191] Continuing simplifications and code cleanup. FossilOrigin-Name: ddf92b5059a9106753fd18b82ba8daa269a62af947561c460790107b83416f0b --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 109 +++++++++++++++++++++----------------------------- 3 files changed, 53 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 71f63d320e..7f065bdc52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sreachable\sALWAYS\sand\sNEVER\smacros. -D 2023-12-04T16:01:39.513 +C Continuing\ssimplifications\sand\scode\scleanup. +D 2023-12-04T17:40:28.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 486643d34e35a9d33c88ba13810119eb08edf8f7585378379b2880ab2f20884f +F src/json.c 0e9fc8f2f2fecb1b4e7904ead93aa98313483ca4fea1d564e252e445f717b226 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52632c92cb06faf0e804654b3490fd6c199521107bd30c8fcbc3a2a5a488098f -R e34d44ceb9e1b582fec5d492940f4291 +P f601de3eeabd85993c1f5ee96b62de6fdabbeae2fe8950e00d08feb48d42c498 +R c9ebf0dbf5357376ee89626a26211c09 U drh -Z 6c24af6356a01e718b1f63d8aebe720b +Z 2f59464d9d28542ea091e094068881ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index feb883af6c..b3d16e9d7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f601de3eeabd85993c1f5ee96b62de6fdabbeae2fe8950e00d08feb48d42c498 \ No newline at end of file +ddf92b5059a9106753fd18b82ba8daa269a62af947561c460790107b83416f0b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 10470e62df..dc884b9fd3 100644 --- a/src/json.c +++ b/src/json.c @@ -2029,7 +2029,7 @@ static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ const u8 *aBlob; int nBlob; JsonParse s; - assert( sqlite3_value_type(pJson)==SQLITE_BLOB ); + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; aBlob = sqlite3_value_blob(pJson); nBlob = sqlite3_value_bytes(pJson); if( nBlob<1 ) return 0; @@ -3555,36 +3555,39 @@ static void jsonRemoveFunc( p = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); if( p==0 ) return; for(i=1; ieEdit = JEDIT_DEL; p->delta = 0; rc = jsonLookupStep(p, 0, zPath+1, 0); - if( rc==JSON_LOOKUP_NOTFOUND ) continue; - if( JSON_LOOKUP_ISERROR(rc) ) goto json_remove_patherror; + if( JSON_LOOKUP_ISERROR(rc) ){ + if( rc==JSON_LOOKUP_NOTFOUND ){ + continue; /* No-op */ + }else if( rc==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_remove_done; + } } jsonReturnParse(ctx, p); jsonParseFree(p); return; json_remove_patherror: - jsonParseFree(p); jsonBadPathError(ctx, zPath); - return; -json_remove_return_null: +json_remove_done: jsonParseFree(p); return; } @@ -3657,18 +3660,8 @@ static void jsonTypeFunc( p = jsonParseFuncArg(ctx, argv[0], 0); if( p==0 ) return; if( argc==2 ){ - if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ - goto json_type_done; - } - if( sqlite3_value_bytes(argv[1])==0 ){ - jsonBadPathError(ctx, ""); - goto json_type_done; - } zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ){ - sqlite3_result_error_nomem(ctx); - goto json_type_done; - } + if( zPath==0 ) goto json_type_done; if( zPath[0]!='$' ){ jsonBadPathError(ctx, zPath); goto json_type_done; @@ -3841,17 +3834,14 @@ static void jsonValidFunc( ** ** If the argument is NULL, return NULL ** -** If the argument is TEXT then try to interpret that text as JSON and +** If the argument is BLOB, do a fast validity check and return non-zero +** if the check fails. The returned value does not indicate where in the +** BLOB the error occurs. +** +** Otherwise interpret the argument is TEXT (even if it is numeric) and ** return the 1-based character position for where the parser first recognized ** that the input was not valid JSON, or return 0 if the input text looks ** ok. JSON-5 extensions are accepted. -** -** If the argument is a BLOB then try to interpret the blob as a JSONB -** and return the 1-based byte offset of the first position that is -** misformatted. Return 0 if the input BLOB seems to be well-formed. -** -** Numeric inputs are converted into text, which is usually valid -** JSON-5, so they should return 0. */ static void jsonErrorFunc( sqlite3_context *ctx, @@ -3863,46 +3853,42 @@ static void jsonErrorFunc( assert( argc==1 ); UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_NULL: { - return; + memset(&s, 0, sizeof(s)); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + JsonString out; + jsonStringInit(&out, 0); + s.aBlob = (u8*)sqlite3_value_blob(argv[0]); + s.nBlob = sqlite3_value_bytes(argv[0]); + jsonXlateBlobToText(&s, 0, &out); + if( out.eErr ){ + iErrPos = (out.eErr & JSTRING_MALFORMED)!=0 ? 1 : -1; } - case SQLITE_BLOB: { - if( !jsonFuncArgMightBeBinary(argv[0]) ) iErrPos = 1; - break; - } - default: { - memset(&s, 0, sizeof(s)); - s.zJson = (char*)sqlite3_value_text(argv[0]); - s.nJson = sqlite3_value_bytes(argv[0]); - if( s.nJson==0 ){ - iErrPos = 1; - break; - } - if( s.zJson==0 ){ - sqlite3_result_error_nomem(ctx); - return; - } - if( jsonConvertTextToBlob(&s,0) ){ - u32 k; - if( s.oom ){ - sqlite3_result_error_nomem(ctx); - jsonParseReset(&s); - return; - } + jsonStringReset(&out); + }else{ + s.zJson = (char*)sqlite3_value_text(argv[0]); + if( s.zJson==0 ) return; /* NULL input or OOM */ + s.nJson = sqlite3_value_bytes(argv[0]); + if( jsonConvertTextToBlob(&s,0) ){ + if( s.oom ){ + iErrPos = -1; + }else{ /* Convert byte-offset s.iErr into a character offset */ - for(k=0; ksParse.nBlob = sqlite3_value_bytes(argv[0]); p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); - if( p->sParse.aBlob==0 ){ - return SQLITE_NOMEM; - } }else{ goto json_each_malformed_input; } From 3dfc063705277f68540fc8aa3d55967481ebf042 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 17:45:33 +0000 Subject: [PATCH 166/191] Fix a problem with the xInstCount() API and "ORDER BY rank" queries. FossilOrigin-Name: 317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 2 +- ext/fts5/fts5_main.c | 3 ++- ext/fts5/test/fts5origintext5.test | 17 ++++++++++++++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4e4385ce2b..9beb26e056 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -787,7 +787,7 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); -int sqlite3Fts5ExprInstToken(Fts5Expr*, int, int, int, int, const char**, int*); +int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*); void sqlite3Fts5ExprClearTokens(Fts5Expr*); /******************************************* diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 6f58cf8735..02d7e78cbe 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3199,6 +3199,7 @@ int sqlite3Fts5ExprQueryToken( */ int sqlite3Fts5ExprInstToken( Fts5Expr *pExpr, + i64 iRowid, int iPhrase, int iCol, int iOff, @@ -3208,7 +3209,6 @@ int sqlite3Fts5ExprInstToken( ){ Fts5ExprPhrase *pPhrase = 0; Fts5IndexIter *pIter = 0; - i64 iRowid = pExpr->pRoot->iRowid; if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ return SQLITE_RANGE; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index e911b0c0f9..c183f3c579 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2364,9 +2364,10 @@ static int fts5ApiInstToken( int iPhrase = pCsr->aInst[iIdx*3]; int iCol = pCsr->aInst[iIdx*3 + 1]; int iOff = pCsr->aInst[iIdx*3 + 2]; + i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5ExprInstToken( - pCsr->pExpr, iPhrase, iCol, iOff, iToken, ppOut, pnOut + pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut ); } } diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test index 9421758f88..b425c582e8 100644 --- a/ext/fts5/test/fts5origintext5.test +++ b/ext/fts5/test/fts5origintext5.test @@ -79,15 +79,17 @@ db func document document #------------------------------------------------------------------------- +expr srand(6) + set NDOC 200 set NLOOP 100 sqlite3_fts5_register_origintext db proc tokens {cmd} { + set ret [list] for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { set txt [$cmd xInstToken $iTok 0] - if {$txt==""} break set txt [string map [list "\0" "."] $txt] lappend ret $txt } @@ -95,6 +97,11 @@ proc tokens {cmd} { } sqlite3_fts5_create_function db tokens tokens +proc rankfunc {cmd} { + $cmd xRowid +} +sqlite3_fts5_create_function db rankfunc rankfunc + proc ctrl_tokens {doc term} { set ret [list] set term [string tolower $term] @@ -114,12 +121,13 @@ db func ctrl_tokens ctrl_tokens do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( - x, tokenize="origintext unicode61", + x, tokenize="origintext unicode61", content=, contentless_delete=1, tokendata=1 ); CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT); INSERT INTO ft(ft, rank) VALUES('pgsz', 64); + INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()'); } do_test 1.1 { for {set ii 0} {$ii < $NDOC} {incr ii} { @@ -136,9 +144,12 @@ proc do_all_vocab_test {tn} { set answer [execsql { SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%' }] - do_execsql_test $tn.$::v { + do_execsql_test $tn.$::v.1 { SELECT rowid, tokens(ft) FROM ft($::v) } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank + } $answer } } diff --git a/manifest b/manifest index 4ecc82d692..00c355a167 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbug\sin\sxInstToken()\scausing\sthe\swrong\stoken\sto\sbe\sreturned. -D 2023-12-04T17:05:37.721 +C Fix\sa\sproblem\swith\sthe\sxInstCount()\sAPI\sand\s"ORDER\sBY\srank"\squeries. +D 2023-12-04T17:45:33.714 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -90,14 +90,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff90acaa97f8e865b66d1177d1b56b8c110fd5548ab5863bab43f055a1d745fe -F ext/fts5/fts5Int.h 1fdbf3d16bdd481fe2ee99927919e4c3db835efae00f8efd7efb5e6a93277459 +F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 5619c3fab45a78eb5ed3021e3b40ec3b435ef3669293e8700354aa8dd3e6c796 +F ext/fts5/fts5_expr.c aec893108d90cc8eae6801283d38d4e705d8c22aba12717fcc56e1910b8e3b32 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10 -F ext/fts5/fts5_main.c 075995302198fe6f591fdbbedd415dfac564a9bfc20aea81e6fa0503b2d94af0 +F ext/fts5/fts5_main.c 1fbadf63a7381fd68753efdccce19683f587668489b1c5a9c7bf53d6e9b64872 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 -F ext/fts5/test/fts5origintext5.test f4377b67debb10e3731030ce51245b9843ffb31f85725615b3b8820bd5912702 +F ext/fts5/test/fts5origintext5.test f9dfb005248d764fdc52859308875f680b523a897258a2cc40100f9e2356b5a9 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a2506b8c9718054912270055638204753c4156bbc115e55194e6df9d7e76cb10 -R 135aeb3f5184e1f0131c446ddc8eab37 +P da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f +R 17d195e142931273edf7c88c21ff2164 U dan -Z 33215e49f8892bbcd31004428ffced69 +Z 7652ccb47cdd7a8e28db861627789490 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 60beff1eed..f1f57381d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f \ No newline at end of file +317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc \ No newline at end of file From 910c77b049f6d64866bc8d0a0745f8ed446e0dfa Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 17:58:56 +0000 Subject: [PATCH 167/191] Fix memory leak in new code on this branch. FossilOrigin-Name: ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f --- ext/fts5/fts5_index.c | 1 - manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 095b945838..a24308534c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2763,7 +2763,6 @@ static void fts5SegIterNextInit( pIter->iLeafPgno = iPg - 1; fts5SegIterNextPage(p, pIter); fts5SegIterSetNext(p, pIter); - fts5SegIterAllocTombstone(p, pIter); } if( pIter->pLeaf ){ const u8 *a = pIter->pLeaf->p; diff --git a/manifest b/manifest index 00c355a167..355143083f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\sxInstCount()\sAPI\sand\s"ORDER\sBY\srank"\squeries. -D 2023-12-04T17:45:33.714 +C Fix\smemory\sleak\sin\snew\scode\son\sthis\sbranch. +D 2023-12-04T17:58:56.324 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c aec893108d90cc8eae6801283d38d4e705d8c22aba12717fcc56e1910b8e3b32 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 072666814be04485445c07e0e75362d13245cef6c66e07aa2060c532190d0c10 +F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba F ext/fts5/fts5_main.c 1fbadf63a7381fd68753efdccce19683f587668489b1c5a9c7bf53d6e9b64872 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da78d07e77cbc783fbc725758911c230fd6a1c1885d9576125de955dcc2bd37f -R 17d195e142931273edf7c88c21ff2164 +P 317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc +R 4cc8f92a68d4e11e53934c982deb6dc3 U dan -Z 7652ccb47cdd7a8e28db861627789490 +Z c8f15e3d33aff662e610fc8982efce75 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f1f57381d1..bb8887198f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc \ No newline at end of file +ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f \ No newline at end of file From 54318b382a982fa08dfe3d034156afa20550aea9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 18:45:14 +0000 Subject: [PATCH 168/191] Fixes for xInstToken() with tokendata=0 tables. And with prefix queries. FossilOrigin-Name: 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1 --- ext/fts5/fts5_expr.c | 18 ++++++++++---- ext/fts5/fts5_main.c | 1 - ext/fts5/test/fts5origintext.test | 39 +++++++++++++++++++++++++++++++ manifest | 16 ++++++------- manifest.uuid | 2 +- 5 files changed, 62 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 02d7e78cbe..c5de6eba6c 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3208,7 +3208,8 @@ int sqlite3Fts5ExprInstToken( int *pnOut ){ Fts5ExprPhrase *pPhrase = 0; - Fts5IndexIter *pIter = 0; + Fts5ExprTerm *pTerm = 0; + int rc = SQLITE_OK; if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ return SQLITE_RANGE; @@ -3217,9 +3218,18 @@ int sqlite3Fts5ExprInstToken( if( iToken<0 || iToken>=pPhrase->nTerm ){ return SQLITE_RANGE; } - pIter = pPhrase->aTerm[iToken].pIter; - - return sqlite3Fts5IterToken(pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut); + pTerm = &pPhrase->aTerm[iToken]; + if( pTerm->bPrefix==0 ){ + if( pExpr->pConfig->bTokendata ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; + } + } + return rc; } /* diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index c183f3c579..d35e998da0 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2365,7 +2365,6 @@ static int fts5ApiInstToken( int iCol = pCsr->aInst[iIdx*3 + 1]; int iOff = pCsr->aInst[iIdx*3 + 2]; i64 iRowid = fts5CursorRowid(pCsr); - rc = sqlite3Fts5ExprInstToken( pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut ); diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 8273b3ca4d..9752f35d34 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -252,6 +252,45 @@ do_execsql_test 5.3 { {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5} } +#------------------------------------------------------------------------- +# Test the xInstToken() API with: +# +# * a non tokendata=1 table. +# * prefix queries. +# +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize='origintext unicode61', detail=%DETAIL% + ); + + INSERT INTO ft VALUES('One Two', 'Three two'); + INSERT INTO ft VALUES('three Three', 'one One'); +} +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +do_execsql_test 6.1 { + SELECT rowid, tokens(ft) FROM ft('One'); +} {1 one.One 2 one.One} + +do_execsql_test 6.2 { + SELECT rowid, tokens(ft) FROM ft('on*'); +} {1 {{}} 2 {{} {}}} + +do_execsql_test 6.3 { + SELECT rowid, tokens(ft) FROM ft('Three*'); +} {1 {{}} 2 {{}}} + } finish_test diff --git a/manifest b/manifest index 355143083f..52485ee8e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smemory\sleak\sin\snew\scode\son\sthis\sbranch. -D 2023-12-04T17:58:56.324 +C Fixes\sfor\sxInstToken()\swith\stokendata=0\stables.\sAnd\swith\sprefix\squeries. +D 2023-12-04T18:45:14.192 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,10 +94,10 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c aec893108d90cc8eae6801283d38d4e705d8c22aba12717fcc56e1910b8e3b32 +F ext/fts5/fts5_expr.c 920516be4aac44eccdd9e747fe26f367442848b9a58971e4b36edb0c8284b6b8 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba -F ext/fts5/fts5_main.c 1fbadf63a7381fd68753efdccce19683f587668489b1c5a9c7bf53d6e9b64872 +F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -190,7 +190,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0703aff5981625b527e62 +F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 @@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc -R 4cc8f92a68d4e11e53934c982deb6dc3 +P ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f +R 4a97cb4432f9a2f4a872da3bc4853a9c U dan -Z c8f15e3d33aff662e610fc8982efce75 +Z d5e27dba49298422b322a3619fa6b5f7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb8887198f..667968c045 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f \ No newline at end of file +78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1 \ No newline at end of file From 3fedb7e59e6a5208f006cad5e02be79abdd2e759 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 18:53:10 +0000 Subject: [PATCH 169/191] Fix errors in rendering JSON5 escape sequences embedded in JSONB. FossilOrigin-Name: f1a51ae3863557526a51c6e98e71fcdf4f1ed14a36212b3c90f7408f926345e4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 12 ++++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f065bdc52..7e8f87147a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continuing\ssimplifications\sand\scode\scleanup. -D 2023-12-04T17:40:28.698 +C Fix\serrors\sin\srendering\sJSON5\sescape\ssequences\sembedded\sin\sJSONB. +D 2023-12-04T18:53:10.232 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 0e9fc8f2f2fecb1b4e7904ead93aa98313483ca4fea1d564e252e445f717b226 +F src/json.c b3a194fc1db52f1b169d9779a0d9155b4c123e8492a921114a5381700ea6c81d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f601de3eeabd85993c1f5ee96b62de6fdabbeae2fe8950e00d08feb48d42c498 -R c9ebf0dbf5357376ee89626a26211c09 +P ddf92b5059a9106753fd18b82ba8daa269a62af947561c460790107b83416f0b +R 2f4c11a457fe80bb549ac0019420b08f U drh -Z 2f59464d9d28542ea091e094068881ef +Z 9def50f1dac0c1957715ee3f6f843ba7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b3d16e9d7b..030ca873ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ddf92b5059a9106753fd18b82ba8daa269a62af947561c460790107b83416f0b \ No newline at end of file +f1a51ae3863557526a51c6e98e71fcdf4f1ed14a36212b3c90f7408f926345e4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index dc884b9fd3..aefd47498f 100644 --- a/src/json.c +++ b/src/json.c @@ -2401,7 +2401,10 @@ static void jsonReturnFromBlob( sqlite3 *db = sqlite3_context_db_handle(pCtx); n = jsonbPayloadSize(pParse, i, &sz); - if( n==0 ) return; + if( n==0 ){ + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; + } switch( pParse->aBlob[i] & 0x0f ){ case JSONB_NULL: { sqlite3_result_null(pCtx); @@ -2483,7 +2486,7 @@ static void jsonReturnFromBlob( }else{ u32 vlo; if( (v&0xfc00)==0xd800 - && i Date: Mon, 4 Dec 2023 19:14:13 +0000 Subject: [PATCH 170/191] Do not make the input JSONB editable in json_remove() if there are no PATH argument. FossilOrigin-Name: 66594544f3ba9977475a3e3f74404eb2b2fb845053b28bd24c2b52c7df94e9d7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7e8f87147a..e729bdb871 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\serrors\sin\srendering\sJSON5\sescape\ssequences\sembedded\sin\sJSONB. -D 2023-12-04T18:53:10.232 +C Do\snot\smake\sthe\sinput\sJSONB\seditable\sin\sjson_remove()\sif\sthere\sare\sno\sPATH\nargument. +D 2023-12-04T19:14:13.899 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c b3a194fc1db52f1b169d9779a0d9155b4c123e8492a921114a5381700ea6c81d +F src/json.c ab1eb1ab2ecedf193e0e1dc2ae62668e74ffd8668c39bf2014d419d175f6aae6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ddf92b5059a9106753fd18b82ba8daa269a62af947561c460790107b83416f0b -R 2f4c11a457fe80bb549ac0019420b08f +P f1a51ae3863557526a51c6e98e71fcdf4f1ed14a36212b3c90f7408f926345e4 +R fedbc77310dd69998d8c7dc42f0e4ac1 U drh -Z 9def50f1dac0c1957715ee3f6f843ba7 +Z e47b61cc62bf0572c657f0a0469cf19a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 030ca873ab..4c9ef95e4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1a51ae3863557526a51c6e98e71fcdf4f1ed14a36212b3c90f7408f926345e4 \ No newline at end of file +66594544f3ba9977475a3e3f74404eb2b2fb845053b28bd24c2b52c7df94e9d7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index aefd47498f..97c65df0bc 100644 --- a/src/json.c +++ b/src/json.c @@ -2802,7 +2802,6 @@ rebuild_from_cache: if( n==0 || sz+n!=p->nBlob || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0) - || sz+n!=p->nBlob ){ goto json_pfa_malformed; } @@ -3556,7 +3555,7 @@ static void jsonRemoveFunc( u32 rc; /* Subroutine return code */ if( argc<1 ) return; - p = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + p = jsonParseFuncArg(ctx, argv[0], argc>1 ? JSON_EDITABLE : 0); if( p==0 ) return; for(i=1; i Date: Mon, 4 Dec 2023 19:32:17 +0000 Subject: [PATCH 171/191] Fixes to error handling in json_array_length(). FossilOrigin-Name: aa85df2d26b74c171c55bde19ef17c4f11f40b8af7181bbf7162f87cdea7e88b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index e729bdb871..c1b6b255f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\smake\sthe\sinput\sJSONB\seditable\sin\sjson_remove()\sif\sthere\sare\sno\sPATH\nargument. -D 2023-12-04T19:14:13.899 +C Fixes\sto\serror\shandling\sin\sjson_array_length(). +D 2023-12-04T19:32:17.504 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c ab1eb1ab2ecedf193e0e1dc2ae62668e74ffd8668c39bf2014d419d175f6aae6 +F src/json.c 326c7149206251639111f4fe2275f66d50345da0725dec4c617d583a280e969d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f1a51ae3863557526a51c6e98e71fcdf4f1ed14a36212b3c90f7408f926345e4 -R fedbc77310dd69998d8c7dc42f0e4ac1 +P 66594544f3ba9977475a3e3f74404eb2b2fb845053b28bd24c2b52c7df94e9d7 +R 07c551e152c91bb13fe1d576cd87540a U drh -Z e47b61cc62bf0572c657f0a0469cf19a +Z 0dc2cc057fc8990b916cb29e965f29a3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4c9ef95e4c..2b59f8602a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66594544f3ba9977475a3e3f74404eb2b2fb845053b28bd24c2b52c7df94e9d7 \ No newline at end of file +aa85df2d26b74c171c55bde19ef17c4f11f40b8af7181bbf7162f87cdea7e88b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 97c65df0bc..6b72d3fdea 100644 --- a/src/json.c +++ b/src/json.c @@ -3115,11 +3115,7 @@ static void jsonArrayLengthFunc( if( (p->aBlob[i] & 0x0f)==JSONB_ARRAY ){ cnt = jsonbArrayCount(p, i); } - if( eErr ){ - if( eErr==2 ) sqlite3_result_error(ctx, "malformed JSON", -1); - }else{ - sqlite3_result_int64(ctx, cnt); - } + if( !eErr ) sqlite3_result_int64(ctx, cnt); jsonParseFree(p); } From 49bfbc1ef37784f9f205f06c272716b058d5be3e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Dec 2023 19:48:08 +0000 Subject: [PATCH 172/191] Add further tests for xInstToken(). FossilOrigin-Name: 8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4 --- ext/fts5/test/fts5origintext5.test | 96 +++++++++++++++++++++++------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test index b425c582e8..89a5d8f39c 100644 --- a/ext/fts5/test/fts5origintext5.test +++ b/ext/fts5/test/fts5origintext5.test @@ -82,7 +82,7 @@ db func document document expr srand(6) set NDOC 200 -set NLOOP 100 +set NLOOP 50 sqlite3_fts5_register_origintext db @@ -102,15 +102,17 @@ proc rankfunc {cmd} { } sqlite3_fts5_create_function db rankfunc rankfunc -proc ctrl_tokens {doc term} { +proc ctrl_tokens {term args} { set ret [list] set term [string tolower $term] - foreach a $doc { - if {[string tolower $a]==$term} { - if {$a==$term} { - lappend ret $a - } else { - lappend ret [string tolower $a].$a + foreach doc $args { + foreach a $doc { + if {[string tolower $a]==$term} { + if {$a==$term} { + lappend ret $a + } else { + lappend ret [string tolower $a].$a + } } } } @@ -118,6 +120,20 @@ proc ctrl_tokens {doc term} { } db func ctrl_tokens ctrl_tokens +proc do_all_vocab_test {tn} { + foreach ::v [vocab] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft) FROM ft($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank + } $answer + } +} + do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( @@ -139,20 +155,6 @@ do_test 1.1 { } } {} -proc do_all_vocab_test {tn} { - foreach ::v [vocab] { - set answer [execsql { - SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%' - }] - do_execsql_test $tn.$::v.1 { - SELECT rowid, tokens(ft) FROM ft($::v) - } $answer - do_execsql_test $tn.$::v.2 { - SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank - } $answer - } -} - #execsql_pp { SELECT * FROM ctrl } #execsql_pp { SELECT * FROM ft } #fts5_aux_test_functions db @@ -173,9 +175,59 @@ for {set ii 0} {$ii < $NLOOP} {incr ii} { do_all_vocab_test 1.3.$ii } +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts5( + x, y, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT); + INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64); + INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()'); +} +do_test 2.1 { + for {set ii 0} {$ii < $NDOC} {incr ii} { + set doc1 [document] + set doc2 [document] + execsql { + INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2); + INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2); + } + } +} {} +proc do_all_vocab_test2 {tn} { + foreach ::v [vocab] { + set answer [execsql { + SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2 + WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%'; + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft2) FROM ft2($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank + } $answer + } +} + +do_all_vocab_test2 2.2 + +for {set ii 0} {$ii < $NLOOP} {incr ii} { + set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }] + foreach r $lRowid { + execsql { DELETE FROM ft2 WHERE rowid = $r } + execsql { DELETE FROM ctrl2 WHERE rowid = $r } + + set doc1 [document] + set doc2 [document] + execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) } + execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) } + } + do_all_vocab_test 2.3.$ii +} finish_test diff --git a/manifest b/manifest index 52485ee8e6..11d57c4176 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sxInstToken()\swith\stokendata=0\stables.\sAnd\swith\sprefix\squeries. -D 2023-12-04T18:45:14.192 +C Add\sfurther\stests\sfor\sxInstToken(). +D 2023-12-04T19:48:08.530 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e6 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 -F ext/fts5/test/fts5origintext5.test f9dfb005248d764fdc52859308875f680b523a897258a2cc40100f9e2356b5a9 +F ext/fts5/test/fts5origintext5.test 067bfb3008323585df640ab29e8ef7c4ca6dec62c597be07a9f896d88f98cd10 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f -R 4a97cb4432f9a2f4a872da3bc4853a9c +P 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1 +R ced1cc2c59284a9ef3026043e080f745 U dan -Z d5e27dba49298422b322a3619fa6b5f7 +Z e2cce7acff968ef3c9472723192ae452 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 667968c045..780ed88175 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1 \ No newline at end of file +8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4 \ No newline at end of file From 3cdb079476530f3ef394bbe2c8c0ce3120f012b2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 4 Dec 2023 23:12:57 +0000 Subject: [PATCH 173/191] Rename the internal routine jsonMergePatchBlob() to just jsonMergePatch(). FossilOrigin-Name: ebf667b616235bb64b83832008342ba5e7b10b2c170d7cebc431f040fef7ecfb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c1b6b255f5..db03cd3807 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\serror\shandling\sin\sjson_array_length(). -D 2023-12-04T19:32:17.504 +C Rename\sthe\sinternal\sroutine\sjsonMergePatchBlob()\sto\sjust\sjsonMergePatch(). +D 2023-12-04T23:12:57.639 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 326c7149206251639111f4fe2275f66d50345da0725dec4c617d583a280e969d +F src/json.c d2385c49df219c1d10cb0a3a817df5aa7bd2ef86772f24f3763e2341b0875b70 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 66594544f3ba9977475a3e3f74404eb2b2fb845053b28bd24c2b52c7df94e9d7 -R 07c551e152c91bb13fe1d576cd87540a +P aa85df2d26b74c171c55bde19ef17c4f11f40b8af7181bbf7162f87cdea7e88b +R 6429070b5e7eb1a8581d28813921879f U drh -Z 0dc2cc057fc8990b916cb29e965f29a3 +Z f2f7e0af6ffb5ce2a2c95b6ed2412ab3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2b59f8602a..7301a291e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa85df2d26b74c171c55bde19ef17c4f11f40b8af7181bbf7162f87cdea7e88b \ No newline at end of file +ebf667b616235bb64b83832008342ba5e7b10b2c170d7cebc431f040fef7ecfb \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6b72d3fdea..e0651130ff 100644 --- a/src/json.c +++ b/src/json.c @@ -3243,7 +3243,7 @@ json_extract_error: } /* -** Return codes for jsonMergePatchBlob() +** Return codes for jsonMergePatch() */ #define JSON_MERGE_OK 0 /* Success */ #define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ @@ -3296,7 +3296,7 @@ json_extract_error: ** | ** ^---- Line numbers referenced in comments in the implementation */ -static int jsonMergePatchBlob( +static int jsonMergePatch( JsonParse *pTarget, /* The JSON parser that contains the TARGET */ u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ const JsonParse *pPatch, /* The PATCH */ @@ -3427,7 +3427,7 @@ static int jsonMergePatchBlob( /* Algorithm line 12 */ int rc, savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatchBlob(pTarget, iTValue, pPatch, iPValue); + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -3448,7 +3448,7 @@ static int jsonMergePatchBlob( pTarget->aBlob[iTEnd+szNew] = 0x00; savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatchBlob(pTarget, iTEnd+szNew,pPatch,iPValue); + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -3479,7 +3479,7 @@ static void jsonPatchFunc( if( pTarget==0 ) return; pPatch = jsonParseFuncArg(ctx, argv[1], 0); if( pPatch ){ - rc = jsonMergePatchBlob(pTarget, 0, pPatch, 0); + rc = jsonMergePatch(pTarget, 0, pPatch, 0); if( rc==JSON_MERGE_OK ){ jsonReturnParse(ctx, pTarget); }else if( rc==JSON_MERGE_OOM ){ From ae2e9728020ecc71d8a47bddb0c0555fdca56a81 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 00:17:17 +0000 Subject: [PATCH 174/191] Fix OOM and corrupt JSONB handling in json_patch(). FossilOrigin-Name: 1910feb0b7d5cc2b810c3322f6cca281d8730182d30d162bd7bb56800979ea91 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index db03cd3807..c0fde4f425 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sthe\sinternal\sroutine\sjsonMergePatchBlob()\sto\sjust\sjsonMergePatch(). -D 2023-12-04T23:12:57.639 +C Fix\sOOM\sand\scorrupt\sJSONB\shandling\sin\sjson_patch(). +D 2023-12-05T00:17:17.234 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c d2385c49df219c1d10cb0a3a817df5aa7bd2ef86772f24f3763e2341b0875b70 +F src/json.c 4dadaf584446be580a7a0054d25c4fc3ae17065952d5d1ffdb7a05a1cc102519 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa85df2d26b74c171c55bde19ef17c4f11f40b8af7181bbf7162f87cdea7e88b -R 6429070b5e7eb1a8581d28813921879f +P ebf667b616235bb64b83832008342ba5e7b10b2c170d7cebc431f040fef7ecfb +R 63dbe3f87f86d8011f70d9756cf66af6 U drh -Z f2f7e0af6ffb5ce2a2c95b6ed2412ab3 +Z f62ad889060f6abf937d9ec41a83acdd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7301a291e0..5223dccf73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebf667b616235bb64b83832008342ba5e7b10b2c170d7cebc431f040fef7ecfb \ No newline at end of file +1910feb0b7d5cc2b810c3322f6cca281d8730182d30d162bd7bb56800979ea91 \ No newline at end of file diff --git a/src/json.c b/src/json.c index e0651130ff..57082cf3f8 100644 --- a/src/json.c +++ b/src/json.c @@ -3348,11 +3348,11 @@ static int jsonMergePatch( pTarget->aBlob[iTarget] = (x & 0xf0) | JSONB_OBJECT; } n = jsonbPayloadSize(pPatch, iPatch, &sz); - if( n==0 ) return JSON_MERGE_BADPATCH; + if( NEVER(n==0) ) return JSON_MERGE_BADPATCH; iPCursor = iPatch+n; iPEnd = iPCursor+sz; n = jsonbPayloadSize(pTarget, iTarget, &sz); - if( n==0 ) return JSON_MERGE_BADTARGET; + if( NEVER(n==0) ) return JSON_MERGE_BADTARGET; iTStart = iTarget+n; iTEndBE = iTStart+sz; @@ -3365,7 +3365,7 @@ static int jsonMergePatch( nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); if( nPLabel==0 ) return JSON_MERGE_BADPATCH; iPValue = iPCursor + nPLabel + szPLabel; - if( iPCursor>=iPEnd ) return JSON_MERGE_BADPATCH; + if( iPValue>=iPEnd ) return JSON_MERGE_BADPATCH; nPValue = jsonbPayloadSize(pPatch, iPValue, &szPValue); if( nPValue==0 ) return JSON_MERGE_BADPATCH; iPCursor = iPValue + nPValue + szPValue; @@ -3397,13 +3397,12 @@ static int jsonMergePatch( }else{ /* Should rarely happen */ JsonString s1, s2; - int isEqual; + int isEqual, isOom; jsonStringInit(&s1, 0); jsonXlateBlobToText(pTarget, iTLabel, &s1); - if( s1.eErr ) return JSON_MERGE_OOM; jsonStringInit(&s2, 0); jsonXlateBlobToText(pPatch, iPLabel, &s2); - if( s2.eErr ) return JSON_MERGE_OOM; + isOom = s1.eErr || s2.eErr; if( s1.nUsed==s2.nUsed && memcmp(s1.zBuf, s2.zBuf, s1.nUsed)==0 ){ isEqual = 1; }else{ @@ -3411,6 +3410,7 @@ static int jsonMergePatch( } jsonStringReset(&s1); jsonStringReset(&s2); + if( isOom ) return JSON_MERGE_OOM; if( isEqual ) break; } iTCursor = iTValue + nTValue + szTValue; @@ -3420,9 +3420,9 @@ static int jsonMergePatch( /* A match was found. Algorithm line 08 */ if( x==0 ){ /* Patch value is NULL. Algorithm line 09 */ - jsonBlobEdit(pTarget, iTLabel, nTLabel+szTLabel+nTValue+szTValue, - 0, 0); - if( pTarget->oom ) return JSON_MERGE_OOM; + jsonBlobEdit(pTarget, iTLabel, nTLabel+szTLabel+nTValue+szTValue, 0,0); + /* vvvvvv----- No OOM on a delete-only edit */ + if( NEVER(pTarget->oom) ) return JSON_MERGE_OOM; }else{ /* Algorithm line 12 */ int rc, savedDelta = pTarget->delta; From fa43e21711422f4d5f1e82b6eda8a96f474c5a99 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 01:44:15 +0000 Subject: [PATCH 175/191] Use an assert() to fix a harmless static analyzer warning. FossilOrigin-Name: a249ca657e624028bc6b3d2c2bcedd7162d118addb7d62ce519920cecebf1860 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c0fde4f425..62d3de3ec6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sOOM\sand\scorrupt\sJSONB\shandling\sin\sjson_patch(). -D 2023-12-05T00:17:17.234 +C Use\san\sassert()\sto\sfix\sa\sharmless\sstatic\sanalyzer\swarning. +D 2023-12-05T01:44:15.015 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4dadaf584446be580a7a0054d25c4fc3ae17065952d5d1ffdb7a05a1cc102519 +F src/json.c 3635645a2a07070a38dade2ba34daa3f3633fd134a800cb05db236d68bc1b5c3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebf667b616235bb64b83832008342ba5e7b10b2c170d7cebc431f040fef7ecfb -R 63dbe3f87f86d8011f70d9756cf66af6 +P 1910feb0b7d5cc2b810c3322f6cca281d8730182d30d162bd7bb56800979ea91 +R a9ebb3d8db9982a53cbb9404704cc04b U drh -Z f62ad889060f6abf937d9ec41a83acdd +Z 17d27d3e2a2e6e4799038f7f740bf1ce # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5223dccf73..0dc59fba28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1910feb0b7d5cc2b810c3322f6cca281d8730182d30d162bd7bb56800979ea91 \ No newline at end of file +a249ca657e624028bc6b3d2c2bcedd7162d118addb7d62ce519920cecebf1860 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 57082cf3f8..7bb409a8eb 100644 --- a/src/json.c +++ b/src/json.c @@ -3873,6 +3873,7 @@ static void jsonErrorFunc( }else{ /* Convert byte-offset s.iErr into a character offset */ u32 k; + assert( s.zJson!=0 ); /* Because s.oom is false */ for(k=0; k Date: Tue, 5 Dec 2023 12:20:58 +0000 Subject: [PATCH 176/191] Clean up the JSONB performance test script. FossilOrigin-Name: 905301075a7fc1010ee7e754867b1b698c9b8576d50e98125def32a5dfb7ee9d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json/json-q1-b.txt | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 62d3de3ec6..ba676b6944 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\san\sassert()\sto\sfix\sa\sharmless\sstatic\sanalyzer\swarning. -D 2023-12-05T01:44:15.015 +C Clean\sup\sthe\sJSONB\sperformance\stest\sscript. +D 2023-12-05T12:20:58.241 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1322,7 +1322,7 @@ F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd7 F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 63e3e589e1df8fd3cc1588ba1faaff659214003f8b77a15af5c6452b35e30ee2 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f -F test/json/json-q1-b.txt 606818a5fba6d9e418c9f4ea7d8418af026775042dad81439b72447a147a462c +F test/json/json-q1-b.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 F test/json/json-speed-check.sh b060a9a6c696c0a807d8929400fa11bd7113edc58b0d66b9795f424f8d0db326 x F test/json101.test 70587d7d35ef9e2126364ba70f0c951f70827cfbd28649d779ff3df7e8f87547 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1910feb0b7d5cc2b810c3322f6cca281d8730182d30d162bd7bb56800979ea91 -R a9ebb3d8db9982a53cbb9404704cc04b +P a249ca657e624028bc6b3d2c2bcedd7162d118addb7d62ce519920cecebf1860 +R 458c3105955a68681dcfc6965be984aa U drh -Z 17d27d3e2a2e6e4799038f7f740bf1ce +Z 523674accd38155d2ed6d77fe8db57c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0dc59fba28..cae3c03bea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a249ca657e624028bc6b3d2c2bcedd7162d118addb7d62ce519920cecebf1860 \ No newline at end of file +905301075a7fc1010ee7e754867b1b698c9b8576d50e98125def32a5dfb7ee9d \ No newline at end of file diff --git a/test/json/json-q1-b.txt b/test/json/json-q1-b.txt index bd32ec7201..e78c63670a 100644 --- a/test/json/json-q1-b.txt +++ b/test/json/json-q1-b.txt @@ -1,6 +1,6 @@ .mode qbox .timer on -.param set $label '$.q87' +.param set $label 'q87' SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL; CREATE TEMP TABLE t2(x JSON TEXT); @@ -14,8 +14,7 @@ WITH RECURSIVE ), c2(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c2 WHERE n<5) INSERT INTO t2(x) - SELECT jsonb(json_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6)) - FROM array1, c2; + SELECT jsonb_object('a',n,'b',n*2,'c',y,'d',3,'e',5,'f',6) FROM array1, c2; CREATE INDEX t2x1 ON t2(x->>'a'); CREATE INDEX t2x2 ON t2(x->>'b'); CREATE INDEX t2x3 ON t2(x->>'e'); From 590aaff9928d7904e16f9868f1822beaeab6d909 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 12:22:05 +0000 Subject: [PATCH 177/191] Small performance gain by unwinding the string literal delimiter search loop in the JSON parser by one more level. FossilOrigin-Name: 4c587feac153e8ebe526559ec3d254f545f81e8d1ed3126f91a5ff25ec4aa72e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ba676b6944..483ed62511 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\sJSONB\sperformance\stest\sscript. -D 2023-12-05T12:20:58.241 +C Small\sperformance\sgain\sby\sunwinding\sthe\sstring\sliteral\sdelimiter\ssearch\nloop\sin\sthe\sJSON\sparser\sby\sone\smore\slevel. +D 2023-12-05T12:22:05.245 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 3635645a2a07070a38dade2ba34daa3f3633fd134a800cb05db236d68bc1b5c3 +F src/json.c 53d9e4ea2e1b33ea0f0dc629d08605748056333c57ff03ea8b6cfdcc585a4dc0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a249ca657e624028bc6b3d2c2bcedd7162d118addb7d62ce519920cecebf1860 -R 458c3105955a68681dcfc6965be984aa +P 905301075a7fc1010ee7e754867b1b698c9b8576d50e98125def32a5dfb7ee9d +R ec589ba3fd8295a6ff09aeae12dc3bd7 U drh -Z 523674accd38155d2ed6d77fe8db57c8 +Z 7a001bd7445e746508caf3ed9f69266e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cae3c03bea..5ad309bbc2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -905301075a7fc1010ee7e754867b1b698c9b8576d50e98125def32a5dfb7ee9d \ No newline at end of file +4c587feac153e8ebe526559ec3d254f545f81e8d1ed3126f91a5ff25ec4aa72e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7bb409a8eb..ff488de4ec 100644 --- a/src/json.c +++ b/src/json.c @@ -1412,11 +1412,13 @@ json_parse_restart: j = i+1; while( 1 /*exit-by-break*/ ){ if( jsonIsOk[(u8)z[j]] ){ - if( jsonIsOk[(u8)z[j+1]] ){ - j += 2; - continue; - }else{ + if( !jsonIsOk[(u8)z[j+1]] ){ j += 1; + }else if( !jsonIsOk[(u8)z[j+2]] ){ + j += 2; + }else{ + j += 3; + continue; } } c = z[j]; From 8eac91fab71bec3638d468330446635ca5640486 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 12:52:13 +0000 Subject: [PATCH 178/191] Use strspn() to accelerate whitespace bypass in the JSON parser. FossilOrigin-Name: 843197df08352bdff4b87be91d160e574572aded0d0c66142fd960000c0b4701 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 15 ++++++++++----- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 483ed62511..e5224d07aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sgain\sby\sunwinding\sthe\sstring\sliteral\sdelimiter\ssearch\nloop\sin\sthe\sJSON\sparser\sby\sone\smore\slevel. -D 2023-12-05T12:22:05.245 +C Use\sstrspn()\sto\saccelerate\swhitespace\sbypass\sin\sthe\sJSON\sparser. +D 2023-12-05T12:52:13.143 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 53d9e4ea2e1b33ea0f0dc629d08605748056333c57ff03ea8b6cfdcc585a4dc0 +F src/json.c 752aabf2c961d5d79143ae4a2b41d1bf60d4ec15110b0b75c5c3de5fe62d4dd1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 905301075a7fc1010ee7e754867b1b698c9b8576d50e98125def32a5dfb7ee9d -R ec589ba3fd8295a6ff09aeae12dc3bd7 +P 4c587feac153e8ebe526559ec3d254f545f81e8d1ed3126f91a5ff25ec4aa72e +R 7ec679fd53493789f3af3780d93617a7 U drh -Z 7a001bd7445e746508caf3ed9f69266e +Z be9b20ef175c9b8d4ac4f39def382e58 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5ad309bbc2..3eb7382c63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c587feac153e8ebe526559ec3d254f545f81e8d1ed3126f91a5ff25ec4aa72e \ No newline at end of file +843197df08352bdff4b87be91d160e574572aded0d0c66142fd960000c0b4701 \ No newline at end of file diff --git a/src/json.c b/src/json.c index ff488de4ec..2fc6c8b74f 100644 --- a/src/json.c +++ b/src/json.c @@ -168,6 +168,12 @@ static const char jsonIsSpace[] = { }; #define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) +/* +** The set of all space characters recognized by jsonIsspace(). +** Useful as an argument to strspn(). +*/ +static const char jsonSpaces[] = "\011\012\015\040"; + /* ** Characters that are special to JSON. Control characters, ** '"' and '\\'. @@ -1296,6 +1302,7 @@ json_parse_restart: j++; }else{ if( jsonIsspace(z[j]) ){ + /* strspn() is not helpful here */ do{ j++; }while( jsonIsspace(z[j]) ); if( z[j]==':' ){ j++; @@ -1322,7 +1329,7 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - do{ j++; }while( jsonIsspace(z[j]) ); + j += 1 + strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ @@ -1374,7 +1381,7 @@ json_parse_restart: break; }else{ if( jsonIsspace(z[j]) ){ - do{ j++; }while( jsonIsspace(z[j]) ); + j += 1 + strspn(&z[j+1], jsonSpaces); if( z[j]==',' ){ continue; }else if( z[j]==']' ){ @@ -1635,9 +1642,7 @@ json_parse_restart: case 0x0a: case 0x0d: case 0x20: { - do{ - i++; - }while( jsonIsspace(z[i]) ); + i += 1 + strspn(&z[i+1], jsonSpaces); goto json_parse_restart; } case 0x0b: From a0de45459e8dcd8594a95684768f14c680570e04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 18:28:15 +0000 Subject: [PATCH 179/191] Miscellaneous comment cleanup and typo fixes. FossilOrigin-Name: 59446dc0bd0091572122a3c8b4653d7a2dc867d16c4a5919f79b81bc3a673ce3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 26 +++++++++++++------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index e5224d07aa..ff6dfffb99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sstrspn()\sto\saccelerate\swhitespace\sbypass\sin\sthe\sJSON\sparser. -D 2023-12-05T12:52:13.143 +C Miscellaneous\scomment\scleanup\sand\stypo\sfixes. +D 2023-12-05T18:28:15.619 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 752aabf2c961d5d79143ae4a2b41d1bf60d4ec15110b0b75c5c3de5fe62d4dd1 +F src/json.c 00480474f134f18d77275de3fe1220b198059af39d41b8549f14af991ee50c64 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4c587feac153e8ebe526559ec3d254f545f81e8d1ed3126f91a5ff25ec4aa72e -R 7ec679fd53493789f3af3780d93617a7 +P 843197df08352bdff4b87be91d160e574572aded0d0c66142fd960000c0b4701 +R 51649fd2031c328be393a9b165de9b28 U drh -Z be9b20ef175c9b8d4ac4f39def382e58 +Z 92ba2a23f6f1deba28c66b8e97e6bff1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3eb7382c63..bdb4e91e40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -843197df08352bdff4b87be91d160e574572aded0d0c66142fd960000c0b4701 \ No newline at end of file +59446dc0bd0091572122a3c8b4653d7a2dc867d16c4a5919f79b81bc3a673ce3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 2fc6c8b74f..15f99d49b2 100644 --- a/src/json.c +++ b/src/json.c @@ -10,7 +10,7 @@ ** ****************************************************************************** ** -** This SQLite JSON functions. +** SQLite JSON functions. ** ** This file began as an extension in ext/misc/json1.c in 2015. That ** extension proved so useful that it has now been moved into the core. @@ -170,7 +170,7 @@ static const char jsonIsSpace[] = { /* ** The set of all space characters recognized by jsonIsspace(). -** Useful as an argument to strspn(). +** Useful as the second argument to strspn(). */ static const char jsonSpaces[] = "\011\012\015\040"; @@ -198,14 +198,6 @@ static const char jsonIsOk[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; -/* Put code used only for testing inside the JSON_VVA() macro. -*/ -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) -# define JSON_VVA(X) -#else -# define JSON_VVA(X) X -#endif - /* Objects */ typedef struct JsonCache JsonCache; typedef struct JsonString JsonString; @@ -263,8 +255,8 @@ struct JsonString { #define JSON_SUBTYPE 74 /* Ascii for "J" */ /* -** Bit values for the flags passed into jsonExtractFunc() or -** jsonSetFunc() via the user-data value. +** Bit values for the flags passed into various SQL function implementations +** via the sqlite3_user_data() value. */ #define JSON_JSON 0x01 /* Result is always JSON */ #define JSON_SQL 0x02 /* Result is always SQL */ @@ -323,7 +315,11 @@ struct JsonParse { ** descent parser. A depth of 1000 is far deeper than any sane JSON ** should go. Historical note: This limit was 2000 prior to version 3.42.0 */ -#define JSON_MAX_DEPTH 1000 +#ifndef SQLITE_JSON_MAX_DEPTH +# define JSON_MAX_DEPTH 1000 +#else +# define JSON_MAX_DEPTH SQLITE_JSON_MAX_DEPTH +#endif /* ** Allowed values for the flgs argument to jsonParseFuncArg(); @@ -807,6 +803,10 @@ static void jsonParseFree(JsonParse *pParse){ } } +/************************************************************************** +** Utility routines for the JSON text parser +**************************************************************************/ + /* ** Translate a single byte of Hex into an integer. ** This routine only gives a correct answer if h really is a valid hexadecimal From fb923fc4ccb79fe27d6daa9477ac2ef55b1da9bc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 5 Dec 2023 18:36:23 +0000 Subject: [PATCH 180/191] Further tests for the new code on this branch. FossilOrigin-Name: 59d008b6c23ab900377bc696ee19381feb7614bac80546eae361e401c3620c4e --- ext/fts5/fts5_expr.c | 6 +- ext/fts5/fts5_index.c | 113 ++++++++++++----------------- ext/fts5/test/fts5faultH.test | 93 ++++++++++++++++++++++++ ext/fts5/test/fts5origintext2.test | 39 ++++++++++ ext/fts5/test/fts5origintext3.test | 21 ++++++ ext/fts5/test/fts5origintext5.test | 44 ++++++++++- manifest | 21 +++--- manifest.uuid | 2 +- 8 files changed, 257 insertions(+), 82 deletions(-) create mode 100644 ext/fts5/test/fts5faultH.test diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index c5de6eba6c..bc7c9741ee 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1768,7 +1768,9 @@ static int fts5ParseTokenize( memset(pTerm, 0, sizeof(Fts5ExprTerm)); pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); pTerm->nFullTerm = pTerm->nQueryTerm = nToken; - if( pCtx->pConfig->bTokendata ) pTerm->nQueryTerm = strlen(pTerm->pTerm); + if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ + pTerm->nQueryTerm = strlen(pTerm->pTerm); + } } } @@ -3027,7 +3029,7 @@ static int fts5ExprPopulatePoslistsCb( int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); - if( rc==SQLITE_OK && pExpr->pConfig->bTokendata ){ + if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){ int iCol = p->iOff>>32; int iTokOff = p->iOff & 0x7FFFFFFF; rc = sqlite3Fts5IndexIterWriteTokendata( diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a24308534c..9a2cc026b9 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6661,57 +6661,6 @@ static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ } } -/* -** The iterator passed as the first argument must be a tokendata=1 iterator -** (pIter->pTokenDataIter!=0). This function is used to access the token -** instance located at offset iOff of column iCol of row iRowid. It is -** returned via output pointers *ppOut and *pnOut. This is used by the -** xInstToken() API. -*/ -static int fts5TokendataIterToken( - Fts5Iter *pIter, - i64 iRowid, - int iCol, int iOff, - const char **ppOut, int *pnOut -){ - Fts5TokenDataIter *pT = pIter->pTokenDataIter; - Fts5TokenDataMap *aMap = pT->aMap; - i64 iPos = (((i64)iCol)<<32) + iOff; - - int i1 = 0; - int i2 = pT->nMap; - int iTest = 0; - - while( i2>i1 ){ - iTest = (i1 + i2) / 2; - - if( aMap[iTest].iRowidiRowid ){ - i2 = iTest; - }else{ - if( aMap[iTest].iPosiPos ){ - i2 = iTest; - }else{ - break; - } - } - } - - if( i2>i1 ){ - Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; - *ppOut = (const char*)pMap->aSeg[0].term.p+1; - *pnOut = pMap->aSeg[0].term.n-1; - } - - return SQLITE_OK; -} - /* ** Append a mapping to the token-map belonging to object pT. */ @@ -6960,9 +6909,7 @@ static Fts5Iter *fts5SetupTokendataIter( memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); memset(pPrevIter, 0, sizeof(Fts5SegIter)); bDone = 1; - }else if( pPrevIter->pLeaf - && pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf - ){ + }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){ fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); bDone = 1; } @@ -7071,7 +7018,7 @@ int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ int iPrefixIdx = 0; /* +1 prefix index */ - int bTokendata = (flags&FTS5INDEX_QUERY_NOTOKENDATA)?0:pConfig->bTokendata; + int bTokendata = pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this @@ -7085,6 +7032,7 @@ int sqlite3Fts5IndexQuery( ** for internal sanity checking by the integrity-check in debug ** mode only. */ #ifdef SQLITE_DEBUG + if( flags & FTS5INDEX_QUERY_NOTOKENDATA ) bTokendata = 0; if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; @@ -7208,7 +7156,8 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ /* ** This is used by xInstToken() to access the token at offset iOff, column ** iCol of row iRowid. The token is returned via output variables *ppOut -** and *pnOut. +** and *pnOut. The iterator passed as the first argument must be a tokendata=1 +** iterator (pIter->pTokenDataIter!=0). */ int sqlite3Fts5IterToken( Fts5IndexIter *pIndexIter, @@ -7218,9 +7167,39 @@ int sqlite3Fts5IterToken( const char **ppOut, int *pnOut ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5TokenDataMap *aMap = pT->aMap; + i64 iPos = (((i64)iCol)<<32) + iOff; - if( pIter->pTokenDataIter ){ - return fts5TokendataIterToken(pIter, iRowid, iCol, iOff, ppOut, pnOut); + int i1 = 0; + int i2 = pT->nMap; + int iTest = 0; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; } return SQLITE_OK; @@ -7252,17 +7231,17 @@ int sqlite3Fts5IndexIterWriteTokendata( Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenDataIter *pT = pIter->pTokenDataIter; Fts5Index *p = pIter->pIndex; + int ii; assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); - if( pT ){ - int ii; - for(ii=0; iinIter; ii++){ - Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; - if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; - } - if( iinIter ){ - fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); - } + assert( pIter->pTokenDataIter ); + + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); } return fts5IndexReturn(p); } diff --git a/ext/fts5/test/fts5faultH.test b/ext/fts5/test/fts5faultH.test new file mode 100644 index 0000000000..9dd4cac0d6 --- /dev/null +++ b/ext/fts5/test/fts5faultH.test @@ -0,0 +1,93 @@ +# 2010 June 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultG + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +set ::testprefix fts5faultH + +sqlite3_fts5_register_origintext db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + + BEGIN; + INSERT INTO t1 VALUES('oNe tWo thRee'); + INSERT INTO t1 VALUES('One Two Three'); + INSERT INTO t1 VALUES('onE twO threE'); + COMMIT; + BEGIN; + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + INSERT INTO t1 VALUES('one two three'); + COMMIT; +} + +do_faultsim_test 1 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('three'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {1 2 3 4 5 6}} +} + + +reset_db +sqlite3_fts5_register_origintext db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5( + x, tokenize="origintext unicode61", tokendata=1 + ); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + + BEGIN; + INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB'); + COMMIT; +} + +do_faultsim_test 2 -faults oom* -prep { +} -body { + execsql { + SELECT rowid FROM t1('BBB AND AAA'); + } +} -test { + faultsim_integrity_check + faultsim_test_result {0 {10 24}} +} + + + +finish_test diff --git a/ext/fts5/test/fts5origintext2.test b/ext/fts5/test/fts5origintext2.test index a27309fe0c..a8c5d4eb50 100644 --- a/ext/fts5/test/fts5origintext2.test +++ b/ext/fts5/test/fts5origintext2.test @@ -103,5 +103,44 @@ do_execsql_test 1.12 { SELECT rowid FROM ft('today'); } {4 5 6} do_execsql_test 1.13 { SELECT rowid FROM ft('world'); } {7 8 9} do_execsql_test 1.14 { SELECT rowid FROM ft('hello') ORDER BY rank; } {1 2 3} +#------------------------------------------------------------------------ +reset_db +sqlite3_fts5_register_origintext db +proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret +} +sqlite3_fts5_create_function db tokens tokens + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE x1 USING fts5( + v, tokenize="origintext unicode61", tokendata=1, detail=none + ); + + INSERT INTO x1 VALUES('xxx Xxx XXX yyy YYY yyy'); + INSERT INTO x1 VALUES('xxx yyy xxx yyy yyy yyy'); +} + +do_execsql_test 2.1 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx xxx.Xxx xxx.XXX} {xxx xxx} +} + +do_execsql_test 2.2 { + UPDATE x1_content SET c0 = 'xxx xxX xxx yyy yyy yyy' WHERE id=1; +} + +do_execsql_test 2.3 { + SELECT tokens(x1) FROM x1('xxx'); +} { + {xxx {} xxx} {xxx xxx} +} + finish_test diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index ac00bfabc0..834844595d 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -74,6 +74,27 @@ foreach_detail_mode $testprefix { do_execsql_test 1.6 { SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank; } {three.THREE 3 three 1 three 2} + + do_execsql_test 1.7 { + INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB'); + INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(14, 'bbb BBB bbb'); + INSERT INTO ft2(rowid, x) VALUES(15, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(16, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(17, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(18, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(19, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(20, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(21, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(22, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(23, 'bbb bbb bbb'); + INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB'); + } + + do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} + do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} + } finish_test diff --git a/ext/fts5/test/fts5origintext5.test b/ext/fts5/test/fts5origintext5.test index 89a5d8f39c..03d5bee215 100644 --- a/ext/fts5/test/fts5origintext5.test +++ b/ext/fts5/test/fts5origintext5.test @@ -121,7 +121,7 @@ proc ctrl_tokens {term args} { db func ctrl_tokens ctrl_tokens proc do_all_vocab_test {tn} { - foreach ::v [vocab] { + foreach ::v [concat [vocab] nnn] { set answer [execsql { SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%' }] @@ -134,7 +134,6 @@ proc do_all_vocab_test {tn} { } } - do_execsql_test 1.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", content=, contentless_delete=1, @@ -229,5 +228,46 @@ for {set ii 0} {$ii < $NLOOP} {incr ii} { do_all_vocab_test 2.3.$ii } +#------------------------------------------------------------------------- + +unset -nocomplain ::expanded_vocab +proc vocab {} { + list abcde fghij klmno +} + +proc do_all_vocab_test3 {tn} { + foreach ::v [concat [vocab] nnn] { + set answer [execsql { + SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%' + }] + do_execsql_test $tn.$::v.1 { + SELECT rowid, tokens(ft3) FROM ft3($::v) + } $answer + do_execsql_test $tn.$::v.2 { + SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank + } $answer + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + w, tokenize="origintext unicode61", content=, contentless_delete=1, + tokendata=1 + ); + INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()'); + CREATE TABLE ctrl3(w); +} + +do_execsql_test 3.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2 + ) + INSERT INTO ctrl3 SELECT document() FROM s; + INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3; +} + +do_all_vocab_test3 3.2 + + finish_test diff --git a/manifest b/manifest index 11d57c4176..91a29a053c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sxInstToken(). -D 2023-12-04T19:48:08.530 +C Further\stests\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2023-12-05T18:36:23.618 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,9 +94,9 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c 920516be4aac44eccdd9e747fe26f367442848b9a58971e4b36edb0c8284b6b8 +F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba +F ext/fts5/fts5_index.c be39b44ff8773cff56bcbc01f74701a83e068c20d773cafd01e8bb2fa0fc1bc5 F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -170,6 +170,7 @@ F ext/fts5/test/fts5faultD.test e7ed7895abfe6bc98a5e853826f6b74956e7ba7f594f1860 F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e4710c77eb8ce7075 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test d2e5a4d9a34e08dcaadcaeafef74d10cbc2abdd11aa2659a18af0294bf2812d3 +F ext/fts5/test/fts5faultH.test d845f45dac3e1a3f20c7e0a2be95280c95d3204c06802f86ab2c110e52ed3d14 F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b996159f6909dc8079 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -191,10 +192,10 @@ F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39 -F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969 -F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a +F ext/fts5/test/fts5origintext2.test f3b9436de540828d01f0672df855b09ebc0863e126d5b56234701d71dfa73634 +F ext/fts5/test/fts5origintext3.test 0d25933506600452a5ab3873cbb418ed5f2de2446c3672b9997b1ea104b0e7f0 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871 -F ext/fts5/test/fts5origintext5.test 067bfb3008323585df640ab29e8ef7c4ca6dec62c597be07a9f896d88f98cd10 +F ext/fts5/test/fts5origintext5.test a037bdf7235a22033c4663837bdb12d9738245464a3ac2f60c71fc40d07ede7d F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2150,8 +2151,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1 -R ced1cc2c59284a9ef3026043e080f745 +P 8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4 +R c18539c880ee946e73119264163283e7 U dan -Z e2cce7acff968ef3c9472723192ae452 +Z 86bd4754d1465fbe5cee9e784de6e900 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 780ed88175..c1e3ee2ba4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4 \ No newline at end of file +59d008b6c23ab900377bc696ee19381feb7614bac80546eae361e401c3620c4e \ No newline at end of file From 5afd67b3c38d9af1fe7ad4a04567e5a8aaf5746f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 5 Dec 2023 19:24:07 +0000 Subject: [PATCH 181/191] Use extra assert() statement to silence harmless static analyzer warnings. FossilOrigin-Name: 174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ff6dfffb99..4206539101 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Miscellaneous\scomment\scleanup\sand\stypo\sfixes. -D 2023-12-05T18:28:15.619 +C Use\sextra\sassert()\sstatement\sto\ssilence\sharmless\sstatic\sanalyzer\swarnings. +D 2023-12-05T19:24:07.192 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 00480474f134f18d77275de3fe1220b198059af39d41b8549f14af991ee50c64 +F src/json.c 0a6095d10a8c8251e1838a1f12abc89125b0b05f82497ad12896a7f714e397ce F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 843197df08352bdff4b87be91d160e574572aded0d0c66142fd960000c0b4701 -R 51649fd2031c328be393a9b165de9b28 +P 59446dc0bd0091572122a3c8b4653d7a2dc867d16c4a5919f79b81bc3a673ce3 +R 901a6c57f6544c02dc1f928ffbc0f678 U drh -Z 92ba2a23f6f1deba28c66b8e97e6bff1 +Z d3a76b74f5e90cbdae4d0a0216cf9ffb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bdb4e91e40..4658b8623b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59446dc0bd0091572122a3c8b4653d7a2dc867d16c4a5919f79b81bc3a673ce3 \ No newline at end of file +174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 15f99d49b2..8165eb4f31 100644 --- a/src/json.c +++ b/src/json.c @@ -1119,6 +1119,7 @@ static void jsonBlobAppendNode( jsonBlobExpandAndAppendNode(pParse,eType,szPayload,aPayload); return; } + assert( pParse->aBlob!=0 ); a = &pParse->aBlob[pParse->nBlob]; if( szPayload<=11 ){ a[0] = eType | (szPayload<<4); @@ -2288,6 +2289,8 @@ static u32 jsonLookupStep( nIns = ix.nBlob + nKey + v.nBlob; jsonBlobEdit(pParse, j, 0, 0, nIns); if( !pParse->oom ){ + assert( pParse->aBlob!=0 ); /* Because pParse->oom!=0 */ + assert( ix.aBlob!=0 ); /* Because pPasre->oom!=0 */ memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); k = j + ix.nBlob; memcpy(&pParse->aBlob[k], zKey, nKey); @@ -4406,7 +4409,7 @@ static int jsonEachColumn( } case JEACH_TYPE: { u32 i = jsonSkipLabel(p); - u8 eType = eType = p->sParse.aBlob[i] & 0x0f; + u8 eType = p->sParse.aBlob[i] & 0x0f; sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); break; } From 1f2d7c43120fe831fd63f7fedcc61b15a17fccee Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 6 Dec 2023 12:30:28 +0000 Subject: [PATCH 182/191] README.md typo fix reported in the forum and update all links from http: to https:. FossilOrigin-Name: 5c48acdbb44185b352b54911a57a6986d6c7e624bdeba2af48b985d29f0292bf --- README.md | 20 ++++++++++---------- manifest | 15 +++++++-------- manifest.uuid | 2 +- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ebb917e218..0975a32d9a 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ extension and only later escaped to the wild as an independent library.) Test scripts and programs are found in the **test/** subdirectory. Additional test code is found in other source repositories. -See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for +See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for additional information. The **ext/** subdirectory contains code for extensions. The @@ -183,7 +183,7 @@ manually-edited files and automatically-generated files. The SQLite interface is defined by the **sqlite3.h** header file, which is generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The -[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. +[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. The manifest.uuid file contains the SHA3 hash of the particular check-in and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file contains the current SQLite version number. The sqlite3.h header is really @@ -250,14 +250,14 @@ individual source file exceeds 32K lines in length. ## How It All Fits Together SQLite is modular in design. -See the [architectural description](http://www.sqlite.org/arch.html) +See the [architectural description](https://www.sqlite.org/arch.html) for details. Other documents that are useful in (helping to understand how SQLite works include the -[file format](http://www.sqlite.org/fileformat2.html) description, -the [virtual machine](http://www.sqlite.org/opcode.html) that runs +[file format](https://www.sqlite.org/fileformat2.html) description, +the [virtual machine](https://www.sqlite.org/opcode.html) that runs prepared statements, the description of -[how transactions work](http://www.sqlite.org/atomiccommit.html), and -the [overview of the query planner](http://www.sqlite.org/optoverview.html). +[how transactions work](https://www.sqlite.org/atomiccommit.html), and +the [overview of the query planner](https://www.sqlite.org/optoverview.html). Years of effort have gone into optimizing SQLite, both for small size and high performance. And optimizations tend to result in @@ -353,7 +353,7 @@ hidden by also modifying the makefiles. ## Contacts -The main SQLite website is [http:/sqlite.org/](http://sqlite.org/) +The main SQLite website is [https://sqlite.org/](https://sqlite.org/) with geographically distributed backups at -[http://www2.sqlite.org/](http://www2.sqlite.org) and -[http://www3.sqlite.org/](http://www3.sqlite.org). +[https://www2.sqlite.org/](https://www2.sqlite.org) and +[https://www3.sqlite.org/](https://www3.sqlite.org). diff --git a/manifest b/manifest index c0dab2ba23..6d47dd3e83 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Rework\sthe\sJSON\sfunctions\sso\sthat\sthey\suse\sthe\sJSONB\sformat\sinternally.\nThe\soriginal\sJsonNode\sparse\stree\sdesign\sis\sremoved.\s\sAll\sJSON\sfunctions\nthat\saccept\stext\sJSON\salso\saccept\sJSONB.\s\sNew\sfunctions\sgenerate\sJSONB. -D 2023-12-05T19:45:09.048 +C README.md\stypo\sfix\sreported\sin\sthe\sforum\sand\supdate\sall\slinks\sfrom\shttp:\sto\shttps:. +D 2023-12-06T12:30:28.174 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 890caf094636c308bc981032baf8f9208bf307755f9197ae4218a9fbcec2e449 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 59bb36dba001f0b38212be0794fb838f25371008b180929bcf08aa799694c168 -F README.md 963d30019abf0cc06b263cd2824bce022893f3f93a531758f6f04ff2194a16a8 +F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION 73573d4545343f001bf5dc5461173a7c78c203dd046cabcf99153878cf25d3a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -2147,9 +2147,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8abc2ccaf8106f20243568cd7fa74174386eb85d7ea381201e97e2fd527033e0 174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 -R 2fc453038906bf8ee420a799d6f3c8a1 -T +closed 174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 -U drh -Z fc61504cc821223c214ac8778fb2caf0 +P 7f0c79b94e8f55e5013e52ba64ba8b32dad1dc4e2224d2099733cbc561de1810 +R e4c4253b529b92ef6212320f71e1570e +U stephan +Z 02e09a8447ded58f933fad2e169fffdc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 634d6e05e0..f935efea29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f0c79b94e8f55e5013e52ba64ba8b32dad1dc4e2224d2099733cbc561de1810 \ No newline at end of file +5c48acdbb44185b352b54911a57a6986d6c7e624bdeba2af48b985d29f0292bf \ No newline at end of file From 91ec00c25acd25bd9a6bc2345436370d23723653 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 14:50:48 +0000 Subject: [PATCH 183/191] Increased rigor in comparisons between object labels in JSON. FossilOrigin-Name: 2bc86d145fccc07107b7753cb1a69122676d4096fe59c454497bd81a6142d45e --- manifest | 16 +-- manifest.uuid | 2 +- src/json.c | 311 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 228 insertions(+), 101 deletions(-) diff --git a/manifest b/manifest index c0dab2ba23..918632105e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sJSON\sfunctions\sso\sthat\sthey\suse\sthe\sJSONB\sformat\sinternally.\nThe\soriginal\sJsonNode\sparse\stree\sdesign\sis\sremoved.\s\sAll\sJSON\sfunctions\nthat\saccept\stext\sJSON\salso\saccept\sJSONB.\s\sNew\sfunctions\sgenerate\sJSONB. -D 2023-12-05T19:45:09.048 +C Increased\srigor\sin\scomparisons\sbetween\sobject\slabels\sin\sJSON. +D 2023-12-06T14:50:48.135 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,7 +690,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 0a6095d10a8c8251e1838a1f12abc89125b0b05f82497ad12896a7f714e397ce +F src/json.c 15efd213cc95bde5b714ec068fdb1f6817b1ed9a253644833fed5f196957445a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2147,9 +2147,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8abc2ccaf8106f20243568cd7fa74174386eb85d7ea381201e97e2fd527033e0 174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 -R 2fc453038906bf8ee420a799d6f3c8a1 -T +closed 174c2b2eef5fecd96a5fc89b81032fe81f7801f12097cea10e7e7f0a02114813 +P 7f0c79b94e8f55e5013e52ba64ba8b32dad1dc4e2224d2099733cbc561de1810 +R 2684d5ad233d4af8d84cd782c9755832 +T *branch * json-label-compare +T *sym-json-label-compare * +T -sym-trunk * U drh -Z fc61504cc821223c214ac8778fb2caf0 +Z 93dccc3939eaec1e85cde17c2d8ca8af # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 634d6e05e0..6aa957e936 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f0c79b94e8f55e5013e52ba64ba8b32dad1dc4e2224d2099733cbc561de1810 \ No newline at end of file +2bc86d145fccc07107b7753cb1a69122676d4096fe59c454497bd81a6142d45e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8165eb4f31..37a0ebfe1a 100644 --- a/src/json.c +++ b/src/json.c @@ -2119,6 +2119,188 @@ static void jsonBlobEdit( if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); } +/* +** Return the number of escaped newlines to be ignored. +** An escaped newline is a one of the following byte sequences: +** +** 0x5c 0x0a +** 0x5c 0x0d +** 0x5c 0x0d 0x0a +** 0x5c 0xe2 0x80 0xa8 +** 0x5c 0xe2 0x80 0xa9 +*/ +static u32 jsonBytesToBypass(const char *z, u32 n){ + u32 i = 0; + while( i+10 ); + assert( z[0]=='\\' ); + if( n<2 ){ + *piOut = 0xFFFD; + return n; + } + switch( (u8)z[1] ){ + case 'u': { + u32 v, vlo; + if( n<6 ){ + *piOut = 0xFFFD; + return n; + } + v = jsonHexToInt4(&z[2]); + if( (v & 0xfc00)==0xd800 + && n>=12 + && z[6]=='\\' + && z[7]=='u' + && ((vlo = jsonHexToInt4(&z[8]))&0xfc00)==0xdc00 + ){ + *piOut = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + return 12; + }else{ + *piOut = v; + return 6; + } + } + case 'b': { *piOut = '\b'; return 2; } + case 'f': { *piOut = '\f'; return 2; } + case 'n': { *piOut = '\n'; return 2; } + case 'r': { *piOut = '\r'; return 2; } + case 't': { *piOut = '\t'; return 2; } + case 'v': { *piOut = '\v'; return 2; } + case '0': { *piOut = 0; return 2; } + case '\'': + case '"': + case '/': + case '\\':{ *piOut = z[1]; return 2; } + case 'x': { + if( n<4 ){ + *piOut = 0xFFFD; + return n; + } + *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); + return 4; + } + case 0xe2: + case '\r': + case '\n': { + u32 nSkip = jsonBytesToBypass(z, n); + if( nSkip==0 ){ + *piOut = 0xFFFD; + return n; + }else if( nSkip==n ){ + *piOut = 0; + return n; + }else if( z[nSkip]=='\\' ){ + return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); + }else{ + *piOut = z[nSkip]; + return nSkip+1; + } + } + default: { + *piOut = 0xFFFD; + return 2; + } + } +} + + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. +** +** In this version, we know that one or the other or both of the +** two comparands contains an escape sequence. +*/ +static SQLITE_NOINLINE int jsonLabelCompareEscaped( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + u32 cLeft, cRight; + assert( rawLeft==0 || rawRight==0 ); + while( nLeft>0 && nRight>0 ){ + if( rawLeft || zLeft[0]!='\\' ){ + cLeft = ((u8*)zLeft)[0]; + zLeft++; + nLeft--; + }else{ + u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); + zLeft += n; + assert( n<=nLeft ); + nLeft -= n; + } + if( rawRight || zRight[0]!='\\' ){ + cRight = ((u8*)zRight)[0]; + zRight++; + nRight--; + }else{ + u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); + zRight += n; + assert( n<=nRight ); + nRight -= n; + } + if( cLeft!=cRight ) return 0; + } + return nLeft==0 && nRight==0; +} + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. Return -1 if an OOM occurs. +*/ +static int jsonLabelCompare( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + if( rawLeft && rawRight ){ + /* Simpliest case: Neither label contains escapes. A simple + ** memcmp() is sufficient. */ + if( nLeft!=nRight ) return 0; + return memcmp(zLeft, zRight, nLeft)==0; + }else{ + return jsonLabelCompareEscaped(zLeft, nLeft, rawLeft, + zRight, nRight, rawRight); + } +} + /* ** Error returns from jsonLookupStep() */ @@ -2224,6 +2406,7 @@ static u32 jsonLookupStep( return iRoot; } if( zPath[0]=='.' ){ + int rawKey = 1; x = pParse->aBlob[iRoot]; zPath++; if( zPath[0]=='"' ){ @@ -2236,6 +2419,7 @@ static u32 jsonLookupStep( return JSON_LOOKUP_PATHERROR; } testcase( nKey==0 ); + rawKey = memchr(zKey, '\\', nKey)==0; }else{ zKey = zPath; for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} @@ -2249,13 +2433,17 @@ static u32 jsonLookupStep( j = iRoot + n; /* j is the index of a label */ iEnd = j+sz; while( jaBlob[j] & 0x0f; if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; k = j+n; /* k is the index of the label text */ if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; - if( sz==nKey && memcmp(&pParse->aBlob[k], zKey, nKey)==0 ){ + zLabel = (const char*)&pParse->aBlob[k]; + rawLabel = x==JSONB_TEXT || x==JSONB_TEXTRAW; + if( jsonLabelCompare(zKey, nKey, rawKey, zLabel, sz, rawLabel) ){ u32 v = k+sz; /* v is the index of the value */ if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; n = jsonbPayloadSize(pParse, v, &sz); @@ -2279,7 +2467,7 @@ static u32 jsonLookupStep( testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); memset(&ix, 0, sizeof(ix)); - jsonBlobAppendNode(&ix,JSONB_TEXTRAW, nKey, 0); + jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); pParse->oom |= ix.oom; rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); if( !JSON_LOOKUP_ISERROR(rc) @@ -2483,72 +2671,27 @@ static void jsonReturnFromBlob( for(iIn=iOut=0; iIn>6)); - zOut[iOut++] = 0x80 | (v&0x3f); - }else{ - u32 vlo; - if( (v&0xfc00)==0xd800 - && iIn>18); - zOut[iOut++] = 0x80 | ((v>>12)&0x3f); - zOut[iOut++] = 0x80 | ((v>>6)&0x3f); - zOut[iOut++] = 0x80 | (v&0x3f); - }else{ - zOut[iOut++] = 0xe0 | (v>>12); - zOut[iOut++] = 0x80 | ((v>>6)&0x3f); - zOut[iOut++] = 0x80 | (v&0x3f); - } - } - continue; - }else if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - }else if( c=='v' ){ - c = '\v'; - }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){ - /* pass through unchanged */ - }else if( c=='0' ){ - c = 0; - }else if( c=='x' ){ - c = (jsonHexToInt(z[iIn+1])<<4) | jsonHexToInt(z[iIn+2]); - iIn += 2; - }else if( c=='\r' && z[i+1]=='\n' ){ - iIn++; - continue; - }else if( 0xe2==(u8)c - && iIn>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v<0x10000 ){ + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); }else{ - continue; + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); } - } /* end if( c=='\\' ) */ - zOut[iOut++] = c; + iIn += szEscape - 1; + }else{ + zOut[iOut++] = c; + } } /* end for() */ zOut[iOut] = 0; sqlite3_result_text(pCtx, zOut, iOut, sqlite3_free); @@ -3384,6 +3527,7 @@ static int jsonMergePatch( iTCursor = iTStart; iTEnd = iTEndBE + pTarget->delta; while( iTCursoraBlob[iTCursor] & 0x0f; if( eTLabelJSONB_TEXTRAW ){ @@ -3396,33 +3540,14 @@ static int jsonMergePatch( nTValue = jsonbPayloadSize(pTarget, iTValue, &szTValue); if( nTValue==0 ) return JSON_MERGE_BADTARGET; if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; - if( eTLabel==ePLabel ){ - /* Common case */ - if( szTLabel==szPLabel - && memcmp(&pTarget->aBlob[iTLabel+nTLabel], - &pPatch->aBlob[iPLabel+nPLabel], szTLabel)==0 - ){ - break; /* Labels match. */ - } - }else{ - /* Should rarely happen */ - JsonString s1, s2; - int isEqual, isOom; - jsonStringInit(&s1, 0); - jsonXlateBlobToText(pTarget, iTLabel, &s1); - jsonStringInit(&s2, 0); - jsonXlateBlobToText(pPatch, iPLabel, &s2); - isOom = s1.eErr || s2.eErr; - if( s1.nUsed==s2.nUsed && memcmp(s1.zBuf, s2.zBuf, s1.nUsed)==0 ){ - isEqual = 1; - }else{ - isEqual = 0; - } - jsonStringReset(&s1); - jsonStringReset(&s2); - if( isOom ) return JSON_MERGE_OOM; - if( isEqual ) break; - } + isEqual = jsonLabelCompare( + (const char*)&pPatch->aBlob[iPLabel+nPLabel], + szPLabel, + (ePLabel==JSONB_TEXT || ePLabel==JSONB_TEXTRAW), + (const char*)&pTarget->aBlob[iTLabel+nTLabel], + szTLabel, + (eTLabel==JSONB_TEXT || eTLabel==JSONB_TEXTRAW)); + if( isEqual ) break; iTCursor = iTValue + nTValue + szTValue; } x = pPatch->aBlob[iPValue] & 0x0f; From 6a8581d828e2c6b940cdf19868c309dde4322ea9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 15:35:38 +0000 Subject: [PATCH 184/191] The rule for the RHS of the ->> and -> operators when the RHS does not begin with $ is that it must be (1) all digits, or (2) all alphanumerics, or (3) contained within [..] or else it will become a quoted label. FossilOrigin-Name: 0e059a546ec11fa5c6d007bd65c249ee2422f1facbdb2792c53e0bc0ccc97e14 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 24 +++++++++++++++++++++--- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 918632105e..173207b80f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increased\srigor\sin\scomparisons\sbetween\sobject\slabels\sin\sJSON. -D 2023-12-06T14:50:48.135 +C The\srule\sfor\sthe\sRHS\sof\sthe\s->>\sand\s->\soperators\swhen\sthe\sRHS\sdoes\snot\sbegin\nwith\s$\sis\sthat\sit\smust\sbe\s(1)\sall\sdigits,\sor\s(2)\sall\salphanumerics,\sor\n(3)\scontained\swithin\s[..]\sor\selse\sit\swill\sbecome\sa\squoted\slabel. +D 2023-12-06T15:35:38.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,7 +690,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 15efd213cc95bde5b714ec068fdb1f6817b1ed9a253644833fed5f196957445a +F src/json.c 9d0b8fa594a931d3f84056020c6581bb631cee02ea5f0530a4fd5876a2cb6560 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2147,11 +2147,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7f0c79b94e8f55e5013e52ba64ba8b32dad1dc4e2224d2099733cbc561de1810 -R 2684d5ad233d4af8d84cd782c9755832 -T *branch * json-label-compare -T *sym-json-label-compare * -T -sym-trunk * +P 2bc86d145fccc07107b7753cb1a69122676d4096fe59c454497bd81a6142d45e +R 5951a4c9f5e0f75219c7e2d971403d09 U drh -Z 93dccc3939eaec1e85cde17c2d8ca8af +Z 93fae7007061a25e3b1f2103eb12e9f0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6aa957e936..121f08ea73 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2bc86d145fccc07107b7753cb1a69122676d4096fe59c454497bd81a6142d45e \ No newline at end of file +0e059a546ec11fa5c6d007bd65c249ee2422f1facbdb2792c53e0bc0ccc97e14 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 37a0ebfe1a..44abe2d164 100644 --- a/src/json.c +++ b/src/json.c @@ -3272,6 +3272,20 @@ static void jsonArrayLengthFunc( jsonParseFree(p); } +/* True if the string is all digits */ +static int jsonAllDigits(const char *z, int n){ + int i; + for(i=0; i"(JSON,PATH) @@ -3329,15 +3343,19 @@ static void jsonExtractFunc( ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience */ jsonStringInit(&jx, ctx); - if( sqlite3Isdigit(zPath[0]) ){ + if( jsonAllDigits(zPath, nPath) ){ jsonAppendRawNZ(&jx, "[", 1); jsonAppendRaw(&jx, zPath, nPath); jsonAppendRawNZ(&jx, "]", 2); - }else if( zPath[0]!='[' ){ + }else if( jsonAllAlphanum(zPath, nPath) ){ jsonAppendRawNZ(&jx, ".", 1); jsonAppendRaw(&jx, zPath, nPath); - }else{ + }else if( zPath[0]=='[' && nPath>=3 && zPath[nPath-1]==']' ){ jsonAppendRaw(&jx, zPath, nPath); + }else{ + jsonAppendRawNZ(&jx, ".\"", 2); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "\"", 1); } jsonStringTerminate(&jx); j = jsonLookupStep(p, 0, jx.zBuf, 0); From 8dfbf4addce7f8bf4d4fa3fd96a41aea11d9b41d Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 15:50:13 +0000 Subject: [PATCH 185/191] Test cases for object label matching with escape sequences. FossilOrigin-Name: c6f2aa38e95b7888650cfa7bb773b18a28e01d883033ac77be6d504ffe417d18 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json502.test | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 173207b80f..0ec0180763 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\srule\sfor\sthe\sRHS\sof\sthe\s->>\sand\s->\soperators\swhen\sthe\sRHS\sdoes\snot\sbegin\nwith\s$\sis\sthat\sit\smust\sbe\s(1)\sall\sdigits,\sor\s(2)\sall\salphanumerics,\sor\n(3)\scontained\swithin\s[..]\sor\selse\sit\swill\sbecome\sa\squoted\slabel. -D 2023-12-06T15:35:38.860 +C Test\scases\sfor\sobject\slabel\smatching\swith\sescape\ssequences. +D 2023-12-06T15:50:13.951 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1333,7 +1333,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 -F test/json502.test 98c38e3c4573841028a1381dfb81d4c3f9b105d39668167da10d055e503f6d0b +F test/json502.test 73dd17721c0b4a1320b163c3a1092d714aae5d100faf46f305515e1b3591c3a6 F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a @@ -2147,8 +2147,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2bc86d145fccc07107b7753cb1a69122676d4096fe59c454497bd81a6142d45e -R 5951a4c9f5e0f75219c7e2d971403d09 +P 0e059a546ec11fa5c6d007bd65c249ee2422f1facbdb2792c53e0bc0ccc97e14 +R 4550d328e20b914ec87aa1544ecc75dd U drh -Z 93fae7007061a25e3b1f2103eb12e9f0 +Z 5fceb780d654fe6791112d65204ddb27 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 121f08ea73..6aa28612b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e059a546ec11fa5c6d007bd65c249ee2422f1facbdb2792c53e0bc0ccc97e14 \ No newline at end of file +c6f2aa38e95b7888650cfa7bb773b18a28e01d883033ac77be6d504ffe417d18 \ No newline at end of file diff --git a/test/json502.test b/test/json502.test index 595bf63318..ed61260e3b 100644 --- a/test/json502.test +++ b/test/json502.test @@ -36,5 +36,26 @@ do_catchsql_test 2.3 { SELECT '{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'->'$h[#-1]'; } {1 {malformed JSON}} +# Verify that escaped label names are compared correctly. +# +do_execsql_test 3.1 { + SELECT '{"a\x62c":123}' ->> 'abc'; +} 123 +do_execsql_test 3.2 { + SELECT '{"abc":123}' ->> 'a\x62c'; +} 123 + +db null null +do_execsql_test 3.3 { + DROP TABLE t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(json_insert('{}','$.a\',111,'$."b\\"',222)); + INSERT INTO t1 VALUES(jsonb_insert('{}','$.a\',111,'$."b\\"',222)); + SELECT x->'$.a\', x->'$.a\\', x->'$."a\\"', x->'$."b\\"' FROM t1; +} {111 null 111 222 111 null 111 222} + +do_execsql_test 3.4 { + SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc'; +} 456 finish_test From 3207199c98f4b876934dbd4d09615175e43e8b37 Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 6 Dec 2023 16:27:29 +0000 Subject: [PATCH 186/191] In CLI, move -interactive flag handling back to arg-loop pass 2. FossilOrigin-Name: 63cb05a862532d2d56e9e81fe32ced09bf58f03146587a118f11c2a84e195e69 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/shell.c.in | 9 ++++----- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 99b9d7e832..484e4397c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\stokendata=1\soption\sand\srelated\sAPIs\sto\sfts5. -D 2023-12-06T14:36:34.858 +C In\sCLI,\smove\s-interactive\sflag\shandling\sback\sto\sarg-loop\spass\s2. +D 2023-12-06T16:27:29.628 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -737,7 +737,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 85857bedd2913d888aa571755b48c54cd2e6e7fcb0087e19b226ee0368cfda1e -F src/shell.c.in 7bb83293775e1a5586d65212997442bc7acc70a2f1b781745da64ec3c2e4ea97 +F src/shell.c.in 9b6c3e641de45651ad0b5e9c26cd2f72efabee28179a5315d15c54239515ee3a F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2153,9 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c48acdbb44185b352b54911a57a6986d6c7e624bdeba2af48b985d29f0292bf 8f46eace86e7b2e556913575aa3cd6f7987ac0efcc880f0af649d42c253aeb81 -R aee08254c1ed5ae187dbc54a7e67d0a2 -T +closed 8f46eace86e7b2e556913575aa3cd6f7987ac0efcc880f0af649d42c253aeb81 -U dan -Z b8398992a8dd36d240de5bcbcb58489b +P a76a636b23c0ebd95d47fdf8358de4729e51a5f68f1a730cd4d89b378e94ac0d +R 66c08e25d4bc58dda9bee0684e3ee757 +U larrybr +Z 3fccc4c8303ad91f158ca989e0b5a086 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c9e8895225..025e6d00a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a76a636b23c0ebd95d47fdf8358de4729e51a5f68f1a730cd4d89b378e94ac0d \ No newline at end of file +63cb05a862532d2d56e9e81fe32ced09bf58f03146587a118f11c2a84e195e69 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 4a4a0e1fef..6d1312d0f8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -12061,10 +12061,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-init")==0 ){ zInitFile = cmdline_option_value(argc, argv, ++i); }else if( cli_strcmp(z,"-interactive")==0 ){ - /* Need to check for interactive override here to so that it can - ** affect console setup (for Windows only) and testing thereof. - */ - stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* Need to check for batch mode here to so we can avoid printing ** informational messages (like from process_sqliterc) before @@ -12338,7 +12334,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*)); return 0; }else if( cli_strcmp(z,"-interactive")==0 ){ - /* already handled */ + /* Need to check for interactive override here to so that it can + ** affect console setup (for Windows only) and testing thereof. + */ + stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* already handled */ }else if( cli_strcmp(z,"-utf8")==0 ){ From 9df01b5ccf78535dd44e1f8c0b83fcee40ea5042 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 16:57:18 +0000 Subject: [PATCH 187/191] Fix the routine that determines the json_tree.path value for the first row so that it correctly takes into account escape sequences in the path argument. FossilOrigin-Name: b9243ee8a37c62eb8848e765bd4af83bc1b3d3eb24fb4268a1357ad1f8b2e1fb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 23 ++++++++++++----------- test/json502.test | 4 ++++ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 0ec0180763..48427cf150 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sfor\sobject\slabel\smatching\swith\sescape\ssequences. -D 2023-12-06T15:50:13.951 +C Fix\sthe\sroutine\sthat\sdetermines\sthe\sjson_tree.path\svalue\sfor\sthe\sfirst\srow\nso\sthat\sit\scorrectly\stakes\sinto\saccount\sescape\ssequences\sin\sthe\spath\nargument. +D 2023-12-06T16:57:18.873 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,7 +690,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 9d0b8fa594a931d3f84056020c6581bb631cee02ea5f0530a4fd5876a2cb6560 +F src/json.c 89ce1dbcd8373a33adb82652842956c8a5d2b5c48d6f2c197d47bcee5fdedfd3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -1333,7 +1333,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test e64a8d73677fbae67886642cd5076e2ef3efe89f8483b87595cf9c030216c9bd F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 -F test/json502.test 73dd17721c0b4a1320b163c3a1092d714aae5d100faf46f305515e1b3591c3a6 +F test/json502.test 3c697e506fc38ccb455b49660b21b6e62e08ede0f2d0c869a7d171e17809093c F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a @@ -2147,8 +2147,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0e059a546ec11fa5c6d007bd65c249ee2422f1facbdb2792c53e0bc0ccc97e14 -R 4550d328e20b914ec87aa1544ecc75dd +P c6f2aa38e95b7888650cfa7bb773b18a28e01d883033ac77be6d504ffe417d18 +R 31c04da1922b93d8052617096f6b59a6 U drh -Z 5fceb780d654fe6791112d65204ddb27 +Z a482eb5d0e0092ccc9abb42b468f8c79 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6aa28612b9..3fd26623b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6f2aa38e95b7888650cfa7bb773b18a28e01d883033ac77be6d504ffe417d18 \ No newline at end of file +b9243ee8a37c62eb8848e765bd4af83bc1b3d3eb24fb4268a1357ad1f8b2e1fb \ No newline at end of file diff --git a/src/json.c b/src/json.c index 44abe2d164..c9014f74a5 100644 --- a/src/json.c +++ b/src/json.c @@ -4489,22 +4489,23 @@ static int jsonEachNext(sqlite3_vtab_cursor *cur){ */ static int jsonEachPathLength(JsonEachCursor *p){ u32 n = p->path.nUsed; + const char *z = p->path.zBuf; if( p->iRowid==0 && p->bRecursive && n>1 ){ - if( p->path.zBuf[n-1]==']' ){ + if( z[n-1]==']' ){ do{ - assert( n>0 ); + assert( n>1 ); n--; - }while( p->path.zBuf[n]!='[' ); + }while( z[n]!='[' ); + }else if( z[n-1]=='"' ){ + do{ + assert( n>1 ); + n--; + }while( z[n]!='.' || z[n+1]!='"' ); }else{ - u32 sz = 0; - jsonbPayloadSize(&p->sParse, p->i, &sz); - if( p->path.zBuf[n-1]=='"' ) sz += 2; - assert( szpath.zBuf[n]!='.' && ALWAYS(n>0) ){ + do{ + assert( n>1 ); n--; - assert( n>0 ); - } + }while( z[n]!='.' ); } } return n; diff --git a/test/json502.test b/test/json502.test index ed61260e3b..48c372c4f8 100644 --- a/test/json502.test +++ b/test/json502.test @@ -58,4 +58,8 @@ do_execsql_test 3.4 { SELECT json_patch('{"a\x62c":123}','{"ab\x63":456}') ->> 'abc'; } 456 +do_execsql_test 4.1 { + SELECT * FROM json_tree('{"\u0017":1}','$."\x17"'); +} {{\x17} 1 integer 1 1 null {$."\x17"} {$}} + finish_test From b597fea89436a57b8b4bc39f24dc7e14f5e92dcb Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 17:39:31 +0000 Subject: [PATCH 188/191] Correctly handle 8-byte sizes in the JSONB format. [forum:/forumpost/283daf08e91183fc|Forum post 283daf08e91183fc]. FossilOrigin-Name: 73d390f39c0bbbc017e01544e4d43c76761f2599bd57f900131c706270dfd202 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 15 ++++++++++++++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 484e4397c6..e532282489 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sCLI,\smove\s-interactive\sflag\shandling\sback\sto\sarg-loop\spass\s2. -D 2023-12-06T16:27:29.628 +C Correctly\shandle\s8-byte\ssizes\sin\sthe\sJSONB\sformat.\n[forum:/forumpost/283daf08e91183fc|Forum\spost\s283daf08e91183fc]. +D 2023-12-06T17:39:31.569 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -696,7 +696,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 0a6095d10a8c8251e1838a1f12abc89125b0b05f82497ad12896a7f714e397ce +F src/json.c 07247c969e80e0a70241235ea4d00bb823830ce6a48f101fbcb1c72e8abf6f91 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a76a636b23c0ebd95d47fdf8358de4729e51a5f68f1a730cd4d89b378e94ac0d -R 66c08e25d4bc58dda9bee0684e3ee757 -U larrybr -Z 3fccc4c8303ad91f158ca989e0b5a086 +P 63cb05a862532d2d56e9e81fe32ced09bf58f03146587a118f11c2a84e195e69 +R 3207d50f882d9851bd1114b702910d8f +U drh +Z 9fdc41964fbd72df8cb3bfd54d04d4f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 025e6d00a4..fdb030fa0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63cb05a862532d2d56e9e81fe32ced09bf58f03146587a118f11c2a84e195e69 \ No newline at end of file +73d390f39c0bbbc017e01544e4d43c76761f2599bd57f900131c706270dfd202 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8165eb4f31..e91c9b86dd 100644 --- a/src/json.c +++ b/src/json.c @@ -1792,7 +1792,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ } sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; n = 3; - }else{ + }else if( x==14 ){ if( i+4>=pParse->nBlob ){ *pSz = 0; return 0; @@ -1800,6 +1800,19 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ sz = (pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; n = 5; + }else{ + if( i+8>=pParse->nBlob + || pParse->aBlob[i+1]!=0 + || pParse->aBlob[i+2]!=0 + || pParse->aBlob[i+3]!=0 + || pParse->aBlob[i+4]!=0 + ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; + n = 9; } if( i+sz+n > pParse->nBlob && i+sz+n > pParse->nBlob-pParse->delta From 83ac79282acb7c78647d1a7474c0061da450f124 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Dec 2023 18:10:50 +0000 Subject: [PATCH 189/191] Update documentation comments in fts5.h. FossilOrigin-Name: 38c50e22c98607e6c1fd78d7615cda534773b6d4fd85c712b54749fcd7af0c83 --- ext/fts5/extract_api_docs.tcl | 4 +++- ext/fts5/fts5.h | 6 +++++- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index 2320d70b7d..6762a036d5 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -223,10 +223,12 @@ proc main {data} { Fts5ExtensionApi { set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"] set map [list] + set lKey [list] foreach {k v} [get_struct_members $data] { if {[string match x* $k]==0} continue - lappend map $k "$k" + lappend lKey $k } + foreach k [lsort -decr $lKey] { lappend map $k "$k" } output [string map $map $struct] } diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 63c9765eb6..c7a5e6b7de 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -274,7 +274,11 @@ struct Fts5PhraseIter { ** ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) ** This is used to access token iToken of phrase hit iIdx within the -** current row. +** current row. Output variable (*ppToken) is set to point to a buffer +** containing the matching document token, and (*pnToken) to the size +** of that buffer in bytes. This API is not available if the specified +** token matches a prefix query term. In that case both output variables +** are always set to 0. ** ** The output text is not a copy of the document text that was tokenized. ** It is the output of the tokenizer module. For tokendata=1 tables, this diff --git a/manifest b/manifest index a803cb2a64..68dc896475 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\scorrect\scomparisons\sbetween\sobject\slabels\sin\sJSON\seven\swhen\sthe\stwo\slabels\ncontain\sdifferent\sJSON\sescapes. -D 2023-12-06T17:50:16.616 +C Update\sdocumentation\scomments\sin\sfts5.h. +D 2023-12-06T18:10:50.884 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -88,8 +88,8 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb -F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h ff90acaa97f8e865b66d1177d1b56b8c110fd5548ab5863bab43f055a1d745fe +F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e +F ext/fts5/fts5.h 9d7867cc63631296462c90bafe57c4881b56e480fa510ffaee6a77049258c714 F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 @@ -2153,9 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73d390f39c0bbbc017e01544e4d43c76761f2599bd57f900131c706270dfd202 b9243ee8a37c62eb8848e765bd4af83bc1b3d3eb24fb4268a1357ad1f8b2e1fb -R fd697485adcd3130674f1a92f5c02f3d -T +closed b9243ee8a37c62eb8848e765bd4af83bc1b3d3eb24fb4268a1357ad1f8b2e1fb -U drh -Z 9804c6a78387d2d6c50275c2413cc6ef +P bda2e30cc22e180b19a7a05824dd345880eb402ae5450b2d2dd954946c3ae135 +R 17f891969e2a492b1525813205840ba9 +U dan +Z b2b93165d3875994ffe922a4d9022c9c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bba984a924..0b4ae40007 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bda2e30cc22e180b19a7a05824dd345880eb402ae5450b2d2dd954946c3ae135 \ No newline at end of file +38c50e22c98607e6c1fd78d7615cda534773b6d4fd85c712b54749fcd7af0c83 \ No newline at end of file From 82fc1b63f6e89afec382e7ddbb2d3ffe3ac8cefe Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 6 Dec 2023 18:25:41 +0000 Subject: [PATCH 190/191] Work around LLVM's newfound hatred of function pointer casts. [forum:/forumpost/1a7d257346636292|Forum post 1a7d257346636292]. FossilOrigin-Name: ec0ae4030968c782af48d1c776351c14b2ada21d40aeb97915f33df30706e18f --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 12 +++++++++--- src/expr.c | 14 ++++++++------ src/select.c | 33 ++++++++++++++------------------- src/sqliteInt.h | 5 +++++ 6 files changed, 47 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 68dc896475..5f7ad7a854 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sdocumentation\scomments\sin\sfts5.h. -D 2023-12-06T18:10:50.884 +C Work\saround\sLLVM's\snewfound\shatred\sof\sfunction\spointer\scasts.\n[forum:/forumpost/1a7d257346636292|Forum\spost\s1a7d257346636292]. +D 2023-12-06T18:25:41.085 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -678,7 +678,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c f3b09c5414de3a11db73e11e1d66f4c5e53c9e89876ff3b531a887ab656ca303 x F src/btree.h 03e3356f5208bcab8eed4e094240fdac4a7f9f5ddf5e91045ce589f67d47c240 F src/btreeInt.h 3e2589726c4f105e653461814f65857465da68be1fac688de340c43b873f4062 -F src/build.c d0bb02989e9e652ee8a012c7780e32087c0dd99643d0cf609971029f78cb738a +F src/build.c 386eadecabe2e99a3783eb802ca01e665f8e0c2af0e0aab28161fd7def219a9d F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 23331529e654be40ca97d171cbbffe9b3d4c71cc53b78fe5501230675952da8b @@ -686,7 +686,7 @@ F src/date.c 3b8d02977d160e128469de38493b4085f7c5cf4073193459909a6af3cf6d7c91 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43 F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500 -F src/expr.c e9a491c7f156e5b25641c28af11b735a424e108a21b9f83b6f3e51c99a8141d9 +F src/expr.c f4dbcca060fa95539f6705b5ec4cb5eeaba5c0d84ff64efca8edab7a1d776807 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00 F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43 @@ -736,12 +736,12 @@ F src/printf.c 18fbdf028345c8fbe6044f5f5bfda5a10d48d6287afef088cc21b0ca57985640 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 85857bedd2913d888aa571755b48c54cd2e6e7fcb0087e19b226ee0368cfda1e +F src/select.c a5058ae54211dc49faade7b78f21c69a3fa3ee652eb270fba6881e28e30e0497 F src/shell.c.in 9b6c3e641de45651ad0b5e9c26cd2f72efabee28179a5315d15c54239515ee3a F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h aab66d149269f15f6f1011081b389f001f00b84045c69a4f9ec96dd68cc3a7d7 +F src/sqliteInt.h 39b6fe0652c763d8ff6bd0a2427c009e5abe05fb70f5a0cb145a34bcc614fb90 F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bda2e30cc22e180b19a7a05824dd345880eb402ae5450b2d2dd954946c3ae135 -R 17f891969e2a492b1525813205840ba9 -U dan -Z b2b93165d3875994ffe922a4d9022c9c +P 38c50e22c98607e6c1fd78d7615cda534773b6d4fd85c712b54749fcd7af0c83 +R 46759d7734ecdddbd46479e0808766c4 +U drh +Z d4b23af7ef5680cfa48b70dc8dcb0c9b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b4ae40007..0e3872e86a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38c50e22c98607e6c1fd78d7615cda534773b6d4fd85c712b54749fcd7af0c83 \ No newline at end of file +ec0ae4030968c782af48d1c776351c14b2ada21d40aeb97915f33df30706e18f \ No newline at end of file diff --git a/src/build.c b/src/build.c index 14639f096e..50ce97b51a 100644 --- a/src/build.c +++ b/src/build.c @@ -873,6 +873,9 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } +void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){ + sqlite3DeleteTable(db, (Table*)pTable); +} /* @@ -1410,7 +1413,8 @@ void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ /* ** Clean up the data structures associated with the RETURNING clause. */ -static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ +static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){ + Returning *pRet = (Returning*)pArg; Hash *pHash; pHash = &(db->aDb[1].pSchema->trigHash); sqlite3HashInsert(pHash, pRet->zName, 0); @@ -1452,8 +1456,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pParse->u1.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); + sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); testcase( pParse->earlyCleanup ); if( db->mallocFailed ) return; sqlite3_snprintf(sizeof(pRet->zName), pRet->zName, @@ -5692,4 +5695,7 @@ void sqlite3WithDelete(sqlite3 *db, With *pWith){ sqlite3DbFree(db, pWith); } } +void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){ + sqlite3WithDelete(db, (With*)pWith); +} #endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/expr.c b/src/expr.c index 8a664ffb9c..047d2b6b39 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1223,9 +1223,7 @@ void sqlite3ExprAddFunctionOrderBy( assert( ExprUseXList(pExpr) ); if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){ /* Ignore ORDER BY on zero-argument aggregates */ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy); return; } if( IsWindowFunc(pExpr) ){ @@ -1406,6 +1404,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){ + if( p ) sqlite3ExprDeleteNN(db, (Expr*)p); +} /* ** Clear both elements of an OnOrUsing object @@ -1431,9 +1432,7 @@ void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ ** pExpr to the pParse->pConstExpr list with a register number of 0. */ void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprDelete, - pExpr); + sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); } /* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the @@ -2239,6 +2238,9 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); } +void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){ + if( pList ) exprListDeleteNN(db, (ExprList*)pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given diff --git a/src/select.c b/src/select.c index e5312c7ee7..f61b34f5fe 100644 --- a/src/select.c +++ b/src/select.c @@ -184,6 +184,9 @@ Select *sqlite3SelectNew( void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } +void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ + if( p ) clearSelect(db, (Select*)p, 1); +} /* ** Return a pointer to the right-most SELECT statement in a compound. @@ -3204,9 +3207,7 @@ multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; if( pDelete ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3SelectDelete, - pDelete); + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } return rc; } @@ -3757,8 +3758,7 @@ static int multiSelectOrderBy( /* Make arrangements to free the 2nd and subsequent arms of the compound ** after the parse has finished */ if( pSplit->pPrior ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior); + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior); } pSplit->pPrior = pPrior; pPrior->pNext = pSplit; @@ -4579,9 +4579,7 @@ static int flattenSubquery( Table *pTabToDel = pSubitem->pTab; if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3ParserAddCleanup(pToplevel, - (void(*)(sqlite3*,void*))sqlite3DeleteTable, - pTabToDel); + sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); testcase( pToplevel->earlyCleanup ); }else{ pTabToDel->nTabRef--; @@ -5628,8 +5626,7 @@ static struct Cte *searchWith( With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ if( pWith ){ if( bFree ){ - pWith = (With*)sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3WithDelete, + pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric, pWith); if( pWith==0 ) return 0; } @@ -7037,7 +7034,8 @@ static SrcItem *isSelfJoinView( /* ** Deallocate a single AggInfo object */ -static void agginfoFree(sqlite3 *db, AggInfo *p){ +static void agginfoFree(sqlite3 *db, void *pArg){ + AggInfo *p = (AggInfo*)pArg; sqlite3DbFree(db, p->aCol); sqlite3DbFree(db, p->aFunc); sqlite3DbFreeNN(db, p); @@ -7291,9 +7289,8 @@ int sqlite3Select( sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); } #endif - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - p->pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + p->pOrderBy); testcase( pParse->earlyCleanup ); p->pOrderBy = 0; } @@ -7485,9 +7482,8 @@ int sqlite3Select( ){ TREETRACE(0x800,pParse,p, ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))sqlite3ExprListDelete, - pSub->pOrderBy); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + pSub->pOrderBy); pSub->pOrderBy = 0; } @@ -8016,8 +8012,7 @@ int sqlite3Select( */ pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); if( pAggInfo ){ - sqlite3ParserAddCleanup(pParse, - (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo); testcase( pParse->earlyCleanup ); } if( db->mallocFailed ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c7ced86999..a0d5962005 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4803,6 +4803,7 @@ void sqlite3ExprOrderByAggregateError(Parse*,Expr*); void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); void sqlite3ExprDelete(sqlite3*, Expr*); +void sqlite3ExprDeleteGeneric(sqlite3*,void*); void sqlite3ExprDeferredDelete(Parse*, Expr*); void sqlite3ExprUnmapAndDelete(Parse*, Expr*); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); @@ -4812,6 +4813,7 @@ void sqlite3ExprListSetSortOrder(ExprList*,int,int); void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int); void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); void sqlite3ExprListDelete(sqlite3*, ExprList*); +void sqlite3ExprListDeleteGeneric(sqlite3*,void*); u32 sqlite3ExprListFlags(const ExprList*); int sqlite3IndexHasDuplicateRootPage(Index*); int sqlite3Init(sqlite3*, char**); @@ -4902,6 +4904,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); void sqlite3DropTable(Parse*, SrcList*, int, int); void sqlite3CodeDropTable(Parse*, Table*, int, int); void sqlite3DeleteTable(sqlite3*, Table*); +void sqlite3DeleteTableGeneric(sqlite3*, void*); void sqlite3FreeIndex(sqlite3*, Index*); #ifndef SQLITE_OMIT_AUTOINCREMENT void sqlite3AutoincrementBegin(Parse *pParse); @@ -4938,6 +4941,7 @@ int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); +void sqlite3SelectDeleteGeneric(sqlite3*,void*); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, Trigger*); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); @@ -5510,6 +5514,7 @@ const char *sqlite3JournalModename(int); void sqlite3CteDelete(sqlite3*,Cte*); With *sqlite3WithAdd(Parse*,With*,Cte*); void sqlite3WithDelete(sqlite3*,With*); + void sqlite3WithDeleteGeneric(sqlite3*,void*); With *sqlite3WithPush(Parse*, With*, u8); #else # define sqlite3CteNew(P,T,E,S) ((void*)0) From 00f3ac5544db5fcb86ae09b791b2d43945c31aac Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Dec 2023 18:34:59 +0000 Subject: [PATCH 191/191] Fix compiler warning about shadowed variable in fts5_index.c. FossilOrigin-Name: ee70e4c1c9c41617850228e48d8df44f105cf2fbbe789340ceca6f27ad6ce5eb --- ext/fts5/fts5_index.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9a2cc026b9..7ec1976d33 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6743,9 +6743,9 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Allocate array of iterators if they are not already allocated. */ if( pT->aPoslistReader==0 ){ - int nByte = pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)); pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( - &pIter->pIndex->rc, nByte + &pIter->pIndex->rc, + pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)) ); if( pT->aPoslistReader==0 ) return; pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; diff --git a/manifest b/manifest index 5f7ad7a854..10b2d5cbfd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sLLVM's\snewfound\shatred\sof\sfunction\spointer\scasts.\n[forum:/forumpost/1a7d257346636292|Forum\spost\s1a7d257346636292]. -D 2023-12-06T18:25:41.085 +C Fix\scompiler\swarning\sabout\sshadowed\svariable\sin\sfts5_index.c. +D 2023-12-06T18:34:59.649 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -96,7 +96,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf F ext/fts5/fts5_expr.c b1ec526371b9ffde82341423a5b9753c42cbea629a41b69f26fa377d13b95a8e F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c be39b44ff8773cff56bcbc01f74701a83e068c20d773cafd01e8bb2fa0fc1bc5 +F ext/fts5/fts5_index.c 111fdb22e4d78a5c3813ac2c782409b3567291a48f19ae834cd50268c8e9fafc F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8 @@ -2153,8 +2153,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38c50e22c98607e6c1fd78d7615cda534773b6d4fd85c712b54749fcd7af0c83 -R 46759d7734ecdddbd46479e0808766c4 -U drh -Z d4b23af7ef5680cfa48b70dc8dcb0c9b +P ec0ae4030968c782af48d1c776351c14b2ada21d40aeb97915f33df30706e18f +R edfe66d6aa99b5210c61e8b6b973888d +U dan +Z 66c944797bd4988ebf2844c99b4718ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0e3872e86a..414f169a76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec0ae4030968c782af48d1c776351c14b2ada21d40aeb97915f33df30706e18f \ No newline at end of file +ee70e4c1c9c41617850228e48d8df44f105cf2fbbe789340ceca6f27ad6ce5eb \ No newline at end of file