paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

The json_valid(*,8) function does a much better check of the validity of

Browse Source 
the JSONB input.  The json_error_position() function returns an approximate
byte offset to the point of the first detected error in the JSONB.

FossilOrigin-Name: 840efb33a3934a9bdccad65ea15c70d5c335a517502c9844aaad8405d16d1604
This commit is contained in:
drh
2023-12-12 17:31:17 +00:00
parent b89e64d822 78e636bba6
commit df25a8fd4b
9 changed files with 279 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest
+13 -12
View File
@@ -1,5 +1,5 @@
C Fix\sa\spotential\suse\sof\suninitialized\svalue\sin\sjson_valid()\swith\s2nd\nargument\sof\s8.
D 2023-12-11T02:39:11.053
C The\sjson_valid(*,8)\sfunction\sdoes\sa\smuch\sbetter\scheck\sof\sthe\svalidity\sof\nthe\sJSONB\sinput.\s\sThe\sjson_error_position()\sfunction\sreturns\san\sapproximate\nbyte\soffset\sto\sthe\spoint\sof\sthe\sfirst\sdetected\serror\sin\sthe\sJSONB.
D 2023-12-12T17:31:17.429
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -690,16 +690,16 @@ F src/expr.c 05278def9c186b5875d6903ea26148c7461b9ce0344f0fd7be9a0dfea0a4538a
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c a47610f0a5c6cb0ad79f8fcef039c01833dec0c751bb695f28dc0ec6a4c3ba00
F src/func.c 472f6dcfa39cf54f89a6aec76c79c225fb880a6c14469c15d361331662b9bf43
F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d
F src/global.c 765a0656d6cbf043cb272ff0ae38f39cc46713539ffe6793258ed3eb4b188b52
F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
F src/json.c 022548736478f10bec4f2766f2c9fe1cc25c06b14cabb0ca64b4a1beb83fbf16
F src/json.c cd270629b4b99d065522b45c3d2b5185491b1ab3544843c6f780aa50755d54dc
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
F src/main.c 1b89f3de98d1b59fec5bac1d66d6ece21f703821b8eaa0d53d9604c35309f6f9
F src/main.c ce714ee501122c76eb2e69b292bebe443aba611fc3b88e6786eb910285515fe4
F src/malloc.c f016922435dc7d1f1f5083a03338a3e91f8c67ce2c5bdcfa4cdef62e612f5fcc
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -737,11 +737,11 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
F src/resolve.c d017bad7ba8e778617701a0e986fdeb393d67d6afa84fb28ef4e8b8ad2acf916
F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
F src/select.c 9f55c9f3307b9888f62abe709eec245e98ff217bd14c044f93d72810bb7dc445
F src/shell.c.in 9b6c3e641de45651ad0b5e9c26cd2f72efabee28179a5315d15c54239515ee3a
F src/sqlite.h.in d93a4821d2f792467a60f7dc81268d1bb8634f40c31694ef254cab4f9921f96a
F src/shell.c.in 0cd2ef4b3c814dded5436625ab664d9a973cbc4266a1768e3aa4cbf11bb41eca
F src/sqlite.h.in adcc7dbfeea1e69d6d487139a7e90db8a48fe998f3f5bb0f85c683e6a6fa68ca
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
F src/sqliteInt.h 39b6fe0652c763d8ff6bd0a2427c009e5abe05fb70f5a0cb145a34bcc614fb90
F src/sqliteInt.h 615524fff083abf9353aa4a3a8adca2d32b3e61e014e62b1a5335c454a386783
F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -1229,7 +1229,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
F test/fuzzcheck.c 69b8549e112fb815931a8c14c7955a0c407ae91a79356eecb82458384f2cb989
F test/fuzzcheck.c 442b0b02146bb905c47c4ee1388949e7fd9d27acf22d9535e9abf216e582b956
F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517
F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -2153,8 +2153,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 1503cba6d17e9bade7a5c103ddd23241ff4741f9a2e3032ffe2987af243dae65
R 10c5202c73d5a29cc977b618b1930c91
P fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
R 6e221a94cd5525fe9f8a7e94ea467b60
T +closed c96ebb086feb89341565cc52b970ae7799ce1327fe1ad4fc790f1b0dcaa6e229
U drh
Z 03628577bea44fb563932e72818a5e19
Z 749a6c45182c15aa056f901d03cf2368
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
fa102036fe46eeb71b7df3e265be1935ae5c78e0b939b08841bcfb8abadbc77a
840efb33a3934a9bdccad65ea15c70d5c335a517502c9844aaad8405d16d1604
src/global.c
+3
View File
@@ -244,6 +244,9 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = {
0, /* bSmallMalloc */
1, /* bExtraSchemaChecks */
sizeof(LONGDOUBLE_TYPE)>8, /* bUseLongDouble */
#ifdef SQLITE_DEBUG
0, /* bJsonSelfcheck */
#endif
0x7ffffffe, /* mxStrlen */
0, /* neverCorrupt */
SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */
src/json.c
+223 -35
View File
@@ -176,7 +176,9 @@ static const char jsonSpaces[] = "\011\012\015\040";
/*
** Characters that are special to JSON. Control characters,
** '"' and '\\'.
** '"' and '\\' and '\''. Actually, '\'' is not special to
** canonical JSON, but it is special in JSON-5, so we include
** it in the set of special characters.
*/
static const char jsonIsOk[256] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -336,6 +338,8 @@ static u32 jsonXlateBlobToText(const JsonParse*,u32,JsonString*);
static void jsonReturnParse(sqlite3_context*,JsonParse*);
static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32);
static void jsonParseFree(JsonParse*);
static u32 jsonbPayloadSize(const JsonParse*, u32, u32*);
static u32 jsonUnescapeOneChar(const char*, u32, u32*);
/**************************************************************************
** Utility routines for dealing with JsonCache objects
@@ -1228,6 +1232,208 @@ static int jsonIs4HexB(const char *z, int *pOp){
return 1;
}
/*
** Check a single element of the JSONB in pParse for validity.
**
** The element to be checked starts at offset i and must end at on the
** last byte before iEnd.
**
** Return 0 if everything is correct. Return the 1-based byte offset of the
** error if a problem is detected. (In other words, if the error is at offset
** 0, return 1).
*/
static u32 jsonbValidityCheck(
const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */
u32 i, /* Start of element as pParse->aBlob[i] */
u32 iEnd, /* One more than the last byte of the element */
u32 iDepth /* Current nesting depth */
){
u32 n, sz, j, k;
const u8 *z;
u8 x;
if( iDepth>JSON_MAX_DEPTH ) return i+1;
sz = 0;
n = jsonbPayloadSize(pParse, i, &sz);
if( NEVER(n==0) ) return i+1; /* Checked by caller */
if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */
z = pParse->aBlob;
x = z[i] & 0x0f;
switch( x ){
case JSONB_NULL:
case JSONB_TRUE:
case JSONB_FALSE: {
return n+sz==1 ? 0 : i+1;
}
case JSONB_INT: {
if( sz<1 ) return i+1;
j = i+n;
if( z[j]=='-' ){
j++;
if( sz<2 ) return i+1;
}
k = i+n+sz;
while( j<k ){
if( sqlite3Isdigit(z[j]) ){
j++;
}else{
return j+1;
}
}
return 0;
}
case JSONB_INT5: {
if( sz<3 ) return i+1;
j = i+n;
if( z[j]=='-' ){
if( sz<4 ) return i+1;
j++;
}
if( z[j]!='0' ) return i+1;
if( z[j+1]!='x' && z[j+1]!='X' ) return j+2;
j += 2;
k = i+n+sz;
while( j<k ){
if( sqlite3Isxdigit(z[j]) ){
j++;
}else{
return j+1;
}
}
return 0;
}
case JSONB_FLOAT:
case JSONB_FLOAT5: {
u8 seen = 0; /* 0: initial. 1: '.' seen 2: 'e' seen */
if( sz<2 ) return i+1;
j = i+n;
k = j+sz;
if( z[j]=='-' ){
j++;
if( sz<3 ) return i+1;
}
if( z[j]=='.' ){
if( x==JSONB_FLOAT ) return j+1;
if( !sqlite3Isdigit(z[j+1]) ) return j+1;
j += 2;
seen = 1;
}else if( z[j]=='0' && x==JSONB_FLOAT ){
if( j+3>k ) return j+1;
if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1;
j++;
}
for(; j<k; j++){
if( sqlite3Isdigit(z[j]) ) continue;
if( z[j]=='.' ){
if( seen>0 ) return j+1;
if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){
return j+1;
}
seen = 1;
continue;
}
if( z[j]=='e' || z[j]=='E' ){
if( seen==2 ) return j+1;
if( j==k-1 ) return j+1;
if( z[j+1]=='+' || z[j+1]=='-' ){
j++;
if( j==k-1 ) return j+1;
}
seen = 2;
continue;
}
return j+1;
}
if( seen==0 ) return i+1;
return 0;
}
case JSONB_TEXT: {
j = i+n;
k = j+sz;
while( j<k ){
if( !jsonIsOk[z[j]] && z[j]!='\'' ) return j+1;
j++;
}
return 0;
}
case JSONB_TEXTJ:
case JSONB_TEXT5: {
j = i+n;
k = j+sz;
while( j<k ){
if( !jsonIsOk[z[j]] && z[j]!='\'' ){
if( z[j]=='"' ){
if( x==JSONB_TEXTJ ) return j+1;
j++;
}else if( z[j]!='\\' || j+1>=k ){
return j+1;
}else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){
j++;
}else if( z[j+1]=='u' ){
if( j+5>=k ) return j+1;
if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1;
j++;
}else if( x!=JSONB_TEXT5 ){
return j+1;
}else{
u32 c = 0;
u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c);
if( c==0xfffd ) return j+1;
j += szC - 1;
}
}
j++;
}
return 0;
}
case JSONB_TEXTRAW: {
return 0;
}
case JSONB_ARRAY: {
u32 sub;
j = i+n;
k = j+sz;
while( j<k ){
sz = 0;
n = jsonbPayloadSize(pParse, j, &sz);
if( n==0 ) return j+1;
if( j+n+sz>k ) return j+1;
sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1);
if( sub ) return sub;
j += n + sz;
}
assert( j==k );
return 0;
}
case JSONB_OBJECT: {
u32 cnt = 0;
u32 sub;
j = i+n;
k = j+sz;
while( j<k ){
sz = 0;
n = jsonbPayloadSize(pParse, j, &sz);
if( n==0 ) return j+1;
if( j+n+sz>k ) return j+1;
if( (cnt & 1)==0 ){
x = z[j] & 0x0f;
if( x<JSONB_TEXT || x>JSONB_TEXTRAW ) return j+1;
}
sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1);
if( sub ) return sub;
cnt++;
j += n + sz;
}
assert( j==k );
if( (cnt & 1)!=0 ) return j+1;
return 0;
}
default: {
return i+1;
}
}
}
/*
** Translate a single element of JSON text at pParse->zJson[i] into
** its equivalent binary JSONB representation. Append the translation into
@@ -1456,6 +1662,8 @@ json_parse_restart:
/* Control characters are not allowed in strings */
pParse->iErr = j;
return -1;
}else if( c=='"' ){
opcode = JSONB_TEXT5;
}
j++;
}
@@ -1713,7 +1921,12 @@ static int jsonConvertTextToBlob(
i = jsonXlateTextToBlob(pParse, 0);
if( pParse->oom ) i = -1;
if( i>0 ){
#ifdef SQLITE_DEBUG
assert( pParse->iDepth==0 );
if( sqlite3Config.bJsonSelfcheck ){
assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 );
}
#endif
while( jsonIsspace(zJson[i]) ) i++;
if( zJson[i] ){
i += json5Whitespace(&zJson[i]);
@@ -1910,13 +2123,13 @@ static u32 jsonXlateBlobToText(
}
break;
}
case JSONB_TEXT:
case JSONB_TEXTJ: {
jsonAppendChar(pOut, '"');
jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz);
jsonAppendChar(pOut, '"');
break;
}
case JSONB_TEXT:
case JSONB_TEXT5: {
const char *zIn;
u32 k;
@@ -3954,38 +4167,19 @@ static void jsonValidFunc(
case SQLITE_BLOB: {
if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){
if( flags & 0x04 ){
/* Superficial checking only - accomplisehd by the
/* Superficial checking only - accomplished by the
** jsonFuncArgMightBeBinary() call above. */
res = 1;
}else{
/* Strict checking. Check by translating BLOB->TEXT->BLOB. If
** no errors occur, call that a "strict check". */
JsonParse px;
JsonString sx;
u8 oom = 0;
u32 iErr;
memset(&px, 0, sizeof(px));
px.aBlob = (u8*)sqlite3_value_blob(argv[0]);
px.nBlob = sqlite3_value_bytes(argv[0]);
jsonStringInit(&sx, 0);
jsonXlateBlobToText(&px, 0, &sx);
jsonParseReset(&px);
if( sx.eErr & JSTRING_OOM ) oom = 1;
if( sx.eErr==0 ){
memset(&px, 0, sizeof(px));
jsonStringTerminate(&sx);
px.zJson = sx.zBuf;
px.nJson = sx.nUsed;
if( jsonXlateTextToBlob(&px, 0)==px.nJson ){
res = 1;
}
oom |= px.oom;
jsonParseReset(&px);
}
jsonStringReset(&sx);
if( oom ){
sqlite3_result_error_nomem(ctx);
return;
}
iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
res = iErr==0;
}
}
break;
@@ -4019,9 +4213,9 @@ static void jsonValidFunc(
**
** If the argument is NULL, return NULL
**
** If the argument is BLOB, do a fast validity check and return non-zero
** if the check fails. The returned value does not indicate where in the
** BLOB the error occurs.
** If the argument is BLOB, do a full validity check and return non-zero
** if the check fails. The return value is the approximate 1-based offset
** to the byte of the element that contains the first error.
**
** Otherwise interpret the argument is TEXT (even if it is numeric) and
** return the 1-based character position for where the parser first recognized
@@ -4040,15 +4234,9 @@ static void jsonErrorFunc(
UNUSED_PARAMETER(argc);
memset(&s, 0, sizeof(s));
if( jsonFuncArgMightBeBinary(argv[0]) ){
JsonString out;
jsonStringInit(&out, 0);
s.aBlob = (u8*)sqlite3_value_blob(argv[0]);
s.nBlob = sqlite3_value_bytes(argv[0]);
jsonXlateBlobToText(&s, 0, &out);
if( out.eErr ){
iErrPos = (out.eErr & JSTRING_MALFORMED)!=0 ? 1 : -1;
}
jsonStringReset(&out);
iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1);
}else{
s.zJson = (char*)sqlite3_value_text(argv[0]);
if( s.zJson==0 ) return; /* NULL input or OOM */
src/main.c
+22
View File
@@ -4659,6 +4659,28 @@ int sqlite3_test_control(int op, ...){
break;
}
#endif
/* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff);
**
** Activate or deactivate validation of JSONB that is generated from
** text. Off by default, as the validation is slow. Validation is
** only available if compiled using SQLITE_DEBUG.
**
** If onOff is initially 1, then turn it on. If onOff is initially
** off, turn it off. If onOff is initially -1, then change onOff
** to be the current setting.
*/
case SQLITE_TESTCTRL_JSON_SELFCHECK: {
#if defined(SQLITE_DEBUG)
int *pOnOff = va_arg(ap, int*);
if( *pOnOff<0 ){
*pOnOff = sqlite3Config.bJsonSelfcheck;
}else{
sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff);
}
#endif
break;
}
}
va_end(ap);
#endif /* SQLITE_UNTESTABLE */
src/shell.c.in
+11
View File
@@ -10756,6 +10756,7 @@ static int do_meta_command(char *zLine, ShellState *p){
{"fk_no_action", SQLITE_TESTCTRL_FK_NO_ACTION, 0, "BOOLEAN" },
{"imposter", SQLITE_TESTCTRL_IMPOSTER,1,"SCHEMA ON/OFF ROOTPAGE"},
{"internal_functions", SQLITE_TESTCTRL_INTERNAL_FUNCTIONS,0,"" },
{"json_selfcheck", SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN" },
{"localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN" },
{"never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN" },
{"optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK" },
@@ -10974,6 +10975,16 @@ static int do_meta_command(char *zLine, ShellState *p){
isOk = 3;
}
break;
case SQLITE_TESTCTRL_JSON_SELFCHECK:
if( nArg==2 ){
rc2 = -1;
isOk = 1;
}else{
rc2 = booleanValue(azArg[2]);
isOk = 3;
}
sqlite3_test_control(testctrl, &rc2);
break;
}
}
if( isOk==0 && iCtrl>=0 ){
src/sqlite.h.in
+1
View File
@@ -8300,6 +8300,7 @@ int sqlite3_test_control(int op, ...);
#define SQLITE_TESTCTRL_ASSERT 12
#define SQLITE_TESTCTRL_ALWAYS 13
#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */
#define SQLITE_TESTCTRL_JSON_SELFCHECK 14
#define SQLITE_TESTCTRL_OPTIMIZATIONS 15
#define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */
#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */
src/sqliteInt.h
+3
View File
@@ -4177,6 +4177,9 @@ struct Sqlite3Config {
u8 bSmallMalloc; /* Avoid large memory allocations if true */
u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */
u8 bUseLongDouble; /* Make use of long double */
#ifdef SQLITE_DEBUG
u8 bJsonSelfcheck; /* Double-check JSON parsing */
#endif
int mxStrlen; /* Maximum string length */
int neverCorrupt; /* Database is always well-formed */
int szLookaside; /* Default lookaside buffer size */
test/fuzzcheck.c
+2
View File
@@ -1267,6 +1267,8 @@ int runCombinedDbSqlInput(
}
sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100);
sqlite3_hard_heap_limit64(heapLimit);
rc = 1;
sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &rc);
if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){
aDb[18] = aDb[19] = 1;