Add the SQLITE_DBCONFIG_LENIENT_JSON configuration option. Modify the

built-in JSON routines such that when this setting is active, arguments that
that ought to be JSON but still give a reasonable result (ex: NULL) rather
than raising an error.

FossilOrigin-Name: 186db57d263be09624cac84c97723d05813d70fe2e88d45e15b9f3b6d8c63935

manifest

@@ -1,5 +1,5 @@
-C Fix\scorner\scases\sin\sUTF8\shandling\sin\sthe\sREGEXP\sextension.\n[forum:/forumpost/3ffe058b04|Forum\spost\s3ffe058b04].
-D 2022-11-17T19:24:39.375
+C Add\sthe\sSQLITE_DBCONFIG_LENIENT_JSON\sconfiguration\soption.\s\sModify\sthe\nbuilt-in\sJSON\sroutines\ssuch\sthat\swhen\sthis\ssetting\sis\sactive,\sarguments\sthat\nthat\sought\sto\sbe\sJSON\sbut\sstill\sgive\sa\sreasonable\sresult\s(ex:\sNULL)\srather\nthan\sraising\san\serror.
+D 2022-11-18T17:50:52.177
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -597,10 +597,10 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 90a32bc7faa755cd5292ade21d2b3c6edba8fd1d70754a364caccabfde2c3bb2
-F src/json.c 7749b98c62f691697c7ee536b570c744c0583cab4a89200fdd0fc2aa8cc8cbd6
+F src/json.c 7634894944baeab438665d792401a159b5e4ae45735337ccfbfa2224f00faf70
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 8086232d10e51e183a7f64199815bad1c579896354db69435347665f62f481e9
-F src/main.c fa53bb2ae09549dab5629271c3cfd681f89059f5192afaaaf5c0d396bb3957fe
+F src/main.c 88e65046323fd5ac7e11278f6a45043e519c4b7c5a1ed561c89fb1f13ec2b013
 F src/malloc.c dfddca1e163496c0a10250cedeafaf56dff47673e0f15888fb0925340a8e3f90
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
@@ -638,16 +638,16 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184
-F src/shell.c.in a0ba4a297f8134fef1a6b618ac57604a6c4f1eadeab4f6950b99a8bc151f3c1f
-F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca
+F src/shell.c.in d912025ccd71060b7344c876f4d5b193407047e3b172f098cd69611b66dd0d55
+F src/sqlite.h.in 3455a8d57cb50353a66f4f8d3566e04ee3959337ce2ab6d1d5ab7362576ee532
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f
-F src/sqliteInt.h 2c24ba38f78e32fe5d7ec136321a6ad827698b33ca98664970a8b7274d69ef7c
+F src/sqliteInt.h d924a3cbfcc146dcc5cde2d2a83eb88201fd4565149502f9c6718ba1d775cdd0
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c 4e64ba300a5a26e0f1170e09032429faeb65e45e8f3d1a7833e8edb69fc2979e
-F src/test1.c 40c9a40975512985762f87b83d0c63e4904833a9fe78cbcca664a37095301b1d
+F src/test1.c e2953a23ad637abe90f60679bb07a261934315451583d079d961ba686cc2c5c8
 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664
@@ -1257,6 +1257,7 @@ F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6
 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe
 F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5
 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d
+F test/json106.test fc906df241664e0b27b68c714a04239116c9e281f9ffaabaef0da23d05c7e45a
 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff
 F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b
 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63
@@ -2055,8 +2056,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7c572d02e60a83b36543ba4d9d45f61e9fc111b61fee085410c2d87558c732d6
-R f93830c4b318c6359968570a2d6be5a6
+P abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288
+R e4559648d28c24dd2cc0a0eb6174583b
+T *branch * lenient-json
+T *sym-lenient-json *
+T -sym-trunk *
 U drh
-Z 502cd80eb5e32a4170e530e0ee906c79
+Z daa8f079a41c86c9792c1e9898a8a53b
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-abb18f61c5cec0f524acc41453b4c06b61c5af51ff46417588837fc0c3967288
+186db57d263be09624cac84c97723d05813d70fe2e88d45e15b9f3b6d8c63935

src/json.c

@@ -916,6 +916,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
   }
 }
 
+/*
+** Return true if we are in lenient-JSON mode
+*/
+static int jsonLenientMode(sqlite3_context *pCtx){
+  sqlite3 *db = sqlite3_context_db_handle(pCtx);
+  return (db->flags & SQLITE_LenientJSON)!=0;
+}
+
 /*
 ** Parse a complete JSON string.  Return 0 on success or non-zero if there
 ** are any errors.  If an error occurs, free all memory associated with
@@ -926,7 +934,8 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
 static int jsonParse(
   JsonParse *pParse,           /* Initialize and fill this JsonParse object */
   sqlite3_context *pCtx,       /* Report errors here */
-  const char *zJson            /* Input JSON text to be parsed */
+  const char *zJson,           /* Input JSON text to be parsed */
+  int noErr                    /* If bad JSON, act as if input was "null" */
 ){
   int i;
   memset(pParse, 0, sizeof(*pParse));
@@ -943,11 +952,15 @@ static int jsonParse(
     if( pCtx!=0 ){
       if( pParse->oom ){
         sqlite3_result_error_nomem(pCtx);
-      }else{
+      }else if( !jsonLenientMode(pCtx) ){
         sqlite3_result_error(pCtx, "malformed JSON", -1);
+        noErr = 0;
       }
     }
     jsonParseReset(pParse);
+    if( noErr ){
+      return jsonParse(pParse, pCtx, "null", 0);
+    }
     return 1;
   }
   return 0;
@@ -1056,7 +1069,7 @@ static JsonParse *jsonParseCached(
   memset(p, 0, sizeof(*p));
   p->zJson = (char*)&p[1];
   memcpy((char*)p->zJson, zJson, nJson+1);
-  if( jsonParse(p, pErrCtx, p->zJson) ){
+  if( jsonParse(p, pErrCtx, p->zJson, 0) ){
     sqlite3_free(p);
     return 0;
   }
@@ -1374,7 +1387,7 @@ static void jsonParseFunc(
   u32 i;
 
   assert( argc==1 );
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 0) ) return;
   jsonParseFindParents(&x);
   jsonInit(&s, ctx);
   for(i=0; i<x.nNode; i++){
@@ -1483,7 +1496,10 @@ static void jsonArrayLengthFunc(
   JsonNode *pNode;
 
   p = jsonParseCached(ctx, argv, ctx);
-  if( p==0 ) return;
+  if( p==0 ){
+    if( jsonLenientMode(ctx) ) goto end_json_array_length;
+    return;
+  }
   assert( p->nNode );
   if( argc==2 ){
     const char *zPath = (const char*)sqlite3_value_text(argv[1]);
@@ -1492,6 +1508,7 @@ static void jsonArrayLengthFunc(
     pNode = p->aNode;
   }
   if( pNode==0 ){
+    if( jsonLenientMode(ctx) ) goto end_json_array_length;
     return;
   }
   if( pNode->eType==JSON_ARRAY ){
@@ -1500,6 +1517,7 @@ static void jsonArrayLengthFunc(
       i += jsonNodeSize(&pNode[i]);
     }
   }
+end_json_array_length:
   sqlite3_result_int64(ctx, n);
 }
 
@@ -1707,8 +1725,8 @@ static void jsonPatchFunc(
   JsonNode *pResult;   /* The result of the merge */
 
   UNUSED_PARAMETER(argc);
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
-  if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){
+  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return;
+  if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1]), 0) ){
     jsonParseReset(&x);
     return;
   }
@@ -1782,7 +1800,7 @@ static void jsonRemoveFunc(
   u32 i;
 
   if( argc<1 ) return;
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 0) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -1819,7 +1837,7 @@ static void jsonReplaceFunc(
     jsonWrongNumArgs(ctx, "replace");
     return;
   }
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -1873,7 +1891,7 @@ static void jsonSetFunc(
     jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert");
     return;
   }
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0]), 1) ) return;
   assert( x.nNode );
   for(i=1; i<(u32)argc; i+=2){
     zPath = (const char*)sqlite3_value_text(argv[i]);
@@ -2521,7 +2539,7 @@ static int jsonEachFilter(
   p->zJson = sqlite3_malloc64( n+1 );
   if( p->zJson==0 ) return SQLITE_NOMEM;
   memcpy(p->zJson, z, (size_t)n+1);
-  if( jsonParse(&p->sParse, 0, p->zJson) ){
+  if( jsonParse(&p->sParse, 0, p->zJson, 0) ){
     int rc = SQLITE_NOMEM;
     if( p->sParse.oom==0 ){
       sqlite3_free(cur->pVtab->zErrMsg);

src/main.c

@@ -954,6 +954,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_DQS_DML,               SQLITE_DqsDML         },
         { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT,    SQLITE_LegacyFileFmt  },
         { SQLITE_DBCONFIG_TRUSTED_SCHEMA,        SQLITE_TrustedSchema  },
+        { SQLITE_DBCONFIG_LENIENT_JSON,          SQLITE_LenientJSON    },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */

src/shell.c.in

@@ -7929,6 +7929,7 @@ static int do_meta_command(char *zLine, ShellState *p){
         { "fts3_tokenizer",     SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER },
         { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE    },
         { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    },
+        { "lenient_json",       SQLITE_DBCONFIG_LENIENT_JSON          },
         { "load_extension",     SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION },
         { "no_ckpt_on_close",   SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE      },
         { "reset_database",     SQLITE_DBCONFIG_RESET_DATABASE        },

src/sqlite.h.in

@@ -2424,6 +2424,19 @@ struct sqlite3_mem_methods {
 ** process a table with generated columns and a descending index.  This is
 ** not considered a bug since SQLite versions 3.3.0 and earlier do not support
 ** either generated columns or decending indexes.
+**
+** [[SQLITE_DBCONFIG_LENIENT_JSON]]
+** <dt>SQLITE_DBCONFIG_LENIENT_JSON</td>
+** <dd>The SQLITE_DBCONFIG_LENIENT_JSON option affects the operation of
+** the [JSON SQL functions].  When SQLITE_DBCONFIG_LENIENT_JSON is disabled
+** (the default) then the JSON functions will typically (depending on the
+** specific function) raise an error when given malformed JSON inputs.
+** When SQLITE_DBCONFIG_LENIENT_JSON is enabled, the JSON functions are more
+** forgiving of malformed inputs and will press on and try to return some kind
+** of sensible answer in spite of the malformed input. The lenient JSON mode
+** is useful when dealing with unvalidated inputs as it avoids having to
+** test every input using [json_valid()] prior to using it, thus greatly
+** simplifying complex queries.
 ** </dd>
 ** </dl>
 */
@@ -2445,7 +2458,8 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_VIEW           1015 /* int int* */
 #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    1016 /* int int* */
 #define SQLITE_DBCONFIG_TRUSTED_SCHEMA        1017 /* int int* */
-#define SQLITE_DBCONFIG_MAX                   1017 /* Largest DBCONFIG */
+#define SQLITE_DBCONFIG_LENIENT_JSON          1018 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1018 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes

src/sqliteInt.h

@@ -1707,8 +1707,7 @@ struct sqlite3 {
 #define SQLITE_ShortColNames  0x00000040  /* Show short columns names */
 #define SQLITE_TrustedSchema  0x00000080  /* Allow unsafe functions and
                                           ** vtabs in the schema definition */
-#define SQLITE_NullCallback   0x00000100  /* Invoke the callback once if the */
-                                          /*   result set is empty */
+#define SQLITE_LenientJSON    0x00000100  /* Be less strict about JSON inputs */
 #define SQLITE_IgnoreChecks   0x00000200  /* Do not enforce check constraints */
 #define SQLITE_ReadUncommit   0x00000400  /* READ UNCOMMITTED in shared-cache */
 #define SQLITE_NoCkptOnClose  0x00000800  /* No checkpoint on close()/DETACH */
@@ -1736,6 +1735,8 @@ struct sqlite3 {
                                           /*   DELETE, or UPDATE and return */
                                           /*   the count using a callback. */
 #define SQLITE_CorruptRdOnly  HI(0x00002) /* Prohibit writes due to error */
+#define SQLITE_NullCallback   HI(0x00004) /* Invoke the callback once if the */
+                                          /*   result set is empty */
 
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG

src/test1.c

@@ -8154,6 +8154,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config(
     { "DQS_DML",            SQLITE_DBCONFIG_DQS_DML },
     { "DQS_DDL",            SQLITE_DBCONFIG_DQS_DDL },
     { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT },
+    { "LENIENT_JSON",       SQLITE_DBCONFIG_LENIENT_JSON },
   };
   int i;
   int v = 0;

test/json106.test

@@ -0,0 +1,90 @@
+# 2022-11-18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for JSON SQL functions extension to the
+# SQLite library.  This file focuses on SQLITE_DBCONFIG_LENIENT_JSON.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+sqlite3_db_config db LENIENT_JSON 0
+do_catchsql_test json108-100 {
+  SELECT json_array_length(NULL);
+} {0 {{}}}
+do_catchsql_test json108-110 {
+  SELECT json_array_length('[1,2,3]');
+} {0 3}
+do_catchsql_test json108-120 {
+  SELECT json_array_length('not-json');
+} {1 {malformed JSON}}
+do_catchsql_test json108-130 {
+  SELECT json_array_length('null');
+} {0 0}
+
+sqlite3_db_config db LENIENT_JSON 1
+do_catchsql_test json108-150 {
+  SELECT json_array_length(NULL);
+} {0 0}
+do_catchsql_test json108-160 {
+  SELECT json_array_length('[1,2,3]');
+} {0 3}
+do_catchsql_test json108-170 {
+  SELECT json_array_length('not-json');
+} {0 0}
+do_catchsql_test json108-180 {
+  SELECT json_array_length('null');
+} {0 0}
+
+sqlite3_db_config db LENIENT_JSON 0
+do_catchsql_test json108-200 {
+ SELECT null ->> 'x';
+} {0 {{}}}
+do_catchsql_test json108-210 {
+ SELECT 'null' ->> 'x';
+} {0 {{}}}
+do_catchsql_test json108-220 {
+ SELECT 'nullx' ->> 'x';
+} {1 {malformed JSON}}
+do_catchsql_test json108-230 {
+ SELECT '{"x":5}' ->> 'x';
+} {0 5}
+
+sqlite3_db_config db LENIENT_JSON 1
+do_catchsql_test json108-250 {
+ SELECT null ->> 'x';
+} {0 {{}}}
+do_catchsql_test json108-260 {
+ SELECT 'null' ->> 'x';
+} {0 {{}}}
+do_catchsql_test json108-270 {
+ SELECT 'nullx' ->> 'x';
+} {0 {{}}}
+do_catchsql_test json108-280 {
+ SELECT '{"x":5}' ->> 'x';
+} {0 5}
+
+sqlite3_db_config db LENIENT_JSON 0
+do_catchsql_test json108-300 {
+ SELECT json_patch('nullx','{"x":5,"y":10}');
+} {1 {malformed JSON}}
+do_catchsql_test json108-310 {
+ SELECT json_patch('null','{"x":5,"y":10}');
+} {0 {{{"x":5,"y":10}}}}
+
+sqlite3_db_config db LENIENT_JSON 1
+do_catchsql_test json108-350 {
+ SELECT json_patch('nullx','{"x":5,"y":10}');
+} {0 {{{"x":5,"y":10}}}}
+do_catchsql_test json108-360 {
+ SELECT json_patch('null','{"x":5,"y":10}');
+} {0 {{{"x":5,"y":10}}}}
+
+finish_test