diff --git a/Makefile.msc b/Makefile.msc index 50f63619b2..ef78eae62f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -580,17 +580,17 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall # <> -TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall # <> -TEST_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl +TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> !ELSE CORE_CCONV_OPTS = diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 40d0e8113d..524aeb6f41 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -502,12 +502,12 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) # !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 !IF "$(PLATFORM)"=="x86" -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE !IFNDEF PLATFORM -CORE_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -SHELL_CCONV_OPTS = -Gz -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall +SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = diff --git a/doc/json-enhancements.md b/doc/json-enhancements.md new file mode 100644 index 0000000000..4c226a2c96 --- /dev/null +++ b/doc/json-enhancements.md @@ -0,0 +1,123 @@ +# Proposed Enhancements To JSON Functions + +## 1.0 New function json_nextract() + +The new function json_nextract() works the same as json_extract() except +when the input JSON is not well-formed. This is what the routines do +when the input JSON in the first argument is not well-formed: + + 1. **json_extract()** → raises an error and aborts the query. + + 2. **json_nextract()** with 2 arguments the second argument is + exactly `'$'` → work like json_quote() and return the first + argument as a JSON quoted string. + + 3. **json_nextract()** otherwise → return NULL. + +If the input is known to be JSON, then json_extract() should work just +fine for all your needs. But sometimes a table might have a column that +sometimes holds JSON and sometimes holds some other content. Suppose, +for example, an application started out holding a single phone number for +each user, but later was enhanced so that the same database file could +hold a JSON array of phone numbers. The USER table might have some entries +that are JSON arrays and some entries which are just text strings containing +phone numbers. The application can use json_nextract() to be robust in +extracting values from that column. + +The feature (2) above is envisioned to be useful for sanitizing table +content. Suppose a table is populated from dirty CSV, and some of the +JSON is mis-formatted. You could convert all entries in a table to use +well-formed JSON using something like this: + +> ~~~ +UPDATE data SET jsonData = json_nextract(jsonData,'$'); +~~~ + +In the query above, well-formed JSON would be unchanged, and mis-formatted +JSON would be converted into a well-formatted JSON string. + +## 2.0 Add the `->` and '->>` operators as aliases for json_extract(). + +Two new binary operators "`->`" and "`->>`" operators are the same +as json_nextract() and json_extract(), respectively. + +> ~~~ +SELECT '{"a":5,"b":17}' -> '$.a', '[4,1,-6]' ->> '$[0]'; +~~~ + +Is equivalent to (and generates the same bytecode as): + +> ~~~ +SELECT json_nextract('{"a":5,"b":17}','$.a'), json_extract('[4,1,-6]','$[0]'); +~~~ + +The ->> operator works the same as the ->> operator in MySQL +and mostly compatible with PostgreSQL (hereafter "PG"). Addition enhancements +in section 3.0 below are required to bring ->> into compatibility with PG. + +The -> operator is mostly compatible with MySQL and PG too. The main +difference is that in MySQL and PG, the result from -> is not a primitive +SQL datatype type but rather more JSON. It is unclear how this would ever +be useful for anything, and so I am unsure why they do this. But that is +the way it is done in those system. + +SQLite strives to be compatible with MySQL and PG with the ->> operator, +but not with the -> operator. + +## 3.0 Abbreviated JSON path specifications for use with -> and ->> + +The "->" and "->>" and operators allow abbreviated +forms of JSON path specs that omit unnecessary $-prefix text. For +example, the following queries are equivalent: + +> ~~~ +SELECT '{"a":17, "b":4.5}' ->> '$.a'; +SELECT '{"a":17, "b":4.5}' ->> 'a'; +~~~ + +Similarly, these queries mean exactly the same thing: + +> ~~~ +SELECT '[17,4.5,"hello",0]' ->> '$[1]'; +SELECT '[17,4.5,"hello",0]' ->> 1; +~~~ + +The abbreviated JSON path specs are allowed with the -> and ->> operators +only. The json_extract() and json_nextract() functions, and all the other +JSON functions, still use the full path spec and will raise an error if +the full path spec is not provided. + +This enhancement provides compatibility with PG. +PG does not support JSON path specs on its ->> operator. With PG, the +right-hand side of ->> must be either an integer (if the left-hand side +is a JSON array) or a text string which is interpreted as a field name +(if the left-hand side is a JSON object). So the ->> operator in PG is +rather limited. With this enhancement, the ->> operator in SQLite +covers all the functionality of PG, plus a lot more. + +MySQL also supports the ->> operator, but it requires a full JSON path +spec on the right-hand side. SQLite also supports this, so SQLite is +compatibility with MySQL as well. Note, however, that MySQL and PG +are incompatible with each other. You can (in theory) write SQL that +uses the ->> operator that is compatible between SQLite and MySQL, +or that is compatible between SQLite and PG, but not that is compatible +with all three. + +## 4.0 New json_ntype() SQL function + +A new function "json_ntype(JSON)" works like the existing one-argument +version of the "json_type(JSON)" function, except that json_ntype(JSON) +returns NULL if the argument is not well-formed JSON, whereas the +existing json_type() function raises an error in that case. + +In other words, "`json_ntype($json)`" is equivalent to +"`CASE WHEN json_valid($json) THEN json_type($json) END`". + +This function is seen as useful for figuring out which rows of a table +have a JSON type in a column and which do not. For example, to find +all rows in a table in which the value of the the "phonenumber" column +contains a JSON array, you could write: + +> ~~~ +SELECT * FROM users WHERE json_ntype(phonenumber) IS 'array'; +~~~ diff --git a/manifest b/manifest index c39afa35cd..b3ac61a7f5 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C An\sattempt\sto\sintegrate\sthe\sJSON\sfunctions\sdirectly\sinto\sthe\sSQLite\score,\nrather\sthan\sholding\sthem\sas\san\sextension. -D 2022-01-06T01:40:09.292 +C Merge\sthe\sJSON\sfunction\senhancements\sfrom\sthe\sjson-enhancements\sbranch\sinto\njson-in-core. +D 2022-01-08T15:37:13.201 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in fd537743957bfe87997dc5727783d8eec82098921e15eab984d6711cd46f001b F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc 8ba680ec26e6d6f348954e0fee719607688075ef6c242e50e20d28e8ded7cc08 +F Makefile.msc 22ce0007874c61c8eb51fc22b84f72af175ce2d7431c242253bdffa39c163da4 F README.md 2dd87a5c1d108b224921f3dd47dea567973f706e1f6959386282a626f459a70c F VERSION 392c2f83569705069415a5d98b1c138ec8fe8a56a663a0d94cea019e806537b2 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am a8d1d24affe52ebf8d7ddcf91aa973fa0316618ab95bb68c87cabf8faf527dc8 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc d146a08ebbdf7f881ba600a49cd8dce40c4c807addcdb4b9b6a507e4b40ce837 +F autoconf/Makefile.msc 262dad10223b39a6853c4faefadb0f71931ca55c742fa285f8ee8d56b7543ddc F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac a8ba2a9e61216f5093d44f3b7d2cb8fe1890d6b7dc330a02f802d8efaa1fdc79 @@ -38,6 +38,7 @@ F configure a2877fe63cc821af0df41abe70f1f7c4e97cb7e23a42e0a1402e8a2f55a88aa2 x F configure.ac 3ef6eeff4387585bfcab76b0c3f6e15a0618587bb90245dd5d44e4378141bb35 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd +F doc/json-enhancements.md b026346e18a18a90d84dbda457e3127282468ad26baaefc005a1656429fa4232 F doc/lemon.html efc0cd2345d66905505d98f862e1c571512def0ceb5b016cb658fd4918eb76a3 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a @@ -494,7 +495,7 @@ F src/btree.c 0e9f84f974e970fb373c15caa1624a281b3c33098cb9dd9021d6801c04eb4fde F src/btree.h 74d64b8f28cfa4a894d14d4ed64fa432cd697b98b61708d4351482ae15913e22 F src/btreeInt.h ee9348c4cb9077243b049edc93a82c1f32ca48baeabf2140d41362b9f9139ff7 F src/build.c 6e16f7b539bfc55149a039bf0cda26b089640339df6147070b072df2d1c4f771 -F src/callback.c 106b585da1edd57d75fa579d823a5218e0bf37f191dbf7417eeb4a8a9a267dbc +F src/callback.c 4c19af69835787bfe790ac560f3071a824eb629f34e41f97b52ce5235c77de1c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c b09ce320b78718d5e0c4a7a59b8705abe8ee4683d9fa4b33768fe347e1b2a42a F src/date.c ab8e01d928f201f5dee0bc6d54d6702fdcec96dff4d58c387447671f6a46d191 @@ -511,7 +512,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h cb1d7e3e1ed94b7aa6fde95ae2c2daccc3df826be26fc9ed7fd90d1750ae6144 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c e528416ff5d86fc5d656ea6a26f03fde39836b6175f93048c32a03cb2ee16743 -F src/json.c 973b36a733cc5d3c8347bbdb3479c1124c0916ee7416341d0126895d055d2447 w ext/misc/json1.c +F src/json.c 726f1901ecc66bc359d785c268e70c2f10fcec22b212613b00d7da5aa561d462 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 95db1fe62c5973f1c5d9c53f6083e21a73ece14cdd47eeca0639691332e85c4d F src/main.c 2b6b0dbfeb14d4bb57e368604b0736b2aa42b51b00339d399b01d6b1fc9b4960 @@ -539,7 +540,7 @@ F src/os_win.c 77d39873836f1831a9b0b91894fec45ab0e9ca8e067dc8c549e1d1eca1566fe9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 55a9a8c745c47a49e6541b1d634f6152e3f72f76c6dbb04fe24413986a928091 F src/pager.h 4bf9b3213a4b2bebbced5eaa8b219cf25d4a82f385d093cd64b7e93e5285f66f -F src/parse.y 761b5d30a7ea9bd2db3b3571438cfcceb5f7dbf4fcad6881c8de65bdda07135a +F src/parse.y 04f61db1cdd7036c6d74baad1c342d3e3110cb0765c48fcfd3bdf4e974a1e5bb F src/pcache.c 084e638432c610f95aea72b8509f0845d2791293f39d1b82f0c0a7e089c3bb6b F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 54881292a9a5db202b2c0ac541c5e3ef9a5e8c4f1c1383adb2601d5499a60e65 @@ -614,7 +615,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c dc5367c6d42336b35d55f96d2f04cd6256e92bc6ecf74ed5d855d24e43343aff +F src/tokenize.c b74d878aa7c82ec8460779468061a96185e22257f68ab785b69abce354b70446 F src/treeview.c 9dfdb7ff7f6645d0a6458dbdf4ffac041c071c4533a6db8bb6e502b979ac67bc F src/trigger.c 40e7c3dcff57a770d5fa38ba21ed4725572fd2e224c58af61eb980598b60f9c8 F src/update.c d6f5c7b9e072660757ac7d58175aca11c07cb95ebbb297ae7f38853700f52328 @@ -1045,7 +1046,7 @@ F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f75 F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9 F test/fts4upfrom.test f25835162c989dffd5e2ef91ec24c4848cc9973093e2d492d1c7b32afac1b49d F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test 3a65ddb6c1998f71aa86492501a6be87904197e62bfb5b70b2493552b558abd1 +F test/func.test 0a305e88a4fbdce6be8c6f4863d6dab9ecb2975b95e94734621e2c900779b712 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 600a632c305a88f3946d38f9a51efe145c989b2e13bd2b2a488db47fe76bab6a F test/func4.test 2285fb5792d593fef442358763f0fd9de806eda47dbc7a5934df57ffdc484c31 @@ -1147,7 +1148,7 @@ F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test bb71538005f2d9e18620bdd3b76839a93ca0be61903eb8d751a64e78cf99b8fb -F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 +F test/json102.test a44a20b7dfa446e67f47d0c02927fe93f80d6ebf3080a5d827757e8d4921c081 F test/json103.test aff6b7a4c17d5a20b487a7bc1a274bfdc63b829413bdfb83bedac42ec7f67e3b F test/json104.test 2cb7ff2cca2c8214d3e5260eeb9ce45faec0926f68b3e40c1aaa6ca247284144 F test/json105.test 45f7d6a9a54c85f8a9589b68d3e7a1f42d02f2359911a8cdbad1f9988f571173 @@ -1937,11 +1938,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b6a82f3c3b9d89fdf628c7f117b6a4a64383a36c84fe84d47c80e845c9bd8a4f -R 512e6db3c5e4ab4d0c8ece9f791ebefa -T *branch * json-in-core -T *sym-json-in-core * -T -sym-trunk * +P 583b47d865fb8d2c9ae4d3a4e70356a8a758978efb0a282f6b19775bf41fb748 4d81425e1bf2cff6fa961d0a7936b5f62d3f8ffe9bffea89c1e8b8ddf8fad6f4 +R 9de2cc0789b4af5e569fa6f6937afbf5 U drh -Z a41c104f7d21d7e5237fb8ae1a000e0b +Z 5d4090391a2a7b92e27e5b55336ae1bf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8d5afef0f4..3d3e816e87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -583b47d865fb8d2c9ae4d3a4e70356a8a758978efb0a282f6b19775bf41fb748 \ No newline at end of file +e116501c2f0e594eb7a3dd804daa943cc508f32ded3078aed21b695ec83bcd4c \ No newline at end of file diff --git a/src/callback.c b/src/callback.c index 936c374abd..7d8f9dcbce 100644 --- a/src/callback.c +++ b/src/callback.c @@ -358,7 +358,6 @@ void sqlite3InsertBuiltinFuncs( const char *zName = aDef[i].zName; int nName = sqlite3Strlen30(zName); int h = SQLITE_FUNC_HASH(zName[0], nName); - assert( zName[0]>='a' && zName[0]<='z' ); assert( aDef[i].funcFlags & SQLITE_FUNC_BUILTIN ); pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ diff --git a/src/json.c b/src/json.c index 5f39902c0e..aad4507767 100644 --- a/src/json.c +++ b/src/json.c @@ -1502,13 +1502,44 @@ static void jsonArrayLengthFunc( sqlite3_result_int64(ctx, n); } +/* +** Bit values for the flags passed into jsonExtractFunc() or +** jsonSetFunc() via the user-data value. +*/ +#define JSON_NULLERR 0x01 /* Return NULL if input is not JSON */ +#define JSON_ABPATH 0x02 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ + /* ** json_extract(JSON, PATH, ...) +** json_nextract(JSON, PATH, ...) +** "->"(JSON,PATH) +** "->>"(JSON,PATH) ** -** Return the element described by PATH. Return NULL if there is no -** PATH element. If there are multiple PATHs, then return a JSON array -** with the result from each path. Throw an error if the JSON or any PATH -** is malformed. +** Return the element described by PATH. Return NULL if that PATH element +** is not found. For leaf nodes of the JSON, the value returned is a pure +** SQL value. In other words, quotes have been removed from strings. +** +** If there are multiple PATHs, then the value returned is a JSON array +** with one entry in the array for each PATH term. +** +** Throw an error if any PATH is malformed. +** +** If JSON is not well-formed JSON then: +** +** (1) raise an error if the JSON_NULLERR flag is not set. +** +** (2) Otherwise (if the JSON_NULLERR flags is set and) if there +** is a single PATH argument with the value '$', simply quote +** the JSON input as if by json_quote(). In other words, treat +** the JSON input as a string and convert it into a valid JSON +** string. +** +** (3) Otherwise (if JSON_NULLERR is set and the PATH is not '$') +** return NULL +** +** If the JSON_ABPATH flag is set and there is only a single PATH, then +** allow abbreviated PATH specs that omit the leading "$". */ static void jsonExtractFunc( sqlite3_context *ctx, @@ -1518,35 +1549,75 @@ static void jsonExtractFunc( JsonParse *p; /* The parse */ JsonNode *pNode; const char *zPath; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); JsonString jx; - int i; if( argc<2 ) return; - p = jsonParseCached(ctx, argv, ctx); - if( p==0 ) return; - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '['); - for(i=1; inErr ) break; - if( argc>2 ){ + p = jsonParseCached(ctx, argv, (flags & JSON_NULLERR)!=0 ? 0 : ctx); + if( p==0 ){ + /* If the form is "json_nextract(IN,'$')" and IN is not well-formed JSON, + ** then return IN as a quoted JSON string. */ + if( (flags & JSON_NULLERR)!=0 + && argc==2 + && (zPath = (const char*)sqlite3_value_text(argv[1]))!=0 + && zPath[0]=='$' && zPath[1]==0 + ){ + jsonQuoteFunc(ctx, argc, argv); + } + return; + } + if( argc==2 ){ + /* With a single PATH argument, the return is the unquoted SQL value */ + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath && zPath[0]!='$' && zPath[0]!=0 && (flags & JSON_ABPATH)!=0 ){ + /* The -> and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + */ + jsonInit(&jx, ctx); + if( sqlite3Isdigit(zPath[0]) ){ + jsonAppendRaw(&jx, "$[", 2); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendRaw(&jx, "]", 2); + }else{ + jsonAppendRaw(&jx, "$.", 1 + (zPath[0]!='[')); + jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); + jsonAppendChar(&jx, 0); + } + pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); + jsonReset(&jx); + }else{ + pNode = jsonLookup(p, zPath, 0, ctx); + } + if( p->nErr ) return; + if( pNode ) jsonReturn(pNode, ctx, 0); + }else{ + /* Two or more PATH arguments results in a JSON array with each + ** element of the array being the value selected by one of the PATHs */ + int i; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; inErr ) break; jsonAppendSeparator(&jx); if( pNode ){ jsonRenderNode(pNode, &jx, 0); }else{ jsonAppendRaw(&jx, "null", 4); } - }else if( pNode ){ - jsonReturn(pNode, ctx, 0); } + if( i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); } - if( argc>2 && i==argc ){ - jsonAppendChar(&jx, ']'); - jsonResult(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); - } - jsonReset(&jx); } /* This is the RFC 7396 MergePatch algorithm. @@ -1779,6 +1850,7 @@ replace_err: jsonParseReset(&x); } + /* ** json_set(JSON, PATH, VALUE, ...) ** @@ -1839,10 +1911,13 @@ jsonSetDone: /* ** json_type(JSON) +** json_ntype(JSON) ** json_type(JSON, PATH) ** -** Return the top-level "type" of a JSON string. Throw an error if -** either the JSON or PATH inputs are not well-formed. +** Return the top-level "type" of a JSON string. json_type() raises an +** error if either the JSON or PATH inputs are not well-formed. json_ntype() +** works like the one-argument version of json_type() except that it +** returns NULL if the JSON argument is not well-formed. */ static void jsonTypeFunc( sqlite3_context *ctx, @@ -1853,7 +1928,7 @@ static void jsonTypeFunc( const char *zPath; JsonNode *pNode; - p = jsonParseCached(ctx, argv, ctx); + p = jsonParseCached(ctx, argv, sqlite3_user_data(ctx)!=0 ? 0 : ctx); if( p==0 ) return; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); @@ -2557,29 +2632,33 @@ static sqlite3_module jsonTreeModule = { void sqlite3RegisterJsonFunctions(void){ #ifndef SQLITE_OMIT_JSON static FuncDef aJsonFunc[] = { - JFUNCTION(json, 1, 0, jsonRemoveFunc), - JFUNCTION(json_array, -1, 0, jsonArrayFunc), - JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), - JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_extract, -1, 0, jsonExtractFunc), - JFUNCTION(json_insert, -1, 0, jsonSetFunc), - JFUNCTION(json_object, -1, 0, jsonObjectFunc), - JFUNCTION(json_patch, 2, 0, jsonPatchFunc), - JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), - JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), - JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), - JFUNCTION(json_set, -1, 1, jsonSetFunc), - JFUNCTION(json_type, 1, 0, jsonTypeFunc), - JFUNCTION(json_type, 2, 0, jsonTypeFunc), - JFUNCTION(json_valid, 1, 0, jsonValidFunc), + JFUNCTION(json, 1, 0, jsonRemoveFunc), + JFUNCTION(json_array, -1, 0, jsonArrayFunc), + JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_extract, -1, 0, jsonExtractFunc), + JFUNCTION(json_nextract, -1, JSON_NULLERR, jsonExtractFunc), + JFUNCTION(->, 2, JSON_NULLERR|JSON_ABPATH, jsonExtractFunc), + JFUNCTION(->>, 2, JSON_ABPATH, jsonExtractFunc), + JFUNCTION(json_insert, -1, 0, jsonSetFunc), + JFUNCTION(json_ntype, 1, JSON_NULLERR, jsonTypeFunc), + JFUNCTION(json_object, -1, 0, jsonObjectFunc), + JFUNCTION(json_patch, 2, 0, jsonPatchFunc), + JFUNCTION(json_quote, 1, 0, jsonQuoteFunc), + JFUNCTION(json_remove, -1, 0, jsonRemoveFunc), + JFUNCTION(json_replace, -1, 0, jsonReplaceFunc), + JFUNCTION(json_set, -1, JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1, 0, jsonTypeFunc), + JFUNCTION(json_type, 2, 0, jsonTypeFunc), + JFUNCTION(json_valid, 1, 0, jsonValidFunc), #if SQLITE_DEBUG - JFUNCTION(json_parse, 1, 0, jsonParseFunc), - JFUNCTION(json_test1, 1, 0, jsonTest1Func), + JFUNCTION(json_parse, 1, 0, jsonParseFunc), + JFUNCTION(json_test1, 1, 0, jsonTest1Func), #endif - WAGGREGATE(json_group_array, 1, 0, 0, + WAGGREGATE(json_group_array, 1, 0, 0, jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS), - WAGGREGATE(json_group_object, 2, 0, 0, + WAGGREGATE(json_group_object, 2, 0, 0, jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, SQLITE_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) }; @@ -2594,8 +2673,8 @@ void sqlite3RegisterJsonFunctions(void){ int sqlite3JsonTableFunctions(sqlite3 *db){ int rc = SQLITE_OK; static const struct { - const char *zName; - sqlite3_module *pModule; + const char *zName; + sqlite3_module *pModule; } aMod[] = { { "json_each", &jsonEachModule }, { "json_tree", &jsonTreeModule }, diff --git a/src/parse.y b/src/parse.y index 559057f9ae..5b83b16b60 100644 --- a/src/parse.y +++ b/src/parse.y @@ -236,7 +236,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // %token ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST. %token CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL. -%token OR AND NOT MATCH LIKE_KW BETWEEN IS IN ISNULL NOTNULL NE EQ. +%token OR AND NOT IS MATCH LIKE_KW BETWEEN IN ISNULL NOTNULL NE EQ. %token GT LE LT GE ESCAPE. // The following directive causes tokens ABORT, AFTER, ASC, etc. to @@ -286,7 +286,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} %left BITAND BITOR LSHIFT RSHIFT. %left PLUS MINUS. %left STAR SLASH REM. -%left CONCAT. +%left CONCAT PTR. %left COLLATE. %right BITNOT. %nonassoc ON. @@ -1235,6 +1235,12 @@ expr(A) ::= PLUS|MINUS(B) expr(X). [BITNOT] { /*A-overwrites-B*/ } +expr(A) ::= expr(B) PTR(C) expr(D). { + ExprList *pList = sqlite3ExprListAppend(pParse, 0, B); + pList = sqlite3ExprListAppend(pParse, pList, D); + A = sqlite3ExprFunction(pParse, pList, &C, 0); +} + %type between_op {int} between_op(A) ::= BETWEEN. {A = 0;} between_op(A) ::= NOT BETWEEN. {A = 1;} diff --git a/src/tokenize.c b/src/tokenize.c index 5f41e5665e..347fd57323 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -290,6 +290,9 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; + }else if( z[1]=='>' ){ + *tokenType = TK_PTR; + return 2 + (z[2]=='>'); } *tokenType = TK_MINUS; return 1; diff --git a/test/func.test b/test/func.test index ca1027f508..008819baa5 100644 --- a/test/func.test +++ b/test/func.test @@ -1507,4 +1507,19 @@ do_execsql_test func-35.200 { PRAGMA integrity_check; } {ok} +# 2021-01-07: The -> and ->> operators. +# +proc ptr1 {a b} { return "$a->$b" } +db func -> ptr1 +proc ptr2 {a b} { return "$a->>$b" } +db func ->> ptr2 +do_execsql_test func-36.100 { + SELECT 123 -> 456 +} {123->456} +do_execsql_test func-36.110 { + SELECT 123 ->> 456 +} {123->>456} + + + finish_test diff --git a/test/json102.test b/test/json102.test index 18a6bbf5cc..7ae99353bf 100644 --- a/test/json102.test +++ b/test/json102.test @@ -71,21 +71,70 @@ do_execsql_test json102-240 { do_execsql_test json102-250 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); } {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-251 { + SELECT json_nextract('{"a":2,"c":[4,5,{"f":7}]}', '$'); +} {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-252 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> '$'; +} {{{"a":2,"c":[4,5,{"f":7}]}}} do_execsql_test json102-260 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); } {{[4,5,{"f":7}]}} +do_execsql_test json102-261 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> '$.c'; +} {{[4,5,{"f":7}]}} +do_execsql_test json102-262 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> 'c'; +} {{[4,5,{"f":7}]}} +do_catchsql_test json102-265 { + SELECT json_extract('[1,2,3', '$[2]'); +} {1 {malformed JSON}} +do_catchsql_test json102-266 { + SELECT json_nextract('[1,2,3', '$[2]'); +} {0 {{}}} +do_catchsql_test json102-267 { + SELECT json_extract('[1,2,3', '$'); +} {1 {malformed JSON}} +do_catchsql_test json102-268 { + SELECT json_nextract('[1,2,3', '$'); +} {0 {{"[1,2,3"}}} +do_catchsql_test json102-269a { + SELECT '[1,2,3' ->> '$'; +} {1 {malformed JSON}} +do_catchsql_test json102-269b { + SELECT '[1,2,3' -> '$'; +} {0 {{"[1,2,3"}}} + do_execsql_test json102-270 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); } {{{"f":7}}} +do_execsql_test json102-271 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> '$.c[2]'; +} {{{"f":7}}} +do_execsql_test json102-272 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> 'c' -> 2; +} {{{"f":7}}} do_execsql_test json102-280 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); } {{7}} +do_execsql_test json102-281 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> 'c' -> 2 -> 'f'; +} {{7}} +do_execsql_test json102-282 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> 'c' -> '[2]' -> 'f'; +} {{7}} do_execsql_test json102-290 { SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); } {{[[4,5],2]}} do_execsql_test json102-300 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); } {{}} +do_execsql_test json102-301 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> 'x'; +} {{}} +do_execsql_test json102-302 { + SELECT '{"a":2,"c":[4,5,{"f":7}]}' -> NULL; +} {{}} do_execsql_test json102-310 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a'); } {{[null,2]}} @@ -149,6 +198,9 @@ do_execsql_test json102-500 { do_execsql_test json102-510 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}'); } {{object}} +do_execsql_test json102-511 { + SELECT json_ntype('{"a":[2,3.5,true,false,null,"x"]}'); +} {{object}} do_execsql_test json102-520 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$'); } {{object}} @@ -182,6 +234,12 @@ do_execsql_test json102-610 { do_execsql_test json102-620 { SELECT json_valid(char(123)||'"x":35'); } {{0}} +do_catchsql_test json102-630 { + SELECT json_type('["a",'); +} {1 {malformed JSON}} +do_catchsql_test json102-631 { + SELECT json_ntype('["a",'); +} {0 {{}}} ifcapable vtab { do_execsql_test json102-1000 {