From 5fa5c10784d6117df1706eb449dcd92b8396939b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Aug 2015 16:49:40 +0000 Subject: [PATCH 01/17] Begin adding an extension that provides JSON SQL functions. FossilOrigin-Name: dde8afdd8dba1d92560326dca7c1cdfedbe5e070 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/json.c | 225 ++++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 25 +++--- manifest.uuid | 2 +- src/test1.c | 2 + test/json1.test | 40 +++++++++ 8 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 ext/misc/json.c create mode 100644 test/json1.test diff --git a/Makefile.in b/Makefile.in index a0f536cb23..4c83d9dfa3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -417,6 +417,7 @@ TESTSRC += \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/json.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/Makefile.msc b/Makefile.msc index 22d3fb523e..52769889aa 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1083,6 +1083,7 @@ TESTEXT = \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\json.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ diff --git a/ext/misc/json.c b/ext/misc/json.c new file mode 100644 index 0000000000..53442a9c6a --- /dev/null +++ b/ext/misc/json.c @@ -0,0 +1,225 @@ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** JSON is pure text. JSONB is a binary encoding that is smaller and easier +** to parse but which holds the equivalent information. Conversions between +** JSON and JSONB are lossless. +** +** Most of the functions here will accept either JSON or JSONB input. The +** input is understood to be JSONB if it a BLOB and JSON if the input is +** of any other type. Functions that begin with the "json_" prefix return +** JSON and functions that begin with "jsonb_" return JSONB. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Unsigned integer types */ +typedef sqlite3_uint64 u64; +typedef unsigned int u32; +typedef unsigned char u8; + +/* An instance of this object represents a JSON string under construction. +*/ +typedef struct Json Json; +struct Json { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON text here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 mallocFailed; /* True if an OOM has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* Set the Json object to an empty string +*/ +static void jsonZero(Json *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the Json object +*/ +static void jsonInit(Json *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->mallocFailed = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the Json object back to its +** initial state. +*/ +static void jsonReset(Json *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(Json *p){ + p->mallocFailed = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(Json *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; + char *zNew; + if( p->bStatic ){ + if( p->mallocFailed ) return SQLITE_NOMEM; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the Json string. +*/ +static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append the N-byte string in zIn to the end of the Json string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(Json *p, const char *zIn, u32 N){ + u32 i; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+1-i > p->nAlloc) && jsonGrow(p,N+1-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; +} + +/* Make pJson the result of the SQL function. +*/ +static void jsonResult(Json *p){ + if( p->mallocFailed==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '['; + + jsonInit(&jx, context); + for(i=0; i Date: Wed, 12 Aug 2015 17:23:34 +0000 Subject: [PATCH 02/17] Add the json_object() function. FossilOrigin-Name: 414a95f3b79359f167858b2325fd2be223569c66 --- ext/misc/json.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++-- manifest | 17 +++++-------- manifest.uuid | 2 +- test/json1.test | 14 ++++++++++ 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 53442a9c6a..8acbb9bf5f 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -194,7 +194,70 @@ static void jsonArrayFunc( } } jsonAppendRaw(&jx, "]", 1); - jsonResult(&jx); + jsonResult(&jx); +} + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '{'; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(context, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, context); + for(i=0; i Date: Thu, 13 Aug 2015 13:54:59 +0000 Subject: [PATCH 03/17] Experimental code (untested) for a JSONB datatype. FossilOrigin-Name: e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 --- ext/misc/json.c | 269 +++++++++++++++++++++++++++++++++++++++++++++++- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 273 insertions(+), 10 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 8acbb9bf5f..4f3e1c02bb 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -23,6 +23,55 @@ ** input is understood to be JSONB if it a BLOB and JSON if the input is ** of any other type. Functions that begin with the "json_" prefix return ** JSON and functions that begin with "jsonb_" return JSONB. +** +** JSONB format: +** +** A JSONB blob is a sequence of terms. Each term begins with a single +** variable length integer X which determines the type and size of the term. +** +** type = X%8 +** size = X>>3 +** +** Term types are 0 through 7 for null, true, false, integer, real, string, +** array, and object. The meaning of size depends on the type. +** +** For null, true, and false terms, the size is always 0. +** +** For integer terms, the size is the number of bytes that contains the +** integer value. The value is stored as big-endian twos-complement. +** +** For real terms, the size is always 8 and the value is a big-ending +** double-precision floating-point number. +** +** For string terms, the size is the number of bytes in the string. The +** string itself immediately follows the X integer. There are no escapes +** and the string is not zero-terminated. The string is always stored as +** UTF8. +** +** For array terms, the size is the number of bytes in content. The +** content consists of zero or more additional terms that are the elements +** of the array. +** +** For object terms, the size is the number of bytes of content. The +** content is zero or more pairs of terms. The first element of each +** pair is a string term which is the label and the second element is +** the value. +** +** Variable Length Integers: +** +** The variable length integer encoding is the 64-bit unsigned integer encoding +** originally developed for SQLite4. The encoding for each integer is between +** 1 and 9 bytes. Call those bytes A0 through A8. The encoding is as follows: +** +** If A0 is between 0 and 240 inclusive, then the value is A0. +** +** If A0 is between 241 and 248 inclusive, then the value is +** 240+256*(A0-241)+A1. +** +** If A0 is 249 then the value is 2288+256*A1+A2. +** +** If A0 is 250 or more then the value is a (A0-247)-byte big-endian +** integer taken from starting at A1. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -34,12 +83,13 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; -/* An instance of this object represents a JSON string under construction. +/* An instance of this object represents a JSON string or +** JSONB blob under construction. */ typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON text here */ + char *zBuf; /* Append JSON or JSONB content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ @@ -47,6 +97,73 @@ struct Json { char zSpace[100]; /* Initial static space */ }; +/* JSONB type values +*/ +#define JSONB_NULL 0 +#define JSONB_TRUE 1 +#define JSONB_FALSE 2 +#define JSONB_INT 3 +#define JSONB_REAL 4 +#define JSONB_STRING 5 +#define JSONB_ARRAY 6 +#define JSONB_OBJECT 7 + +#if 0 +/* +** Decode the varint in the first n bytes z[]. Write the integer value +** into *pResult and return the number of bytes in the varint. +** +** If the decode fails because there are not enough bytes in z[] then +** return 0; +*/ +static int jsonGetVarint64( + const unsigned char *z, + int n, + u64 *pResult +){ + unsigned int x; + if( n<1 ) return 0; + if( z[0]<=240 ){ + *pResult = z[0]; + return 1; + } + if( z[0]<=248 ){ + if( n<2 ) return 0; + *pResult = (z[0]-241)*256 + z[1] + 240; + return 2; + } + if( nzBuf[p->nUsed++] = '"'; } -/* Make pJson the result of the SQL function. +/* +** Write a 32-bit unsigned integer as 4 big-endian bytes. +*/ +static void jsonPutInt32(unsigned char *z, unsigned int y){ + z[0] = (unsigned char)(y>>24); + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); +} + + +/* Write integer X as a variable-length integer into the buffer z[]. +** z[] is guaranteed to be at least 9 bytes in length. Return the +** number of bytes written. +*/ +int jsonPutVarint64(char *zIn, u64 x){ + unsigned char *z = (unsigned char*)zIn; + unsigned int w, y; + if( x<=240 ){ + z[0] = (unsigned char)x; + return 1; + } + if( x<=2287 ){ + y = (unsigned int)(x - 240); + z[0] = (unsigned char)(y/256 + 241); + z[1] = (unsigned char)(y%256); + return 2; + } + if( x<=67823 ){ + y = (unsigned int)(x - 2288); + z[0] = 249; + z[1] = (unsigned char)(y/256); + z[2] = (unsigned char)(y%256); + return 3; + } + y = (unsigned int)x; + w = (unsigned int)(x>>32); + if( w==0 ){ + if( y<=16777215 ){ + z[0] = 250; + z[1] = (unsigned char)(y>>16); + z[2] = (unsigned char)(y>>8); + z[3] = (unsigned char)(y); + return 4; + } + z[0] = 251; + jsonPutInt32(z+1, y); + return 5; + } + if( w<=255 ){ + z[0] = 252; + z[1] = (unsigned char)w; + jsonPutInt32(z+2, y); + return 6; + } + if( w<=65535 ){ + z[0] = 253; + z[1] = (unsigned char)(w>>8); + z[2] = (unsigned char)w; + jsonPutInt32(z+3, y); + return 7; + } + if( w<=16777215 ){ + z[0] = 254; + z[1] = (unsigned char)(w>>16); + z[2] = (unsigned char)(w>>8); + z[3] = (unsigned char)w; + jsonPutInt32(z+4, y); + return 8; + } + z[0] = 255; + jsonPutInt32(z+1, w); + jsonPutInt32(z+5, y); + return 9; +} + + +/* Append integer X as a variable-length integer on the JSONB currently +** under construction in p. +*/ +static void jsonAppendVarint(Json *p, u64 X){ + if( (p->nUsed+9 > p->nAlloc) && jsonGrow(p,9)!=0 ) return; + p->nUsed += jsonPutVarint64(p->zBuf+p->nUsed, X); +} + +/* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ if( p->mallocFailed==0 ){ @@ -150,6 +352,17 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } +/* Make the JSONB in p the result of the SQL function. +*/ +static void jsonbResult(Json *p){ + if( p->mallocFailed==0 ){ + sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + jsonZero(p); + } + assert( p->bStatic ); +} + /* ** Implementation of the json_array(VALUE,...) function. Return a JSON ** array that contains all values given in arguments. Or if any argument @@ -197,6 +410,55 @@ static void jsonArrayFunc( jsonResult(&jx); } +/* +** Implementation of the jsonb_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonbArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + + jsonInit(&jx, context); + jx.nUsed = 5; + for(i=0; i Date: Sat, 15 Aug 2015 21:25:36 +0000 Subject: [PATCH 04/17] Add a prototype JSON parser. FossilOrigin-Name: 789ba487000aa73621a41d115ad5de455ea8ea31 --- ext/misc/json.c | 302 +++++++++++++++++++++++++++++++++++++++++++++--- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 291 insertions(+), 25 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 4f3e1c02bb..2fce79610c 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -77,6 +77,7 @@ SQLITE_EXTENSION_INIT1 #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -93,20 +94,41 @@ struct Json { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 mallocFailed; /* True if an OOM has been encountered */ + u8 oom; /* True if an OOM has been encountered */ char zSpace[100]; /* Initial static space */ }; -/* JSONB type values +/* JSON type values */ -#define JSONB_NULL 0 -#define JSONB_TRUE 1 -#define JSONB_FALSE 2 -#define JSONB_INT 3 -#define JSONB_REAL 4 -#define JSONB_STRING 5 -#define JSONB_ARRAY 6 -#define JSONB_OBJECT 7 +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* A single node of parsed JSON +*/ +typedef struct JsonNode JsonNode; +struct JsonNode { + u32 eType; /* One of the JSON_ type values */ + u32 n; /* Bytes of content, or number of sub-nodes */ + const char *zContent; /* Content for JSON_INT, JSON_REAL, or JSON_STRING */ +}; + +/* A completely parsed JSON string +*/ +typedef struct JsonParse JsonParse; +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u8 oom; /* Set to true if out of memory */ +}; + #if 0 /* @@ -177,7 +199,7 @@ static void jsonZero(Json *p){ */ static void jsonInit(Json *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->mallocFailed = 0; + p->oom = 0; jsonZero(p); } @@ -194,7 +216,7 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ static void jsonOom(Json *p){ - p->mallocFailed = 1; + p->oom = 1; sqlite3_result_error_nomem(p->pCtx); jsonReset(p); } @@ -206,7 +228,7 @@ static int jsonGrow(Json *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; char *zNew; if( p->bStatic ){ - if( p->mallocFailed ) return SQLITE_NOMEM; + if( p->oom ) return SQLITE_NOMEM; zNew = sqlite3_malloc64(nTotal); if( zNew==0 ){ jsonOom(p); @@ -235,6 +257,12 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ p->nUsed += N; } +/* Append the zero-terminated string zIn +*/ +static void jsonAppend(Json *p, const char *zIn){ + jsonAppendRaw(p, zIn, (u32)strlen(zIn)); +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -343,7 +371,7 @@ static void jsonAppendVarint(Json *p, u64 X){ /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ - if( p->mallocFailed==0 ){ + if( p->oom==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); @@ -355,7 +383,7 @@ static void jsonResult(Json *p){ /* Make the JSONB in p the result of the SQL function. */ static void jsonbResult(Json *p){ - if( p->mallocFailed==0 ){ + if( p->oom==0 ){ sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); jsonZero(p); @@ -428,7 +456,7 @@ static void jsonbArrayFunc( for(i=0; iaNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + u32 nNew; + JsonNode *pNew; + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + if( nNew<=pParse->nNode ){ + pParse->oom = 1; + return -1; + } + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + } + p = &pParse->aNode[pParse->nNode]; + p->eType = eType; + p->n = n; + p->zContent = zContent; + return pParse->nNode++; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + u32 iThis; + int x; + while( isspace(pParse->zJson[i]) ){ i++; } + if( (c = pParse->zJson[i])==0 ) return 0; + if( c=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-2) && pParse->nNode==iThis+1 ) return j+1; + return -1; + } + if( pParse->aNode[pParse->nNode-1].eType!=JSON_STRING ) return -1; + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + if( pParse->zJson[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + if( x<0 ) return -1; + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( isspace(pParse->zJson[j]) ){ j++; } + x = jsonParseValue(pParse, j); + if( x<0 ){ + if( x==(-3) && pParse->nNode==iThis+1 ) return j+1; + return -1; + } + j = x; + while( isspace(pParse->zJson[j]) ){ j++; } + c = pParse->zJson[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + j = i+1; + for(;;){ + c = pParse->zJson[j]; + if( c==0 ) return -1; + if( c=='\\' ){ + c = pParse->zJson[++j]; + if( c==0 ) return -1; + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); + return j+1; + }else if( c=='n' + && strncmp(pParse->zJson+i,"null",4)==0 + && !isalnum(pParse->zJson[i+5]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(pParse->zJson+i,"true",4)==0 + && !isalnum(pParse->zJson[i+5]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(pParse->zJson+i,"false",5)==0 + && !isalnum(pParse->zJson[i+6]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + j = i+1; + for(;; j++){ + c = pParse->zJson[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( pParse->zJson[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( pParse->zJson[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = pParse->zJson[j+1]; + if( c=='+' || c=='-' ) j++; + continue; + } + break; + } + if( pParse->zJson[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &pParse->zJson[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse(JsonParse *pParse, const char *zJson){ + int i; + if( zJson==0 ) return 1; + memset(pParse, 0, sizeof(*pParse)); + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( i>0 ){ + while( isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<0 ){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + return 1; + } + return 0; +} + +/* +** The json_debug(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Json s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + char zBuf[50]; + static const char *azType[] = { + "NULL", "TRUE", "FALSE", "INT", "REAL", "STRING", "ARRAY", "OBJECT" + }; + + assert( argc==1 ); + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonInit(&s, context); + for(i=0; i=JSON_INT ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); + jsonAppend(&s, zBuf); + } + if( x.aNode[i].zContent!=0 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", + (u32)(x.aNode[i].zContent - x.zJson)); + jsonAppend(&s, zBuf); + jsonAppendRaw(&s, " text: ", 8); + jsonAppendRaw(&s, x.aNode[i].zContent, x.aNode[i].n); + jsonAppendRaw(&s, "\n", 1); + } + } + sqlite3_free(x.aNode); + jsonResult(&s); +} + #ifdef _WIN32 __declspec(dllexport) #endif @@ -540,6 +805,7 @@ int sqlite3_json_init( { "json_array", -1, jsonArrayFunc }, { "jsonb_array", -1, jsonbArrayFunc }, { "json_object", -1, jsonObjectFunc }, + { "json_debug", 1, jsonDebugFunc }, }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 1d41c76aaf..acc6cbf136 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode\s(untested)\sfor\sa\sJSONB\sdatatype. -D 2015-08-13T13:54:59.587 +C Add\sa\sprototype\sJSON\sparser. +D 2015-08-15T21:25:37.000 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 748c5bffa8840f1b0bdc4b5ff672949c31c7f436 +F ext/misc/json.c f4fa58cb8c6a7e13ae9a357e8c5783f43c60f595 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 414a95f3b79359f167858b2325fd2be223569c66 -R 45e5df2bce81d8e555551bffb25d0cb6 +P e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 +R 45c6bd1bef56b998d07e7b3125f794b3 U drh -Z 62911abad9701038f8848b5004ef4c38 +Z e2e8ec51bb6080feb16e3eda35039c7d diff --git a/manifest.uuid b/manifest.uuid index 58f3872eaf..7cd175e6dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 \ No newline at end of file +789ba487000aa73621a41d115ad5de455ea8ea31 \ No newline at end of file From b2cd10eaa920e143df093432caa0ce97859a4839 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Aug 2015 21:29:14 +0000 Subject: [PATCH 05/17] Fix off-by-one error when parsing primitive JSON types "true", "false", and "null". FossilOrigin-Name: 42c15c1e36b5077646fef99028cf12e587a45023 --- ext/misc/json.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 2fce79610c..f81cbf75a7 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -666,17 +666,17 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 - && !isalnum(pParse->zJson[i+5]) ){ + && !isalnum(pParse->zJson[i+4]) ){ jsonParseAddNode(pParse, JSON_NULL, 0, 0); return i+4; }else if( c=='t' && strncmp(pParse->zJson+i,"true",4)==0 - && !isalnum(pParse->zJson[i+5]) ){ + && !isalnum(pParse->zJson[i+4]) ){ jsonParseAddNode(pParse, JSON_TRUE, 0, 0); return i+4; }else if( c=='f' && strncmp(pParse->zJson+i,"false",5)==0 - && !isalnum(pParse->zJson[i+6]) ){ + && !isalnum(pParse->zJson[i+5]) ){ jsonParseAddNode(pParse, JSON_FALSE, 0, 0); return i+5; }else if( c=='-' || (c>='0' && c<='9') ){ diff --git a/manifest b/manifest index acc6cbf136..40126ac50b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprototype\sJSON\sparser. -D 2015-08-15T21:25:37.000 +C Fix\soff-by-one\serror\swhen\sparsing\sprimitive\sJSON\stypes\s"true",\s"false",\sand\n"null". +D 2015-08-15T21:29:14.831 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c f4fa58cb8c6a7e13ae9a357e8c5783f43c60f595 +F ext/misc/json.c 4f2ea491dd4f24d7a6be299af71093e6008b2f75 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3596ac7b1dd5bde3f9cae5781a6806d8d9f7166 -R 45c6bd1bef56b998d07e7b3125f794b3 +P 789ba487000aa73621a41d115ad5de455ea8ea31 +R 11d4a720790fbd16eb396e67acd5adf6 U drh -Z e2e8ec51bb6080feb16e3eda35039c7d +Z 4f36882f0704e134b10a4112fe9bc870 diff --git a/manifest.uuid b/manifest.uuid index 7cd175e6dd..76fe7c8b07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -789ba487000aa73621a41d115ad5de455ea8ea31 \ No newline at end of file +42c15c1e36b5077646fef99028cf12e587a45023 \ No newline at end of file From 5634cc05e40abcefd891c641a48e4cafea61b128 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 11:28:03 +0000 Subject: [PATCH 06/17] Abandon the JSONB format for now. (We may return to it in the future.) Add a function to render a JSON parse. FossilOrigin-Name: 9703c0aa18ae43375af876474b818e504e1c10a5 --- ext/misc/json.c | 428 ++++++++++++++++++------------------------------ manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 166 insertions(+), 276 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index f81cbf75a7..2e37835708 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -15,63 +15,9 @@ ** ** https://dev.mysql.com/doc/refman/5.7/en/json.html ** -** JSON is pure text. JSONB is a binary encoding that is smaller and easier -** to parse but which holds the equivalent information. Conversions between -** JSON and JSONB are lossless. -** -** Most of the functions here will accept either JSON or JSONB input. The -** input is understood to be JSONB if it a BLOB and JSON if the input is -** of any other type. Functions that begin with the "json_" prefix return -** JSON and functions that begin with "jsonb_" return JSONB. -** -** JSONB format: -** -** A JSONB blob is a sequence of terms. Each term begins with a single -** variable length integer X which determines the type and size of the term. -** -** type = X%8 -** size = X>>3 -** -** Term types are 0 through 7 for null, true, false, integer, real, string, -** array, and object. The meaning of size depends on the type. -** -** For null, true, and false terms, the size is always 0. -** -** For integer terms, the size is the number of bytes that contains the -** integer value. The value is stored as big-endian twos-complement. -** -** For real terms, the size is always 8 and the value is a big-ending -** double-precision floating-point number. -** -** For string terms, the size is the number of bytes in the string. The -** string itself immediately follows the X integer. There are no escapes -** and the string is not zero-terminated. The string is always stored as -** UTF8. -** -** For array terms, the size is the number of bytes in content. The -** content consists of zero or more additional terms that are the elements -** of the array. -** -** For object terms, the size is the number of bytes of content. The -** content is zero or more pairs of terms. The first element of each -** pair is a string term which is the label and the second element is -** the value. -** -** Variable Length Integers: -** -** The variable length integer encoding is the 64-bit unsigned integer encoding -** originally developed for SQLite4. The encoding for each integer is between -** 1 and 9 bytes. Call those bytes A0 through A8. The encoding is as follows: -** -** If A0 is between 0 and 240 inclusive, then the value is A0. -** -** If A0 is between 241 and 248 inclusive, then the value is -** 240+256*(A0-241)+A1. -** -** If A0 is 249 then the value is 2288+256*A1+A2. -** -** If A0 is 250 or more then the value is a (A0-247)-byte big-endian -** integer taken from starting at A1. +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation.) */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -84,13 +30,14 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; -/* An instance of this object represents a JSON string or -** JSONB blob under construction. +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. */ typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON or JSONB content here */ + char *zBuf; /* Append JSON content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ @@ -113,9 +60,10 @@ struct Json { */ typedef struct JsonNode JsonNode; struct JsonNode { - u32 eType; /* One of the JSON_ type values */ + u8 eType; /* One of the JSON_ type values */ + u8 bRaw; /* Content is raw, rather than JSON encoded */ u32 n; /* Bytes of content, or number of sub-nodes */ - const char *zContent; /* Content for JSON_INT, JSON_REAL, or JSON_STRING */ + const char *zJContent; /* JSON content */ }; /* A completely parsed JSON string @@ -129,63 +77,6 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ }; - -#if 0 -/* -** Decode the varint in the first n bytes z[]. Write the integer value -** into *pResult and return the number of bytes in the varint. -** -** If the decode fails because there are not enough bytes in z[] then -** return 0; -*/ -static int jsonGetVarint64( - const unsigned char *z, - int n, - u64 *pResult -){ - unsigned int x; - if( n<1 ) return 0; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - if( n<2 ) return 0; - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( nnUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -283,91 +181,6 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; } -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void jsonPutInt32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - - -/* Write integer X as a variable-length integer into the buffer z[]. -** z[] is guaranteed to be at least 9 bytes in length. Return the -** number of bytes written. -*/ -int jsonPutVarint64(char *zIn, u64 x){ - unsigned char *z = (unsigned char*)zIn; - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - jsonPutInt32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - jsonPutInt32(z+2, y); - return 6; - } - if( w<=65535 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - jsonPutInt32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - jsonPutInt32(z+4, y); - return 8; - } - z[0] = 255; - jsonPutInt32(z+1, w); - jsonPutInt32(z+5, y); - return 9; -} - - -/* Append integer X as a variable-length integer on the JSONB currently -** under construction in p. -*/ -static void jsonAppendVarint(Json *p, u64 X){ - if( (p->nUsed+9 > p->nAlloc) && jsonGrow(p,9)!=0 ) return; - p->nUsed += jsonPutVarint64(p->zBuf+p->nUsed, X); -} - /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ @@ -380,15 +193,109 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } -/* Make the JSONB in p the result of the SQL function. +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. */ -static void jsonbResult(Json *p){ - if( p->oom==0 ){ - sqlite3_result_blob(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - jsonZero(p); +static int jsonRenderNode(JsonNode *pNode, Json *pOut){ + u32 j = 0; + switch( pNode->eType ){ + case JSON_NULL: { + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->bRaw ){ + jsonAppendString(pOut, pNode->zJContent, pNode->n); + break; + } + /* Fall through into the next case */ + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + jsonAppendChar(pOut, '['); + j = 0; + while( jn ){ + if( j>0 ) jsonAppendChar(pOut, ','); + j += jsonRenderNode(&pNode[j+1], pOut); + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + jsonAppendChar(pOut, '{'); + j = 0; + while( jn ){ + if( j>0 ) jsonAppendChar(pOut, ','); + j += jsonRenderNode(&pNode[j+1], pOut); + jsonAppendChar(pOut, ':'); + j += jsonRenderNode(&pNode[j+1], pOut); + } + jsonAppendChar(pOut, '}'); + break; + } + } + return j+1; +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ + switch( pNode->eType ){ + case JSON_NULL: { + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + + /* FIXME: We really want to do text->numeric conversion on these. + ** Doing so would be easy if these were internal routines, but the + ** necessary interfaces are not exposed for doing it as a loadable + ** extension. */ + case JSON_REAL: + case JSON_INT: { + sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + break; + } + + case JSON_STRING: { + if( pNode->bRaw ){ + sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + Json s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s); + jsonResult(&s); + break; + } } - assert( p->bStatic ); } /* @@ -438,55 +345,6 @@ static void jsonArrayFunc( jsonResult(&jx); } -/* -** Implementation of the jsonb_array(VALUE,...) function. Return a JSON -** array that contains all values given in arguments. Or if any argument -** is a BLOB, throw an error. -*/ -static void jsonbArrayFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - Json jx; - - jsonInit(&jx, context); - jx.nUsed = 5; - for(i=0; iaNode = pNew; } p = &pParse->aNode[pParse->nNode]; - p->eType = eType; + p->eType = (u8)eType; + p->bRaw = 0; p->n = n; - p->zContent = zContent; + p->zJContent = zContent; return pParse->nNode++; } @@ -744,11 +603,11 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } /* -** The json_debug(JSON) function returns a string which describes +** The json_parse(JSON) function returns a string which describes ** a parse of the JSON provided. Or it returns NULL if JSON is not ** well-formed. */ -static void jsonDebugFunc( +static void jsonParseFunc( sqlite3_context *context, int argc, sqlite3_value **argv @@ -774,12 +633,12 @@ static void jsonDebugFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); jsonAppend(&s, zBuf); } - if( x.aNode[i].zContent!=0 ){ + if( x.aNode[i].zJContent!=0 ){ sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", - (u32)(x.aNode[i].zContent - x.zJson)); + (u32)(x.aNode[i].zJContent - x.zJson)); jsonAppend(&s, zBuf); jsonAppendRaw(&s, " text: ", 8); - jsonAppendRaw(&s, x.aNode[i].zContent, x.aNode[i].n); + jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } } @@ -787,6 +646,36 @@ static void jsonDebugFunc( jsonResult(&s); } +/* +** The json_test1(JSON) function parses and rebuilds the JSON string. +*/ +static void jsonTest1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonReturn(x.aNode, context); + sqlite3_free(x.aNode); +} + +/* +** The json_nodecount(JSON) function returns the number of nodes in the +** input JSON string. +*/ +static void jsonNodeCountFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + sqlite3_result_int64(context, x.nNode); + sqlite3_free(x.aNode); +} + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -803,9 +692,10 @@ int sqlite3_json_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "json_array", -1, jsonArrayFunc }, - { "jsonb_array", -1, jsonbArrayFunc }, { "json_object", -1, jsonObjectFunc }, - { "json_debug", 1, jsonDebugFunc }, + { "json_parse", 1, jsonParseFunc }, /* DEBUG */ + { "json_test1", 1, jsonTest1Func }, /* DEBUG */ + { "json_nodecount", 1, jsonNodeCountFunc }, /* DEBUG */ }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 40126ac50b..a89a28af9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\soff-by-one\serror\swhen\sparsing\sprimitive\sJSON\stypes\s"true",\s"false",\sand\n"null". -D 2015-08-15T21:29:14.831 +C Abandon\sthe\sJSONB\sformat\sfor\snow.\s\s(We\smay\sreturn\sto\sit\sin\sthe\sfuture.)\s\sAdd\na\sfunction\sto\srender\sa\sJSON\sparse. +D 2015-08-17T11:28:03.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 4f2ea491dd4f24d7a6be299af71093e6008b2f75 +F ext/misc/json.c fce2fee3ac62dd53cc502cf2673b2ee5947d702d F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 789ba487000aa73621a41d115ad5de455ea8ea31 -R 11d4a720790fbd16eb396e67acd5adf6 +P 42c15c1e36b5077646fef99028cf12e587a45023 +R 4717cf337d2cdc65a44f3f6bb72a9c97 U drh -Z 4f36882f0704e134b10a4112fe9bc870 +Z 1ecceeb77cb551a38df879c773148819 diff --git a/manifest.uuid b/manifest.uuid index 76fe7c8b07..669996d2c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42c15c1e36b5077646fef99028cf12e587a45023 \ No newline at end of file +9703c0aa18ae43375af876474b818e504e1c10a5 \ No newline at end of file From 987eb1fa9b4a26ea00bf105c9725b20d27dbde51 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 15:17:37 +0000 Subject: [PATCH 07/17] Initial implementation for json_array_length(), json_extract(), and json_type(). FossilOrigin-Name: 39983204515837e7bd574cf47918e493acc03d1f --- ext/misc/json.c | 501 +++++++++++++++++++++++++++++++++++------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 380 insertions(+), 135 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 2e37835708..1b2b6ff5c2 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -24,6 +24,7 @@ SQLITE_EXTENSION_INIT1 #include #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -56,12 +57,21 @@ struct Json { #define JSON_ARRAY 6 #define JSON_OBJECT 7 +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + + /* A single node of parsed JSON */ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 bRaw; /* Content is raw, rather than JSON encoded */ + u8 bBackslash; /* Formatted JSON_STRING contains \ escapes */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -268,22 +278,89 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ sqlite3_result_int(pCtx, 0); break; } - - /* FIXME: We really want to do text->numeric conversion on these. - ** Doing so would be easy if these were internal routines, but the - ** necessary interfaces are not exposed for doing it as a loadable - ** extension. */ - case JSON_REAL: - case JSON_INT: { - sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + case JSON_REAL: { + double r = strtod(pNode->zJContent, 0); + sqlite3_result_double(pCtx, r); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ i = i*10 + *(z++) - '0'; } + if( pNode->zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); break; } - case JSON_STRING: { if( pNode->bRaw ){ sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + }else if( !pNode->bBackslash ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i='0' && c<='9' ) v = v*16 + c - '0'; + else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; + else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; + else break; + z++; + } + if( v<=0x7f ){ + zOut[j++] = v; + }else if( v<=0x7ff ){ + zOut[j++] = 0xc0 | (v>>6); + zOut[j++] = 0x80 | (v&0x3f); + }else if( v<=0xffff ){ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else if( v<=0x10ffff ){ + zOut[j++] = 0xf0 | (v>>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); } break; } @@ -298,116 +375,6 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ } } -/* -** Implementation of the json_array(VALUE,...) function. Return a JSON -** array that contains all values given in arguments. Or if any argument -** is a BLOB, throw an error. -*/ -static void jsonArrayFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - int i; - Json jx; - char cSep = '['; - - jsonInit(&jx, context); - for(i=0; iaNode[] of the @@ -440,6 +407,7 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->bRaw = 0; + p->bBackslash = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -509,6 +477,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='"' ){ /* Parse string */ + u8 bBackslash = 0; j = i+1; for(;;){ c = pParse->zJson[j]; @@ -516,12 +485,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='\\' ){ c = pParse->zJson[++j]; if( c==0 ) return -1; + bBackslash = 1; }else if( c=='"' ){ break; } j++; } jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); + if( bBackslash ) pParse->aNode[pParse->nNode-1].bBackslash = 1; return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 @@ -601,6 +572,60 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } return 0; } +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +*/ +static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ + u32 i, j; + if( zPath[0]==0 ) return pRoot; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + for(i=0; isalnum(zPath[i]); i++){} + if( i==0 ) return 0; + j = 1; + while( j<=pRoot->n ){ + if( pRoot[j].n==i+2 + && strncmp(&pRoot[j].zJContent[1],zPath,i)==0 + ){ + return jsonLookup(&pRoot[j+1], &zPath[i]); + } + j++; + if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ + j += pRoot[j].n; + } + j++; + } + }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ + if( pRoot->eType!=JSON_ARRAY ) return 0; + i = 0; + zPath++; + while( isdigit(zPath[0]) ){ + i = i + zPath[0] - '0'; + zPath++; + } + if( zPath[0]!=']' ) return 0; + zPath++; + j = 1; + while( i>0 && j<=pRoot->n ){ + if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ + j += pRoot[j].n; + } + j++; + i--; + } + if( j<=pRoot->n ){ + return jsonLookup(&pRoot[j], zPath); + } + } + return 0; +} + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ /* ** The json_parse(JSON) function returns a string which describes @@ -616,9 +641,6 @@ static void jsonParseFunc( JsonParse x; /* The parse */ u32 i; char zBuf[50]; - static const char *azType[] = { - "NULL", "TRUE", "FALSE", "INT", "REAL", "STRING", "ARRAY", "OBJECT" - }; assert( argc==1 ); if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; @@ -627,7 +649,7 @@ static void jsonParseFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, "node %u:\n", i); jsonAppend(&s, zBuf); sqlite3_snprintf(sizeof(zBuf), zBuf, " type: %s\n", - azType[x.aNode[i].eType]); + jsonType[x.aNode[i].eType]); jsonAppend(&s, zBuf); if( x.aNode[i].eType>=JSON_INT ){ sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); @@ -675,6 +697,222 @@ static void jsonNodeCountFunc( sqlite3_free(x.aNode); } +/**************************************************************************** +** SQL function implementations +****************************************************************************/ + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '['; + + jsonInit(&jx, context); + for(i=0; ieType==JSON_ARRAY ){ + for(i=1; i<=pNode->n; i++, n++){ + if( pNode[i].eType==JSON_ARRAY || pNode[i].eType==JSON_OBJECT ){ + i += pNode[i].n; + } + } + } + } + sqlite3_free(x.aNode); + } + sqlite3_result_int64(context, n); +} + +/* +** json_extract(JSON, PATH) +** +** Return the element described by PATH. Return NULL if JSON is not +** valid JSON or if there is no PATH element or if PATH is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + assert( argc==2 ); + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath==0 ) return; + if( zPath[0]!='$' ) return; + zPath++; + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + pNode = jsonLookup(x.aNode, zPath); + if( pNode ){ + jsonReturn(pNode, context); + } + sqlite3_free(x.aNode); +} + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + Json jx; + char cSep = '{'; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(context, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, context); + for(i=0; ieType], -1, SQLITE_STATIC); + } + sqlite3_free(x.aNode); +} #ifdef _WIN32 __declspec(dllexport) @@ -691,11 +929,18 @@ int sqlite3_json_init( int nArg; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "json_array", -1, jsonArrayFunc }, - { "json_object", -1, jsonObjectFunc }, - { "json_parse", 1, jsonParseFunc }, /* DEBUG */ - { "json_test1", 1, jsonTest1Func }, /* DEBUG */ - { "json_nodecount", 1, jsonNodeCountFunc }, /* DEBUG */ + { "json_array", -1, jsonArrayFunc }, + { "json_array_length", 1, jsonArrayLengthFunc }, + { "json_array_length", 2, jsonArrayLengthFunc }, + { "json_extract", 2, jsonExtractFunc }, + { "json_object", -1, jsonObjectFunc }, + { "json_type", 1, jsonTypeFunc }, + { "json_type", 2, jsonTypeFunc }, + + /* DEBUG and TESTING functions */ + { "json_parse", 1, jsonParseFunc }, + { "json_test1", 1, jsonTest1Func }, + { "json_nodecount", 1, jsonNodeCountFunc }, }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index a89a28af9e..67b31769fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Abandon\sthe\sJSONB\sformat\sfor\snow.\s\s(We\smay\sreturn\sto\sit\sin\sthe\sfuture.)\s\sAdd\na\sfunction\sto\srender\sa\sJSON\sparse. -D 2015-08-17T11:28:03.070 +C Initial\simplementation\sfor\sjson_array_length(),\sjson_extract(),\sand\njson_type(). +D 2015-08-17T15:17:37.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c fce2fee3ac62dd53cc502cf2673b2ee5947d702d +F ext/misc/json.c f26cbaa8ba1e396b3bf1e29aa116abed2a27ef95 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42c15c1e36b5077646fef99028cf12e587a45023 -R 4717cf337d2cdc65a44f3f6bb72a9c97 +P 9703c0aa18ae43375af876474b818e504e1c10a5 +R cfeb418e7f67bb21a131fd01c852e6d6 U drh -Z 1ecceeb77cb551a38df879c773148819 +Z 903f29a5c61013c958dfe3ed9e5ebef0 diff --git a/manifest.uuid b/manifest.uuid index 669996d2c6..be625468c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9703c0aa18ae43375af876474b818e504e1c10a5 \ No newline at end of file +39983204515837e7bd574cf47918e493acc03d1f \ No newline at end of file From 301eeccaa700ea954abf0c63c2f8526f93d6578e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 20:14:19 +0000 Subject: [PATCH 08/17] Add an initial implementation for json_remove(). FossilOrigin-Name: 2a8267209dbda36a37c1b5f65000b2f763c62341 --- ext/misc/json.c | 148 ++++++++++++++++++++++++++++++++---------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 107 insertions(+), 55 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 1b2b6ff5c2..dc27349c0f 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -64,14 +64,19 @@ static const char * const jsonType[] = { "null", "true", "false", "integer", "real", "text", "array", "object" }; +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ + /* A single node of parsed JSON */ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ - u8 bRaw; /* Content is raw, rather than JSON encoded */ - u8 bBackslash; /* Formatted JSON_STRING contains \ escapes */ + u8 jnFlags; /* JNODE flags */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -87,6 +92,15 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ }; +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +*/ +static u32 jsonSizeof(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + /* Set the Json object to an empty string */ static void jsonZero(Json *p){ @@ -126,7 +140,7 @@ static void jsonOom(Json *p){ ** Return zero on success. Return non-zero on an OOM error */ static int jsonGrow(Json *p, u32 N){ - u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+100; + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ if( p->oom ) return SQLITE_NOMEM; @@ -171,6 +185,16 @@ static void jsonAppendChar(Json *p, char c){ p->zBuf[p->nUsed++] = c; } +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(Json *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + /* Append the N-byte string in zIn to the end of the Json string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the @@ -209,7 +233,7 @@ static void jsonResult(Json *p){ ** the number of JsonNode objects that are encoded. */ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ - u32 j = 0; + u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { jsonAppendRaw(pOut, "null", 4); @@ -224,7 +248,7 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ break; } case JSON_STRING: { - if( pNode->bRaw ){ + if( pNode->jnFlags & JNODE_RAW ){ jsonAppendString(pOut, pNode->zJContent, pNode->n); break; } @@ -237,28 +261,34 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ } case JSON_ARRAY: { jsonAppendChar(pOut, '['); - j = 0; - while( jn ){ - if( j>0 ) jsonAppendChar(pOut, ','); - j += jsonRenderNode(&pNode[j+1], pOut); + while( j<=pNode->n ){ + if( pNode[j].jnFlags & JNODE_REMOVE ){ + j += jsonSizeof(&pNode[j]); + }else{ + jsonAppendSeparator(pOut); + j += jsonRenderNode(&pNode[j], pOut); + } } jsonAppendChar(pOut, ']'); break; } case JSON_OBJECT: { jsonAppendChar(pOut, '{'); - j = 0; - while( jn ){ - if( j>0 ) jsonAppendChar(pOut, ','); - j += jsonRenderNode(&pNode[j+1], pOut); - jsonAppendChar(pOut, ':'); - j += jsonRenderNode(&pNode[j+1], pOut); + while( j<=pNode->n ){ + if( pNode[j+1].jnFlags & JNODE_REMOVE ){ + j += 1 + jsonSizeof(&pNode[j+1]); + }else{ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut); + jsonAppendChar(pOut, ':'); + j += 1 + jsonRenderNode(&pNode[j+1], pOut); + } } jsonAppendChar(pOut, '}'); break; } } - return j+1; + return j; } /* @@ -293,9 +323,9 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ break; } case JSON_STRING: { - if( pNode->bRaw ){ + if( pNode->jnFlags & JNODE_RAW ){ sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); - }else if( !pNode->bBackslash ){ + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, SQLITE_TRANSIENT); @@ -406,8 +436,7 @@ static int jsonParseAddNode( } p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; - p->bRaw = 0; - p->bBackslash = 0; + p->jnFlags = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -477,7 +506,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return j+1; }else if( c=='"' ){ /* Parse string */ - u8 bBackslash = 0; + u8 jnFlags = 0; j = i+1; for(;;){ c = pParse->zJson[j]; @@ -485,14 +514,14 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='\\' ){ c = pParse->zJson[++j]; if( c==0 ) return -1; - bBackslash = 1; + jnFlags = JNODE_ESCAPE; }else if( c=='"' ){ break; } j++; } jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); - if( bBackslash ) pParse->aNode[pParse->nNode-1].bBackslash = 1; + pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; return j+1; }else if( c=='n' && strncmp(pParse->zJson+i,"null",4)==0 @@ -572,6 +601,7 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ } return 0; } + /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such @@ -593,10 +623,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ return jsonLookup(&pRoot[j+1], &zPath[i]); } j++; - if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ - j += pRoot[j].n; - } - j++; + j += jsonSizeof(&pRoot[j]); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -610,10 +637,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ zPath++; j = 1; while( i>0 && j<=pRoot->n ){ - if( pRoot[j].eType==JSON_ARRAY || pRoot[j].eType==JSON_OBJECT ){ - j += pRoot[j].n; - } - j++; + j += jsonSizeof(&pRoot[j]); i--; } if( j<=pRoot->n ){ @@ -627,6 +651,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ ** SQL functions used for testing and debugging ****************************************************************************/ +#ifdef SQLITE_DEBUG /* ** The json_parse(JSON) function returns a string which describes ** a parse of the JSON provided. Or it returns NULL if JSON is not @@ -640,26 +665,17 @@ static void jsonParseFunc( Json s; /* Output string - not real JSON */ JsonParse x; /* The parse */ u32 i; - char zBuf[50]; + char zBuf[100]; assert( argc==1 ); if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; jsonInit(&s, context); for(i=0; i=JSON_INT ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, " n: %u\n", x.aNode[i].n); - jsonAppend(&s, zBuf); - } if( x.aNode[i].zJContent!=0 ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, " ofst: %u\n", - (u32)(x.aNode[i].zJContent - x.zJson)); - jsonAppend(&s, zBuf); - jsonAppendRaw(&s, " text: ", 8); + jsonAppendRaw(&s, " text: ", 10); jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } @@ -696,6 +712,7 @@ static void jsonNodeCountFunc( sqlite3_result_int64(context, x.nNode); sqlite3_free(x.aNode); } +#endif /* SQLITE_DEBUG */ /**************************************************************************** ** SQL function implementations @@ -779,10 +796,8 @@ static void jsonArrayLengthFunc( JsonNode *pNode = x.aNode; if( zPath ) pNode = jsonLookup(pNode, zPath); if( pNode->eType==JSON_ARRAY ){ - for(i=1; i<=pNode->n; i++, n++){ - if( pNode[i].eType==JSON_ARRAY || pNode[i].eType==JSON_OBJECT ){ - i += pNode[i].n; - } + for(i=1; i<=pNode->n; n++){ + i += jsonSizeof(&pNode[i]); } } } @@ -882,6 +897,40 @@ static void jsonObjectFunc( } +/* +** json_remove(JSON, PATH, ...) +** +** Remove the named elements from JSON and return the result. Ill-formed +** PATH arguments are silently ignored. If JSON is ill-formed, then NULL +** is returned. +*/ +static void jsonRemoveFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturn(x.aNode, context); + } + } + sqlite3_free(x.aNode); +} + /* ** json_type(JSON) ** json_type(JSON, PATH) @@ -934,13 +983,16 @@ int sqlite3_json_init( { "json_array_length", 2, jsonArrayLengthFunc }, { "json_extract", 2, jsonExtractFunc }, { "json_object", -1, jsonObjectFunc }, + { "json_remove", -1, jsonRemoveFunc }, { "json_type", 1, jsonTypeFunc }, { "json_type", 2, jsonTypeFunc }, +#if SQLITE_DEBUG /* DEBUG and TESTING functions */ { "json_parse", 1, jsonParseFunc }, { "json_test1", 1, jsonTest1Func }, { "json_nodecount", 1, jsonNodeCountFunc }, +#endif }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ diff --git a/manifest b/manifest index 67b31769fb..4fc2dcacd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sfor\sjson_array_length(),\sjson_extract(),\sand\njson_type(). -D 2015-08-17T15:17:37.780 +C Add\san\sinitial\simplementation\sfor\sjson_remove(). +D 2015-08-17T20:14:19.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c f26cbaa8ba1e396b3bf1e29aa116abed2a27ef95 +F ext/misc/json.c 30fd85ea1fba24031952aa0c635156cfd8ca02ea F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9703c0aa18ae43375af876474b818e504e1c10a5 -R cfeb418e7f67bb21a131fd01c852e6d6 +P 39983204515837e7bd574cf47918e493acc03d1f +R 9ed2e57314d456e476aa0e6edd9d8c55 U drh -Z 903f29a5c61013c958dfe3ed9e5ebef0 +Z 25ffed392c0fb3616f3b027e2b1bb1db diff --git a/manifest.uuid b/manifest.uuid index be625468c7..3cd093bc94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39983204515837e7bd574cf47918e493acc03d1f \ No newline at end of file +2a8267209dbda36a37c1b5f65000b2f763c62341 \ No newline at end of file From d096059fca8f84284b8fa66c2e0ad6fa32197750 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Aug 2015 21:22:32 +0000 Subject: [PATCH 09/17] Initial implementation of json_replace(). FossilOrigin-Name: 3c4bee65d93efc7f03f0f11817a068b02068d37c --- ext/misc/json.c | 218 ++++++++++++++++++++++++++++++------------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 145 insertions(+), 87 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index dc27349c0f..d865fb6152 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -42,7 +42,7 @@ struct Json { u64 nAlloc; /* Bytes of storage available in zBuf[] */ u64 nUsed; /* Bytes of zBuf[] currently used */ u8 bStatic; /* True if zBuf is static space */ - u8 oom; /* True if an OOM has been encountered */ + u8 bErr; /* True if an error has been encountered */ char zSpace[100]; /* Initial static space */ }; @@ -69,6 +69,7 @@ static const char * const jsonType[] = { #define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ /* A single node of parsed JSON @@ -77,6 +78,7 @@ typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ + u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ const char *zJContent; /* JSON content */ }; @@ -97,7 +99,7 @@ struct JsonParse { ** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and ** OBJECT types, the number might be larger. */ -static u32 jsonSizeof(JsonNode *pNode){ +static u32 jsonSize(JsonNode *pNode){ return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; } @@ -114,7 +116,7 @@ static void jsonZero(Json *p){ */ static void jsonInit(Json *p, sqlite3_context *pCtx){ p->pCtx = pCtx; - p->oom = 0; + p->bErr = 0; jsonZero(p); } @@ -131,9 +133,11 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ static void jsonOom(Json *p){ - p->oom = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); + if( !p->bErr ){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); + } } /* Enlarge pJson->zBuf so that it can hold at least N more bytes. @@ -143,7 +147,7 @@ static int jsonGrow(Json *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ - if( p->oom ) return SQLITE_NOMEM; + if( p->bErr ) return 1; zNew = sqlite3_malloc64(nTotal); if( zNew==0 ){ jsonOom(p); @@ -172,11 +176,13 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ p->nUsed += N; } +#ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ static void jsonAppend(Json *p, const char *zIn){ jsonAppendRaw(p, zIn, (u32)strlen(zIn)); } +#endif /* Append a single character */ @@ -215,10 +221,48 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ p->zBuf[p->nUsed++] = '"'; } +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + Json *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendString(p, z, n); + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 1; + jsonReset(p); + } + break; + } + } +} + + /* Make the JSON in p the result of the SQL function. */ static void jsonResult(Json *p){ - if( p->oom==0 ){ + if( p->bErr==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); @@ -232,7 +276,11 @@ static void jsonResult(Json *p){ ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static int jsonRenderNode(JsonNode *pNode, Json *pOut){ +static int jsonRenderNode( + JsonNode *pNode, /* The node to render */ + Json *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { @@ -262,11 +310,15 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ case JSON_ARRAY: { jsonAppendChar(pOut, '['); while( j<=pNode->n ){ - if( pNode[j].jnFlags & JNODE_REMOVE ){ - j += jsonSizeof(&pNode[j]); + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ + if( pNode[j].jnFlags & JNODE_REPLACE ){ + jsonAppendSeparator(pOut); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + } + j += jsonSize(&pNode[j]); }else{ jsonAppendSeparator(pOut); - j += jsonRenderNode(&pNode[j], pOut); + j += jsonRenderNode(&pNode[j], pOut, aReplace); } } jsonAppendChar(pOut, ']'); @@ -276,12 +328,17 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ jsonAppendChar(pOut, '{'); while( j<=pNode->n ){ if( pNode[j+1].jnFlags & JNODE_REMOVE ){ - j += 1 + jsonSizeof(&pNode[j+1]); + j += 1 + jsonSize(&pNode[j+1]); }else{ jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); - j += 1 + jsonRenderNode(&pNode[j+1], pOut); + if( pNode[j+1].jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + j += 1 + jsonSize(&pNode[j+1]); + }else{ + j += 1 + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } } } jsonAppendChar(pOut, '}'); @@ -294,7 +351,11 @@ static int jsonRenderNode(JsonNode *pNode, Json *pOut){ /* ** Make the JsonNode the return value of the function. */ -static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ switch( pNode->eType ){ case JSON_NULL: { sqlite3_result_null(pCtx); @@ -398,7 +459,7 @@ static void jsonReturn(JsonNode *pNode, sqlite3_context *pCtx){ case JSON_OBJECT: { Json s; jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s); + jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); break; } @@ -437,6 +498,7 @@ static int jsonParseAddNode( p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; + p->iVal = 0; p->n = n; p->zJContent = zContent; return pParse->nNode++; @@ -623,7 +685,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ return jsonLookup(&pRoot[j+1], &zPath[i]); } j++; - j += jsonSizeof(&pRoot[j]); + j += jsonSize(&pRoot[j]); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -637,7 +699,7 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ zPath++; j = 1; while( i>0 && j<=pRoot->n ){ - j += jsonSizeof(&pRoot[j]); + j += jsonSize(&pRoot[j]); i--; } if( j<=pRoot->n ){ @@ -694,7 +756,7 @@ static void jsonTest1Func( ){ JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonReturn(x.aNode, context); + jsonReturn(x.aNode, context, 0); sqlite3_free(x.aNode); } @@ -730,38 +792,14 @@ static void jsonArrayFunc( ){ int i; Json jx; - char cSep = '['; jsonInit(&jx, context); + jsonAppendChar(&jx, '['); for(i=0; ieType==JSON_ARRAY ){ for(i=1; i<=pNode->n; n++){ - i += jsonSizeof(&pNode[i]); + i += jsonSize(&pNode[i]); } } } @@ -828,7 +866,7 @@ static void jsonExtractFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; pNode = jsonLookup(x.aNode, zPath); if( pNode ){ - jsonReturn(pNode, context); + jsonReturn(pNode, context, 0); } sqlite3_free(x.aNode); } @@ -845,7 +883,6 @@ static void jsonObjectFunc( ){ int i; Json jx; - char cSep = '{'; const char *z; u32 n; @@ -855,44 +892,21 @@ static void jsonObjectFunc( return; } jsonInit(&jx, context); + jsonAppendChar(&jx, '{'); for(i=0; ijnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturn(x.aNode, context); + jsonReturn(x.aNode, context, 0); + } + } + sqlite3_free(x.aNode); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON is ill-formed, return NULL. +*/ +static void jsonReplaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + sqlite3_result_error(context, + "json_replace() needs an odd number of arguments", -1); + return; + } + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REPLACE; + pNode->iVal = i+1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(context, argv[x.aNode[0].iVal]); + }else{ + jsonReturn(x.aNode, context, argv); } } sqlite3_free(x.aNode); @@ -984,6 +1041,7 @@ int sqlite3_json_init( { "json_extract", 2, jsonExtractFunc }, { "json_object", -1, jsonObjectFunc }, { "json_remove", -1, jsonRemoveFunc }, + { "json_replace", -1, jsonReplaceFunc }, { "json_type", 1, jsonTypeFunc }, { "json_type", 2, jsonTypeFunc }, diff --git a/manifest b/manifest index 4fc2dcacd6..7d15376185 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinitial\simplementation\sfor\sjson_remove(). -D 2015-08-17T20:14:19.276 +C Initial\simplementation\sof\sjson_replace(). +D 2015-08-17T21:22:32.495 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 30fd85ea1fba24031952aa0c635156cfd8ca02ea +F ext/misc/json.c d96116de8aafdb117b99712b2a83144d86755350 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 39983204515837e7bd574cf47918e493acc03d1f -R 9ed2e57314d456e476aa0e6edd9d8c55 +P 2a8267209dbda36a37c1b5f65000b2f763c62341 +R f98c8d887493a196dac50851b7b4599f U drh -Z 25ffed392c0fb3616f3b027e2b1bb1db +Z 432e296535146c125ab741ef125c263a diff --git a/manifest.uuid b/manifest.uuid index 3cd093bc94..e0120e1b2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a8267209dbda36a37c1b5f65000b2f763c62341 \ No newline at end of file +3c4bee65d93efc7f03f0f11817a068b02068d37c \ No newline at end of file From 52216adf7efa4b7ebb3306c9a7a92c20e087015c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Aug 2015 02:28:03 +0000 Subject: [PATCH 10/17] Initial implementation of json_set() and json_insert(). FossilOrigin-Name: 4aa49656d98e2894f2faa8963f79462ee6165d40 --- ext/misc/json.c | 276 ++++++++++++++++++++++++++++++++++++------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 215 insertions(+), 75 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index d865fb6152..e980b52de6 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -31,11 +31,15 @@ typedef sqlite3_uint64 u64; typedef unsigned int u32; typedef unsigned char u8; +/* Objects */ +typedef struct Json Json; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + /* An instance of this object represents a JSON string ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. */ -typedef struct Json Json; struct Json { sqlite3_context *pCtx; /* Function context - put error messages here */ char *zBuf; /* Append JSON content here */ @@ -70,22 +74,24 @@ static const char * const jsonType[] = { #define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ /* A single node of parsed JSON */ -typedef struct JsonNode JsonNode; struct JsonNode { u8 eType; /* One of the JSON_ type values */ u8 jnFlags; /* JNODE flags */ u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ - const char *zJContent; /* JSON content */ + union { + const char *zJContent; /* JSON content */ + u32 iAppend; /* Appended content */ + } u; }; /* A completely parsed JSON string */ -typedef struct JsonParse JsonParse; struct JsonParse { u32 nNode; /* Number of slots of aNode[] used */ u32 nAlloc; /* Number of slots of aNode[] allocated */ @@ -98,6 +104,10 @@ struct JsonParse { ** Return the number of consecutive JsonNode slots need to represent ** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and ** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. */ static u32 jsonSize(JsonNode *pNode){ return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; @@ -276,12 +286,11 @@ static void jsonResult(Json *p){ ** append to pOut. Subsubstructure is also included. Return ** the number of JsonNode objects that are encoded. */ -static int jsonRenderNode( +static void jsonRenderNode( JsonNode *pNode, /* The node to render */ Json *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ - u32 j = 1; switch( pNode->eType ){ case JSON_NULL: { jsonAppendRaw(pOut, "null", 4); @@ -297,55 +306,64 @@ static int jsonRenderNode( } case JSON_STRING: { if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->zJContent, pNode->n); + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); break; } /* Fall through into the next case */ } case JSON_REAL: case JSON_INT: { - jsonAppendRaw(pOut, pNode->zJContent, pNode->n); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); break; } case JSON_ARRAY: { + u32 j = 1; jsonAppendChar(pOut, '['); - while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ + for(;;){ + while( j<=pNode->n ){ + if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ + if( pNode[j].jnFlags & JNODE_REPLACE ){ + jsonAppendSeparator(pOut); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + } + }else{ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); + jsonRenderNode(&pNode[j], pOut, aReplace); } j += jsonSize(&pNode[j]); - }else{ - jsonAppendSeparator(pOut); - j += jsonRenderNode(&pNode[j], pOut, aReplace); } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; } jsonAppendChar(pOut, ']'); break; } case JSON_OBJECT: { + u32 j = 1; jsonAppendChar(pOut, '{'); - while( j<=pNode->n ){ - if( pNode[j+1].jnFlags & JNODE_REMOVE ){ - j += 1 + jsonSize(&pNode[j+1]); - }else{ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - j += 1 + jsonSize(&pNode[j+1]); - }else{ - j += 1 + jsonRenderNode(&pNode[j+1], pOut, aReplace); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + if( pNode[j+1].jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); + }else{ + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } } + j += 1 + jsonSize(&pNode[j+1]); } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; } jsonAppendChar(pOut, '}'); break; } } - return j; } /* @@ -370,31 +388,32 @@ static void jsonReturn( break; } case JSON_REAL: { - double r = strtod(pNode->zJContent, 0); + double r = strtod(pNode->u.zJContent, 0); sqlite3_result_double(pCtx, r); break; } case JSON_INT: { sqlite3_int64 i = 0; - const char *z = pNode->zJContent; + const char *z = pNode->u.zJContent; if( z[0]=='-' ){ z++; } while( z[0]>='0' && z[0]<='9' ){ i = i*10 + *(z++) - '0'; } - if( pNode->zJContent[0]=='-' ){ i = -i; } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } sqlite3_result_int64(pCtx, i); break; } case JSON_STRING: { if( pNode->jnFlags & JNODE_RAW ){ - sqlite3_result_text(pCtx, pNode->zJContent, pNode->n, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ - sqlite3_result_text(pCtx, pNode->zJContent+1, pNode->n-2, + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, SQLITE_TRANSIENT); }else{ /* Translate JSON formatted string into raw text */ u32 i; u32 n = pNode->n; - const char *z = pNode->zJContent; + const char *z = pNode->u.zJContent; char *zOut; u32 j; zOut = sqlite3_malloc( n+1 ); @@ -500,7 +519,7 @@ static int jsonParseAddNode( p->jnFlags = 0; p->iVal = 0; p->n = n; - p->zJContent = zContent; + p->u.zJContent = zContent; return pParse->nNode++; } @@ -664,13 +683,26 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ return 0; } +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); + /* ** Search along zPath to find the node specified. Return a pointer ** to that node, or NULL if zPath is malformed or if there is no such ** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. */ -static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ - u32 i, j; +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd /* Append nodes to complete path if not NULL */ +){ + u32 i, j, k; + JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; @@ -678,14 +710,29 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ for(i=0; isalnum(zPath[i]); i++){} if( i==0 ) return 0; j = 1; - while( j<=pRoot->n ){ - if( pRoot[j].n==i+2 - && strncmp(&pRoot[j].zJContent[1],zPath,i)==0 - ){ - return jsonLookup(&pRoot[j+1], &zPath[i]); + for(;;){ + while( j<=pRoot->n ){ + if( pRoot[j].n==i+2 + && strncmp(&pRoot[j].u.zJContent[1],zPath,i)==0 + ){ + return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); + } + j++; + j += jsonSize(&pRoot[j]); } - j++; - j += jsonSize(&pRoot[j]); + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + k = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + pRoot->u.iAppend = k - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + k = jsonParseAddNode(pParse, JSON_STRING, i, zPath); + if( !pParse->oom ) pParse->aNode[k].jnFlags |= JNODE_RAW; + zPath += i; + return jsonLookupAppend(pParse, zPath, pApnd); } }else if( zPath[0]=='[' && isdigit(zPath[1]) ){ if( pRoot->eType!=JSON_ARRAY ) return 0; @@ -698,17 +745,54 @@ static JsonNode *jsonLookup(JsonNode *pRoot, const char *zPath){ if( zPath[0]!=']' ) return 0; zPath++; j = 1; - while( i>0 && j<=pRoot->n ){ - j += jsonSize(&pRoot[j]); - i--; + for(;;){ + while( i>0 && j<=pRoot->n ){ + j += jsonSize(&pRoot[j]); + i--; + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; } if( j<=pRoot->n ){ - return jsonLookup(&pRoot[j], zPath); + return jsonLookup(pParse, iRoot+j, zPath, pApnd); + } + if( i==0 && pApnd ){ + k = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pRoot->u.iAppend = k - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + return jsonLookupAppend(pParse, zPath, pApnd); } } return 0; } +/* +** Append content to pParse that will complete zPath. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd /* Set this flag to 1 */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookup(pParse, pParse->nNode-1, zPath, pApnd); +} + + /**************************************************************************** ** SQL functions used for testing and debugging ****************************************************************************/ @@ -736,9 +820,9 @@ static void jsonParseFunc( sqlite3_snprintf(sizeof(zBuf), zBuf, "node %3u: %7s n=%d\n", i, jsonType[x.aNode[i].eType], x.aNode[i].n); jsonAppend(&s, zBuf); - if( x.aNode[i].zJContent!=0 ){ + if( x.aNode[i].u.zJContent!=0 ){ jsonAppendRaw(&s, " text: ", 10); - jsonAppendRaw(&s, x.aNode[i].zJContent, x.aNode[i].n); + jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); jsonAppendRaw(&s, "\n", 1); } } @@ -832,8 +916,9 @@ static void jsonArrayLengthFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0]))==0 ){ if( x.nNode ){ JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(pNode, zPath); + if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); for(i=1; i<=pNode->n; n++){ i += jsonSize(&pNode[i]); } @@ -864,7 +949,7 @@ static void jsonExtractFunc( if( zPath[0]!='$' ) return; zPath++; if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - pNode = jsonLookup(x.aNode, zPath); + pNode = jsonLookup(&x, 0, zPath, 0); if( pNode ){ jsonReturn(pNode, context, 0); } @@ -935,7 +1020,7 @@ static void jsonRemoveFunc( zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) continue; if( zPath[0]!='$' ) continue; - pNode = jsonLookup(x.aNode, &zPath[1]); + pNode = jsonLookup(&x, 0, &zPath[1], 0); if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ @@ -973,7 +1058,7 @@ static void jsonReplaceFunc( zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) continue; if( zPath[0]!='$' ) continue; - pNode = jsonLookup(x.aNode, &zPath[1]); + pNode = jsonLookup(&x, 0, &zPath[1], 0); if( pNode ){ pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; @@ -987,6 +1072,57 @@ static void jsonReplaceFunc( } sqlite3_free(x.aNode); } +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON is ill-formed, return NULL. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON is ill-formed, return NULL. +*/ +static void jsonSetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(context); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + sqlite3_result_error(context, + "json_set() needs an odd number of arguments", -1); + return; + } + if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; + if( x.nNode ){ + for(i=1; ijnFlags |= JNODE_REPLACE; + pNode->iVal = i+1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(context, argv[x.aNode[0].iVal]); + }else{ + jsonReturn(x.aNode, context, argv); + } + } + sqlite3_free(x.aNode); +} /* ** json_type(JSON) @@ -1014,7 +1150,7 @@ static void jsonTypeFunc( if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ JsonNode *pNode = x.aNode; - if( zPath ) pNode = jsonLookup(pNode, zPath); + if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); sqlite3_result_text(context, jsonType[pNode->eType], -1, SQLITE_STATIC); } sqlite3_free(x.aNode); @@ -1033,30 +1169,34 @@ int sqlite3_json_init( static const struct { const char *zName; int nArg; + int flag; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { - { "json_array", -1, jsonArrayFunc }, - { "json_array_length", 1, jsonArrayLengthFunc }, - { "json_array_length", 2, jsonArrayLengthFunc }, - { "json_extract", 2, jsonExtractFunc }, - { "json_object", -1, jsonObjectFunc }, - { "json_remove", -1, jsonRemoveFunc }, - { "json_replace", -1, jsonReplaceFunc }, - { "json_type", 1, jsonTypeFunc }, - { "json_type", 2, jsonTypeFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", 2, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, #if SQLITE_DEBUG /* DEBUG and TESTING functions */ - { "json_parse", 1, jsonParseFunc }, - { "json_test1", 1, jsonTest1Func }, - { "json_nodecount", 1, jsonNodeCountFunc }, + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, + { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; i Date: Tue, 18 Aug 2015 12:59:58 +0000 Subject: [PATCH 11/17] Comment clarification. No changes to code. FossilOrigin-Name: 71a966952cf8f1d699cf8362ea3f6204c6e16384 --- ext/misc/json.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index e980b52de6..274749726d 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -85,8 +85,8 @@ struct JsonNode { u8 iVal; /* Replacement value when JNODE_REPLACE */ u32 n; /* Bytes of content, or number of sub-nodes */ union { - const char *zJContent; /* JSON content */ - u32 iAppend; /* Appended content */ + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ } u; }; diff --git a/manifest b/manifest index 9469d276c7..c69e955bf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\sjson_set()\sand\sjson_insert(). -D 2015-08-18T02:28:03.829 +C Comment\sclarification.\s\sNo\schanges\sto\scode. +D 2015-08-18T12:59:58.393 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7669f34c487f5b328de6b508f374ee1e56558bb0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c d4e2e960549ec023e56f74ee72343416f5f9ef17 +F ext/misc/json.c 3b265167a9f8f006042bf0226a838c773a098c03 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1374,7 +1374,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c4bee65d93efc7f03f0f11817a068b02068d37c -R 1a1455dd49cebbaac366ccfaac1c0acb +P 4aa49656d98e2894f2faa8963f79462ee6165d40 +R c3c28209f7dd1255986fe872c35f7d65 U drh -Z 7cdd28284db3ca2451d1af500f69309d +Z 63f1a876b018989d4396ee5737a9cb33 diff --git a/manifest.uuid b/manifest.uuid index bb7757d569..787f49ef02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4aa49656d98e2894f2faa8963f79462ee6165d40 \ No newline at end of file +71a966952cf8f1d699cf8362ea3f6204c6e16384 \ No newline at end of file From cb6c6c6f45955918e2b73653ae7c6676288de0b2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Aug 2015 22:47:17 +0000 Subject: [PATCH 12/17] Add the json_each(JSON,PATH) table-valued-function. FossilOrigin-Name: 3335ac17bbcb09dc915173d69bf42048f84ad563 --- ext/misc/json.c | 277 +++++++++++++++++++++++++++++++++++++++++++++++- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 283 insertions(+), 8 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 274749726d..7e1d210e37 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -17,7 +17,9 @@ ** ** For the time being, all JSON is stored as pure text. (We might add ** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation.) +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -1156,6 +1158,276 @@ static void jsonTypeFunc( sqlite3_free(x.aNode); } +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + char *zJson; /* Input json */ + char *zPath; /* Path by which to filter zJson */ + JsonParse sParse; /* The input json */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_JSON 2 +#define JEACH_PATH 3 + + sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,json hidden,path hidden)"); + memset(pNew, 0, sizeof(*pNew)); + return SQLITE_OK; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object. */ +static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zPath); + sqlite3_free(p->sParse.aNode); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->zJson = 0; + p->zPath = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next top-level element of the current +** JSON string */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + return SQLITE_OK; +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + switch( i ){ + case JEACH_KEY: { + if( p->eType==JSON_OBJECT ){ + jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + }else{ + sqlite3_result_int64(ctx, p->iRowid); + } + break; + } + case JEACH_VALUE: { + if( p->eType==JSON_OBJECT ){ + jsonReturn(&p->sParse.aNode[p->i+1], ctx, 0); + }else{ + jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + } + break; + } + case JEACH_PATH: { + const char *zPath = p->zPath; + if( zPath==0 ) zPath = "$"; + sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); + break; + } + default: { + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zPath are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int jsonIdx = -1; + int pathIdx = -1; + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case JEACH_JSON: jsonIdx = i; break; + case JEACH_PATH: pathIdx = i; break; + default: /* no-op */ break; + } + } + if( jsonIdx<0 ){ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)2000000000; + }else{ + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; + if( pathIdx<0 ){ + pIdxInfo->idxNum = 1; + }else{ + pIdxInfo->aConstraintUsage[pathIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[pathIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zPath; + sqlite3_int64 n; + + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + if( idxNum&2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath==0 || zPath[0]!='$' ) return SQLITE_OK; + } + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, n+1); + if( jsonParse(&p->sParse, p->zJson) ){ + jsonEachCursorReset(p); + }else{ + JsonNode *pNode; + if( idxNum==3 ){ + n = sqlite3_value_bytes(argv[1]); + p->zPath = sqlite3_malloc( n+1 ); + if( p->zPath==0 ) return SQLITE_NOMEM; + memcpy(p->zPath, zPath, n+1); + pNode = jsonLookup(&p->sParse, 0, p->zPath+1, 0); + if( pNode==0 ){ + jsonEachCursorReset(p); + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + p->i++; + p->iEnd = p->i + pNode->n; + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpen, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -1199,5 +1471,8 @@ int sqlite3_json_init( (void*)&aFunc[i].flag, aFunc[i].xFunc, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "json_each", &jsonEachModule, 0); + } return rc; } diff --git a/manifest b/manifest index fa1a720a00..d5dc07e75e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\stable-valued-function\srowid\sfix. -D 2015-08-19T19:26:13.526 +C Add\sthe\sjson_each(JSON,PATH)\stable-valued-function. +D 2015-08-19T22:47:17.476 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 3b265167a9f8f006042bf0226a838c773a098c03 +F ext/misc/json.c 57a9f747b2813edc5f481eb808889d304a1c5f79 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96a5d44d9fcb7b159c04630ad2d956fe27df5a43 a325a08599759471047e234ef9cfcc3cb110aafd -R dcfe96af8a1b65c426e62b553bd60db8 +P a06a6392bd48baa8b9bac2624869c0cc7da7e779 +R 43d8c9f353202a0c5a0252660436f9ac U drh -Z 00eb910bc67976561bafd5a76ff4be3d +Z 41b78b71d0425818407424da18ee9720 diff --git a/manifest.uuid b/manifest.uuid index ab65dd6ddb..a1d2dfbb42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a06a6392bd48baa8b9bac2624869c0cc7da7e779 \ No newline at end of file +3335ac17bbcb09dc915173d69bf42048f84ad563 \ No newline at end of file From 6b43cc85318e15be7e42e4da8fab7024fd626ed3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Aug 2015 23:02:49 +0000 Subject: [PATCH 13/17] Fix the path lookup for objects so that it can handle quoted identifier names and non-alphanumerics in the identifier. FossilOrigin-Name: 87f5873004f69396baa7c67937342b4e32f79bda --- ext/misc/json.c | 20 +++++++++++++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 7e1d210e37..77bc6d6794 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -703,19 +703,29 @@ static JsonNode *jsonLookup( const char *zPath, /* The path to search */ int *pApnd /* Append nodes to complete path if not NULL */ ){ - u32 i, j, k; + u32 i, j, k, nKey; + const char *zKey; JsonNode *pRoot = &pParse->aNode[iRoot]; if( zPath[0]==0 ) return pRoot; if( zPath[0]=='.' ){ if( pRoot->eType!=JSON_OBJECT ) return 0; zPath++; - for(i=0; isalnum(zPath[i]); i++){} - if( i==0 ) return 0; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ) i++; + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ) return 0; j = 1; for(;;){ while( j<=pRoot->n ){ - if( pRoot[j].n==i+2 - && strncmp(&pRoot[j].u.zJContent[1],zPath,i)==0 + if( pRoot[j].n==nKey+2 + && strncmp(&pRoot[j].u.zJContent[1],zKey,nKey)==0 ){ return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); } diff --git a/manifest b/manifest index d5dc07e75e..28efbe5685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sjson_each(JSON,PATH)\stable-valued-function. -D 2015-08-19T22:47:17.476 +C Fix\sthe\spath\slookup\sfor\sobjects\sso\sthat\sit\scan\shandle\squoted\sidentifier\nnames\sand\snon-alphanumerics\sin\sthe\sidentifier. +D 2015-08-19T23:02:49.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 57a9f747b2813edc5f481eb808889d304a1c5f79 +F ext/misc/json.c 8654245a5d5aeb499fb0c00abce53f3836d52c20 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a06a6392bd48baa8b9bac2624869c0cc7da7e779 -R 43d8c9f353202a0c5a0252660436f9ac +P 3335ac17bbcb09dc915173d69bf42048f84ad563 +R bd9ac56614e747436a95e2948b6eea19 U drh -Z 41b78b71d0425818407424da18ee9720 +Z b6a64887dfaaae356da3f82cf4115061 diff --git a/manifest.uuid b/manifest.uuid index a1d2dfbb42..40c8eba985 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3335ac17bbcb09dc915173d69bf42048f84ad563 \ No newline at end of file +87f5873004f69396baa7c67937342b4e32f79bda \ No newline at end of file From 505ad2ca41b1b7026ab74680d11179824d69c653 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 17:33:11 +0000 Subject: [PATCH 14/17] Add the json_tree() virtual table. FossilOrigin-Name: 08c36e45f0d3a7b89caf823652d7543b76ac802a --- ext/misc/json.c | 348 +++++++++++++++++++++++++++++++++++------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 263 insertions(+), 99 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 77bc6d6794..26cde61056 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -34,7 +34,7 @@ typedef unsigned int u32; typedef unsigned char u8; /* Objects */ -typedef struct Json Json; +typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; @@ -42,7 +42,7 @@ typedef struct JsonParse JsonParse; ** under construction. Really, this is a generic string accumulator ** that can be and is used to create strings other than JSON. */ -struct Json { +struct JsonString { sqlite3_context *pCtx; /* Function context - put error messages here */ char *zBuf; /* Append JSON content here */ u64 nAlloc; /* Bytes of storage available in zBuf[] */ @@ -89,6 +89,7 @@ struct JsonNode { union { const char *zJContent; /* Content for INT, REAL, and STRING */ u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ } u; }; @@ -99,44 +100,36 @@ struct JsonParse { u32 nAlloc; /* Number of slots of aNode[] allocated */ JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ u8 oom; /* Set to true if out of memory */ }; -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ -/* Set the Json object to an empty string +/* Set the JsonString object to an empty string */ -static void jsonZero(Json *p){ +static void jsonZero(JsonString *p){ p->zBuf = p->zSpace; p->nAlloc = sizeof(p->zSpace); p->nUsed = 0; p->bStatic = 1; } -/* Initialize the Json object +/* Initialize the JsonString object */ -static void jsonInit(Json *p, sqlite3_context *pCtx){ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ p->pCtx = pCtx; p->bErr = 0; jsonZero(p); } -/* Free all allocated memory and reset the Json object back to its +/* Free all allocated memory and reset the JsonString object back to its ** initial state. */ -static void jsonReset(Json *p){ +static void jsonReset(JsonString *p){ if( !p->bStatic ) sqlite3_free(p->zBuf); jsonZero(p); } @@ -144,7 +137,7 @@ static void jsonReset(Json *p){ /* Report an out-of-memory (OOM) condition */ -static void jsonOom(Json *p){ +static void jsonOom(JsonString *p){ if( !p->bErr ){ p->bErr = 1; sqlite3_result_error_nomem(p->pCtx); @@ -155,7 +148,7 @@ static void jsonOom(Json *p){ /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ -static int jsonGrow(Json *p, u32 N){ +static int jsonGrow(JsonString *p, u32 N){ u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; char *zNew; if( p->bStatic ){ @@ -180,9 +173,9 @@ static int jsonGrow(Json *p, u32 N){ return SQLITE_OK; } -/* Append N bytes from zIn onto the end of the Json string. +/* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; @@ -191,14 +184,14 @@ static void jsonAppendRaw(Json *p, const char *zIn, u32 N){ #ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ -static void jsonAppend(Json *p, const char *zIn){ +static void jsonAppend(JsonString *p, const char *zIn){ jsonAppendRaw(p, zIn, (u32)strlen(zIn)); } #endif /* Append a single character */ -static void jsonAppendChar(Json *p, char c){ +static void jsonAppendChar(JsonString *p, char c){ if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; p->zBuf[p->nUsed++] = c; } @@ -206,19 +199,19 @@ static void jsonAppendChar(Json *p, char c){ /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. */ -static void jsonAppendSeparator(Json *p){ +static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; if( c!='[' && c!='{' ) jsonAppendChar(p, ','); } -/* Append the N-byte string in zIn to the end of the Json string +/* Append the N-byte string in zIn to the end of the JsonString string ** under construction. Enclose the string in "..." and escape ** any double-quotes or backslash characters contained within the ** string. */ -static void jsonAppendString(Json *p, const char *zIn, u32 N){ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ u32 i; if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; @@ -238,7 +231,7 @@ static void jsonAppendString(Json *p, const char *zIn, u32 N){ ** construction. */ static void jsonAppendValue( - Json *p, /* Append to this JSON string */ + JsonString *p, /* Append to this JSON string */ sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ @@ -273,7 +266,7 @@ static void jsonAppendValue( /* Make the JSON in p the result of the SQL function. */ -static void jsonResult(Json *p){ +static void jsonResult(JsonString *p){ if( p->bErr==0 ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, @@ -283,6 +276,36 @@ static void jsonResult(Json *p){ assert( p->bStatic ); } +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + /* ** Convert the JsonNode pNode into a pure JSON string and ** append to pOut. Subsubstructure is also included. Return @@ -290,7 +313,7 @@ static void jsonResult(Json *p){ */ static void jsonRenderNode( JsonNode *pNode, /* The node to render */ - Json *pOut, /* Write JSON here */ + JsonString *pOut, /* Write JSON here */ sqlite3_value **aReplace /* Replacement values */ ){ switch( pNode->eType ){ @@ -332,7 +355,7 @@ static void jsonRenderNode( jsonAppendSeparator(pOut); jsonRenderNode(&pNode[j], pOut, aReplace); } - j += jsonSize(&pNode[j]); + j += jsonNodeSize(&pNode[j]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; pNode = &pNode[pNode->u.iAppend]; @@ -356,7 +379,7 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j+1], pOut, aReplace); } } - j += 1 + jsonSize(&pNode[j+1]); + j += 1 + jsonNodeSize(&pNode[j+1]); } if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; pNode = &pNode[pNode->u.iAppend]; @@ -478,7 +501,7 @@ static void jsonReturn( } case JSON_ARRAY: case JSON_OBJECT: { - Json s; + JsonString s; jsonInit(&s, pCtx); jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); @@ -676,15 +699,51 @@ static int jsonParse(JsonParse *pParse, const char *zJson){ if( zJson[i] ) i = -1; } if( i<0 ){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; + jsonParseReset(pParse); return 1; } return 0; } +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); + if( aUp==0 ) return SQLITE_NOMEM; + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + /* forward declaration */ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*); @@ -730,7 +789,7 @@ static JsonNode *jsonLookup( return jsonLookup(pParse, iRoot+j+1, &zPath[i], pApnd); } j++; - j += jsonSize(&pRoot[j]); + j += jsonNodeSize(&pRoot[j]); } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; iRoot += pRoot->u.iAppend; @@ -759,7 +818,7 @@ static JsonNode *jsonLookup( j = 1; for(;;){ while( i>0 && j<=pRoot->n ){ - j += jsonSize(&pRoot[j]); + j += jsonNodeSize(&pRoot[j]); i--; } if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; @@ -820,8 +879,8 @@ static void jsonParseFunc( int argc, sqlite3_value **argv ){ - Json s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ u32 i; char zBuf[100]; @@ -838,7 +897,7 @@ static void jsonParseFunc( jsonAppendRaw(&s, "\n", 1); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); jsonResult(&s); } @@ -853,7 +912,7 @@ static void jsonTest1Func( JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; jsonReturn(x.aNode, context, 0); - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -868,7 +927,7 @@ static void jsonNodeCountFunc( JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; sqlite3_result_int64(context, x.nNode); - sqlite3_free(x.aNode); + jsonParseReset(&x); } #endif /* SQLITE_DEBUG */ @@ -887,7 +946,7 @@ static void jsonArrayFunc( sqlite3_value **argv ){ int i; - Json jx; + JsonString jx; jsonInit(&jx, context); jsonAppendChar(&jx, '['); @@ -932,11 +991,11 @@ static void jsonArrayLengthFunc( if( pNode->eType==JSON_ARRAY ){ assert( (pNode->jnFlags & JNODE_APPEND)==0 ); for(i=1; i<=pNode->n; n++){ - i += jsonSize(&pNode[i]); + i += jsonNodeSize(&pNode[i]); } } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } sqlite3_result_int64(context, n); } @@ -965,7 +1024,7 @@ static void jsonExtractFunc( if( pNode ){ jsonReturn(pNode, context, 0); } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -979,7 +1038,7 @@ static void jsonObjectFunc( sqlite3_value **argv ){ int i; - Json jx; + JsonString jx; const char *z; u32 n; @@ -1039,7 +1098,7 @@ static void jsonRemoveFunc( jsonReturn(x.aNode, context, 0); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -1082,8 +1141,9 @@ static void jsonReplaceFunc( jsonReturn(x.aNode, context, argv); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } + /* ** json_set(JSON, PATH, VALUE, ...) ** @@ -1133,7 +1193,7 @@ static void jsonSetFunc( jsonReturn(x.aNode, context, argv); } } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /* @@ -1165,7 +1225,7 @@ static void jsonTypeFunc( if( zPath ) pNode = jsonLookup(&x, 0, zPath, 0); sqlite3_result_text(context, jsonType[pNode->eType], -1, SQLITE_STATIC); } - sqlite3_free(x.aNode); + jsonParseReset(&x); } /**************************************************************************** @@ -1174,13 +1234,14 @@ static void jsonTypeFunc( typedef struct JsonEachCursor JsonEachCursor; struct JsonEachCursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - u32 iRowid; /* The rowid */ - u32 i; /* Index in sParse.aNode[] of current row */ - u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ - char *zJson; /* Input json */ - char *zPath; /* Path by which to filter zJson */ - JsonParse sParse; /* The input json */ + u32 iRowid; /* The rowid */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zPath; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ }; /* Constructor for the json_each virtual table */ @@ -1192,18 +1253,26 @@ static int jsonEachConnect( char **pzErr ){ sqlite3_vtab *pNew; - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; + int rc; /* Column numbers */ #define JEACH_KEY 0 #define JEACH_VALUE 1 -#define JEACH_JSON 2 -#define JEACH_PATH 3 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_JSON 6 +#define JEACH_PATH 7 - sqlite3_declare_vtab(db, "CREATE TABLE x(key,value,json hidden,path hidden)"); - memset(pNew, 0, sizeof(*pNew)); - return SQLITE_OK; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,json hidden,path hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; } /* destructor for json_each virtual table */ @@ -1212,8 +1281,8 @@ static int jsonEachDisconnect(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* constructor for a JsonEachCursor object. */ -static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ JsonEachCursor *pCur; pCur = sqlite3_malloc( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; @@ -1222,17 +1291,26 @@ static int jsonEachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + /* Reset a JsonEachCursor back to its original state. Free any memory ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zJson); sqlite3_free(p->zPath); - sqlite3_free(p->sParse.aNode); + jsonParseReset(&p->sParse); p->iRowid = 0; p->i = 0; p->iEnd = 0; p->eType = 0; - memset(&p->sParse, 0, sizeof(p->sParse)); p->zJson = 0; p->zPath = 0; } @@ -1252,18 +1330,39 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ return p->i >= p->iEnd; } -/* Advance the cursor to the next top-level element of the current -** JSON string */ -static int jsonEachNext(sqlite3_vtab_cursor *cur){ +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNextTree(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->i==0 ){ + p->i = 1; + }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ + p->i += 2; + }else{ + p->i++; + } + p->iRowid++; + if( p->isParse.nNode ){ + JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; + if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ + p->sParse.aNode[p->i].u.iKey = 0; + } + } + return SQLITE_OK; +} + +/* Advance the cursor to the next element for json_each() */ +static int jsonEachNextEach(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; switch( p->eType ){ case JSON_ARRAY: { - p->i += jsonSize(&p->sParse.aNode[p->i]); + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); p->iRowid++; break; } case JSON_OBJECT: { - p->i += 1 + jsonSize(&p->sParse.aNode[p->i+1]); + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); p->iRowid++; break; } @@ -1282,20 +1381,46 @@ static int jsonEachColumn( int i /* Which column to return */ ){ JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; switch( i ){ case JEACH_KEY: { if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse.aNode[p->i], ctx, 0); - }else{ - sqlite3_result_int64(ctx, p->iRowid); + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey - 1; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, iKey); } break; } case JEACH_VALUE: { - if( p->eType==JSON_OBJECT ){ - jsonReturn(&p->sParse.aNode[p->i+1], ctx, 0); - }else{ - jsonReturn(&p->sParse.aNode[p->i], ctx, 0); + if( p->eType==JSON_OBJECT ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( p->eType==JSON_OBJECT ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( p->eType==JSON_OBJECT ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, p->i + (p->eType==JSON_OBJECT)); + break; + } + case JEACH_PARENT: { + if( p->i>0 && p->bRecursive ){ + sqlite3_result_int64(ctx, p->sParse.aUp[p->i]); } break; } @@ -1306,6 +1431,7 @@ static int jsonEachColumn( break; } default: { + assert( i==JEACH_JSON ); sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); break; } @@ -1345,9 +1471,9 @@ static int jsonEachBestIndex( } if( jsonIdx<0 ){ pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = (double)2000000000; + pIdxInfo->estimatedCost = 1e99; }else{ - pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedCost = 1.0; pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; if( pathIdx<0 ){ @@ -1384,7 +1510,9 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, n+1); - if( jsonParse(&p->sParse, p->zJson) ){ + if( jsonParse(&p->sParse, p->zJson) + || (p->bRecursive && jsonParseFindParents(&p->sParse)) + ){ jsonEachCursorReset(p); }else{ JsonNode *pNode; @@ -1421,10 +1549,10 @@ static sqlite3_module jsonEachModule = { jsonEachBestIndex, /* xBestIndex */ jsonEachDisconnect, /* xDisconnect */ 0, /* xDestroy */ - jsonEachOpen, /* xOpen - open a cursor */ + jsonEachOpenEach, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ + jsonEachNextEach, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ @@ -1437,6 +1565,35 @@ static sqlite3_module jsonEachModule = { 0, /* xRename */ }; +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNextTree, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ +}; + +/**************************************************************************** +** The following routine is the only publically visible identifier in this +** file. Call the following routine in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ #ifdef _WIN32 __declspec(dllexport) @@ -1473,6 +1630,13 @@ int sqlite3_json_init( { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ for(i=0; i Date: Fri, 21 Aug 2015 20:02:48 +0000 Subject: [PATCH 15/17] Add the fullkey column to both json_each() and json_tree(). FossilOrigin-Name: 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 --- ext/misc/json.c | 158 ++++++++++++++++++++++++++++++++++-------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 120 insertions(+), 52 deletions(-) diff --git a/ext/misc/json.c b/ext/misc/json.c index 26cde61056..9d9bf63896 100644 --- a/ext/misc/json.c +++ b/ext/misc/json.c @@ -27,6 +27,7 @@ SQLITE_EXTENSION_INIT1 #include #include #include +#include /* Unsigned integer types */ typedef sqlite3_uint64 u64; @@ -181,6 +182,17 @@ static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ p->nUsed += N; } +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + #ifdef SQLITE_DEBUG /* Append the zero-terminated string zIn */ @@ -1256,17 +1268,19 @@ static int jsonEachConnect( int rc; /* Column numbers */ -#define JEACH_KEY 0 -#define JEACH_VALUE 1 -#define JEACH_TYPE 2 -#define JEACH_ATOM 3 -#define JEACH_ID 4 -#define JEACH_PARENT 5 -#define JEACH_JSON 6 -#define JEACH_PATH 7 +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_JSON 7 +#define JEACH_PATH 8 rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,json hidden,path hidden)"); + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey," + "json HIDDEN,path HIDDEN)"); if( rc==SQLITE_OK ){ pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; @@ -1331,47 +1345,71 @@ static int jsonEachEof(sqlite3_vtab_cursor *cur){ } /* Advance the cursor to the next element for json_tree() */ -static int jsonEachNextTree(sqlite3_vtab_cursor *cur){ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->i==0 ){ - p->i = 1; - }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ - p->i += 2; + if( p->bRecursive ){ + if( p->i==0 ){ + p->i = 1; + }else if( p->sParse.aNode[p->sParse.aUp[p->i]].eType==JSON_OBJECT ){ + p->i += 2; + }else{ + p->i++; + } + p->iRowid++; + if( p->isParse.nNode ){ + JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; + if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ + p->sParse.aNode[p->i].u.iKey = 0; + } + } }else{ - p->i++; - } - p->iRowid++; - if( p->isParse.nNode ){ - JsonNode *pUp = &p->sParse.aNode[p->sParse.aUp[p->i]]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ) pUp->u.iKey++; - if( p->sParse.aNode[p->i].eType==JSON_ARRAY ){ - p->sParse.aNode[p->i].u.iKey = 0; + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } } } return SQLITE_OK; } -/* Advance the cursor to the next element for json_each() */ -static int jsonEachNextEach(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( pNode->eType>=JSON_ARRAY ) pNode--; + assert( pNode->eType==JSON_STRING ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); } - return SQLITE_OK; } /* Return the value of a column */ @@ -1424,9 +1462,38 @@ static int jsonEachColumn( } break; } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zPath ){ + jsonAppendRaw(&x, p->zPath, (int)strlen(p->zPath)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else{ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } case JEACH_PATH: { const char *zPath = p->zPath; - if( zPath==0 ) zPath = "$"; + if( zPath==0 ){ + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + zPath = "$"; + } sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); break; } @@ -1517,6 +1584,7 @@ static int jsonEachFilter( }else{ JsonNode *pNode; if( idxNum==3 ){ + p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); p->zPath = sqlite3_malloc( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; @@ -1552,7 +1620,7 @@ static sqlite3_module jsonEachModule = { jsonEachOpenEach, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNextEach, /* xNext - advance a cursor */ + jsonEachNext, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ @@ -1576,7 +1644,7 @@ static sqlite3_module jsonTreeModule = { jsonEachOpenTree, /* xOpen - open a cursor */ jsonEachClose, /* xClose - close a cursor */ jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNextTree, /* xNext - advance a cursor */ + jsonEachNext, /* xNext - advance a cursor */ jsonEachEof, /* xEof - check for end of scan */ jsonEachColumn, /* xColumn - read data */ jsonEachRowid, /* xRowid - read data */ diff --git a/manifest b/manifest index 6e096c9ed7..a20995d28b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sheader\sfile\sfixes\sfrom\strunk. -D 2015-08-21T19:56:45.243 +C Add\sthe\sfullkey\scolumn\sto\sboth\sjson_each()\sand\sjson_tree(). +D 2015-08-21T20:02:48.424 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 522d833e6c4ae55b345cf25776aab335323d2f59 +F ext/misc/json.c 92bb4e5fe2956564d23f933e2140ce8e086d988b F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 08c36e45f0d3a7b89caf823652d7543b76ac802a da3c9df09c46564353218d0163e378b880a3ce62 -R 7de9031da9783d58b8a952bc530228c5 +P 7c2713e98ffb5f0d96eb7de9514eab43f0712011 +R a1c5d56719dd38b21713ad204a82abd5 U drh -Z 5f4a9a3778df860327f346518d686e5d +Z 8567373ad141fc34581702b17b3fb646 diff --git a/manifest.uuid b/manifest.uuid index ded172cdf0..7f4f0b9760 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c2713e98ffb5f0d96eb7de9514eab43f0712011 \ No newline at end of file +15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 \ No newline at end of file From f28ed795dbda2cba597368fc4c85b3ee487bf531 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 20:12:43 +0000 Subject: [PATCH 16/17] Change the name of the json loadable extension to "json1.c", in anticipation of having future major changes to the interface. FossilOrigin-Name: d0d4bec9e3d8829a2d488f2742f1650214fa716a --- ext/misc/{json.c => json1.c} | 0 manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename ext/misc/{json.c => json1.c} (100%) diff --git a/ext/misc/json.c b/ext/misc/json1.c similarity index 100% rename from ext/misc/json.c rename to ext/misc/json1.c diff --git a/manifest b/manifest index a20995d28b..25d78bc23f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfullkey\scolumn\sto\sboth\sjson_each()\sand\sjson_tree(). -D 2015-08-21T20:02:48.424 +C Change\sthe\sname\sof\sthe\sjson\sloadable\sextension\sto\s"json1.c",\sin\santicipation\nof\shaving\sfuture\smajor\schanges\sto\sthe\sinterface. +D 2015-08-21T20:12:43.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json.c 92bb4e5fe2956564d23f933e2140ce8e086d988b +F ext/misc/json1.c 92bb4e5fe2956564d23f933e2140ce8e086d988b w ext/misc/json.c F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7c2713e98ffb5f0d96eb7de9514eab43f0712011 -R a1c5d56719dd38b21713ad204a82abd5 +P 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 +R 46781c36cc64ae92318031e0c6494ecf U drh -Z 8567373ad141fc34581702b17b3fb646 +Z cf5f4e21b0acc856e5deddedc237451f diff --git a/manifest.uuid b/manifest.uuid index 7f4f0b9760..02752a564b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 \ No newline at end of file +d0d4bec9e3d8829a2d488f2742f1650214fa716a \ No newline at end of file From 6fd5c1e0e859f08977defd48ea9385036c055d7c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Aug 2015 20:37:12 +0000 Subject: [PATCH 17/17] Fixes for compiler warnings and errors in the makefiles. Rename the one test script to json101.test. FossilOrigin-Name: 9ff6ccde5f26f18073587c320290570854ffc833 --- Makefile.in | 2 +- Makefile.msc | 2 +- ext/misc/json1.c | 46 +++++++++++++++++++++---------- main.mk | 2 +- manifest | 20 +++++++------- manifest.uuid | 2 +- test/{json1.test => json101.test} | 0 7 files changed, 45 insertions(+), 29 deletions(-) rename test/{json1.test => json101.test} (100%) diff --git a/Makefile.in b/Makefile.in index 38df655816..8f3d6f7dfe 100644 --- a/Makefile.in +++ b/Makefile.in @@ -417,7 +417,7 @@ TESTSRC += \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json.c \ + $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/Makefile.msc b/Makefile.msc index 5f64dfcc5e..441c499df8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1083,7 +1083,7 @@ TESTEXT = \ $(TOP)\ext\fts5\fts5_tcl.c \ $(TOP)\ext\fts5\fts5_test_mi.c \ $(TOP)\ext\misc\ieee754.c \ - $(TOP)\ext\misc\json.c \ + $(TOP)\ext\misc\json1.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 9d9bf63896..66173482c4 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -29,6 +29,8 @@ SQLITE_EXTENSION_INIT1 #include #include +#define UNUSED_PARAM(X) (void)(X) + /* Unsigned integer types */ typedef sqlite3_uint64 u64; typedef unsigned int u32; @@ -159,7 +161,7 @@ static int jsonGrow(JsonString *p, u32 N){ jsonOom(p); return SQLITE_NOMEM; } - memcpy(zNew, p->zBuf, p->nUsed); + memcpy(zNew, p->zBuf, (size_t)p->nUsed); p->zBuf = zNew; p->bStatic = 0; }else{ @@ -578,7 +580,6 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c=='{' ){ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); @@ -605,7 +606,6 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ }else if( c=='[' ){ /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; for(j=i+1;;j++){ while( isspace(pParse->zJson[j]) ){ j++; } x = jsonParseValue(pParse, j); @@ -938,7 +938,7 @@ static void jsonNodeCountFunc( ){ JsonParse x; /* The parse */ if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; - sqlite3_result_int64(context, x.nNode); + sqlite3_result_int64(context, (sqlite3_int64)x.nNode); jsonParseReset(&x); } #endif /* SQLITE_DEBUG */ @@ -1099,7 +1099,7 @@ static void jsonRemoveFunc( if( argc<1 ) return; if( jsonParse(&x, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ - for(i=1; iiRowid; } - sqlite3_result_int64(ctx, iKey); + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); } break; } @@ -1453,12 +1459,12 @@ static int jsonEachColumn( break; } case JEACH_ID: { - sqlite3_result_int64(ctx, p->i + (p->eType==JSON_OBJECT)); + sqlite3_result_int64(ctx, (sqlite3_int64)p->i + (p->eType==JSON_OBJECT)); break; } case JEACH_PARENT: { if( p->i>0 && p->bRecursive ){ - sqlite3_result_int64(ctx, p->sParse.aUp[p->i]); + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); } break; } @@ -1526,6 +1532,8 @@ static int jsonEachBestIndex( int jsonIdx = -1; int pathIdx = -1; const struct sqlite3_index_constraint *pConstraint; + + UNUSED_PARAM(tab); pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ if( pConstraint->usable==0 ) continue; @@ -1565,6 +1573,8 @@ static int jsonEachFilter( const char *zPath; sqlite3_int64 n; + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); jsonEachCursorReset(p); if( idxNum==0 ) return SQLITE_OK; z = (const char*)sqlite3_value_text(argv[0]); @@ -1574,9 +1584,9 @@ static int jsonEachFilter( if( zPath==0 || zPath[0]!='$' ) return SQLITE_OK; } n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc( n+1 ); + p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, n+1); + memcpy(p->zJson, z, (size_t)n+1); if( jsonParse(&p->sParse, p->zJson) || (p->bRecursive && jsonParseFindParents(&p->sParse)) ){ @@ -1586,9 +1596,9 @@ static int jsonEachFilter( if( idxNum==3 ){ p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); - p->zPath = sqlite3_malloc( n+1 ); + p->zPath = sqlite3_malloc64( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; - memcpy(p->zPath, zPath, n+1); + memcpy(p->zPath, zPath, (size_t)n+1); pNode = jsonLookup(&p->sParse, 0, p->zPath+1, 0); if( pNode==0 ){ jsonEachCursorReset(p); @@ -1631,6 +1641,9 @@ static sqlite3_module jsonEachModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; /* The methods of the json_tree virtual table. */ @@ -1655,6 +1668,9 @@ static sqlite3_module jsonTreeModule = { 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ }; /**************************************************************************** @@ -1672,7 +1688,7 @@ int sqlite3_json_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; - int i; + unsigned int i; static const struct { const char *zName; int nArg; diff --git a/main.mk b/main.mk index e18370a3a1..9988fe2775 100644 --- a/main.mk +++ b/main.mk @@ -297,7 +297,7 @@ TESTSRC += \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json.c \ + $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ diff --git a/manifest b/manifest index 25d78bc23f..3c1e7613e4 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Change\sthe\sname\sof\sthe\sjson\sloadable\sextension\sto\s"json1.c",\sin\santicipation\nof\shaving\sfuture\smajor\schanges\sto\sthe\sinterface. -D 2015-08-21T20:12:43.689 +C Fixes\sfor\scompiler\swarnings\sand\serrors\sin\sthe\smakefiles.\s\sRename\sthe\none\stest\sscript\sto\sjson101.test. +D 2015-08-21T20:37:12.208 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 90f3097efb9a53f5fc59a4f8a08be07cf9f52c02 +F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc f6ed2ce438f711641dc63d762bb0d41aa659518f +F Makefile.msc 10af19cc089862481d49b347acd99c02635ddc49 F Makefile.vxworks e1b65dea203f054e71653415bd8f96dcaed47858 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION ccfc4d1576dbfdeece0a4372a2e6a2e37d3e7975 @@ -192,7 +192,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 92bb4e5fe2956564d23f933e2140ce8e086d988b w ext/misc/json.c +F ext/misc/json1.c f83f02ec4d7cc18794bd7ac046f1bb5905805b44 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -258,7 +258,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a551a3b8a398e1e64fefbb14bc142aa9584238fc +F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -810,7 +810,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json1.test 950ed4e8deb8ad4c10bd4fbc858eb54143de9867 +F test/json101.test 950ed4e8deb8ad4c10bd4fbc858eb54143de9867 w test/json1.test F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1378,7 +1378,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 15dd99431e9ddd0fbdbb8dcc921687b0c6d26a29 -R 46781c36cc64ae92318031e0c6494ecf +P d0d4bec9e3d8829a2d488f2742f1650214fa716a +R 0dbca2db2baae780df721a60ce2b8302 U drh -Z cf5f4e21b0acc856e5deddedc237451f +Z f736e312f15e55183b503d223fff9c6b diff --git a/manifest.uuid b/manifest.uuid index 02752a564b..a805312be8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0d4bec9e3d8829a2d488f2742f1650214fa716a \ No newline at end of file +9ff6ccde5f26f18073587c320290570854ffc833 \ No newline at end of file diff --git a/test/json1.test b/test/json101.test similarity index 100% rename from test/json1.test rename to test/json101.test