Cache the JSON parse used by json_extract().

FossilOrigin-Name: 44ca6c2c4639f3c50ae9233ee299ff0fc4566462c31f28d8676f8de7ffdcd7f0

ext/misc/json1.c

@@ -171,6 +171,7 @@ struct JsonParse {
   u8 oom;            /* Set to true if out of memory */
   u8 nErr;           /* Number of errors seen */
   u16 iDepth;        /* Nesting depth */
+  int nJson;         /* Length of the zJson string in bytes */
 };
 
 /*
@@ -413,6 +414,14 @@ static void jsonParseReset(JsonParse *pParse){
   pParse->aUp = 0;
 }
 
+/*
+** Free a JsonParse object that was obtained from sqlite3_malloc().
+*/
+static void jsonParseFree(JsonParse *pParse){
+  jsonParseReset(pParse);
+  sqlite3_free(pParse);
+}
+
 /*
 ** Convert the JsonNode pNode into a pure JSON string and
 ** append to pOut.  Subsubstructure is also included.  Return
@@ -964,6 +973,49 @@ static int jsonParseFindParents(JsonParse *pParse){
   return SQLITE_OK;
 }
 
+/*
+** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
+*/
+#define JSON_CACHE_ID  (-429938)
+
+/*
+** Obtain a complete parse of the JSON found in the first argument
+** of the argv array.  Use the sqlite3_get_auxdata() cache for this
+** parse if it is available.  If the cache is not available or if it
+** is no longer valid, parse the JSON again and return the new parse,
+** and also register the new parse so that it will be available for
+** future sqlite3_get_auxdata() calls.
+*/
+static JsonParse *jsonParseCached(
+  sqlite3_context *pCtx,
+  sqlite3_value **argv
+){
+  const char *zJson = (const char*)sqlite3_value_text(argv[0]);
+  int nJson = sqlite3_value_bytes(argv[0]);
+  JsonParse *p;
+  if( zJson==0 ) return 0;
+  p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+  if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
+    p->nErr = 0;
+    return p; /* The cached entry matches, so return it */
+  }
+  p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
+  if( p==0 ){
+    sqlite3_result_error_nomem(pCtx);
+    return 0;
+  }
+  memset(p, 0, sizeof(*p));
+  p->zJson = (char*)&p[1];
+  memcpy((char*)p->zJson, zJson, nJson+1);
+  if( jsonParse(p, pCtx, p->zJson) ){
+    sqlite3_free(p);
+    return 0;
+  }
+  p->nJson = nJson;
+  sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
+  return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+}
+
 /*
 ** Compare the OBJECT label at pNode against zKey,nKey.  Return true on
 ** a match.
@@ -1329,29 +1381,30 @@ static void jsonArrayLengthFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
   sqlite3_int64 n = 0;
   u32 i;
   JsonNode *pNode;
 
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
-  assert( x.nNode );
+  p = jsonParseCached(ctx, argv);
+  if( p==0 ) return;
+  assert( p->nNode );
   if( argc==2 ){
     const char *zPath = (const char*)sqlite3_value_text(argv[1]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
+    pNode = jsonLookup(p, zPath, 0, ctx);
   }else{
-    pNode = x.aNode;
+    pNode = p->aNode;
   }
   if( pNode==0 ){
-    x.nErr = 1;
-  }else if( pNode->eType==JSON_ARRAY ){
+    return;
+  }
+  if( pNode->eType==JSON_ARRAY ){
     assert( (pNode->jnFlags & JNODE_APPEND)==0 );
     for(i=1; i<=pNode->n; n++){
       i += jsonNodeSize(&pNode[i]);
     }
   }
-  if( x.nErr==0 ) sqlite3_result_int64(ctx, n);
-  jsonParseReset(&x);
+  sqlite3_result_int64(ctx, n);
 }
 
 /*
@@ -1367,20 +1420,21 @@ static void jsonExtractFunc(
   int argc,
   sqlite3_value **argv
 ){
-  JsonParse x;          /* The parse */
+  JsonParse *p;          /* The parse */
   JsonNode *pNode;
   const char *zPath;
   JsonString jx;
   int i;
 
   if( argc<2 ) return;
-  if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return;
+  p = jsonParseCached(ctx, argv);
+  if( p==0 ) return;
   jsonInit(&jx, ctx);
   jsonAppendChar(&jx, '[');
   for(i=1; i<argc; i++){
     zPath = (const char*)sqlite3_value_text(argv[i]);
-    pNode = jsonLookup(&x, zPath, 0, ctx);
-    if( x.nErr ) break;
+    pNode = jsonLookup(p, zPath, 0, ctx);
+    if( p->nErr ) break;
     if( argc>2 ){
       jsonAppendSeparator(&jx);
       if( pNode ){
@@ -1398,7 +1452,6 @@ static void jsonExtractFunc(
     sqlite3_result_subtype(ctx, JSON_SUBTYPE);
   }
   jsonReset(&jx);
-  jsonParseReset(&x);
 }
 
 /* This is the RFC 7396 MergePatch algorithm.

manifest

@@ -1,5 +1,5 @@
-C Do\snot\sinvoke\scodec\smacros\sfor\sin-memory\ssubjournals.
-D 2017-05-10T12:49:50.041
+C Cache\sthe\sJSON\sparse\sused\sby\sjson_extract().
+D 2017-05-11T16:49:59.245
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -219,7 +219,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
-F ext/misc/json1.c bd3bbdefb6024c5e40287d61cdf876587c18a6e60fbe8cd6bcdde10eefd14bb1
+F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
 F ext/misc/memvfs.c e5225bc22e79dde6b28380f3a068ddf600683a33
 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342
 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e
@@ -341,7 +341,7 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F src/alter.c 3b23977620ce9662ac54443f65b87ba996e36121
 F src/analyze.c 0d0ccf7520a201d8747ea2f02c92c26e26f801bc161f714f27b9f7630dde0421
 F src/attach.c 8c476f8bd5d2afe11d925f890d30e527e5b0ce43
-F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
+F src/auth.c 79f96c6f33bf0e5da8d1c282cee5ebb1852bb8a6ccca3e485d7c459b035d9c3c
 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca
@@ -354,7 +354,7 @@ F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c
 F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7
 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d
-F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c
+F src/delete.c 665e705641e5815c3f32d05820d1a5aa630274e568af73f377fdbc614fcf40b4
 F src/expr.c c980b2c9291a12a0f1de1e1e1aaa72c4579ded716e1e3a3ac1c3d898ba0df0a1
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c db65492ae549c3b548c9ef1f279ce1684f1c473b116e1c56a90878cd5dcf968d
@@ -402,16 +402,16 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c 4f0adefaa5e9417459b07757e0f6060cac97930a86f0fba9797bab233ced66c0
+F src/select.c d74b1cde1d9ca6d08bec50b60a5be19440273646bc8ae16648d748c38161d5b7
 F src/shell.c a37d96b20b3644d0eb905df5aa7a0fcf9f6e73c15898337230c760a24a8df794
-F src/sqlite.h.in eeb1da70a61d52e1d58e5b55446b85bbac571699421d3cf857421c56214013ce
+F src/sqlite.h.in 8dd468837a4f6d76713e3a4cc65bea48095009038593d41040ab46c1b351197f
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28
 F src/sqliteInt.h aea3aa1b81e0d07d5b1c39b8c5a54a1dc5e4f10136cb63da392aef9eb2a5108b
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
-F src/tclsqlite.c 6c2151b6d8d98e183a04466d40df8889c0574d79
+F src/tclsqlite.c c8cf60d0c5411d5e70e7c136470d29dbe760d250f55198b71682c67086524e4a
 F src/test1.c c99f0442918a7a5d5b68a95d6024c211989e6c782c15ced5a558994baaf76a5e
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee
@@ -471,9 +471,9 @@ F src/util.c fc081ec6f63448dcd80d3dfad35baecfa104823254a815b081a4d9fe76e1db23
 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569
 F src/vdbe.c 9bac2bc2313ed682e6f48ccff6644d3263341885bfcbb3cdea7b720c722be2d5
 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848
-F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade
-F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860
-F src/vdbeaux.c b4999c744e59deba7ab8733640219ecbc771721b362d7e26ce4c57db575ad80b
+F src/vdbeInt.h 1ecdacc1322fdd3241ec30c32a480e328a6f864e532dc53fae8e0ab68121aebf
+F src/vdbeapi.c dc904b3c5e459727993c2421e653e29d63223846d129fae98adc782b0a996481
+F src/vdbeaux.c 01dcf59b2a96bd3cc9db8c0d7f266518d113587459a2b3316279c4f9c90f28a9
 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9
 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89
 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c
@@ -526,9 +526,9 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d
 F test/attach3.test c59d92791070c59272e00183b7353eeb94915976
 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c
 F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0
-F test/auth.test c6ede04bee65637ff354b43fc1235aa560c0863e
+F test/auth.test 32ee0e98593c9ea73870d9b0c8e50c2f43371d9fede388c09e9477c6bf5f8aab
 F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1
-F test/auth3.test 0d48b901cf111c14b4b1b5205c7d28f1a278190f
+F test/auth3.test db21405b95257c24d29273b6b31d0efc59e1d337e3d5804ba2d1fd4897b1ae49
 F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec
 F test/autoinc.test 6ae8fb69c9f656962464ae4e6667045d0dfc3b46
 F test/autoindex1.test 14b63a9f1e405fe6d5bfc8c8d00249c2ebaf13ea
@@ -791,13 +791,13 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
 F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3
 F test/fts3expr5.test f9abfffbf5e53d48a33e12a1e8f8ba2c551c9b49
-F test/fts3fault.test 268e9589f44f49d6694ef39a293f0e0f80c6230fb01cc6f34325412acceb99ae
+F test/fts3fault.test 3764ecffb3d341c5b05b3abe64153f385880035e67706ca2fc719e5d3352aedb
 F test/fts3fault2.test 536bbe01fe2946ec24b063a5eee813e8fd90354a6ca0b8f941d299c405edd17e
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3join.test 34750f3ce1e29b2749eaf0f1be2fa6301c5d50da
 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6
 F test/fts3matchinfo.test ce864e0bd92429df8008f31cf557269ba172482a
-F test/fts3misc.test f481128013b9555babdf3bc04c58ab59d59bebc24b5f780f50342b9ffe05b547
+F test/fts3misc.test 66e7b59576ce2c795f0baff6d47f7f6f57e6f41101cf85fad05989e43bb060dd
 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905
 F test/fts3offsets.test b85fd382abdc78ebce721d8117bd552dfb75094c
 F test/fts3prefix.test fa794eaab0bdae466494947b0b153d7844478ab2
@@ -1579,7 +1579,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2c145ee6c9e7916f022331453384cbe61ee3654c08a1b88467f85235b5bc18c4
-R 036cd868b45040b2c72bb4968e138d5b
+P ff5306752e83e760255a10f20168c0f090929a4fee2a5f720dfab36f0ee72fae
+R cae1848a63e8e13e207beadd6dc14b62
 U drh
-Z 70f7b9e85ac1b4ba86dc2c4c0ab20dce
+Z e9ec1c79d7987d6b2eda6f4466e2dc4b

manifest.uuid

@@ -1 +1 @@
-d2bb0066f7c8413ef9992e6b07641cdf40ad260778074bd83cc22dcaba87860b
+44ca6c2c4639f3c50ae9233ee299ff0fc4566462c31f28d8676f8de7ffdcd7f0

src/auth.c

@@ -216,6 +216,18 @@ int sqlite3AuthCheck(
   if( db->xAuth==0 ){
     return SQLITE_OK;
   }
+
+  /* EVIDENCE-OF: R-43249-19882 The third through sixth parameters to the
+  ** callback are either NULL pointers or zero-terminated strings that
+  ** contain additional details about the action to be authorized.
+  **
+  ** The following testcase() macros show that any of the 3rd through 6th
+  ** parameters can be either NULL or a string. */
+  testcase( zArg1==0 );
+  testcase( zArg2==0 );
+  testcase( zArg3==0 );
+  testcase( pParse->zAuthContext==0 );
+
   rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext
 #ifdef SQLITE_USER_AUTHENTICATION
                  ,db->auth.zAuthUser

src/delete.c

@@ -350,7 +350,14 @@ void sqlite3DeleteFrom(
   /* Special case: A DELETE without a WHERE clause deletes everything.
   ** It is easier just to erase the whole table. Prior to version 3.6.5,
   ** this optimization caused the row change count (the value returned by 
-  ** API function sqlite3_count_changes) to be set incorrectly.  */
+  ** API function sqlite3_count_changes) to be set incorrectly.
+  **
+  ** The "rcauth==SQLITE_OK" terms is the
+  ** IMPLEMENATION-OF: R-17228-37124 If the action code is SQLITE_DELETE and
+  ** the callback returns SQLITE_IGNORE then the DELETE operation proceeds but
+  ** the truncate optimization is disabled and all rows are deleted
+  ** individually.
+  */
   if( rcauth==SQLITE_OK
    && pWhere==0
    && !bComplex

src/select.c

@@ -5115,13 +5115,38 @@ int sqlite3Select(
   }
 #endif
 
-  /* Generate code for all sub-queries in the FROM clause
+  /* For each term in the FROM clause, do two things:
+  ** (1) Authorized unreferenced tables
+  ** (2) Generate code for all sub-queries
   */
-#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
   for(i=0; i<pTabList->nSrc; i++){
     struct SrcList_item *pItem = &pTabList->a[i];
     SelectDest dest;
-    Select *pSub = pItem->pSelect;
+    Select *pSub;
+
+    /* Issue SQLITE_READ authorizations with a fake column name for any tables that
+    ** are referenced but from which no values are extracted. Examples of where these
+    ** kinds of null SQLITE_READ authorizations would occur:
+    **
+    **     SELECT count(*) FROM t1;   -- SQLITE_READ t1.""
+    **     SELECT t1.* FROM t1, t2;   -- SQLITE_READ t2.""
+    **
+    ** The fake column name is an empty string.  It is possible for a table to
+    ** have a column named by the empty string, in which case there is no way to
+    ** distinguish between an unreferenced table and an actual reference to the
+    ** "" column.  The original design was for the fake column name to be a NULL,
+    ** which would be unambiguous.  But legacy authorization callbacks might
+    ** assume the column name is non-NULL and segfault.  The use of an empty string
+    ** for the fake column name seems safer.
+    */
+    if( pItem->colUsed==0 ){
+      sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase);
+    }
+
+#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
+    /* Generate code for all sub-queries in the FROM clause
+    */
+    pSub = pItem->pSelect;
     if( pSub==0 ) continue;
 
     /* Sometimes the code for a subquery will be generated more than
@@ -5242,8 +5267,8 @@ int sqlite3Select(
     }
     if( db->mallocFailed ) goto select_end;
     pParse->nHeight -= sqlite3SelectExprHeight(p);
-  }
 #endif
+  }
 
   /* Various elements of the SELECT copied into local variables for
   ** convenience */

src/sqlite.h.in

@@ -2673,6 +2673,7 @@ void sqlite3_randomness(int N, void *P);
 /*
 ** CAPI3REF: Compile-Time Authorization Callbacks
 ** METHOD: sqlite3
+** KEYWORDS: {authorizer callback}
 **
 ** ^This routine registers an authorizer callback with a particular
 ** [database connection], supplied in the first argument.
@@ -2700,8 +2701,10 @@ void sqlite3_randomness(int N, void *P);
 ** parameter to the sqlite3_set_authorizer() interface. ^The second parameter
 ** to the callback is an integer [SQLITE_COPY | action code] that specifies
 ** the particular action to be authorized. ^The third through sixth parameters
-** to the callback are zero-terminated strings that contain additional
-** details about the action to be authorized.
+** to the callback are either NULL pointers or zero-terminated strings
+** that contain additional details about the action to be authorized.
+** Applications must always be prepared to encounter a NULL pointer in any
+** of the third through the sixth parameters of the authorization callback.
 **
 ** ^If the action code is [SQLITE_READ]
 ** and the callback returns [SQLITE_IGNORE] then the
@@ -2710,6 +2713,10 @@ void sqlite3_randomness(int N, void *P);
 ** been read if [SQLITE_OK] had been returned.  The [SQLITE_IGNORE]
 ** return can be used to deny an untrusted user access to individual
 ** columns of a table.
+** ^When a table is referenced by a [SELECT] but no column values are
+** extracted from that table (for example in a query like
+** "SELECT count(*) FROM tab") then the [SQLITE_READ] authorizer callback
+** is invoked once for that table with a column name that is an empty string.
 ** ^If the action code is [SQLITE_DELETE] and the callback returns
 ** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the
 ** [truncate optimization] is disabled and all rows are deleted individually.
@@ -4756,10 +4763,11 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 ** the compiled regular expression can be reused on multiple
 ** invocations of the same function.
 **
-** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata
-** associated by the sqlite3_set_auxdata() function with the Nth argument
-** value to the application-defined function. ^If there is no metadata
-** associated with the function argument, this sqlite3_get_auxdata() interface
+** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata
+** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument
+** value to the application-defined function.  ^N is zero for the left-most
+** function argument.  ^If there is no metadata
+** associated with the function argument, the sqlite3_get_auxdata(C,N) interface
 ** returns a NULL pointer.
 **
 ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th
@@ -4790,6 +4798,10 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 ** function parameters that are compile-time constants, including literal
 ** values and [parameters] and expressions composed from the same.)^
 **
+** The value of the N parameter to these interfaces should be non-negative.
+** Future enhancements may make use of negative N values to define new
+** kinds of function caching behavior.
+**
 ** These routines must be called from the same thread in which
 ** the SQL function is running.
 */

src/tclsqlite.c

@@ -1033,9 +1033,16 @@ static int auth_callback(
   Tcl_DString str;
   int rc;
   const char *zReply;
+  /* EVIDENCE-OF: R-38590-62769 The first parameter to the authorizer
+  ** callback is a copy of the third parameter to the
+  ** sqlite3_set_authorizer() interface.
+  */
   SqliteDb *pDb = (SqliteDb*)pArg;
   if( pDb->disableAuth ) return SQLITE_OK;
 
+  /* EVIDENCE-OF: R-56518-44310 The second parameter to the callback is an
+  ** integer action code that specifies the particular action to be
+  ** authorized. */
   switch( code ){
     case SQLITE_COPY              : zCode="SQLITE_COPY"; break;
     case SQLITE_CREATE_INDEX      : zCode="SQLITE_CREATE_INDEX"; break;

src/vdbeInt.h

@@ -287,11 +287,11 @@ struct sqlite3_value {
 ** when the VM is halted (if not before).
 */
 struct AuxData {
-  int iOp;                        /* Instruction number of OP_Function opcode */
-  int iArg;                       /* Index of function argument. */
+  int iAuxOp;                     /* Instruction number of OP_Function opcode */
+  int iAuxArg;                    /* Index of function argument. */
   void *pAux;                     /* Aux data pointer */
-  void (*xDelete)(void *);        /* Destructor for the aux data */
-  AuxData *pNext;                 /* Next element in list */
+  void (*xDeleteAux)(void*);      /* Destructor for the aux data */
+  AuxData *pNextAux;              /* Next element in list */
 };
 
 /*

src/vdbeapi.c

@@ -804,6 +804,12 @@ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){
 /*
 ** Return the auxiliary data pointer, if any, for the iArg'th argument to
 ** the user-function defined by pCtx.
+**
+** The left-most argument is 0.
+**
+** Undocumented behavior:  If iArg is negative then access a cache of
+** auxiliary data pointers that is available to all functions within a
+** single prepared statement.  The iArg values must match.
 */
 void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
   AuxData *pAuxData;
@@ -814,17 +820,24 @@ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){
 #else
   assert( pCtx->pVdbe!=0 );
 #endif
-  for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){
-    if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break;
+  for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){
+    if(  pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){
+      return pAuxData->pAux;
+    }
   }
-
-  return (pAuxData ? pAuxData->pAux : 0);
+  return 0;
 }
 
 /*
 ** Set the auxiliary data pointer and delete function, for the iArg'th
 ** argument to the user-function defined by pCtx. Any previous value is
 ** deleted by calling the delete function specified when it was set.
+**
+** The left-most argument is 0.
+**
+** Undocumented behavior:  If iArg is negative then make the data available
+** to all functions within the current prepared statement using iArg as an
+** access code.
 */
 void sqlite3_set_auxdata(
   sqlite3_context *pCtx, 
@@ -836,33 +849,34 @@ void sqlite3_set_auxdata(
   Vdbe *pVdbe = pCtx->pVdbe;
 
   assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) );
-  if( iArg<0 ) goto failed;
 #ifdef SQLITE_ENABLE_STAT3_OR_STAT4
   if( pVdbe==0 ) goto failed;
 #else
   assert( pVdbe!=0 );
 #endif
 
-  for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){
-    if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break;
+  for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){
+    if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){
+      break;
+    }
   }
   if( pAuxData==0 ){
     pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData));
     if( !pAuxData ) goto failed;
-    pAuxData->iOp = pCtx->iOp;
-    pAuxData->iArg = iArg;
-    pAuxData->pNext = pVdbe->pAuxData;
+    pAuxData->iAuxOp = pCtx->iOp;
+    pAuxData->iAuxArg = iArg;
+    pAuxData->pNextAux = pVdbe->pAuxData;
     pVdbe->pAuxData = pAuxData;
     if( pCtx->fErrorOrAux==0 ){
       pCtx->isError = 0;
       pCtx->fErrorOrAux = 1;
     }
-  }else if( pAuxData->xDelete ){
-    pAuxData->xDelete(pAuxData->pAux);
+  }else if( pAuxData->xDeleteAux ){
+    pAuxData->xDeleteAux(pAuxData->pAux);
   }
 
   pAuxData->pAux = pAux;
-  pAuxData->xDelete = xDelete;
+  pAuxData->xDeleteAux = xDelete;
   return;
 
 failed:

src/vdbeaux.c

@@ -2968,16 +2968,18 @@ void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){
   while( *pp ){
     AuxData *pAux = *pp;
     if( (iOp<0)
-     || (pAux->iOp==iOp && (pAux->iArg>31 || !(mask & MASKBIT32(pAux->iArg))))
+     || (pAux->iAuxOp==iOp
+          && pAux->iAuxArg>=0
+          && (pAux->iAuxArg>31 || !(mask & MASKBIT32(pAux->iAuxArg))))
     ){
-      testcase( pAux->iArg==31 );
-      if( pAux->xDelete ){
-        pAux->xDelete(pAux->pAux);
+      testcase( pAux->iAuxArg==31 );
+      if( pAux->xDeleteAux ){
+        pAux->xDeleteAux(pAux->pAux);
       }
-      *pp = pAux->pNext;
+      *pp = pAux->pNextAux;
       sqlite3DbFree(db, pAux);
     }else{
-      pp= &pAux->pNext;
+      pp= &pAux->pNextAux;
     }
   }
 }

test/auth.test

@@ -36,12 +36,20 @@ proc_real proc {name arguments script} {
 do_test auth-1.1.1 {
   db close
   set ::DB [sqlite3 db test.db]
+  proc authx {code arg1 arg2 arg3 arg4 args} {return SQLITE_DENY}
   proc auth {code arg1 arg2 arg3 arg4 args} {
     if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} {
       return SQLITE_DENY
     }
     return SQLITE_OK
   }
+  db authorizer ::authx
+  # EVIDENCE-OF: R-03993-24285 Only a single authorizer can be in place on
+  # a database connection at a time. Each call to sqlite3_set_authorizer
+  # overrides the previous call.
+  #
+  # The authx authorizer above is overridden by the auth authorizer below
+  # authx is never invoked.
   db authorizer ::auth
   catchsql {CREATE TABLE t1(a,b,c)}
 } {1 {not authorized}}
@@ -60,6 +68,9 @@ do_test auth-1.1.4 {
 do_test auth-1.2 {
   execsql {SELECT name FROM sqlite_master}
 } {}
+# EVIDENCE-OF: R-04452-49349 When the callback returns SQLITE_DENY, the
+# sqlite3_prepare_v2() or equivalent call that triggered the authorizer
+# will fail with an error message explaining that access is denied.
 do_test auth-1.3.1 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
     if {$code=="SQLITE_CREATE_TABLE"} {
@@ -312,6 +323,10 @@ ifcapable attach {
   } {1 {access to two.t2.b is prohibited}}
   execsql {DETACH DATABASE two}
 }
+# EVIDENCE-OF: R-38392-49970 If the action code is SQLITE_READ and the
+# callback returns SQLITE_IGNORE then the prepared statement statement
+# is constructed to substitute a NULL value in place of the table column
+# that would have been read if SQLITE_OK had been returned.
 do_test auth-1.36 {
   proc auth {code arg1 arg2 arg3 arg4 args} {
     if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} {
@@ -1606,6 +1621,8 @@ do_test auth-1.248 {
   set ::authargs
 } {COMMIT {} {} {}}
 do_test auth-1.249 {
+  # EVIDENCE-OF: R-52112-44167 Disable the authorizer by installing a NULL
+  # callback.
   db authorizer {}
   catchsql {ROLLBACK}
 } {0 {}}
@@ -2478,6 +2495,35 @@ do_test auth-7.4 {
   SQLITE_READ t7 c main {}       \
 ]
 
+# If a table is referenced but no columns are read from the table,
+# that causes a single SQLITE_READ authorization with a NULL column
+# name.
+#
+# EVIDENCE-OF: R-31520-16302 When a table is referenced by a SELECT but
+# no column values are extracted from that table (for example in a query
+# like "SELECT count(*) FROM tab") then the SQLITE_READ authorizer
+# callback is invoked once for that table with a column name that is an
+# empty string.
+#
+set ::authargs [list]
+do_test auth-8.1 {
+  execsql {SELECT count(*) FROM t7}
+  set ::authargs
+} [list \
+  SQLITE_SELECT {} {} {} {}          \
+  SQLITE_FUNCTION {} count {} {}     \
+  SQLITE_READ t7 {} {} {}            \
+  ]
+set ::authargs [list]
+
+do_test auth-8.2 {
+  execsql {SELECT t6.a FROM t6, t7}
+  set ::authargs
+} [list \
+  SQLITE_SELECT {} {} {} {}          \
+  SQLITE_READ t6 a main {}           \
+  SQLITE_READ t7 {} {} {}            \
+  ]
 
 rename proc {}
 rename proc_real proc

test/auth3.test

@@ -53,6 +53,10 @@ do_test auth3.1.2 {
   set ::authcode SQLITE_DENY
   catchsql { DELETE FROM t1 }
 } {1 {not authorized}}
+# EVIDENCE-OF: R-64962-58611 If the authorizer callback returns any
+# value other than SQLITE_IGNORE, SQLITE_OK, or SQLITE_DENY then the
+# sqlite3_prepare_v2() or equivalent call that triggered the authorizer
+# will fail with an error message.
 do_test auth3.1.3 {
   set ::authcode SQLITE_INVALID
   catchsql { DELETE FROM t1 }

test/fts3fault.test

@@ -177,13 +177,15 @@ do_test 8.0 {
   faultsim_save_and_close
 } {}
 
-do_faultsim_test 8.1 -faults oom-t* -prep { 
-  faultsim_restore_and_reopen
-  db func mit mit
-} -body {
-  execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
-} -test {
-  faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
+ifcapable fts4_deferred {
+  do_faultsim_test 8.1 -faults oom-t* -prep { 
+    faultsim_restore_and_reopen
+    db func mit mit
+  } -body {
+    execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
+  } -test {
+    faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
+  }
 }
 
 do_faultsim_test 8.2 -faults oom-t* -prep { 

test/fts3misc.test

@@ -147,30 +147,32 @@ do_execsql_test 3.1.5 {
 #-------------------------------------------------------------------------
 #
 reset_db
-do_execsql_test 4.0 {
-  PRAGMA page_size = 512;
-  CREATE VIRTUAL TABLE t4 USING fts4;
-  WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 )
-  INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s;
+ifcapable fts4_deferred {
+  do_execsql_test 4.0 {
+    PRAGMA page_size = 512;
+    CREATE VIRTUAL TABLE t4 USING fts4;
+    WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<8000 )
+    INSERT INTO t4 SELECT 'a b c a b c a b c' FROM s;
+  }
+  do_execsql_test 4.1 {
+    SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+  } {8000}
+  do_execsql_test 4.2 {
+    SELECT quote(value) from t4_stat where id=0
+  } {X'C03EC0B204C0A608'}
+  do_execsql_test 4.3 {
+    UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0;
+  }
+  do_catchsql_test 4.4 {
+    SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+  } {1 {database disk image is malformed}}
+  do_execsql_test 4.5 {
+    UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0;
+  }
+  do_catchsql_test 4.6 {
+    SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
+  } {1 {database disk image is malformed}}
 }
-do_execsql_test 4.1 {
-  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
-} {8000}
-do_execsql_test 4.2 {
-  SELECT quote(value) from t4_stat where id=0
-} {X'C03EC0B204C0A608'}
-do_execsql_test 4.3 {
-  UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0;
-}
-do_catchsql_test 4.4 {
-  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
-} {1 {database disk image is malformed}}
-do_execsql_test 4.5 {
-  UPDATE t4_stat SET value = X'00C03EC0B204C0A608' WHERE id=0;
-}
-do_catchsql_test 4.6 {
-  SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
-} {1 {database disk image is malformed}}
 
 #-------------------------------------------------------------------------
 #