Experimental changes to (optionally) allow double-quoted strings to be checked against known identifiers.

FossilOrigin-Name: 73a6b8c1b9c282b9d28c2ce131fc2f3545aaef8b9357a4ae17b46059e473c2d6

manifest

@@ -1,5 +1,5 @@
-C When\smasking\sbits\soff\sof\ssqlite3.flags,\smake\ssure\sthe\smask\sis\s64\sbits\nin\ssize\sso\sas\snot\sto\saccidentally\smask\sof\shigh-order\sbits.
-D 2018-12-06T17:06:02.839
+C Experimental\schanges\sto\s(optionally)\sallow\sdouble-quoted\sstrings\sto\sbe\schecked\sagainst\sknown\sidentifiers.
+D 2018-12-06T20:18:43.697
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c
@@ -451,7 +451,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c 4377d0d9a0b969c30b2bc343a12140a53ba6ab0dbf34c1686f232e67d87a557b
 F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c ef9d7dc73e40dd9d10c28848343e21e8bc1baaab92cfb75eda893fff4fbf6b55
+F src/build.c fce47a9789704e65c63299b01be8153745faee7919f5137d3f29b7c3c0b549bd
 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b
@@ -502,17 +502,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586
 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc
 F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0
 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13
-F src/prepare.c 5fe8eb586d0e9d4009609490866525f242a3d0df01e39d967bf2611dc8081088
+F src/prepare.c 91cc20f27cebacaf3abb1ae585d41222f982e00e02abdc9601761d466201e1bc
 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c 06a122decd9268c747e9110448b0c84249a4623ca9d78a73618ce648f7af8869
 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d
-F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4
+F src/sqlite.h.in 448c5b2e3666249d82b7a604fefbe53b3918b0f1cc2eec366099f269e1d56f03
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
-F src/sqliteInt.h 97adda953e7f118d47b8135f76f88c6420ff6707285782616f393a9ea255d577
+F src/sqliteInt.h 904156295f6d196fd59abc2c8a0d15ee160de5452728b91778dbe7c7bda10de6
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -581,8 +581,8 @@ F src/vacuum.c c67085526fb51e141a85be3f7c9b91a2d38705bbb0ef911d8dec1e2d36cbdce1
 F src/vdbe.c c7312c909df5032b4ed27996c2ed8f5bf948f6a982458f85398520aaba0ccf81
 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907
 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980
-F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947
-F src/vdbeaux.c d71429f70e95e21aaf15ee287a78beefcda4bdc7d1b6d3c704dfeba820a792e0
+F src/vdbeapi.c affdfc7436c688e4f25626e2be27d40a69e39177d9cd8ba28bdb3b0eed48ed36
+F src/vdbeaux.c 33f61089dd3796db2a0b6a9beaf2d538496f06049bad743b8c0612514f24fba2
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9
 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
@@ -1142,7 +1142,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
-F test/normalize.test 7fe7674d83e37c635ea2e88a27b54a0fe1afddb192d636d5672b1639859800f1
+F test/normalize.test 2b0a6938bdb73e9eb10b1f0e82c21852a50ead546560c2cf3806cb56dd6bd666
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
@@ -1782,7 +1782,12 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076
-R b1b5c125b5cc5dd847ef51923c461312
-U drh
-Z 9423d60b113ec33e61947be8b42c640e
+P 53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3
+Q -3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de
+Q -e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4
+R 9ac1fdbbcc6858f34d6b5ce3ad1b78f7
+T *branch * normalize_v4
+T *sym-normalize_v4 *
+T -sym-trunk *
+U mistachkin
+Z 76bb8a3e588c994ab0e58f4e6ed733c6

manifest.uuid

@@ -1 +1 @@
-53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3
+73a6b8c1b9c282b9d28c2ce131fc2f3545aaef8b9357a4ae17b46059e473c2d6

src/build.c

@@ -636,6 +636,12 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
 
   /* Delete the Table structure itself.
   */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  if( pTable->pColHash ){
+    sqlite3HashClear(pTable->pColHash);
+    sqlite3_free(pTable->pColHash);
+  }
+#endif
   sqlite3DeleteColumnNames(db, pTable);
   sqlite3DbFree(db, pTable->zName);
   sqlite3DbFree(db, pTable->zColAff);

src/prepare.c

@@ -710,6 +710,96 @@ static int sqlite3LockAndPrepare(
 }
 
 #ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Checks if the specified token is a table, column, or function name,
+** based on the databases associated with the statement being prepared.
+** If the function fails, zero is returned and pRc is filled with the
+** error code.
+*/
+static int shouldTreatAsIdentifier(
+  sqlite3 *db,        /* Database handle. */
+  const char *zToken, /* Pointer to start of token to be checked */
+  int nToken,         /* Length of token to be checked */
+  int *pRc            /* Pointer to error code upon failure */
+){
+  int bFound = 0;     /* Non-zero if token is an identifier name. */
+  int i, j;           /* Database and column loop indexes. */
+  Schema *pSchema;    /* Schema for current database. */
+  Hash *pHash;        /* Hash table of tables for current database. */
+  HashElem *e;        /* Hash element for hash table iteration. */
+  Table *pTab;        /* Database table for columns being checked. */
+  char *zId;          /* Zero terminated name of the identifier */
+  char zSpace[65];    /* Static space for the zero-terminated name */
+
+  if( nToken<sizeof(zSpace) ){
+    memcpy(zSpace, zToken, nToken);
+    zSpace[nToken] = 0;
+    zId = zSpace;
+  }else{
+    zId = sqlite3DbStrNDup(db, zToken, nToken);
+    if( zId==0 ){
+      *pRc = SQLITE_NOMEM_BKPT;
+      return 0;
+    }
+  }
+  if( sqlite3IsRowid(zId) ){
+    bFound = 1;
+    goto done1;
+  }
+  if( nToken>0 ){
+    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
+    if( sqlite3FunctionSearch(hash, zId) ){
+      bFound = 1;
+      goto done1;
+    }
+  }
+  assert( db!=0 );
+  sqlite3_mutex_enter(db->mutex);
+  sqlite3BtreeEnterAll(db);
+  for(i=0; i<db->nDb; i++){
+    pHash = &db->aFunc;
+    if( sqlite3HashFind(pHash, zId) ){
+      bFound = 1;
+      break;
+    }
+    pSchema = db->aDb[i].pSchema;
+    if( pSchema==0 ) continue;
+    pHash = &pSchema->tblHash;
+    if( sqlite3HashFind(pHash, zId) ){
+      bFound = 1;
+      break;
+    }
+    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
+      pTab = sqliteHashData(e);
+      if( pTab==0 ) continue;
+      pHash = pTab->pColHash;
+      if( pHash==0 ){
+        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
+        if( pHash ){
+          sqlite3HashInit(pHash);
+          for(j=0; j<pTab->nCol; j++){
+            Column *pCol = &pTab->aCol[j];
+            sqlite3HashInsert(pHash, pCol->zName, pCol);
+          }
+        }else{
+          *pRc = SQLITE_NOMEM_BKPT;
+          bFound = 0;
+          goto done2;
+        }
+      }
+      if( pHash && sqlite3HashFind(pHash, zId) ){
+        bFound = 1;
+        goto done2;
+      }
+    }
+  }
+done2:
+  sqlite3BtreeLeaveAll(db);
+  sqlite3_mutex_leave(db->mutex);
+done1:
+  if( zId!=zSpace ) sqlite3DbFree(db, zId);
+  return bFound;
+}
 
 /*
 ** Attempt to estimate the final output buffer size needed for the fully
@@ -782,7 +872,8 @@ static void copyNormalizedToken(
 char *sqlite3Normalize(
   Vdbe *pVdbe,      /* VM being reprepared */
   const char *zSql, /* The original SQL string */
-  int nSql          /* Size of the input string in bytes */
+  int nSql,         /* Size of the input string in bytes */
+  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
 ){
   sqlite3 *db;           /* Database handle. */
   char *z;               /* The output string */
@@ -885,6 +976,16 @@ char *sqlite3Normalize(
           int i2 = i, n2 = n;
           if( nParen==nParenAtIN ) iStartIN = 0;
           if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
+          if( (prepFlags & SQLITE_PREPARE_CHKIDENTS)!=0 ){
+            int rc = SQLITE_OK;
+            if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
+              if( rc!=SQLITE_OK ) goto normalizeError;
+              if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
+                z[j++] = '?';
+                break;
+              }
+            }
+          }
         }
         copyNormalizedToken(zSql, i, n, flags, z, &j);
         break;

src/sqlite.h.in

@@ -3641,6 +3641,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 */
 #define SQLITE_PREPARE_PERSISTENT              0x01
 #define SQLITE_PREPARE_NORMALIZE               0x02
+#define SQLITE_PREPARE_CHKIDENTS               0x04
 
 /*
 ** CAPI3REF: Compiling An SQL Statement

src/sqliteInt.h

@@ -1957,6 +1957,9 @@ struct VTable {
 struct Table {
   char *zName;         /* Name of the table or view */
   Column *aCol;        /* Information about each column */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  Hash *pColHash;      /* All columns indexed by name */
+#endif
   Index *pIndex;       /* List of SQL indexes on this table. */
   Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
   FKey *pFKey;         /* Linked list of all foreign keys in this table */
@@ -4416,7 +4419,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
 void sqlite3ParserReset(Parse*);
 #ifdef SQLITE_ENABLE_NORMALIZE
-char *sqlite3Normalize(Vdbe*, const char*, int);
+char *sqlite3Normalize(Vdbe*, const char*, int, u8);
 #endif
 int sqlite3Reprepare(Vdbe*);
 void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);

src/vdbeapi.c

@@ -1715,7 +1715,7 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
   if( p==0 ) return 0;
   if( p->zNormSql==0 && p->zSql!=0 ){
     sqlite3_mutex_enter(p->db->mutex);
-    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql));
+    p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql), 0);
     sqlite3_mutex_leave(p->db->mutex);
   }
   return p->zNormSql;

src/vdbeaux.c

@@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){
 #ifdef SQLITE_ENABLE_NORMALIZE
   assert( p->zNormSql==0 );
   if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
-    p->zNormSql = sqlite3Normalize(p, p->zSql, n);
+    p->zNormSql = sqlite3Normalize(p, p->zSql, n, prepFlags);
     assert( p->zNormSql!=0 || p->db->mallocFailed );
   }
 #endif

test/normalize.test

@@ -209,6 +209,11 @@ foreach {tnum sql flags norm} {
   0x2
   {0 {SELECT"a"FROM t1 WHERE"x"IN("1","2",?);}}
 
+  431
+  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
+  0x6
+  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}
+
   440
   {SELECT 'a' FROM t1 WHERE 'x';}
   0x2