paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor the sqlite3_normalized_sql() implementation. This is a

Browse Source 
work-in-progress. There are still issues.

FossilOrigin-Name: a4c890b0af9786295e6df05022009d8946550adb873535c610be805c2b7a4083
This commit is contained in:
drh
2018-12-10 16:00:57 +00:00
parent fb34559eb1
commit 643d855da9
7 changed files with 148 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/prepare.c
-198
View File
@@ -709,204 +709,6 @@ static int sqlite3LockAndPrepare(
return rc;
}
#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Attempt to estimate the final output buffer size needed for the fully
** normalized version of the specified SQL string. This should take into
** account any potential expansion that could occur (e.g. via IN clauses
** being expanded, etc). This size returned is the total number of bytes
** including the NUL terminator.
*/
static int estimateNormalizedSize(
const char *zSql, /* The original SQL string */
int nSql /* Length of original SQL string */
){
int nOut = nSql + 4;
const char *z = zSql;
while( nOut<nSql*5 ){
while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
if( z[0]==0 ) break;
z++;
if( z[0]!='N' && z[0]!='n' ) break;
z++;
while( sqlite3Isspace(z[0]) ){ z++; }
if( z[0]!='(' ) break;
z++;
nOut += 5; /* ?,?,? */
}
return nOut;
}
/*
** Copy the current token into the output buffer while dealing with quoted
** identifiers. By default, all letters will be converted into lowercase.
** If the bUpper flag is set, uppercase will be used. The piOut argument
** will be used to update the target index into the output string.
*/
static void copyNormalizedToken(
const char *zSql, /* The original SQL string */
int iIn, /* Current index into the original SQL string */
int nToken, /* Number of bytes in the current token */
int tokenFlags, /* Flags returned by the tokenizer */
char *zOut, /* The output string */
int *piOut /* Pointer to target index into the output string */
){
int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
int j = *piOut, k = 0;
for(; k<nToken; k++){
if( bQuoted ){
if( k==0 && iIn>0 ){
zOut[j++] = '"';
continue;
}else if( k==nToken-1 ){
zOut[j++] = '"';
continue;
}
}
if( bKeyword ){
zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
}else{
zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
}
}
*piOut = j;
}
/*
** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return
** the normalization in space obtained from sqlite3DbMalloc(). Or return
** NULL if anything goes wrong or if zSql is NULL.
*/
char *sqlite3Normalize(
Vdbe *pVdbe, /* VM being reprepared */
const char *zSql, /* The original SQL string */
int nSql /* Size of the input string in bytes */
){
sqlite3 *db; /* Database handle. */
char *z; /* The output string */
int nZ; /* Size of the output string in bytes */
int i; /* Next character to read from zSql[] */
int j; /* Next character to fill in on z[] */
int tokenType = 0; /* Type of the next token */
int prevTokenType = 0; /* Type of the previous token, except spaces */
int n; /* Size of the next token */
int nParen = 0; /* Nesting level of parenthesis */
int iStartIN = 0; /* Start of RHS of IN operator in z[] */
int nParenAtIN = 0; /* Value of nParent at start of RHS of IN operator */
db = sqlite3VdbeDb(pVdbe);
assert( db!=0 );
if( zSql==0 ) return 0;
nZ = estimateNormalizedSize(zSql, nSql);
z = sqlite3DbMallocRawNN(db, nZ);
if( z==0 ) goto normalizeError;
for(i=j=0; i<nSql && zSql[i]; i+=n){
int flags = 0;
if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
switch( tokenType ){
case TK_SPACE: {
break;
}
case TK_ILLEGAL: {
goto normalizeError;
}
case TK_STRING:
case TK_INTEGER:
case TK_FLOAT:
case TK_VARIABLE:
case TK_BLOB: {
z[j++] = '?';
break;
}
case TK_LP:
case TK_RP: {
if( tokenType==TK_LP ){
nParen++;
if( prevTokenType==TK_IN ){
iStartIN = j;
nParenAtIN = nParen;
}
}else{
if( iStartIN>0 && nParen==nParenAtIN ){
assert( iStartIN+6<nZ );
memcpy(z+iStartIN+1, "?,?,?", 5);
j = iStartIN+6;
assert( nZ-1-j>=0 );
assert( nZ-1-j<nZ );
memset(z+j, 0, nZ-1-j);
iStartIN = 0;
}
nParen--;
}
assert( nParen>=0 );
/* Fall through */
}
case TK_MINUS:
case TK_SEMI:
case TK_PLUS:
case TK_STAR:
case TK_SLASH:
case TK_REM:
case TK_EQ:
case TK_LE:
case TK_NE:
case TK_LSHIFT:
case TK_LT:
case TK_RSHIFT:
case TK_GT:
case TK_GE:
case TK_BITOR:
case TK_CONCAT:
case TK_COMMA:
case TK_BITAND:
case TK_BITNOT:
case TK_DOT:
case TK_IN:
case TK_IS:
case TK_NOT:
case TK_NULL:
case TK_ID: {
if( tokenType==TK_NULL ){
if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
/* NULL is a keyword in this case, not a literal value */
}else{
/* Here the NULL is a literal value */
z[j++] = '?';
break;
}
}
if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
z[j++] = ' ';
}
if( tokenType==TK_ID ){
if( zSql[i]=='"'
&& sqlite3VdbeUsesDoubleQuotedString(db,pVdbe,zSql+i,n)
){
z[j++] = '?';
break;
}
if( nParen==nParenAtIN ) iStartIN = 0;
}
copyNormalizedToken(zSql, i, n, flags, z, &j);
break;
}
}
}
assert( j<nZ && "one" );
while( j>0 && z[j-1]==' ' ){ j--; }
if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
z[j] = 0;
assert( j<nZ && "two" );
return z;
normalizeError:
sqlite3DbFree(db, z);
return 0;
}
#endif /* SQLITE_ENABLE_NORMALIZE */
/*
** Rerun the compilation of a statement after a schema change.
src/sqliteInt.h
-3
View File
@@ -4255,9 +4255,6 @@ void sqlite3AlterFunctions(void);
void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
int sqlite3GetToken(const unsigned char *, int *);
#ifdef SQLITE_ENABLE_NORMALIZE
int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
#endif
void sqlite3NestedParse(Parse*, const char*, ...);
void sqlite3ExpirePreparedStatements(sqlite3*, int);
int sqlite3CodeSubselect(Parse*, Expr *, int, int);
src/tokenize.c
+130 -67
View File
@@ -545,73 +545,6 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
return i;
}
#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Return the length (in bytes) of the token that begins at z[0].
** Store the token type in *tokenType before returning. If flags has
** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
** for keywords. Add SQLITE_TOKEN_QUOTED to flags if the token was
** actually a quoted identifier. Add SQLITE_TOKEN_KEYWORD to flags
** if the token was recognized as a keyword; this is useful when the
** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
** to differentiate between a keyword being treated as an identifier
** (for normalization purposes) and an actual identifier.
*/
int sqlite3GetTokenNormalized(
const unsigned char *z,
int *tokenType,
int *flags
){
int n;
unsigned char iClass = aiClass[*z];
if( iClass==CC_KYWD ){
int i;
for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
if( IdChar(z[i]) ){
/* This token started out using characters that can appear in keywords,
** but z[i] is a character not allowed within keywords, so this must
** be an identifier instead */
i++;
while( IdChar(z[i]) ){ i++; }
*tokenType = TK_ID;
return i;
}
*tokenType = TK_ID;
n = keywordCode((char*)z, i, tokenType);
/* If the token is no longer considered to be an identifier, then it is a
** keyword of some kind. Make the token back into an identifier and then
** set the SQLITE_TOKEN_KEYWORD flag. Several non-identifier tokens are
** used verbatim, including IN, IS, NOT, and NULL. */
switch( *tokenType ){
case TK_ID: {
/* do nothing, handled by caller */
break;
}
case TK_IN:
case TK_IS:
case TK_NOT:
case TK_NULL: {
*flags |= SQLITE_TOKEN_KEYWORD;
break;
}
default: {
*tokenType = TK_ID;
*flags |= SQLITE_TOKEN_KEYWORD;
break;
}
}
}else{
n = sqlite3GetToken(z, tokenType);
/* If the token is considered to be an identifier and the character class
** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
*flags |= SQLITE_TOKEN_QUOTED;
}
}
return n;
}
#endif /* SQLITE_ENABLE_NORMALIZE */
/*
** Run the parser on the given SQL string. The parser structure is
** passed in. An SQLITE_ status code is returned. If an error occurs
@@ -781,3 +714,133 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
assert( nErr==0 || pParse->rc!=SQLITE_OK );
return nErr;
}
#ifdef SQLITE_ENABLE_NORMALIZE
/*
** Insert a single space character into pStr if the current string
** ends with an identifier
*/
static void addSpaceSeparator(sqlite3_str *pStr){
if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
sqlite3_str_append(pStr, " ", 1);
}
}
/*
** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return
** the normalization in space obtained from sqlite3DbMalloc(). Or return
** NULL if anything goes wrong or if zSql is NULL.
*/
char *sqlite3Normalize(
Vdbe *pVdbe, /* VM being reprepared */
const char *zSql, /* The original SQL string */
int nSql /* Size of the input string in bytes */
){
sqlite3 *db; /* The database connection */
int i; /* Next unread byte of zSql[] */
int n; /* length of current token */
int tokenType; /* type of current token */
int prevType; /* Previous non-whitespace token */
int nParen; /* Number of nested levels of parentheses */
int iStartIN; /* Start of RHS of IN operator in z[] */
int nParenAtIN; /* Value of nParent at start of RHS of IN operator */
int j; /* Bytes of normalized SQL generated so far */
sqlite3_str *pStr; /* The normalized SQL string under construction */
if( zSql==0 || nSql==0 ) return 0;
db = sqlite3VdbeDb(pVdbe);
tokenType = -1;
nParen = iStartIN = nParenAtIN = 0;
pStr = sqlite3_str_new(db);
for(i=0; i<nSql && pStr->accError==0; i+=n){
if( tokenType!=TK_SPACE ){
prevType = tokenType;
}
n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
if( NEVER(n<=0) ) break;
switch( tokenType ){
case TK_SPACE: {
break;
}
case TK_NULL: {
if( prevType==TK_IS || prevType==TK_NOT ){
sqlite3_str_append(pStr, " NULL", 5);
break;
}
/* Fall through */
}
case TK_STRING:
case TK_INTEGER:
case TK_FLOAT:
case TK_VARIABLE:
case TK_BLOB: {
sqlite3_str_append(pStr, "?", 1);
break;
}
case TK_LP: {
nParen++;
if( prevType==TK_IN ){
iStartIN = pStr->nChar;
nParenAtIN = nParen;
}
sqlite3_str_append(pStr, "(", 1);
break;
}
case TK_RP: {
if( iStartIN>0 && nParen==nParenAtIN ){
assert( pStr->nChar>=iStartIN );
pStr->nChar = iStartIN+1;
sqlite3_str_append(pStr, "?,?,?", 5);
iStartIN = 0;
}
nParen--;
sqlite3_str_append(pStr, ")", 1);
break;
}
case TK_ID: {
j = pStr->nChar;
if( sqlite3Isquote(zSql[i]) ){
char *zId = sqlite3DbStrNDup(db, zSql+i, n);
int nId;
int eType = 0;
if( zId==0 ) break;
sqlite3Dequote(zId);
if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
sqlite3_str_append(pStr, "?", 1);
sqlite3DbFree(db, zId);
break;
}
nId = sqlite3Strlen30(zId);
if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
addSpaceSeparator(pStr);
sqlite3_str_append(pStr, zId, nId);
}else{
sqlite3_str_appendf(pStr, "\"%w\"", zId);
}
sqlite3DbFree(db, zId);
}else{
addSpaceSeparator(pStr);
sqlite3_str_append(pStr, zSql+i, n);
}
while( j<pStr->nChar ){
pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
j++;
}
break;
}
default: {
if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
j = pStr->nChar;
sqlite3_str_append(pStr, zSql+i, n);
while( j<pStr->nChar ){
pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
j++;
}
break;
}
}
}
return sqlite3_str_finish(pStr);
}
#endif /* SQLITE_ENABLE_NORMALIZE */
src/vdbe.h
+1 -1
View File
@@ -253,7 +253,7 @@ u8 sqlite3VdbePrepareFlags(Vdbe*);
void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8);
#ifdef SQLITE_ENABLE_NORMALIZE
void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*);
int sqlite3VdbeUsesDoubleQuotedString(sqlite3*,Vdbe*,const char*,int);
int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*);
#endif
void sqlite3VdbeSwap(Vdbe*,Vdbe*);
VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*);
src/vdbeaux.c
+3 -13
View File
@@ -97,26 +97,16 @@ void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){
** that identifier is really used as a string literal.
*/
int sqlite3VdbeUsesDoubleQuotedString(
sqlite3 *db, /* Used for transient malloc */
Vdbe *pVdbe, /* The prepared statement */
const char *zId, /* The double-quoted identifier */
int nId /* Bytes in zId, which is not zero-terminated */
const char *zId /* The double-quoted identifier, already dequoted */
){
char *z;
DblquoteStr *pStr;
assert( zId!=0 );
assert( zId[0]=='"' );
assert( nId>=2 );
assert( zId[nId-1]=='"' );
if( pVdbe->pDblStr==0 ) return 0;
z = sqlite3DbStrNDup(db, zId, nId);
if( z==0 ) return 0;
sqlite3Dequote(z);
for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){
if( strcmp(z, pStr->z)==0 ) break;
if( strcmp(zId, pStr->z)==0 ) return 1;
}
sqlite3DbFree(db, z);
return pStr!=0;
return 0;
}
#endif