From 83e63dc3857dbb2220090ef0cd3541cef2189aa3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Apr 2015 19:41:18 +0000 Subject: [PATCH 1/3] Work toward adding the --changeset option to the sqldiff utility program. Changes are incomplete. This is an incremental check-in. FossilOrigin-Name: 463e38d765f9d055b63792a8ea15c3782657b07f --- manifest | 17 ++++--- manifest.uuid | 2 +- tool/sqldiff.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a4b1b68367..c7c3d03725 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\ssqlite3_declare_vtab(),\savoid\saccessing\sthe\sdatabase\sstructure\suntil\safter\sthe\s"api-armour"\ssafety-check\shas\scompleted\sand\sthe\sdb\smutex\shas\sbeen\sobtained. -D 2015-04-10T16:05:33.033 +C Work\stoward\sadding\sthe\s--changeset\soption\sto\sthe\ssqldiff\sutility\sprogram.\nChanges\sare\sincomplete.\s\sThis\sis\san\sincremental\scheck-in. +D 2015-04-10T19:41:18.818 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f78b1ab81b64e7c57a75d170832443e66c0880a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1239,7 +1239,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 050763654cb28d23c4d9516deb348c8632e432cd +F tool/sqldiff.c 3e6f54359a070089ed0d11456e8868dcd3f20e94 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f @@ -1250,7 +1250,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ed3cbaab6ad49b0cb5b17e44def26c866919387a -R f962720dd8c90d63a1a200cd065fa312 -U dan -Z 666766483214acd3a759f5cb43f45fc3 +P 860e4f8a94901d451fac3954960c1d2f589e8882 +R 6c8da3b8e133d42c3c0048f2d2a47a98 +T *branch * sqldiff-changeset +T *sym-sqldiff-changeset * +T -sym-trunk * +U drh +Z b58ac33ed90e0dd7edcdd56b18b68a1d diff --git a/manifest.uuid b/manifest.uuid index 909c5925c5..324240a04b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -860e4f8a94901d451fac3954960c1d2f589e8882 \ No newline at end of file +463e38d765f9d055b63792a8ea15c3782657b07f \ No newline at end of file diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 53c5977ed4..2b46353535 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -714,6 +714,113 @@ end_diff_one_table: return; } +/* +** Generate a CHANGESET for all differences from main.zTab to aux.zTab. +*/ +static void changeset_one_table(const char *zTab, FILE *out){ + sqlite3_stmt *pStmt; /* SQL statment */ + char *zId = safeId(zTab); /* Escaped name of the table */ + char **azCol = 0; /* List of escaped column names */ + int nCol = 0; /* Number of columns */ + int *aiFlg = 0; /* 0 if column is not part of PK */ + int *aiPk = 0; /* Column numbers for each PK column */ + int nPk = 0; /* Number of PRIMARY KEY columns */ + Str sql; /* SQL for the diff query */ + int i; /* Loop counter */ + const char *zSep; /* List separator */ + + pStmt = db_prepare( + "SELECT A.sql=B.sql FROM main.sqlite_master A, aux.sqlite_master B" + " WHERE A.name=%Q AND B.name=%Q", zTab, zTab + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_int(pStmt,0)==0 ){ + runtimeError("schema changes for table %s", safeId(zTab)); + } + }else{ + runtimeError("table %s missing from one or both databases", safeId(zTab)); + } + sqlite3_finalize(pStmt); + pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + nCol++; + azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); + if( azCol==0 ) runtimeError("out of memory"); + aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); + if( aiFlg==0 ) runtimeError("out of memory"); + azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); + aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); + if( i>0 ){ + if( i>nPk ){ + nPk = i; + aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); + if( aiPk==0 ) runtimeError("out of memory"); + } + aiPk[i-1] = nCol-1; + } + } + sqlite3_finalize(pStmt); + if( nPk==0 ) goto end_changeset_one_table; + strInit(&sql); + if( nCol>nPk ){ + strPrintf(&sql, "SELECT 1"); /* Changes to non-PK columns */ + for(i=0; i0 ) sqlite3_free(azCol[--nCol]); + sqlite3_free(azCol); + sqlite3_free(aiPk); + sqlite3_free(zId); +} + /* ** Print sketchy documentation for this utility program */ @@ -722,6 +829,7 @@ static void showHelp(void){ printf( "Output SQL text that would transform DB1 into DB2.\n" "Options:\n" +" --changeset FILE Write a CHANGESET into FILE\n" " --primarykey Use schema-defined PRIMARY KEYs\n" " --schema Show only differences in the schema\n" " --table TAB Show only differences in table TAB\n" @@ -737,6 +845,7 @@ int main(int argc, char **argv){ char *zSql; sqlite3_stmt *pStmt; char *zTab = 0; + FILE *out = 0; g.zArgv0 = argv[0]; for(i=1; i Date: Sat, 11 Apr 2015 12:07:40 +0000 Subject: [PATCH 2/3] First complete attempt to generate a working changeset. Still contains bugs. FossilOrigin-Name: 5611fa9bd5b8fd762d16ce9b0853c2e779a1a1b7 --- manifest | 15 ++++----- manifest.uuid | 2 +- tool/sqldiff.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c7c3d03725..f2c0b806d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\sadding\sthe\s--changeset\soption\sto\sthe\ssqldiff\sutility\sprogram.\nChanges\sare\sincomplete.\s\sThis\sis\san\sincremental\scheck-in. -D 2015-04-10T19:41:18.818 +C First\scomplete\sattempt\sto\sgenerate\sa\sworking\schangeset.\s\sStill\scontains\sbugs. +D 2015-04-11T12:07:40.414 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f78b1ab81b64e7c57a75d170832443e66c0880a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1239,7 +1239,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 3e6f54359a070089ed0d11456e8868dcd3f20e94 +F tool/sqldiff.c ed945df30d962d15d26188e33f7c5431cf197487 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f @@ -1250,10 +1250,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 860e4f8a94901d451fac3954960c1d2f589e8882 -R 6c8da3b8e133d42c3c0048f2d2a47a98 -T *branch * sqldiff-changeset -T *sym-sqldiff-changeset * -T -sym-trunk * +P 463e38d765f9d055b63792a8ea15c3782657b07f +R c7ffcaee222b675f0ec3e43f31931e14 U drh -Z b58ac33ed90e0dd7edcdd56b18b68a1d +Z c4472641aefb50d8b597b0b444fecd27 diff --git a/manifest.uuid b/manifest.uuid index 324240a04b..4657a0c7e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -463e38d765f9d055b63792a8ea15c3782657b07f \ No newline at end of file +5611fa9bd5b8fd762d16ce9b0853c2e779a1a1b7 \ No newline at end of file diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 2b46353535..d166174440 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -714,6 +714,31 @@ end_diff_one_table: return; } +/* +** Write a 64-bit signed integer as a varint onto out +*/ +static void putsVarint(FILE *out, sqlite3_uint64 v){ + int i, n; + unsigned char buf[12], p[12]; + if( v & (((sqlite3_uint64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + fwrite(p, 8, 1, out); + }else{ + n = 9; + do{ + p[n--] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[9] &= 0x7f; + fwrite(buf+n+1, 9-n, 1, out); + } +} + /* ** Generate a CHANGESET for all differences from main.zTab to aux.zTab. */ @@ -726,7 +751,7 @@ static void changeset_one_table(const char *zTab, FILE *out){ int *aiPk = 0; /* Column numbers for each PK column */ int nPk = 0; /* Number of PRIMARY KEY columns */ Str sql; /* SQL for the diff query */ - int i; /* Loop counter */ + int i, j; /* Loop counters */ const char *zSep; /* List separator */ pStmt = db_prepare( @@ -763,7 +788,7 @@ static void changeset_one_table(const char *zTab, FILE *out){ if( nPk==0 ) goto end_changeset_one_table; strInit(&sql); if( nCol>nPk ){ - strPrintf(&sql, "SELECT 1"); /* Changes to non-PK columns */ + strPrintf(&sql, "SELECT %d", SQLITE_UPDATE); for(i=0; i=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_FLOAT: + rX = sqlite3_column_int64(pStmt,i); + memcpy(&uX, &rX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_TEXT: + iX = sqlite3_column_bytes(pStmt,i); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_column_text(pStmt,i),1,iX,out); + break; + case SQLITE_BLOB: + iX = sqlite3_column_bytes(pStmt,i); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_column_blob(pStmt,i),1,iX,out); + break; + case SQLITE_NULL: + break; + } + } + } + sqlite3_finalize(pStmt); end_changeset_one_table: while( nCol>0 ) sqlite3_free(azCol[--nCol]); From 6e42ce44db4d367c64c704f953e575d0f42310e5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Apr 2015 13:48:01 +0000 Subject: [PATCH 3/3] The --changeset option now appears to be working. FossilOrigin-Name: 1a2e2803920dcf64190d81d8a487d6c3c9bb28ee --- manifest | 12 ++-- manifest.uuid | 2 +- tool/sqldiff.c | 161 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 124 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index f2c0b806d6..336a1877e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\scomplete\sattempt\sto\sgenerate\sa\sworking\schangeset.\s\sStill\scontains\sbugs. -D 2015-04-11T12:07:40.414 +C The\s--changeset\soption\snow\sappears\sto\sbe\sworking. +D 2015-04-11T13:48:01.693 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5f78b1ab81b64e7c57a75d170832443e66c0880a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1239,7 +1239,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c ed945df30d962d15d26188e33f7c5431cf197487 +F tool/sqldiff.c 51c05cc1435507736b8b5a41a0498016041b3e48 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f @@ -1250,7 +1250,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 463e38d765f9d055b63792a8ea15c3782657b07f -R c7ffcaee222b675f0ec3e43f31931e14 +P 5611fa9bd5b8fd762d16ce9b0853c2e779a1a1b7 +R 907cf695d8fcffe3003a321f9aace7fc U drh -Z c4472641aefb50d8b597b0b444fecd27 +Z 3b5f839006775d8054dcfb8aab08f0fa diff --git a/manifest.uuid b/manifest.uuid index 4657a0c7e9..a5255d3652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5611fa9bd5b8fd762d16ce9b0853c2e779a1a1b7 \ No newline at end of file +1a2e2803920dcf64190d81d8a487d6c3c9bb28ee \ No newline at end of file diff --git a/tool/sqldiff.c b/tool/sqldiff.c index d166174440..4455c582ef 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -719,7 +719,7 @@ end_diff_one_table: */ static void putsVarint(FILE *out, sqlite3_uint64 v){ int i, n; - unsigned char buf[12], p[12]; + unsigned char p[12]; if( v & (((sqlite3_uint64)0xff000000)<<32) ){ p[8] = (unsigned char)v; v >>= 8; @@ -734,8 +734,45 @@ static void putsVarint(FILE *out, sqlite3_uint64 v){ p[n--] = (unsigned char)((v & 0x7f) | 0x80); v >>= 7; }while( v!=0 ); - buf[9] &= 0x7f; - fwrite(buf+n+1, 9-n, 1, out); + p[9] &= 0x7f; + fwrite(p+n+1, 9-n, 1, out); + } +} + +/* +** Write an SQLite value onto out. +*/ +static void putValue(FILE *out, sqlite3_value *pVal){ + int iDType = sqlite3_value_type(pVal); + sqlite3_int64 iX; + double rX; + sqlite3_uint64 uX; + int j; + + putc(iDType, out); + switch( iDType ){ + case SQLITE_INTEGER: + iX = sqlite3_value_int64(pVal); + memcpy(&uX, &iX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_FLOAT: + rX = sqlite3_value_int64(pVal); + memcpy(&uX, &rX, 8); + for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); + break; + case SQLITE_TEXT: + iX = sqlite3_value_bytes(pVal); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_value_text(pVal),1,iX,out); + break; + case SQLITE_BLOB: + iX = sqlite3_value_bytes(pVal); + putsVarint(out, (sqlite3_uint64)iX); + fwrite(sqlite3_value_blob(pVal),1,iX,out); + break; + case SQLITE_NULL: + break; } } @@ -751,7 +788,7 @@ static void changeset_one_table(const char *zTab, FILE *out){ int *aiPk = 0; /* Column numbers for each PK column */ int nPk = 0; /* Number of PRIMARY KEY columns */ Str sql; /* SQL for the diff query */ - int i, j; /* Loop counters */ + int i, k; /* Loop counters */ const char *zSep; /* List separator */ pStmt = db_prepare( @@ -789,8 +826,14 @@ static void changeset_one_table(const char *zTab, FILE *out){ strInit(&sql); if( nCol>nPk ){ strPrintf(&sql, "SELECT %d", SQLITE_UPDATE); - for(i=0; i=0; j-=8) putc((uX>>j)&0xff, out); - break; - case SQLITE_FLOAT: - rX = sqlite3_column_int64(pStmt,i); - memcpy(&uX, &rX, 8); - for(j=56; j>=0; j-=8) putc((uX>>j)&0xff, out); - break; - case SQLITE_TEXT: - iX = sqlite3_column_bytes(pStmt,i); - putsVarint(out, (sqlite3_uint64)iX); - fwrite(sqlite3_column_text(pStmt,i),1,iX,out); - break; - case SQLITE_BLOB: - iX = sqlite3_column_bytes(pStmt,i); - putsVarint(out, (sqlite3_uint64)iX); - fwrite(sqlite3_column_blob(pStmt,i),1,iX,out); - break; - case SQLITE_NULL: - break; + switch( sqlite3_column_int(pStmt,0) ){ + case SQLITE_UPDATE: { + for(k=1, i=0; i