From ff4917e9a537deeb0bdbbec7c9ea3849e8a095c6 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:32:26 +0000 Subject: [PATCH 1/9] Create new branch named "base_convert" FossilOrigin-Name: 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 1fd11cbf21..dabd88714c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\soptimization\sin\sbtree.c. -D 2022-11-19T00:22:12.991 +C Create\snew\sbranch\snamed\s"base_convert" +D 2022-11-19T02:32:26.249 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2055,8 +2055,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff494449efd475878c549728cc22ee9b12d13674068781747fc042a0c1bd09c8 +P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff R 8e4332decceeea7e1973b8e5ac88e8a4 -U drh -Z 80da1bc9f69008afc02dd88003bee918 +T *branch * base_convert +T *sym-base_convert * +T -sym-trunk * +U larrybr +Z 21ba949f5c7ccf39d76b47dd8fe1f905 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e33296fde..bc34d0feed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff \ No newline at end of file +0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file From fb2e0bfa8b540fb6e7be5a74beaadd06b60570c1 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 19 Nov 2022 02:39:16 +0000 Subject: [PATCH 2/9] New extensions for base85 and base64 conversion UDFs FossilOrigin-Name: 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 --- ext/misc/base64.c | 233 +++++++++++++++++++++++++++++ ext/misc/base85.c | 366 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 15 +- manifest.uuid | 2 +- 4 files changed, 607 insertions(+), 9 deletions(-) create mode 100644 ext/misc/base64.c create mode 100644 ext/misc/base85.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c new file mode 100644 index 0000000000..1ed1a4f174 --- /dev/null +++ b/ext/misc/base64.c @@ -0,0 +1,233 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes being the same as ASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that. +** But the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ + +typedef unsigned char ubyte; + +static const ubyte b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) +#define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[dv]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define DARK_MAX 72 + +/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4 }; + int nbi = (nbIn > 3)? 3 : nbIn; + signed char nc; + int nbe; + unsigned long qv = (ubyte)*pIn++; + for( nbe=1; nbe<3; ++nbe ){ + ubyte b = (nbe=0; --nbe ){ + char ce = (nbe0 && *pIn!='=' ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn<=0 ) break; + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + ubyte bdp = BX_DV_PROTO(c); + --ncIn; + switch( bdp ){ + case WS: + case ND: + nti = 0; + break; + case PC: + bdp = 0; + /* fall thru */ + default: /* It's the digit value. */ + qv = qv<<6 | bdp; + --nti; + break; + } + } + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + ubyte *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (ubyte*)sqlite3_value_blob(av[0]); + nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} diff --git a/ext/misc/base85.c b/ext/misc/base85.c new file mode 100644 index 0000000000..365db80f2b --- /dev/null +++ b/ext/misc/base85.c @@ -0,0 +1,366 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane ASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit ASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +/* Classify c according to interval within ASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static unsigned char b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static unsigned char base85DigitValue( char c ){ + unsigned char dv = (unsigned char)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +static char * skipNonB85( char *s ){ + char c; + while( (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +static char base85Numeral( unsigned char b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} + +static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + *pOut = 0; + while( nbIn > 0 ){ + static signed char ncio[] = { 0, 2, 3, 4, 5 }; + int nbi = (nbIn > 4)? 4 : nbIn; + unsigned long qv = 0L; + int nbe = 0; + signed char nco; + while( nbe++ < nbi ){ + qv = (qv<<8) | *pIn++; + } + nco = ncio[nbi]; + nbIn -= nbi; + while( nco > 0 ){ + unsigned char dv = (unsigned char)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += ncio[nbi]; + if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + char *p = pSep; + while( *p ) *pOut++ = *p++; + nCol = 0; + } + *pOut = 0; + } + return pOut; +} + +static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + if( ncIn==0 ) break; + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + while( nti>0 ){ + char c = *pIn++; + unsigned char cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + c - cdo; + --nti; + } + nbo -= nti; + while( nbo-- > 0 ){ + *pOut++ = (qv >> (8*nbo))&0xff; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL.", -1); + break; + } +} +# endif + +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + unsigned char *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big.", -1); + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + bBuf = (unsigned char*)sqlite3_value_blob(av[0]); + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big.", -1); + }else if( nb<1 ){ + nb = 1; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + cBuf = (char *)sqlite3_value_text(av[0]); + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + break; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + unsigned char bBuf[64]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif diff --git a/manifest b/manifest index dabd88714c..faf2f3f49e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sbranch\snamed\s"base_convert" -D 2022-11-19T02:32:26.249 +C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs +D 2022-11-19T02:39:16.296 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,6 +289,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 +F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 +F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2055,11 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f710cce13577788cf3b95ed7089c3af2854271ff53f0a0b7b0619f315e331eff -R 8e4332decceeea7e1973b8e5ac88e8a4 -T *branch * base_convert -T *sym-base_convert * -T -sym-trunk * +P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea +R 4ecad003b3a6f60dd6b11d354f8edbe2 U larrybr -Z 21ba949f5c7ccf39d76b47dd8fe1f905 +Z d182470ca73df70df09238ac5a5e4382 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bc34d0feed..b47c8a0962 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea \ No newline at end of file +5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file From 423003dca91a84409950bfd4e99ee68dc4f831ee Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 00:11:09 +0000 Subject: [PATCH 3/9] Fix a base64 decode bug. Provide for convenient inclusion of extension(s) built into the CLI, to simplify testing and for its own sake. Improve comments. Cure collision between base64.c and base85.c when both are in the same translation unit. FossilOrigin-Name: 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f --- ext/misc/base64.c | 55 ++++++++++++++++++++++++++-------------- ext/misc/base85.c | 50 ++++++++++++++++++++++++++++++------ ext/misc/basexx.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +++++++------ manifest.uuid | 2 +- src/shell.c.in | 33 ++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 ext/misc/basexx.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 1ed1a4f174..d6f7834a85 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -12,7 +12,7 @@ ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** @@ -37,10 +37,10 @@ ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the -** meaning of the first 128 (7-bit) codes being the same as ASCII. +** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. -** Because it is table-driven, it could be enhanced to handle that. -** But the world and SQLite have moved on from that anachronism. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR= @@ -52,12 +52,16 @@ #include +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ #include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #define PC 0x80 /* pad character */ #define WS 0x81 /* whitespace */ #define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' typedef unsigned char ubyte; @@ -89,10 +93,10 @@ static const char b64Numerals[64] #define IS_BX_PAD(bdp) ((bdp)==PC) #define BX_NUMERAL(dv) (b64Numerals[dv]) /* Width of base64 lines. Should be an integer multiple of 4. */ -#define DARK_MAX 72 +#define B64_DARK_MAX 72 /* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to DARK_MAX +** to be appended to encoded groups to limit their length to B64_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ @@ -111,12 +115,12 @@ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ nc = ncio[nbi]; nbIn -= nbi; for( nbe=3; nbe>=0; --nbe ){ - char ce = (nbe0 && *pIn!='=' ){ + while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; char *pUse = skipNonB64(pIn); unsigned long qv = 0L; - int nti, nbo; + int nti, nbo, nac; ncIn -= (pUse - pIn); - if( ncIn<=0 ) break; pIn = pUse; nti = (ncIn>4)? 4 : ncIn; nbo = nboi[nti]; - while( nti>0 ){ - char c = *pIn++; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac<=nti)? *pIn++ : PAD_CHAR; ubyte bdp = BX_DV_PROTO(c); --ncIn; switch( bdp ){ @@ -157,15 +161,16 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; case PC: bdp = 0; + --nbo; /* fall thru */ - default: /* It's the digit value. */ + default: /* bdp is the digit value. */ qv = qv<<6 | bdp; - --nti; break; } } + nti = 2; while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nbo))&0xff; + *pOut++ = (qv >> (8*nti--))&0xff; } } return pOut; @@ -183,7 +188,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ case SQLITE_BLOB: nb = nv; nc = 4*(nv+2/3); /* quads needed */ - nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big.", -1); } @@ -219,11 +224,15 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ /* ** Establish linkage to running SQLite library. */ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; return sqlite3_create_function @@ -231,3 +240,11 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, 0, base64, 0, 0); } + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 365db80f2b..dfa9bd0e31 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -14,7 +14,7 @@ ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** @@ -24,7 +24,7 @@ ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** -** Base85 numerals are taken from the set of 7-bit ASCII codes, +** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** @@ -32,8 +32,11 @@ ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields -** the simplest transition to byte groups smaller than 4 bytes.) +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite @@ -82,7 +85,10 @@ #ifndef BASE85_STANDALONE +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ # include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #else @@ -107,7 +113,7 @@ static void sayHelp(){ } #endif -/* Classify c according to interval within ASCII set w.r.t. base85 +/* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) @@ -127,16 +133,25 @@ static unsigned char base85DigitValue( char c ){ } #endif +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + static char * skipNonB85( char *s ){ char c; while( (c = *s) && !IS_B85(c) ) ++s; return s; } +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/ static char base85Numeral( unsigned char b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ int nCol = 0; *pOut = 0; @@ -157,7 +172,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ pOut[--nco] = base85Numeral(dv); } pOut += ncio[nbi]; - if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){ char *p = pSep; while( *p ) *pOut++ = *p++; nCol = 0; @@ -167,6 +182,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ return pOut; } +/* Decode base85 text into a byte buffer. */ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ @@ -196,6 +212,7 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ } #ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ @@ -208,6 +225,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ @@ -228,6 +246,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ } # endif +/* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ int nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), @@ -272,11 +291,18 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "base85 OOM", -1); } +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; # ifndef OMIT_BASE85_CHECKER @@ -294,12 +320,20 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, 0, base85, 0, 0); } +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; - unsigned char bBuf[64]; + unsigned char bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000..700a99409c --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,64 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + int rc = SQLITE_OK; + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/manifest b/manifest index faf2f3f49e..f02b4834b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs -D 2022-11-19T02:39:16.296 +C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. +D 2022-11-21T00:11:09.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,8 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 -F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 +F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 +F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -640,7 +641,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 49b6aeaf950b9b7a74aca7b80e6d004ed32f0a16d2c9471e34587cc4755d5eaa F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea -R 4ecad003b3a6f60dd6b11d354f8edbe2 +P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 +R 0260a27c5340f9b412a0f6e0981a0d6d U larrybr -Z d182470ca73df70df09238ac5a5e4382 +Z 18487c3a1bcdc27a89418b38ad3e3f97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b47c8a0962..41284b2a7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file +07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..478c055357 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1070,6 +1070,9 @@ INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -5115,6 +5118,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -5137,6 +5141,34 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -5157,6 +5189,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename); From b5aaf0f28507a27a2ea8e18f642625d8727e9c48 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 21 Nov 2022 02:18:12 +0000 Subject: [PATCH 4/9] Get clean builds for MS tools. Add build examples for extension glommer (basexx.c) Make sure extensions build for dynamic load and built into shell. FossilOrigin-Name: da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b --- ext/misc/base64.c | 2 +- ext/misc/basexx.c | 22 +++++++++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index d6f7834a85..1ab0dc4ead 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -84,7 +84,7 @@ static const ubyte b64DigitValues[128] = { 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND }; -static const char b64Numerals[64] +static const char b64Numerals[64+1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; #define BX_DV_PROTO(c) ((((ubyte)(c))<0x80)? b64DigitValues[c] : 0x80) diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c index 700a99409c..a113134c85 100644 --- a/ext/misc/basexx.c +++ b/ext/misc/basexx.c @@ -16,6 +16,23 @@ ** ** Presently, it combines the base64.c and base85.c extensions. However, ** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% */ #ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ @@ -44,7 +61,10 @@ static void init_api_ptr(const sqlite3_api_routines *pApi){ #define sqlite3_base_init sqlite3_base85_init #include "base85.c" -static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ init_api_ptr(pApi); int rc1 = BASE64_INIT(db); diff --git a/manifest b/manifest index f02b4834b7..6d28b2a340 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. -D 2022-11-21T00:11:09.323 +C Get\sclean\sbuilds\sfor\sMS\stools.\sAdd\sbuild\sexamples\sfor\sextension\sglommer\s(basexx.c)\nMake\ssure\sextensions\sbuild\sfor\sdynamic\sload\sand\sbuilt\sinto\sshell. +D 2022-11-21T02:18:12.614 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,9 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base64.c e611e8ce552b16251c1691191d95912f9e03400637311dbd458062d2d0b95e2a F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 -F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 +F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 -R 0260a27c5340f9b412a0f6e0981a0d6d +P 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f +R 1b52ba3002b9af5fda34f27079570bff U larrybr -Z 18487c3a1bcdc27a89418b38ad3e3f97 +Z 7a2c313c3d99c0b1eaaafc6ea380893a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41284b2a7c..81f4745948 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file +da940d507e9e31a84daedede44a56f743198d7258a9e83892511fe8dd4c78d2b \ No newline at end of file From 9b9017def4edc224e0f4f773b775cfa575cf198b Mon Sep 17 00:00:00 2001 From: larrybr Date: Thu, 24 Nov 2022 02:59:33 +0000 Subject: [PATCH 5/9] Speed up base64 conversions, and add test with more data for the baseNN conversion to grind. FossilOrigin-Name: 6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c --- ext/misc/base64.c | 70 +++++++++++++++++++++++++++-------------------- manifest | 14 +++++----- manifest.uuid | 2 +- test/basexx1.test | 14 ++++++++++ 4 files changed, 62 insertions(+), 38 deletions(-) mode change 100644 => 100755 ext/misc/base64.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c old mode 100644 new mode 100755 index 6e3a6b5cb7..7bb7683d1f --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -92,42 +92,46 @@ static const char b64Numerals[64+1] #define IS_BX_DIGIT(bdp) (((ubyte)(bdp))<0x80) #define IS_BX_WS(bdp) ((bdp)==WS) #define IS_BX_PAD(bdp) ((bdp)==PC) -#define BX_NUMERAL(dv) (b64Numerals[dv]) +#define BX_NUMERAL(dv) (b64Numerals[(ubyte)(dv)]) /* Width of base64 lines. Should be an integer multiple of 4. */ #define B64_DARK_MAX 72 -/* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to B64_DARK_MAX -** or to terminate the last group (to aid concatenation.) +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. */ -static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ +static char* toBase64( ubyte *pIn, int nbIn, char *pOut ){ int nCol = 0; - *pOut = 0; - while( nbIn > 0 ){ - static signed char ncio[] = { 0, 2, 3, 4 }; - int nbi = (nbIn > 3)? 3 : nbIn; - signed char nc; - int nbe; - unsigned long qv = (ubyte)*pIn++; - for( nbe=1; nbe<3; ++nbe ){ - ubyte b = (nbe= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ - char ce = (nbe>= 6; pOut[nbe] = ce; } pOut += 4; - if( pSep && ((nCol += 4)>=B64_DARK_MAX || nbIn<=0) ){ - char *p = pSep; - while( *p ) *pOut++ = *p++; - nCol = 0; - } - *pOut = 0; + *pOut++ = '\n'; } + *pOut = 0; return pOut; } @@ -157,11 +161,12 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ ubyte bdp = BX_DV_PROTO(c); switch( bdp ){ case ND: - /* Treat non-digits as pad, but they terminate decode too. */ + /* Treat dark non-digits as pad, but they terminate decode too. */ ncIn = 0; /* fall thru */ case WS: - /* Treat whitespace as pad */ + /* Treat whitespace as pad and terminate this group.*/ + nti = nac; /* fall thru */ case PC: bdp = 0; @@ -172,10 +177,15 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; } } - nti = 2; - while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nti--))&0xff; + switch( nbo ){ + case 3: + pOut[2] = (qv) & 0xff; + case 2: + pOut[1] = (qv>>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; } + pOut += nbo; } return pOut; } @@ -200,7 +210,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; bBuf = (ubyte*)sqlite3_value_blob(av[0]); - nc = (int)(toBase64(bBuf, nb, cBuf, "\n") - cBuf); + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; case SQLITE_TEXT: diff --git a/manifest b/manifest index 6d54a98758..56f984f662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\sbaseNN\stests,\sget\soversize\serror\strapping\sworking,\sand\ssync\sw/trunk -D 2022-11-22T22:46:41.866 +C Speed\sup\sbase64\sconversions,\sand\sadd\stest\swith\smore\sdata\sfor\sthe\sbaseNN\sconversion\sto\sgrind. +D 2022-11-24T02:59:33.748 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,7 +289,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 0472f388e1a6c168912a363dfbe88d2d588325fba7ce6101f726d01b54fe6d3b +F ext/misc/base64.c 6333194e5c2e85b0748116ad4004bf3e070347cc09984aaa8d462fb3fc0566b6 x F ext/misc/base85.c 9005549904fc06ec2f3ff96970709f92f76e2d9ec2b785553ac32908ddc1baa0 F ext/misc/basexx.c 678dcc83894f78c26fd3662b322886777cc26bf2b40809236cd2abdad532a33c F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a @@ -816,7 +816,7 @@ F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399 -F test/basexx1.test 9b12557d2b5bd017f9f8a8698239438ced5899c3ee55d9a549f74d90b16e51a6 +F test/basexx1.test d8a50f0744b93dca656625597bcd3499ff4b9a4ea2a82432b119b7d46e3e0c08 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0 @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ff67460e1a3d21c9ca7cbd171fbc5e6cbdb3797de359887f851776b73b732fdf 9ec923b5dc24d6082da8d42bc0ee8ab1c418912625c0c56de9627be2c818ef98 -R 36565c16629af64673142ebb24820a85 +P 03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 +R f94c977ea1969bc60efbcf2158a80ba4 U larrybr -Z 3e47c0e49ec42d7b0a58333b563577b4 +Z 312bc23355f0a0c3b5b7524b3c0fa709 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 825fd93e96..11257f9dab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03819e9368fd9f78f351147a1dc865743f9634893e43a9d1e3d7cbaf4c966069 \ No newline at end of file +6c84ae4ba83713c751fddff8be41686bbcb525ac8135e1520436c62d0bc23d2c \ No newline at end of file diff --git a/test/basexx1.test b/test/basexx1.test index 8855a78634..947a5678f3 100644 --- a/test/basexx1.test +++ b/test/basexx1.test @@ -138,4 +138,18 @@ do_catchsql_test 116 { SELECT is_base85(x'00'); } {1 {is_base85 accepts only text or NULL}} +# Round-trip many bigger random blobs. + +do_execsql_test 117 { + CREATE TABLE bs(b blob, num); + INSERT INTO bs SELECT randomblob(4000 + n%3), n + FROM ( + WITH RECURSIVE seq(n) AS ( + VALUES(1) UNION ALL SELECT n+1 + FROM seq WHERE n<100 + ) SELECT n FROM seq); + SELECT num FROM bs WHERE base64(base64(b))!=b; + SELECT num FROM bs WHERE base85(base85(b))!=b; +} {} + finish_test From 217e77d24008f5bd813c719e4a08da93500a2218 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 26 Nov 2022 20:48:56 +0000 Subject: [PATCH 6/9] Show which AggInfo.aCol an Expr node represents in the tree trace debugging logic. Does not affect production builds. FossilOrigin-Name: b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/treeview.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a2d2f52121..54ca01a2ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassertion-fault\sin\sthe\s"TREETRACE"\sdebugging\slogic\son\nselect.c.\s\sThe\sproblem\sdoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:12:02.257 +C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. +D 2022-11-26T20:48:56.633 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b -F src/treeview.c 07787f67cd297a6d09d04b8d70c06769c60c9c1d9080378f93929c16f8fd3298 +F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9542f9ce9e023b489e2d93661f719fb0751c1e28f72fded9d3c2156d5777e7b1 -R 581898efb4aefcef8555315298dbd7f7 +P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e +R 77aba841340d0be9ea0f0036dfeb9ac7 U drh -Z f6493a79b992a3696ede08a261d497d4 +Z cf507f961f29b58691309417f6fc3d94 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4e96fca06..4df4cac51e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e \ No newline at end of file +b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 89a128dff6..9df16a12bf 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -487,7 +487,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ StrAccum x; sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); sqlite3_str_appendf(&x, " fg.af=%x.%c", @@ -504,6 +504,9 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ sqlite3_str_appendf(&x, " IMMUTABLE"); } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; From ca9309cc565da99c6a8e8dea84852b9c4ee478a4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 27 Nov 2022 00:57:56 +0000 Subject: [PATCH 7/9] Document an OPFS API change in Chrome v108 which does not break our code but does change several formerly async methods to synchronous. No code changes. FossilOrigin-Name: e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 16 ++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 8a3db9c648..8e60969bc6 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -29,6 +29,22 @@ This file represents an implementation detail of a larger piece of code, and not a public interface. Its details may change at any time and are not intended to be used by any client-level code. + + 2022-11-27: Chrome v108 changes some async methods to synchronous, as + documented at: + + https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + + We cannot change to the sync forms at this point without breaking + clients who use Chrome v104-ish or higher. truncate(), getSize(), + flush(), and close() are now (as of v108) synchronous. Calling them + with an "await", as we have to for the async forms, is still legal + with the sync forms but is superfluous. Calling the async forms with + theFunc().then(...) is not compatible with the change to + synchronous, but we do do not use those APIs that way. i.e. we don't + _need_ to change anything for this, but at some point (after Chrome + versions (approximately) 104-107 are extinct) should change our + usage of those methods to remove the "await". */ "use strict"; const toss = function(...args){throw new Error(args.join(' '))}; diff --git a/manifest b/manifest index 54ca01a2ed..37ec0eef0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\swhich\sAggInfo.aCol\san\sExpr\snode\srepresents\sin\sthe\stree\strace\sdebugging\nlogic.\s\sDoes\snot\saffect\sproduction\sbuilds. -D 2022-11-26T20:48:56.633 +C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. +D 2022-11-27T00:57:56.426 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js e98a8bd67dea8c20b0ec17332698b44658a6fbc4be18d F ext/wasm/api/sqlite3-api-prologue.js 7fce4c6a138ec3d7c285b7c125cee809e6b668d2cb0d2328a1b790b7037765bd F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 -F ext/wasm/api/sqlite3-opfs-async-proxy.js d933d4a3c84e6dc614b57355fc9029645cfcdfbfde096ed42f4c979a6f60c18a +F ext/wasm/api/sqlite3-opfs-async-proxy.js 798383f6b46fd5dac122d6e35962d25b10401ddb825b5c66df1d21e6b1d8aacc F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 8b32787a3b6bb2990cbaba2304bd5b75a9652acbc8d29909b3279019b6cbaef5 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f11bb42292d94e7b7687bd2134f72afe1353182238cb4837fc0a6d78a021dd7e -R 77aba841340d0be9ea0f0036dfeb9ac7 -U drh -Z cf507f961f29b58691309417f6fc3d94 +P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 +R 4c0998f1bfd61979c1006462c1fac102 +U stephan +Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4df4cac51e..852ac7c5b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 \ No newline at end of file +e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file From c1ca183abfeb54a5735f4f3bd4c0e1994645c3f7 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 28 Nov 2022 02:28:44 +0000 Subject: [PATCH 8/9] Conform CLI .trace arg handling to its help. FossilOrigin-Name: 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 2 +- test/shell4.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 37ec0eef0c..107bf9f15b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\san\sOPFS\sAPI\schange\sin\sChrome\sv108\swhich\sdoes\snot\sbreak\sour\scode\sbut\sdoes\schange\sseveral\sformerly\sasync\smethods\sto\ssynchronous.\sNo\scode\schanges. -D 2022-11-27T00:57:56.426 +C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. +D 2022-11-28T02:28:44.889 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -642,7 +642,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c1eb8f3ee25152327f2e7e87db8cea549e57c104b63638bff4fc584d479c33f0 -F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35 +F src/shell.c.in 334b1661abb6453e4f5d1778ce9c80dddef83470867f1fba07e5ee400792adbe F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -1497,7 +1497,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test e4b4de56f454708e0747b52915135baa2cbfec4965406d6eaf02a4a5c22a9880 F test/shell2.test c536c2aab4852608f8a606262330797abc4d964a4c2c782a7760f54ea1f17a6a F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a -F test/shell4.test 7dc8a515705bc093d8ffe381670e8fa7a969661e8ed177c35c847e3c6dfc35e2 +F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b8076f4dc55810855f668dfd4221bdb555cc78c61df200cf0cb87eb137e22850 -R 4c0998f1bfd61979c1006462c1fac102 -U stephan -Z 9e17173a8e8b4b20b4b9a5a815e1b3d6 +P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 +R ce11368d09e97cb20c99cf53926be781 +U larrybr +Z 2b24f85901c30dde8df5f7afa27a353b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 852ac7c5b3..2d0d9deb13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 \ No newline at end of file +31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e1f0ecfcd4..ab47dba8c9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10658,7 +10658,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); + p->traceOut = output_file_open(z, 0); } } if( p->traceOut==0 ){ diff --git a/test/shell4.test b/test/shell4.test index 068072202d..193dc8b69b 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -125,6 +125,12 @@ SELECT * FROM t1;}} do_test shell4-2.5 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace stdout\nSELECT * FROM t1;" } {0 {SELECT * FROM t1;}} +do_test shell4-2.6 { + catchcmd ":memory:" { +CREATE TABLE t1(x); +.trace --stmt stdout +SELECT * FROM t1;} +} {0 {SELECT * FROM t1;}} } do_test shell4-3.1 { From abbfa7aa17307f31011751072454850e6da4d435 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Nov 2022 13:47:27 +0000 Subject: [PATCH 9/9] Ensure the RETURNING clause is honoured when a row of a temp table is updated by an ON CONFLICT clause. FossilOrigin-Name: a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/returning1.test | 24 ++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 107bf9f15b..91afe6c7c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Conform\sCLI\s.trace\sarg\shandling\sto\sits\shelp. -D 2022-11-28T02:28:44.889 +C Ensure\sthe\sRETURNING\sclause\sis\shonoured\swhen\sa\srow\sof\sa\stemp\stable\sis\supdated\sby\san\sON\sCONFLICT\sclause. +D 2022-11-28T13:47:27.194 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c 29b1dc7e0f84ba090734febe27393d4719682af0cae1b902d5ebf0236ecebea4 -F src/trigger.c 4163ada044af89d51caba1cb713a73165347b2ec05fe84a283737c134d61fcd5 +F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 F src/update.c 5b0302c47cf31b533d5dff04c497ca1d8b9d89c39727e633fbe7b882fd5ac5aa F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1421,7 +1421,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl 0db8aee0c348090fd06da47020ab4ed8ec692e0723427b2f3947d4dfb806f3b0 F test/resetdb.test 8062cf10a09d8c048f8de7711e94571c38b38168db0e5877ba7561789e5eeb2b F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test c43b8370a351f77aec6d71f4a2cde59b849369ed1933261a2c2c69e23e34ff5e +F test/returning1.test 1366e04566cfe1a082d17b1e0f195ec64473c79374b3a5d4ae00c43d885dea31 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9dd87e28a96eb6560943b9a2cb3cb0ff14698c96fe63865944dcfed73b67bd3 -R ce11368d09e97cb20c99cf53926be781 -U larrybr -Z 2b24f85901c30dde8df5f7afa27a353b +P 31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c +R cab683454f2ed9db2c50c1dc1f9667b9 +U dan +Z f330cc4347042764fbf120efeed28ded # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d0d9deb13..0bd7375f33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31546ea320e8daa020f8a9f491718ffc3dde0e32954018afed88e8494a6aff6c \ No newline at end of file +a2449bcc2c71d0f4c3289621fbf1cb97f0f407c9f7b5bf18245b7854a07c6cfa \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index ca64d6145b..02d8540237 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -61,7 +61,7 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ if( pTrig->pTabSchema==pTab->pSchema && pTrig->table && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - && pTrig->pTabSchema!=pTmpSchema + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) ){ pTrig->pNext = pList; pList = pTrig; diff --git a/test/returning1.test b/test/returning1.test index fb058a6438..92e10ee9f4 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -376,4 +376,28 @@ do_execsql_test 16.1 { SELECT * FROM t2; } {1 2 3 a b c} + +foreach {tn temp} { + 1 "" + 2 TEMP +} { + reset_db + do_execsql_test 17.$tn.0 " + CREATE $temp TABLE foo ( + fooid INTEGER PRIMARY KEY, + fooval INTEGER NOT NULL UNIQUE, + refcnt INTEGER NOT NULL DEFAULT 1 + ); + " + do_execsql_test 17.$tn.1 { + INSERT INTO foo (fooval) VALUES (17), (4711), (17) + ON CONFLICT DO + UPDATE SET refcnt = refcnt+1 + RETURNING fooid; + } { + 1 2 1 + } +} + + finish_test