paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for the octet_length() SQL function, with low-level optimizations

Browse Source 
that avoid loading large string content from disk when the argument to
octet_length() is a table column.

FossilOrigin-Name: fa5f77862c0fe0189aa4246a1e55bb7c537c28c436ec10b75f5fa141e5e4aff0
This commit is contained in:
drh
2023-06-23 11:10:13 +00:00
parent d4170ac0eb 077efc2730
commit cc19bed8b1
7 changed files with 90 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
manifest
+11 -10
View File
@@ -1,5 +1,5 @@
C Simplification\sto\sthe\sgetAndInitPage()\sroutine\sthat\sresults\sin\simproved\nperformance.
D 2023-06-22T01:03:39.798
C Add\ssupport\sfor\sthe\soctet_length()\sSQL\sfunction,\swith\slow-level\soptimizations\nthat\savoid\sloading\slarge\sstring\scontent\sfrom\sdisk\swhen\sthe\sargument\sto\noctet_length()\sis\sa\stable\scolumn.
D 2023-06-23T11:10:13.280
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -587,10 +587,10 @@ F src/date.c f73f203b3877cef866c60ab402aec2bf89597219b60635cf50cbe3c5e4533e94
F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387
F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef
F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be
F src/expr.c 36f6a47c8a2c20ec3c267a60fc598857876edd60af0cb40caf7b69b651fd73bf
F src/expr.c 8d1656b65e26af3e34f78e947ac423f0d20c214ed25a67486e433bf16ca6b543
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36
F src/func.c 6303e1ccb80dbd0d9b52f902a01d3b105981486fdfd66f9e1ddfd74aaf3032fc
F src/func.c ee064b524ac7077939febc2d57a5b4243c9bf0ab69893b0d336ec15d101b96b6
F src/global.c bd0892ade7289f6e20bff44c07d06371f2ff9b53cea359e7854b9b72f65adc30
F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
@@ -642,7 +642,7 @@ F src/shell.c.in bdd1fdfc77a67651cdc5a158bc9107cf3c2cf3ddb62d7a4da06c6eaaa5e7203
F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
F src/sqliteInt.h 8974b60740b108269f51e833e85191be6bf9f06f317ee34a53b7ec215762cf8c
F src/sqliteInt.h 44353825a42fec1f2740a2a54de328c46d3dc3014be139341d42b4ad9ca2b7d9
F src/sqliteLimit.h 33b1c9baba578d34efe7dfdb43193b366111cdf41476b1e82699e14c11ee1fb6
F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -707,7 +707,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
F src/util.c 6f9d2f278dcc8d41c618980cd3cfe88e1bafc0626209b917c6773d8202d29ef6
F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
F src/vdbe.c 6c0de640ef3be08cf2992d588a7501aee0f1003027bc952a6916a35f6e33b4cf
F src/vdbe.c c993304c609326cf625b4ad30cbb0e15a3f64c941cf2c9713d0c360b4abbaa98
F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0
F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae
F src/vdbeapi.c de9703f8705afc393cc2864669ce28cf9516983c8331d59aa2b978de01634365
@@ -1107,7 +1107,7 @@ F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f75
F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9
F test/fts4upfrom.test f25835162c989dffd5e2ef91ec24c4848cc9973093e2d492d1c7b32afac1b49d
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test 4be8bed4be235e333f1e0ea31e32f5be3c9f456c30780363e7fcb15e3ff3e6bc
F test/func.test cdebc080122807254173b942bd0aa9464eb859f7a057a0791136072462fb5bcb
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 600a632c305a88f3946d38f9a51efe145c989b2e13bd2b2a488db47fe76bab6a
F test/func4.test 2285fb5792d593fef442358763f0fd9de806eda47dbc7a5934df57ffdc484c31
@@ -2041,8 +2041,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P dc468cfdb825083b3a4b6cb95c913961e9312e22103c5a0cd923b75c83c65e13
R 61e892d0fc1ac1905da44ab1762c00e7
P 2e9734c2335d8c06fedc9f4cca02baaf326f7fa276bd464f3214f383715a48d6 8b8ea4e3f52d96cc217bd1fb27ca4a83489ef1f250756ed2f790d1b0ee529a3a
R a17349c989070702394759c35633bc6c
T +closed 8b8ea4e3f52d96cc217bd1fb27ca4a83489ef1f250756ed2f790d1b0ee529a3a
U drh
Z c0e995c2a64328b6123221956a5cefda
Z 65f3a5c19545a1086348c6238df80a6f
# Remove this line to create a well-formed Fossil manifest.
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
2e9734c2335d8c06fedc9f4cca02baaf326f7fa276bd464f3214f383715a48d6
fa5f77862c0fe0189aa4246a1e55bb7c537c28c436ec10b75f5fa141e5e4aff0
src/expr.c
+9 -6
View File
@@ -4668,10 +4668,10 @@ expr_code_doover:
r1 = sqlite3GetTempRange(pParse, nFarg);
}
/* For length() and typeof() functions with a column argument,
/* For length() and typeof() and octet_length() functions,
** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG
** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data
** loading.
** or OPFLAG_TYPEOFARG or OPFLAG_BYTELENARG respectively, to avoid
** unnecessary data loading.
*/
if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){
u8 exprOp;
@@ -4681,9 +4681,12 @@ expr_code_doover:
if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){
assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG );
assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG );
testcase( pDef->funcFlags & OPFLAG_LENGTHARG );
pFarg->a[0].pExpr->op2 =
pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG);
assert( SQLITE_FUNC_BYTELEN==OPFLAG_BYTELENARG );
assert( (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG)==OPFLAG_BYTELENARG );
testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG );
testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG );
testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG);
pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG;
}
}
src/func.c
+37
View File
@@ -149,6 +149,42 @@ static void lengthFunc(
}
}
/*
** Implementation of the octet_length() function
*/
static void bytelengthFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
assert( argc==1 );
UNUSED_PARAMETER(argc);
switch( sqlite3_value_type(argv[0]) ){
case SQLITE_BLOB: {
sqlite3_result_int(context, sqlite3_value_bytes(argv[0]));
break;
}
case SQLITE_INTEGER:
case SQLITE_FLOAT: {
i64 m = sqlite3_context_db_handle(context)->enc<=SQLITE_UTF8 ? 1 : 2;
sqlite3_result_int64(context, sqlite3_value_bytes(argv[0])*m);
break;
}
case SQLITE_TEXT: {
if( sqlite3_value_encoding(argv[0])<=SQLITE_UTF8 ){
sqlite3_result_int(context, sqlite3_value_bytes(argv[0]));
}else{
sqlite3_result_int(context, sqlite3_value_bytes16(argv[0]));
}
break;
}
default: {
sqlite3_result_null(context);
break;
}
}
}
/*
** Implementation of the abs() function.
**
@@ -2376,6 +2412,7 @@ void sqlite3RegisterBuiltinFunctions(void){
FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF),
FUNCTION2(subtype, 1, 0, 0, subtypeFunc, SQLITE_FUNC_TYPEOF),
FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH),
FUNCTION2(octet_length, 1, 0, 0, bytelengthFunc,SQLITE_FUNC_BYTELEN),
FUNCTION(instr, 2, 0, 0, instrFunc ),
FUNCTION(printf, -1, 0, 0, printfFunc ),
FUNCTION(format, -1, 0, 0, printfFunc ),
src/sqliteInt.h
+3
View File
@@ -1930,6 +1930,7 @@ struct FuncDestructor {
** SQLITE_FUNC_ANYORDER == NC_OrderAgg == SF_OrderByReqd
** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG
** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG
** SQLITE_FUNC_BYTELEN == OPFLAG_BYTELENARG
** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API
** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API
** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS -- opposite meanings!!!
@@ -1948,6 +1949,7 @@ struct FuncDestructor {
#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/
#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */
#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */
#define SQLITE_FUNC_BYTELEN 0x00c0 /* Built-in octet_length() function */
#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */
/* 0x0200 -- available for reuse */
#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */
@@ -3891,6 +3893,7 @@ struct AuthContext {
#define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */
#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */
#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */
#define OPFLAG_BYTELENARG 0xc0 /* OP_Column only for octet_length() */
#define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */
#define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */
#define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */
src/vdbe.c
+6 -2
View File
@@ -3056,10 +3056,14 @@ op_column_restart:
pDest->flags = aFlag[t&1];
}
}else{
u8 p5;
pDest->enc = encoding;
/* This branch happens only when content is on overflow pages */
if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0
&& ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0))
if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0
&& (p5==OPFLAG_TYPEOFARG
|| (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG))
)
)
|| (len = sqlite3VdbeSerialTypeLen(t))==0
){
/* Content is irrelevant for
test/func.test
+23
View File
@@ -42,6 +42,9 @@ do_test func-0.1 {
do_test func-1.0 {
execsql {SELECT length(t1) FROM tbl1 ORDER BY t1}
} {4 2 7 8 4}
do_execsql_test func-1.0b {
SELECT octet_length(t1) FROM tbl1 ORDER BY t1;
} {4 2 7 8 4}
do_test func-1.1 {
set r [catch {execsql {SELECT length(*) FROM tbl1 ORDER BY t1}} msg]
lappend r $msg
@@ -57,9 +60,29 @@ do_test func-1.3 {
do_test func-1.4 {
execsql {SELECT coalesce(length(a),-1) FROM t2}
} {1 -1 3 -1 5}
do_execsql_test func-1.5 {
SELECT octet_length(12345);
} {5}
db null NULL
do_execsql_test func-1.6 {
SELECT octet_length(NULL);
} {NULL}
do_execsql_test func-1.7 {
SELECT octet_length(7.5);
} {3}
do_execsql_test func-1.8 {
SELECT octet_length(x'30313233');
} {4}
do_execsql_test func-1.9 {
WITH c(x) AS (VALUES(char(350,351,352,353,354)))
SELECT length(x), octet_length(x) FROM c;
} {5 10}
# Check out the substr() function
#
db null {}
do_test func-2.0 {
execsql {SELECT substr(t1,1,2) FROM tbl1 ORDER BY t1}
} {fr is pr so th}