From 90fa4c7b2c7653cb64a4662bc98ea73d12143b43 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 31 Aug 2024 14:31:17 +0000 Subject: [PATCH 1/5] Enhance the percentile() and median() extension functions so that they can be window functions. FossilOrigin-Name: 4d0e3df4b9c609755977b8a462126242d2be1310c0122a8d4ba76d98d32a7230 --- Makefile.in | 3 +- Makefile.msc | 1 + ext/misc/percentile.c | 96 +++++++++++++++++++++++++++++++++++++------ main.mk | 1 + manifest | 23 ++++++----- manifest.uuid | 2 +- src/func.c | 8 +++- 7 files changed, 108 insertions(+), 26 deletions(-) diff --git a/Makefile.in b/Makefile.in index 188f94d701..9fcf87f941 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1182,6 +1182,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/memtrace.c \ $(TOP)/ext/misc/pcachetrace.c \ + $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/shathree.c \ @@ -1390,7 +1391,7 @@ srctree-check: $(TOP)/tool/srctree-check.tcl # Testing for a release # -releasetest: srctree-check has_tclsh85 +releasetest: srctree-check has_tclsh85 verify-source $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # Minimal testing that runs in less than 3 minutes diff --git a/Makefile.msc b/Makefile.msc index bf707edfd6..27513da12f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2313,6 +2313,7 @@ SHELL_DEP = \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\memtrace.c \ $(TOP)\ext\misc\pcachetrace.c \ + $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\shathree.c \ diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index cccbaf0253..99b1430171 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -77,6 +77,7 @@ typedef struct Percentile Percentile; struct Percentile { unsigned nAlloc; /* Number of slots allocated for a[] */ unsigned nUsed; /* Number of slots actually used in a[] */ + char bSorted; /* True if a[] is already in sorted order */ double rPct; /* 1.0 more than the value for P */ double *a; /* Array of Y values */ }; @@ -183,10 +184,65 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ p->a = a; } p->a[p->nUsed++] = y; + assert( p->nUsed>=1 ); + if( p->nUsed==1 ){ + p->bSorted = 1; + }else if( p->bSorted && p->a[p->nUsed-2]>y ){ + p->bSorted = 0; + } +} + +/* +** The "inverse" function for percentile(Y,P) is called to remove a +** row that was previously inserted by "step". +*/ +static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ + Percentile *p; + int eType; + double y; + int i; + assert( argc==2 || argc==1 ); + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + return; + } + + /* Ignore the Y value if it is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( isInfinity(y) ){ + return; + } + + /* Find and remove the row */ + for(i=0; inUsed && p->a[i]!=y; i++){} + if( inUsed ){ + p->a[i] = p->a[p->nUsed-1]; + p->nUsed--; + } + p->bSorted = p->nUsed<=1; } /* ** Sort an array of doubles. +** +** Algorithm: quicksort +** +** This is implemented separately rather than using the qsort() routine +** from the standard library because: +** +** (1) To avoid a dependency on qsort() +** (2) To avoid the function call to the comparison routine for each +** comparison. */ static void sortDoubles(double *a, int n){ int iLt; /* Entries with index less than iLt are less than rPivot */ @@ -238,7 +294,7 @@ static void sortDoubles(double *a, int n){ ** Called to compute the final output of percentile() and to clean ** up all allocated memory. */ -static void percentFinal(sqlite3_context *pCtx){ +static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ Percentile *p; unsigned i1, i2; double v1, v2; @@ -247,7 +303,10 @@ static void percentFinal(sqlite3_context *pCtx){ if( p==0 ) return; if( p->a==0 ) return; if( p->nUsed ){ - sortDoubles(p->a, p->nUsed); + if( p->bSorted==0 ){ + sortDoubles(p->a, p->nUsed); + p->bSorted = 1; + } ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; i1 = (unsigned)ix; i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; @@ -256,9 +315,18 @@ static void percentFinal(sqlite3_context *pCtx){ vx = v1 + (v2-v1)*(ix-i1); sqlite3_result_double(pCtx, vx); } - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); + if( bIsFinal ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + } } +static void percentFinal(sqlite3_context *pCtx){ + percentCompute(pCtx, 1); +} +static void percentValue(sqlite3_context *pCtx){ + percentCompute(pCtx, 0); +} + #ifdef _WIN32 @@ -272,18 +340,22 @@ int sqlite3_percentile_init( int rc = SQLITE_OK; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ - rc = sqlite3_create_function(db, "percentile", 2, - SQLITE_UTF8|SQLITE_INNOCUOUS, 0, - 0, percentStep, percentFinal); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "median", 1, + rc = sqlite3_create_window_function(db, "percentile", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, 0, - 0, percentStep, percentFinal); + percentStep, percentFinal, + percentValue, percentInverse, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "median", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + percentStep, percentFinal, + percentValue, percentInverse, 0); + } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "percentile_cont", 2, + rc = sqlite3_create_window_function(db, "percentile_cont", 2, SQLITE_UTF8|SQLITE_INNOCUOUS, &percentStep, - 0, percentStep, percentFinal); + percentStep, percentFinal, + percentValue, percentInverse, 0); } return rc; } diff --git a/main.mk b/main.mk index b1c86a1b76..1f32c22bdf 100644 --- a/main.mk +++ b/main.mk @@ -764,6 +764,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/memtrace.c \ $(TOP)/ext/misc/pcachetrace.c \ + $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/shathree.c \ diff --git a/manifest b/manifest index 25793c4244..4190a7f8da 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Do\snot\sfail\sthe\somittest\sif\sICU\slibraries\sare\snot\sinstalled. -D 2024-08-30T17:33:25.286 +C Enhance\sthe\spercentile()\sand\smedian()\sextension\sfunctions\sso\sthat\sthey\scan\sbe\nwindow\sfunctions. +D 2024-08-31T14:31:17.092 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 77627cbeeffe23606da5663458c0f8bbb86d28e2aca270a5d9b0db8e39a54bb2 +F Makefile.in 394e8c4b17d7a25b26a379573cb74bac569e6e4e9b5f1e2cf6efd53866867054 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 47dfd26a401c5a33e237ecfe126c996977f34ac44e44a1a1ebf2204abb89d483 +F Makefile.msc 8dc55cc46fabd6a01afca3a5829c4cb69b99537e42143ca6f0bf6e811b9f5068 F README.md 5b678e264236788390d11991f2c0052bd73f19790173883fc56d638bcb849154 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -410,7 +410,7 @@ F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58 F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 -F ext/misc/percentile.c af1941dc87d45dd0c2698a3087fbfe9ee0d157e5e72da521430c4b784abcbe81 +F ext/misc/percentile.c 89416b108569171be1d8dda4fa2687ad116ea969b4d129d02cf3dc1fd67fc87e F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed @@ -685,7 +685,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 40f8cd07372d7eed2fe33403a42b82c3f5d2d46b59480444384326db70bb33f1 +F main.mk de83918046afeb0ac4239e88511623f3ef2a0c41aab0e2b401e0275f4fd24a8e F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -718,7 +718,7 @@ F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f -F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f +F src/func.c df400a1d3f4625997d4dd8a81951c303e066277c29b861d37e03cd152d7858dd F src/global.c 61a419dd9e993b9be0f91de4c4ccf322b053eb829868e089f0321dd669be3b90 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -2211,8 +2211,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e48add02695a41b26a04e7942b5333e2bf4dc5598e363367aea3a4690982667d -R 3b9cf08a862640c366f07bc714e510b5 +P ddc55efd2d59df3f20743b0533550436da945453c069025a3f871d28d40e13d4 +R 5abe90c6c09c8e29fb97dabd43182a07 +T *branch * percentile-window +T *sym-percentile-window * +T -sym-trunk * U drh -Z ce5d41834ddc15c40aed5b812cb5e6fa +Z 85a1ed3baf7191136421a56e2c79709a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0044aadb49..ac02b4544f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ddc55efd2d59df3f20743b0533550436da945453c069025a3f871d28d40e13d4 +4d0e3df4b9c609755977b8a462126242d2be1310c0122a8d4ba76d98d32a7230 diff --git a/src/func.c b/src/func.c index 8fcda11dc0..a634a7fbea 100644 --- a/src/func.c +++ b/src/func.c @@ -2049,7 +2049,11 @@ static void minMaxFinalize(sqlite3_context *context){ ** group_concat(EXPR, ?SEPARATOR?) ** string_agg(EXPR, SEPARATOR) ** -** The SEPARATOR goes before the EXPR string. This is tragic. The +** Content is accumulated in GroupConcatCtx.str with the SEPARATOR +** coming before the EXPR value, except for the first entry which +** omits the SEPARATOR. +** +** It is tragic that the SEPARATOR goes before the EXPR string. The ** groupConcatInverse() implementation would have been easier if the ** SEPARATOR were appended after EXPR. And the order is undocumented, ** so we could change it, in theory. But the old behavior has been @@ -2153,7 +2157,7 @@ static void groupConcatInverse( /* pGCC is always non-NULL since groupConcatStep() will have always ** run first to initialize it */ if( ALWAYS(pGCC) ){ - int nVS; + int nVS; /* Number of characters to remove */ /* Must call sqlite3_value_text() to convert the argument into text prior ** to invoking sqlite3_value_bytes(), in case the text encoding is UTF16 */ (void)sqlite3_value_text(argv[0]); From 129767958ae41bf0da17b69643ca061a8d4f5620 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 31 Aug 2024 15:02:07 +0000 Subject: [PATCH 2/5] Test cases added. FossilOrigin-Name: 25e68229843cc84978955817285550085d1306ba4ce3b0517dd00e5d05b9ae0a --- manifest | 15 +++++------- manifest.uuid | 2 +- test/percentile.test | 58 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4190a7f8da..e61e0275b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\spercentile()\sand\smedian()\sextension\sfunctions\sso\sthat\sthey\scan\sbe\nwindow\sfunctions. -D 2024-08-31T14:31:17.092 +C Test\scases\sadded. +D 2024-08-31T15:02:07.805 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1517,7 +1517,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 -F test/percentile.test 74e383216a075251512d6ba98beb9dccd6da465e3f73817fc438379e3a628de7 +F test/percentile.test b30d92480943de8236c1522cace227ad354512ade699789636a4f50ba8a790a1 F test/permutations.test 405542f1d659942994a6b38a9e024cf5cfd23eaa68c806aeb24a72d7c9186e80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test 11cb9310c42f921918f7f563e3c0b6e70f9f9c3a6a1cf12af8fccb6c574f3882 @@ -2211,11 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ddc55efd2d59df3f20743b0533550436da945453c069025a3f871d28d40e13d4 -R 5abe90c6c09c8e29fb97dabd43182a07 -T *branch * percentile-window -T *sym-percentile-window * -T -sym-trunk * +P 4d0e3df4b9c609755977b8a462126242d2be1310c0122a8d4ba76d98d32a7230 +R 06b03a21c6b126e146ea802b7b43139b U drh -Z 85a1ed3baf7191136421a56e2c79709a +Z a8a75faed57aa0d8766eb4f3377323de # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac02b4544f..301d4eacfd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d0e3df4b9c609755977b8a462126242d2be1310c0122a8d4ba76d98d32a7230 +25e68229843cc84978955817285550085d1306ba4ce3b0517dd00e5d05b9ae0a diff --git a/test/percentile.test b/test/percentile.test index ab5b4883fd..92c2010f4b 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -229,4 +229,62 @@ ifcapable vtab { } } +# median() as a window function. (2024-08-31) +# +do_execsql_test percentile-3.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + INSERT INTO t1 VALUES (1, 'A', 'one', 8.4), + (2, 'B', 'two', 7.1), + (3, 'C', 'three', 5.9), + (4, 'D', 'one', 11.0), + (5, 'E', 'two', 12.5), + (6, 'F', 'three', 0.0), + (7, 'G', 'one', 2.7); +} +do_execsql_test percentile-3.1 { +SELECT a, b, c, d, + group_concat(b,'.') OVER W1 AS 'elements', + median(d) OVER W1 AS 'median' + FROM t1 + WINDOW W1 AS (ORDER BY c, a ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) +} { + 1 A one 8.4 A.D 9.7 + 4 D one 11.0 A.D.G 8.4 + 7 G one 2.7 D.G.C 5.9 + 3 C three 5.9 G.C.F 2.7 + 6 F three 0.0 C.F.B 5.9 + 2 B two 7.1 F.B.E 7.1 + 5 E two 12.5 B.E 9.8 +} +do_execsql_test percentile-3.2 { +SELECT a, b, c, d, + group_concat(b,'.') OVER W1 AS 'elements', + median(d) OVER W1 AS 'median' + FROM t1 + WINDOW W1 AS (ORDER BY c, a ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) +} { + 1 A one 8.4 A.D 9.7 + 4 D one 11.0 A.D.G 8.4 + 7 G one 2.7 A.D.G.C 7.15 + 3 C three 5.9 A.D.G.C.F 5.9 + 6 F three 0.0 A.D.G.C.F.B 6.5 + 2 B two 7.1 A.D.G.C.F.B.E 7.1 + 5 E two 12.5 A.D.G.C.F.B.E 7.1 +} +do_execsql_test percentile-3.3 { +SELECT a, b, c, d, + group_concat(b,'.') OVER W1 AS 'elements', + median(d) OVER W1 AS 'median' + FROM t1 + WINDOW W1 AS (ORDER BY c, a ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) +} { + 1 A one 8.4 A.D.G.C.F.B.E 7.1 + 4 D one 11.0 A.D.G.C.F.B.E 7.1 + 7 G one 2.7 D.G.C.F.B.E 6.5 + 3 C three 5.9 G.C.F.B.E 5.9 + 6 F three 0.0 C.F.B.E 6.5 + 2 B two 7.1 F.B.E 7.1 + 5 E two 12.5 B.E 9.8 +} finish_test From ad8ec9db63f0da294625211d6f261d99685fb971 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 31 Aug 2024 16:55:14 +0000 Subject: [PATCH 3/5] Avoid unnecessary sort operations when running one of the percentile aggregates as a window function. FossilOrigin-Name: 5d311536211eb1e3c887ceb7e6516d3900e6eebbccc8c445dd43cdd556182728 --- ext/misc/percentile.c | 196 ++++++++++++++++++++++++++++++------------ manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 149 insertions(+), 61 deletions(-) diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index 99b1430171..e427497878 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -62,6 +62,29 @@ ** ** (14) A separate percentile_cond(Y,X) function is the equivalent of ** percentile(Y,X*100.0). +** +** (15) All three SQL functions implemented by this module can also be +** used as window-functions. +** +** Implementation notes as of 2024-08-31: +** +** * The regular aggregate-function versions of the merge(), percentile(), +** and percentile_cond() routines work by accumulating all values in +** an array of doubles, then sorting that array using a quicksort +** before computing the answer. Thus the runtime is O(NlogN) where +** N is the number of rows of input. +** +** * For the window-function versions of these routines, the array of +** inputs is sorted as soon as the first value is computed. Thereafter, +** the array is kept in sorted order using an insert-sort. This +** results in O(N*K) performance where K is the size of the window. +** One can devise alternative implementations that give O(N*logN*logK) +** performance, but they require more complex logic and data structures. +** The developers have elected to keep the asymptotically slower +** algorithm for now, for simplicity, under the theory that window +** functions are seldom used and when they are, the window size K is +** often small. The developers might revisit that decision later, +** should the need arise. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -78,6 +101,7 @@ struct Percentile { unsigned nAlloc; /* Number of slots allocated for a[] */ unsigned nUsed; /* Number of slots actually used in a[] */ char bSorted; /* True if a[] is already in sorted order */ + char bKeepSorted; /* True if advantageous to keep a[] sorted */ double rPct; /* 1.0 more than the value for P */ double *a; /* Array of Y values */ }; @@ -85,7 +109,7 @@ struct Percentile { /* ** Return TRUE if the input floating-point number is an infinity. */ -static int isInfinity(double r){ +static int percentIsInfinity(double r){ sqlite3_uint64 u; assert( sizeof(u)==sizeof(r) ); memcpy(&u, &r, sizeof(u)); @@ -93,13 +117,55 @@ static int isInfinity(double r){ } /* -** Return TRUE if two doubles differ by 0.001 or less +** Return TRUE if two doubles differ by 0.001 or less. */ -static int sameValue(double a, double b){ +static int percentSameValue(double a, double b){ a -= b; return a>=-0.001 && a<=0.001; } +#if 0 +/* Verify that the elements of the Percentile p are in fact sorted. +** Used for testing and debugging only. +*/ +static void percentAssertSorted(Percentile *p){ + int i; + for(i=p->nUsed-2; i>=0 && p->a[i]<=p->a[i+1]; i--){} + assert( i<0 ); +} +#else +# define percentAssertSorted(X) +#endif + +/* +** Search p (which must have p->bSorted) looking for an entry with +** value y. Return the index of that entry. +** +** If bExact is true, return -1 if the entry is not found. +** +** If bExact is false, return the index at which a new entry with +** value y should be insert in order to keep the values in sorted +** order. The smallest return value in this case will be 0, and +** the largest return value will be p->nUsed. +*/ +static int percentBinarySearch(Percentile *p, double y, int bExact){ + int iFirst = 0; /* First element of search range */ + int iLast = p->nUsed - 1; /* Last element of search range */ + while( iLast>=iFirst ){ + int iMid = (iFirst+iLast)/2; + double x = p->a[iMid]; + if( xy ){ + iLast = iMid - 1; + }else{ + return iMid; + } + } + if( bExact ) return -1; + return iFirst; +} + /* ** The "step" function for percentile(Y,P) is called once for each ** input row. @@ -145,7 +211,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ ** from any prior row, per Requirement (2). */ if( p->rPct==0.0 ){ p->rPct = rPct+1.0; - }else if( !sameValue(p->rPct,rPct+1.0) ){ + }else if( !percentSameValue(p->rPct,rPct+1.0) ){ sqlite3_result_error(pCtx, "2nd argument to percentile() is not the " "same for all input rows", -1); return; @@ -165,7 +231,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ /* Throw an error if the Y value is infinity or NaN */ y = sqlite3_value_double(argv[0]); - if( isInfinity(y) ){ + if( percentIsInfinity(y) ){ sqlite3_result_error(pCtx, "Inf input to percentile()", -1); return; } @@ -183,55 +249,26 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ p->nAlloc = n; p->a = a; } - p->a[p->nUsed++] = y; - assert( p->nUsed>=1 ); - if( p->nUsed==1 ){ + if( p->nUsed==0 ){ + p->a[p->nUsed++] = y; p->bSorted = 1; - }else if( p->bSorted && p->a[p->nUsed-2]>y ){ + }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){ + p->a[p->nUsed++] = y; + }else if( p->bKeepSorted ){ + int i; + percentAssertSorted(p); + i = percentBinarySearch(p, y, 0); + if( inUsed ){ + memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); + } + p->a[i] = y; + p->nUsed++; + }else{ + p->a[p->nUsed++] = y; p->bSorted = 0; } } -/* -** The "inverse" function for percentile(Y,P) is called to remove a -** row that was previously inserted by "step". -*/ -static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ - Percentile *p; - int eType; - double y; - int i; - assert( argc==2 || argc==1 ); - - /* Allocate the session context. */ - p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); - assert( p!=0 ); - - /* Ignore rows for which Y is NULL */ - eType = sqlite3_value_type(argv[0]); - if( eType==SQLITE_NULL ) return; - - /* If not NULL, then Y must be numeric. Otherwise throw an error. - ** Requirement 4 */ - if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ - return; - } - - /* Ignore the Y value if it is infinity or NaN */ - y = sqlite3_value_double(argv[0]); - if( isInfinity(y) ){ - return; - } - - /* Find and remove the row */ - for(i=0; inUsed && p->a[i]!=y; i++){} - if( inUsed ){ - p->a[i] = p->a[p->nUsed-1]; - p->nUsed--; - } - p->bSorted = p->nUsed<=1; -} - /* ** Sort an array of doubles. ** @@ -290,9 +327,59 @@ static void sortDoubles(double *a, int n){ #endif } + /* -** Called to compute the final output of percentile() and to clean -** up all allocated memory. +** The "inverse" function for percentile(Y,P) is called to remove a +** row that was previously inserted by "step". +*/ +static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ + Percentile *p; + int eType; + double y; + int i; + assert( argc==2 || argc==1 ); + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + assert( p!=0 ); + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + return; + } + + /* Ignore the Y value if it is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( percentIsInfinity(y) ){ + return; + } + if( p->bSorted==0 ){ + sortDoubles(p->a, p->nUsed); + p->bSorted = 1; + }else{ + percentAssertSorted(p); + } + p->bKeepSorted = 1; + + /* Find and remove the row */ + i = percentBinarySearch(p, y, 1); + if( i>=0 ){ + p->nUsed--; + if( inUsed ){ + memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); + } + } + percentAssertSorted(p); +} + +/* +** Compute the final output of percentile(). Clean up all allocated +** memory if and only if bIsFinal is true. */ static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ Percentile *p; @@ -306,6 +393,8 @@ static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ if( p->bSorted==0 ){ sortDoubles(p->a, p->nUsed); p->bSorted = 1; + }else{ + percentAssertSorted(p); } ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; i1 = (unsigned)ix; @@ -318,6 +407,8 @@ static void percentCompute(sqlite3_context *pCtx, int bIsFinal){ if( bIsFinal ){ sqlite3_free(p->a); memset(p, 0, sizeof(*p)); + }else{ + p->bKeepSorted = 1; } } static void percentFinal(sqlite3_context *pCtx){ @@ -327,8 +418,6 @@ static void percentValue(sqlite3_context *pCtx){ percentCompute(pCtx, 0); } - - #ifdef _WIN32 __declspec(dllexport) #endif @@ -349,7 +438,6 @@ int sqlite3_percentile_init( SQLITE_UTF8|SQLITE_INNOCUOUS, 0, percentStep, percentFinal, percentValue, percentInverse, 0); - } if( rc==SQLITE_OK ){ rc = sqlite3_create_window_function(db, "percentile_cont", 2, diff --git a/manifest b/manifest index e61e0275b4..e4debfefcd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sadded. -D 2024-08-31T15:02:07.805 +C Avoid\sunnecessary\ssort\soperations\swhen\srunning\sone\sof\sthe\spercentile\naggregates\sas\sa\swindow\sfunction. +D 2024-08-31T16:55:14.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -410,7 +410,7 @@ F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58 F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 -F ext/misc/percentile.c 89416b108569171be1d8dda4fa2687ad116ea969b4d129d02cf3dc1fd67fc87e +F ext/misc/percentile.c 46627b7495c69344d384f667bb6c80ba2c4aeb779997a4e22fea1a39cd20beb9 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d0e3df4b9c609755977b8a462126242d2be1310c0122a8d4ba76d98d32a7230 -R 06b03a21c6b126e146ea802b7b43139b +P 25e68229843cc84978955817285550085d1306ba4ce3b0517dd00e5d05b9ae0a +R 1b58d4d1990506055e066a3b56b6ff9a U drh -Z a8a75faed57aa0d8766eb4f3377323de +Z 10ad6a710202821993e7562cc60a54b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 301d4eacfd..1413d7e77e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25e68229843cc84978955817285550085d1306ba4ce3b0517dd00e5d05b9ae0a +5d311536211eb1e3c887ceb7e6516d3900e6eebbccc8c445dd43cdd556182728 From f138110dfdfac0e17f0539521e40e5d9550602cd Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 31 Aug 2024 17:27:55 +0000 Subject: [PATCH 4/5] Add more test cases. FossilOrigin-Name: 0d0e5456793b4bef673ebc7fcc1c393a6d3c817363d948ddfe06d60eab56dc24 --- manifest | 12 +++++------ manifest.uuid | 2 +- test/percentile.test | 47 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e4debfefcd..df07cca990 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\ssort\soperations\swhen\srunning\sone\sof\sthe\spercentile\naggregates\sas\sa\swindow\sfunction. -D 2024-08-31T16:55:14.747 +C Add\smore\stest\scases. +D 2024-08-31T17:27:55.143 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1517,7 +1517,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 -F test/percentile.test b30d92480943de8236c1522cace227ad354512ade699789636a4f50ba8a790a1 +F test/percentile.test 4c84880c6d5ca216206a92b6cb33c2fb27c972bec6afc6f5a2e53fb895020647 F test/permutations.test 405542f1d659942994a6b38a9e024cf5cfd23eaa68c806aeb24a72d7c9186e80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test 11cb9310c42f921918f7f563e3c0b6e70f9f9c3a6a1cf12af8fccb6c574f3882 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25e68229843cc84978955817285550085d1306ba4ce3b0517dd00e5d05b9ae0a -R 1b58d4d1990506055e066a3b56b6ff9a +P 5d311536211eb1e3c887ceb7e6516d3900e6eebbccc8c445dd43cdd556182728 +R 8645e4703962687ea28510b503521695 U drh -Z 10ad6a710202821993e7562cc60a54b3 +Z c1cee7eb4fe675aa957233112b675c53 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1413d7e77e..5c6fc432f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d311536211eb1e3c887ceb7e6516d3900e6eebbccc8c445dd43cdd556182728 +0d0e5456793b4bef673ebc7fcc1c393a6d3c817363d948ddfe06d60eab56dc24 diff --git a/test/percentile.test b/test/percentile.test index 92c2010f4b..c2bc5633d2 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -287,4 +287,51 @@ SELECT a, b, c, d, 2 B two 7.1 F.B.E 7.1 5 E two 12.5 B.E 9.8 } + +# Test case adapted from examples shown at +# https://database.guide/3-functions-to-calculate-the-median-in-sql/ +# +do_execsql_test percential-4.0 { + CREATE TABLE products( + vendorId INT, + productId INTEGER PRIMARY KEY, + productName REAL, + price REAL + ); + INSERT INTO products VALUES + (1001, 17, 'Left-handed screwdriver', 25.99), + (1001, 49, 'Right-handed screwdriver', 25.99), + (1001, 216, 'Long weight (blue)', 14.75), + (1001, 31, 'Long weight (green)', 11.99), + (1002, 37, 'Sledge hammer', 33.49), + (1003, 7, 'Chainsaw', 245.00), + (1003, 8, 'Straw dog box', 55.99), + (1003, 12, 'Hammock', 11.01), + (1004, 113, 'Teapot', 12.45), + (1004, 117, 'Bottomless coffee mug', 9.99); +} +do_execsql_test percentile-4.1 { + SELECT VendorId, ProductId, /* ProductName,*/ Price, + avg(price) OVER (PARTITION BY vendorId) AS "Average", + median(price) OVER (PARTITION BY vendorId) AS "Median" + FROM products + ORDER BY vendorId, productId; +} { + 1001 17 25.99 19.68 20.37 + 1001 31 11.99 19.68 20.37 + 1001 49 25.99 19.68 20.37 + 1001 216 14.75 19.68 20.37 + 1002 37 33.49 33.49 33.49 + 1003 7 245.0 104.0 55.99 + 1003 8 55.99 104.0 55.99 + 1003 12 11.01 104.0 55.99 + 1004 113 12.45 11.22 11.22 + 1004 117 9.99 11.22 11.22 +} +do_execsql_test percentile-4.2 { + SELECT vendorId, median(price) FROM products + GROUP BY 1 ORDER BY 1; +} {1001 20.37 1002 33.49 1003 55.99 1004 11.22} + + finish_test From a5315a6ecef14cd356915f7f19c0bda390d33c36 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 31 Aug 2024 17:50:06 +0000 Subject: [PATCH 5/5] Still more test cases. FossilOrigin-Name: f09904608195dac38172b0dd4dcab3190f33c116d468beff27f913a7433b400e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/percentile.test | 46 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index df07cca990..21b3220b47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smore\stest\scases. -D 2024-08-31T17:27:55.143 +C Still\smore\stest\scases. +D 2024-08-31T17:50:06.875 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1517,7 +1517,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 -F test/percentile.test 4c84880c6d5ca216206a92b6cb33c2fb27c972bec6afc6f5a2e53fb895020647 +F test/percentile.test 9ae96346c6f5f000eeeca023cabf85efefd744f7b66031147354a4da6dcb50d6 F test/permutations.test 405542f1d659942994a6b38a9e024cf5cfd23eaa68c806aeb24a72d7c9186e80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test 11cb9310c42f921918f7f563e3c0b6e70f9f9c3a6a1cf12af8fccb6c574f3882 @@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d311536211eb1e3c887ceb7e6516d3900e6eebbccc8c445dd43cdd556182728 -R 8645e4703962687ea28510b503521695 +P 0d0e5456793b4bef673ebc7fcc1c393a6d3c817363d948ddfe06d60eab56dc24 +R 55d55ff8a79d8904a0d256c79a96924e U drh -Z c1cee7eb4fe675aa957233112b675c53 +Z 79dbcf8ff805bbacf34d3a6772c416df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c6fc432f4..06468111d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d0e5456793b4bef673ebc7fcc1c393a6d3c817363d948ddfe06d60eab56dc24 +f09904608195dac38172b0dd4dcab3190f33c116d468beff27f913a7433b400e diff --git a/test/percentile.test b/test/percentile.test index c2bc5633d2..c7a41bbced 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -333,5 +333,51 @@ do_execsql_test percentile-4.2 { GROUP BY 1 ORDER BY 1; } {1001 20.37 1002 33.49 1003 55.99 1004 11.22} +do_execsql_test percentile-5.0 { + CREATE TABLE user(name TEXT, class TEXT, cost REAL); + INSERT INTO user VALUES + ('Alice', 'Y', 3578.27), + ('Bob', 'X', 3399.99), + ('Cindy', 'Z', 699.10), + ('Dave', 'Y', 3078.27), + ('Emma', 'Z', 2319.99), + ('Fred', 'Y', 539.99), + ('Gina', 'X', 2320.49), + ('Hank', 'W', 24.99), + ('Irma', 'W', 24.99), + ('Jake', 'X', 2234.99), + ('Kim', 'Y', 4319.99), + ('Liam', 'X', 4968.59), + ('Mia', 'W', 59.53), + ('Nate', 'W', 23.50); +} +do_execsql_test percentile-5.1 { + SELECT name, class, cost, + percentile_cont(cost, 0.00) OVER w1 AS 'P0', + percentile_cont(cost, 0.25) OVER w1 AS 'P1', + percentile_cont(cost, 0.50) OVER w1 AS 'P2', + percentile_cont(cost, 0.75) OVER w1 AS 'P3', + percentile_cont(cost, 1.00) OVER w1 AS 'P4' + FROM user + WINDOW w1 AS (PARTITION BY class) + ORDER BY class, cost; +} { + Nate W 23.5 23.5 24.6175 24.99 33.625 59.53 + Hank W 24.99 23.5 24.6175 24.99 33.625 59.53 + Irma W 24.99 23.5 24.6175 24.99 33.625 59.53 + Mia W 59.53 23.5 24.6175 24.99 33.625 59.53 + Jake X 2234.99 2234.99 2299.115 2860.24 3792.14 4968.59 + Gina X 2320.49 2234.99 2299.115 2860.24 3792.14 4968.59 + Bob X 3399.99 2234.99 2299.115 2860.24 3792.14 4968.59 + Liam X 4968.59 2234.99 2299.115 2860.24 3792.14 4968.59 + Fred Y 539.99 539.99 2443.7 3328.27 3763.7 4319.99 + Dave Y 3078.27 539.99 2443.7 3328.27 3763.7 4319.99 + Alice Y 3578.27 539.99 2443.7 3328.27 3763.7 4319.99 + Kim Y 4319.99 539.99 2443.7 3328.27 3763.7 4319.99 + Cindy Z 699.1 699.1 1104.3225 1509.545 1914.7675 2319.99 + Emma Z 2319.99 699.1 1104.3225 1509.545 1914.7675 2319.99 +} + + finish_test