sdmxlib 0.64.0__tar.gz → 0.65.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/PKG-INFO +1 -1
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/pyproject.toml +1 -1
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/pyproject.toml.orig +1 -1
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/__init__.py +24 -1
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/writer.py +6 -5
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_data_common.py +6 -2
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/sql.py +69 -2
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/dataset.py +14 -2
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/polars.py +4 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_compile.py +32 -5
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_expr.py +12 -6
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/observations.py +21 -3
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/query.py +65 -4
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/README.md +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_duckdb.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_singleflight.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_time.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/_freshness.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/client.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/federated.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/filters.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/policy.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/providers.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/query.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/registry.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/session.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/contentconstraint.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/metadata_annotation.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/data_store.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/_coerce.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/_index.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/data_format.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/gaps.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_dto.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_resolve.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_structure_dto.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_structure_ingest.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v1.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v2.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/metadata.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/structure_values.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json_1_0/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json_1_0/writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_common.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_v3_reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_v3_writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/_synthetic.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/namespaces.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/metadata.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/namespaces.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/namespaces.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/reader.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/writer.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/data_store.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/registry.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_interning.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_items.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_label_match.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_refshape.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/annotations.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/base.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/binding.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/category.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/codelist.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/collections.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/concept.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/constraint.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/convert.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/dataflow.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/datastructure.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/errors.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/expr.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/hierarchy.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/in_memory_registry.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/istring.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/mapping.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/message.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadata_provision.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadataflow.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadataset.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadatastructure.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/organisation.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/provision.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/ref.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/registry.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/representation.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/rest_api.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/rollup.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/urn.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/validation.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/version.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/py.typed +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_closure.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_common.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_edges.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_execute.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_joins.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_plan.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_pushdown.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_rebase.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_rest.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/accessors.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/raw.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/schema.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/rest.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/session.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/sql.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/__init__.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/_kinds.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/_projections.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/lazy.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/readers.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/resolve.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/schema.py +0 -0
- {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/writers.py +0 -0
|
@@ -19,7 +19,7 @@ from sdmxlib.session import Session, SessionHook, ValueSet
|
|
|
19
19
|
from sdmxlib.query import RawSql, RawSqlError, schema
|
|
20
20
|
from sdmxlib.query.accessors import ArtefactRow, CodeRow, HCodeRow, ReportedAttributeRow, q
|
|
21
21
|
from sdmxlib.query._compile import CompiledQuery
|
|
22
|
-
from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Win, WinFn, WinPred
|
|
22
|
+
from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Untyped, Win, WinFn, WinPred
|
|
23
23
|
from sdmxlib.query.observations import ObsQuery, ObsRow
|
|
24
24
|
from sdmxlib.query.query import Frame, Grouped, Page, Query, union
|
|
25
25
|
from sdmxlib.sql import SqlRelation
|
|
@@ -136,6 +136,27 @@ from sdmxlib.model import (
|
|
|
136
136
|
artefact_urn,
|
|
137
137
|
)
|
|
138
138
|
|
|
139
|
+
|
|
140
|
+
def len[R: RowKind]() -> Agg[R]: # noqa: A001 - polars' `pl.len()`; nothing in this module calls the builtin
|
|
141
|
+
"""The number of rows, as an aggregate — `count(*)`, named for polars' `pl.len()`.
|
|
142
|
+
|
|
143
|
+
For `Query.agg`, which folds a whole query into one row, so the row count
|
|
144
|
+
can come out of the same scan as the other aggregates (#414):
|
|
145
|
+
|
|
146
|
+
obs.agg(SEX=obs.col("SEX").unique(), n=sl.len())
|
|
147
|
+
|
|
148
|
+
A function rather than a column method because it counts *rows*, not the
|
|
149
|
+
values of any column — the same reason `group_by(...).len()` is a method on
|
|
150
|
+
the grouping rather than on a column. The row kind is taken from where it is
|
|
151
|
+
used, so it fits any query's `agg`.
|
|
152
|
+
|
|
153
|
+
Shadows the builtin only as `sl.len`; this module never calls `len()`
|
|
154
|
+
itself, which is why it is defined here rather than re-exported from a
|
|
155
|
+
module that would need a second suppression.
|
|
156
|
+
"""
|
|
157
|
+
return Agg[R]("count", ())
|
|
158
|
+
|
|
159
|
+
|
|
139
160
|
__all__ = [
|
|
140
161
|
"Agency",
|
|
141
162
|
"AgencyScheme",
|
|
@@ -278,6 +299,7 @@ __all__ = [
|
|
|
278
299
|
"SupportsUrn",
|
|
279
300
|
"TimeDimension",
|
|
280
301
|
"UnresolvedRef",
|
|
302
|
+
"Untyped",
|
|
281
303
|
"ValidationError",
|
|
282
304
|
"ValidationIssue",
|
|
283
305
|
"ValueMap",
|
|
@@ -290,6 +312,7 @@ __all__ = [
|
|
|
290
312
|
"dim",
|
|
291
313
|
"from_dict",
|
|
292
314
|
"item",
|
|
315
|
+
"len",
|
|
293
316
|
"parse",
|
|
294
317
|
"parse_components",
|
|
295
318
|
"q",
|
|
@@ -23,7 +23,7 @@ column maps in their dict-insertion order, which matches the order in
|
|
|
23
23
|
which they were declared on the structure.
|
|
24
24
|
|
|
25
25
|
Performance: rows are serialised via Polars' Rust CSV writer
|
|
26
|
-
(``pl.
|
|
26
|
+
(``pl.DataFrame(batch).write_csv``) rather than Python's `csv`,
|
|
27
27
|
which makes the writer fast enough that bulk SDMX-CSV downloads are
|
|
28
28
|
network-bound rather than writer-bound (#61).
|
|
29
29
|
|
|
@@ -43,7 +43,7 @@ Example:
|
|
|
43
43
|
from __future__ import annotations
|
|
44
44
|
|
|
45
45
|
import io
|
|
46
|
-
from typing import TYPE_CHECKING
|
|
46
|
+
from typing import TYPE_CHECKING
|
|
47
47
|
|
|
48
48
|
import polars as pl
|
|
49
49
|
|
|
@@ -143,10 +143,11 @@ def write_sdmx_csv(
|
|
|
143
143
|
)
|
|
144
144
|
raise KeyError(msg)
|
|
145
145
|
|
|
146
|
-
#
|
|
147
|
-
#
|
|
146
|
+
# The constructor rather than `pl.from_arrow`, which is typed
|
|
147
|
+
# `DataFrame | Series` and since polars 2.0 really does return a Series
|
|
148
|
+
# for stream-only inputs — a `cast` here would have hidden that.
|
|
148
149
|
df = (
|
|
149
|
-
|
|
150
|
+
pl.DataFrame(batch)
|
|
150
151
|
.select(data_cols)
|
|
151
152
|
.with_columns(
|
|
152
153
|
pl.lit(structure).alias("STRUCTURE"),
|
|
@@ -14,7 +14,7 @@ from __future__ import annotations
|
|
|
14
14
|
import calendar
|
|
15
15
|
import re
|
|
16
16
|
from datetime import datetime, timedelta
|
|
17
|
-
from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict
|
|
17
|
+
from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict
|
|
18
18
|
|
|
19
19
|
import msgspec
|
|
20
20
|
import polars as pl
|
|
@@ -640,7 +640,11 @@ def encode_batch(
|
|
|
640
640
|
)
|
|
641
641
|
raise KeyError(msg)
|
|
642
642
|
|
|
643
|
-
|
|
643
|
+
# The constructor, not `pl.from_arrow`: the same frame for a RecordBatch, but
|
|
644
|
+
# typed `DataFrame` rather than `DataFrame | Series` — and polars 2.0 made
|
|
645
|
+
# `from_arrow` return a Series for stream-only inputs, which a `cast` would
|
|
646
|
+
# have passed through unchecked.
|
|
647
|
+
df = pl.DataFrame(batch)
|
|
644
648
|
n = df.height
|
|
645
649
|
if n == 0:
|
|
646
650
|
return b""
|
|
@@ -59,6 +59,20 @@ if TYPE_CHECKING:
|
|
|
59
59
|
from sdmxlib.model.binding import DataflowBinding
|
|
60
60
|
|
|
61
61
|
|
|
62
|
+
class SidecarNotUniqueError(ValueError):
|
|
63
|
+
"""A series-attributes sidecar holds more than one row per join key (#412).
|
|
64
|
+
|
|
65
|
+
The sidecar is joined as `LEFT JOIN … USING (<keys>)`, so every extra row
|
|
66
|
+
per key multiplies each observation it matches. Nothing errors downstream:
|
|
67
|
+
`count()`, `collect()` and every aggregate over a sidecar column come back
|
|
68
|
+
inflated by the fan-out factor, and every row looks valid.
|
|
69
|
+
|
|
70
|
+
A `ValueError` because it is the same kind of refusal `plan` already makes
|
|
71
|
+
for a sidecar sharing no columns — the binding names a table that cannot
|
|
72
|
+
serve as its series attributes — so a caller catching that catches this.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
|
|
62
76
|
@frozen(kw_only=True)
|
|
63
77
|
class QueryPlan:
|
|
64
78
|
"""The resolved ``FROM`` side of a `DataflowBinding`'s query.
|
|
@@ -109,6 +123,10 @@ class SchemaInspector:
|
|
|
109
123
|
def __init__(self, conn: duckdb.DuckDBPyConnection) -> None:
|
|
110
124
|
self._conn = conn
|
|
111
125
|
self._columns_cache: dict[str, tuple[str, ...]] = {}
|
|
126
|
+
#: (table, keys) -> (rows, distinct keys), for `key_counts`. Shares
|
|
127
|
+
#: `_columns_cache`'s lock and lifecycle: both describe a table as it
|
|
128
|
+
#: stands, and `invalidate` is the one signal that it changed.
|
|
129
|
+
self._key_counts_cache: dict[tuple[str, tuple[str, ...]], tuple[int, int]] = {}
|
|
112
130
|
self._lock = threading.Lock()
|
|
113
131
|
|
|
114
132
|
def columns(self, table: str) -> tuple[str, ...]:
|
|
@@ -139,15 +157,50 @@ class SchemaInspector:
|
|
|
139
157
|
cols_b = set(self.columns(table_b))
|
|
140
158
|
return tuple(c for c in self.columns(table_a) if c in cols_b)
|
|
141
159
|
|
|
160
|
+
def key_counts(self, table: str, keys: tuple[str, ...]) -> tuple[int, int]:
|
|
161
|
+
"""``(rows, distinct key combinations)`` of ``table`` over ``keys`` (cached).
|
|
162
|
+
|
|
163
|
+
Rows with a NULL in any key are left out of both counts: `USING` never
|
|
164
|
+
matches NULL, so repeated NULL-keyed rows cannot multiply a join, and
|
|
165
|
+
refusing a table for them would be refusing a harmless one.
|
|
166
|
+
|
|
167
|
+
A scan of the table — ~5 ms on cenprof-2021-csip's 619-row geography
|
|
168
|
+
sidecar, ~37 ms cold — so memoised single-flight beside `columns`, and
|
|
169
|
+
dropped by `invalidate` with it.
|
|
170
|
+
|
|
171
|
+
Args:
|
|
172
|
+
table: The table to count, as `columns` takes it.
|
|
173
|
+
keys: The columns whose combinations are counted.
|
|
174
|
+
"""
|
|
175
|
+
cache_key = (table, keys)
|
|
176
|
+
cached = self._key_counts_cache.get(cache_key)
|
|
177
|
+
if cached is not None:
|
|
178
|
+
return cached
|
|
179
|
+
with self._lock:
|
|
180
|
+
cached = self._key_counts_cache.get(cache_key)
|
|
181
|
+
if cached is not None:
|
|
182
|
+
return cached
|
|
183
|
+
tuple_sql = ", ".join(keys)
|
|
184
|
+
not_null = " AND ".join(f"{k} IS NOT NULL" for k in keys)
|
|
185
|
+
row = self._conn.execute(
|
|
186
|
+
f"SELECT count(*), count(DISTINCT ({tuple_sql})) FROM {table} WHERE {not_null}"
|
|
187
|
+
).fetchone()
|
|
188
|
+
counts = (int(row[0]), int(row[1])) if row else (0, 0)
|
|
189
|
+
self._key_counts_cache[cache_key] = counts
|
|
190
|
+
return counts
|
|
191
|
+
|
|
142
192
|
def invalidate(self, table: str) -> None:
|
|
143
|
-
"""Drop
|
|
193
|
+
"""Drop everything cached about ``table`` (e.g. after a rewrite)."""
|
|
144
194
|
with self._lock:
|
|
145
195
|
self._columns_cache.pop(table, None)
|
|
196
|
+
for cache_key in [k for k in self._key_counts_cache if k[0] == table]:
|
|
197
|
+
del self._key_counts_cache[cache_key]
|
|
146
198
|
|
|
147
199
|
def invalidate_all(self) -> None:
|
|
148
|
-
"""Drop
|
|
200
|
+
"""Drop everything cached about every table."""
|
|
149
201
|
with self._lock:
|
|
150
202
|
self._columns_cache.clear()
|
|
203
|
+
self._key_counts_cache.clear()
|
|
151
204
|
|
|
152
205
|
def plan(self, binding: DataflowBinding) -> QueryPlan:
|
|
153
206
|
"""Resolve ``binding``'s main/sidecar tables into a `QueryPlan`.
|
|
@@ -163,6 +216,9 @@ class SchemaInspector:
|
|
|
163
216
|
Raises:
|
|
164
217
|
ValueError: If the binding names a sidecar that shares no columns
|
|
165
218
|
with the main table (no derivable join keys).
|
|
219
|
+
SidecarNotUniqueError: If the sidecar holds more than one row for
|
|
220
|
+
some combination of those keys, which would multiply every
|
|
221
|
+
observation the join matches (#412).
|
|
166
222
|
"""
|
|
167
223
|
main = f"{binding.database_role}.{binding.table_location}"
|
|
168
224
|
sidecar_loc = binding.series_attributes_location
|
|
@@ -176,4 +232,15 @@ class SchemaInspector:
|
|
|
176
232
|
f"shares no columns with {main!r}; cannot derive JOIN keys"
|
|
177
233
|
)
|
|
178
234
|
raise ValueError(msg)
|
|
235
|
+
rows, distinct = self.key_counts(sidecar, join_keys)
|
|
236
|
+
if rows != distinct:
|
|
237
|
+
keys = ", ".join(join_keys)
|
|
238
|
+
msg = (
|
|
239
|
+
f"series-attributes sidecar {sidecar!r} holds {rows:,} rows but only {distinct:,} distinct "
|
|
240
|
+
f"({keys}) — joined on those keys, every observation of a repeated series would be counted "
|
|
241
|
+
f"once per sidecar row, so counts and aggregates over its columns come back inflated (#412). "
|
|
242
|
+
f"The keys are the columns the two tables share; make the sidecar unique on them, or drop the "
|
|
243
|
+
f"shared columns that are not part of the series key."
|
|
244
|
+
)
|
|
245
|
+
raise SidecarNotUniqueError(msg)
|
|
179
246
|
return QueryPlan(main=main, sidecar=sidecar, join_keys=join_keys)
|
|
@@ -156,8 +156,16 @@ class Dataset:
|
|
|
156
156
|
if hint_cl is not None:
|
|
157
157
|
on_columns = hint_cl.code_ids()
|
|
158
158
|
else:
|
|
159
|
-
# Codelist not resolved — discover column values from the data
|
|
160
|
-
|
|
159
|
+
# Codelist not resolved — discover column values from the data.
|
|
160
|
+
# Sorted, because `unique` promises no order and the wide
|
|
161
|
+
# frame's columns would otherwise change from run to run.
|
|
162
|
+
on_columns = (
|
|
163
|
+
self.data.select(pl.col(pivot_id).cast(pl.String))
|
|
164
|
+
.unique()
|
|
165
|
+
.sort(pivot_id)
|
|
166
|
+
.collect()[pivot_id]
|
|
167
|
+
.to_list()
|
|
168
|
+
)
|
|
161
169
|
index = [d.id for d in self.structure.dimensions if d.id != pivot_id]
|
|
162
170
|
|
|
163
171
|
else:
|
|
@@ -187,11 +195,15 @@ class Dataset:
|
|
|
187
195
|
index.append(self.structure.time_dimension.id)
|
|
188
196
|
measures = self.measures
|
|
189
197
|
values = measures[0] if len(measures) == 1 else measures
|
|
198
|
+
# `maintain_order`: rows in the order their series first appear, which
|
|
199
|
+
# is `DataFrame.pivot`'s contract. The lazy pivot defaults to no order,
|
|
200
|
+
# and at a few thousand series that order differs run to run.
|
|
190
201
|
return self.data.pivot(
|
|
191
202
|
on=pivot_id,
|
|
192
203
|
on_columns=pl.Series(on_columns),
|
|
193
204
|
index=index,
|
|
194
205
|
values=values,
|
|
206
|
+
maintain_order=True,
|
|
195
207
|
)
|
|
196
208
|
|
|
197
209
|
# ── SDMX-aware expressions ────────────────────────────────────────────────
|
|
@@ -208,6 +208,10 @@ def validation_report(
|
|
|
208
208
|
.rename({comp.id: "invalid_value"})
|
|
209
209
|
.with_columns(pl.lit(comp.id).alias("column"))
|
|
210
210
|
.select("column", "invalid_value", "count")
|
|
211
|
+
# `group_by` promises no order, and at a few thousand values the
|
|
212
|
+
# order it gives differs run to run — on polars 1.x and 2.0 alike.
|
|
213
|
+
# Components stay in DSD order (the loop); values within one sort.
|
|
214
|
+
.sort("invalid_value")
|
|
211
215
|
.collect()
|
|
212
216
|
)
|
|
213
217
|
if not bad.is_empty():
|
|
@@ -843,12 +843,28 @@ class _Scope:
|
|
|
843
843
|
for windows, applied a third time.
|
|
844
844
|
"""
|
|
845
845
|
if isinstance(item, AnyAgg):
|
|
846
|
-
|
|
847
|
-
body = self._agg_body_from(item, refs)
|
|
846
|
+
body = self._grouped_agg_body(item, context="in a grouped aggregate")
|
|
848
847
|
return Sql(f"{body.text} AS {name}", body.params)
|
|
849
848
|
ref = self._flat_reference(item, context="as a group key")
|
|
850
849
|
return Sql(f"{ref.text} AS {name}", ref.params)
|
|
851
850
|
|
|
851
|
+
def _grouped_agg_body(self, agg: AnyAgg, *, context: str) -> Sql:
|
|
852
|
+
"""A plain aggregate over the group's rows, honouring the column's `.sort()`.
|
|
853
|
+
|
|
854
|
+
The order was dropped here until #414: `col.sort().unique()` compiled to
|
|
855
|
+
`list(DISTINCT col ORDER BY col)` through a fan-out but to an unordered
|
|
856
|
+
`list(DISTINCT col)` in a `group_by().agg()` — the caller asked for an
|
|
857
|
+
order and silently did not get one. Read off the column exactly as
|
|
858
|
+
`_agg_subquery` reads it, so the two readings of an `Agg` agree.
|
|
859
|
+
"""
|
|
860
|
+
refs = [self._flat_reference(column, context=context) for column in agg.columns]
|
|
861
|
+
order = agg.columns[0]._order if agg.columns else None
|
|
862
|
+
order_sql: Sql | None = None
|
|
863
|
+
if order is not None:
|
|
864
|
+
key_ref = self._flat_reference(order[0], context=context)
|
|
865
|
+
order_sql = Sql(f"{key_ref.text}{' DESC' if order[1] else ''}", key_ref.params)
|
|
866
|
+
return self._agg_body_from(agg, refs, order_sql)
|
|
867
|
+
|
|
852
868
|
def grouped_having(self, pred: AggPred) -> Sql:
|
|
853
869
|
"""One `HAVING` term — an aggregate compared to a value.
|
|
854
870
|
|
|
@@ -861,8 +877,7 @@ class _Scope:
|
|
|
861
877
|
means the correlated-subquery reading instead, which is why the two
|
|
862
878
|
clauses compile it through different methods rather than sharing one.
|
|
863
879
|
"""
|
|
864
|
-
|
|
865
|
-
body = self._agg_body_from(pred.agg, refs)
|
|
880
|
+
body = self._grouped_agg_body(pred.agg, context="in a HAVING clause")
|
|
866
881
|
return Sql(f"{body.text} {pred.op} ?", (*body.params, pred.value))
|
|
867
882
|
|
|
868
883
|
def _aggregate(self, name: str, agg: AnyAgg) -> Sql:
|
|
@@ -1433,6 +1448,7 @@ def compile_projection(
|
|
|
1433
1448
|
distinct: bool = False,
|
|
1434
1449
|
qualify: Sequence[AnyWinPred] = (),
|
|
1435
1450
|
group_keys: Sequence[AnyCol] = (),
|
|
1451
|
+
aggregated: bool = False,
|
|
1436
1452
|
having: Sequence[AggPred] = (),
|
|
1437
1453
|
positional_order: Sequence[tuple[int, bool]] = (),
|
|
1438
1454
|
fetch_scope: str | None = None,
|
|
@@ -1454,7 +1470,13 @@ def compile_projection(
|
|
|
1454
1470
|
# front of `items`, so `_projected_prefixes` has already joined them.
|
|
1455
1471
|
# Measured — adding one was a redundant call no mutation could kill.
|
|
1456
1472
|
scope = _Scope(table, predicates, _Namespace(), extra_paths=extra, schema=root.__schema__, joins=root.__joins__)
|
|
1457
|
-
|
|
1473
|
+
# `aggregated` rather than `group_keys` decides how an item compiles (#414):
|
|
1474
|
+
# a root-level `Query.agg` aggregates over every row with no keys at all,
|
|
1475
|
+
# and keying this on the keys sent it down the per-row projection path,
|
|
1476
|
+
# which correlates each aggregate over a fan-out and refuses at the root.
|
|
1477
|
+
# The keys still decide `GROUP BY` and the tiebreak below.
|
|
1478
|
+
grouped = aggregated or bool(group_keys)
|
|
1479
|
+
projected = [scope.grouped(name, item) if grouped else scope.projection(name, item) for name, item in items]
|
|
1458
1480
|
constraint = _constraint_clause(root.__constraint__, scope.alias)
|
|
1459
1481
|
where = _conjoin(
|
|
1460
1482
|
[constraint, _scope_clause(fetch_scope, scope.alias, root.__schema__), *(scope.compile(p) for p in predicates)]
|
|
@@ -1503,6 +1525,11 @@ def compile_projection(
|
|
|
1503
1525
|
# would name columns the projection may have dropped.
|
|
1504
1526
|
keys = [f"{position} {'ASC' if ascending else 'DESC'} NULLS LAST" for position, ascending in positional_order]
|
|
1505
1527
|
ordered = Sql("\nORDER BY " + ", ".join([*keys, *(tiebreak or [])]))
|
|
1528
|
+
elif aggregated and not group_keys:
|
|
1529
|
+
# One row, so nothing to order — and the root-key fallback below would
|
|
1530
|
+
# name row columns that are no longer in an aggregate's scope, which
|
|
1531
|
+
# DuckDB rejects with a GROUP BY binder error.
|
|
1532
|
+
ordered = Sql("")
|
|
1506
1533
|
else:
|
|
1507
1534
|
ordered = _order_sql(root, scope, order, tiebreak=tiebreak)
|
|
1508
1535
|
# `from_sql` last: it emits only the root joins something above referenced.
|
|
@@ -522,29 +522,35 @@ class Agg[R: RowKind](AnyAgg):
|
|
|
522
522
|
#
|
|
523
523
|
# Approved suppression, on the grounds that it mirrors `Win.__eq__`
|
|
524
524
|
# exactly — same override, same reason, same pair of checkers.
|
|
525
|
+
#
|
|
526
|
+
# Declared `AggCompare[R]`, which is what all six return (#414). They said
|
|
527
|
+
# `Pred[R]`, so `having(col.count() > 1)` — whose parameter is `AggPred` —
|
|
528
|
+
# was a type error for every typed caller; the existing `having` tests
|
|
529
|
+
# passed only because their fixture is untyped. `AggCompare` is both an
|
|
530
|
+
# `AggPred` and a `Pred[R]`, so `filter` still takes it unchanged.
|
|
525
531
|
def __hash__(self) -> int:
|
|
526
532
|
return id(self)
|
|
527
533
|
|
|
528
|
-
def __eq__(self, other: object) ->
|
|
534
|
+
def __eq__(self, other: object) -> AggCompare[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
|
|
529
535
|
return AggCompare(self, "=", other)
|
|
530
536
|
|
|
531
|
-
def __ne__(self, other: object) ->
|
|
537
|
+
def __ne__(self, other: object) -> AggCompare[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
|
|
532
538
|
return AggCompare(self, "!=", other)
|
|
533
539
|
|
|
534
540
|
def __bool__(self) -> bool:
|
|
535
541
|
msg = _ambiguous_truth("an aggregate")
|
|
536
542
|
raise TypeError(msg)
|
|
537
543
|
|
|
538
|
-
def __gt__(self, other: object) ->
|
|
544
|
+
def __gt__(self, other: object) -> AggCompare[R]:
|
|
539
545
|
return AggCompare(self, ">", other)
|
|
540
546
|
|
|
541
|
-
def __ge__(self, other: object) ->
|
|
547
|
+
def __ge__(self, other: object) -> AggCompare[R]:
|
|
542
548
|
return AggCompare(self, ">=", other)
|
|
543
549
|
|
|
544
|
-
def __lt__(self, other: object) ->
|
|
550
|
+
def __lt__(self, other: object) -> AggCompare[R]:
|
|
545
551
|
return AggCompare(self, "<", other)
|
|
546
552
|
|
|
547
|
-
def __le__(self, other: object) ->
|
|
553
|
+
def __le__(self, other: object) -> AggCompare[R]:
|
|
548
554
|
return AggCompare(self, "<=", other)
|
|
549
555
|
|
|
550
556
|
def over(self, *partition: Col[R]) -> Win[R]:
|
|
@@ -44,7 +44,7 @@ from typing import TYPE_CHECKING, Self
|
|
|
44
44
|
import attrs
|
|
45
45
|
|
|
46
46
|
from sdmxlib.model.datastructure import ComponentRole
|
|
47
|
-
from sdmxlib.query._expr import
|
|
47
|
+
from sdmxlib.query._expr import Cols, RowKind, Untyped
|
|
48
48
|
from sdmxlib.query._joins import RootJoin
|
|
49
49
|
from sdmxlib.query.query import Query
|
|
50
50
|
|
|
@@ -69,10 +69,19 @@ class Sidecar:
|
|
|
69
69
|
changes the *plan* and never the spelling: `obs["GEO_NAME"]` means the same
|
|
70
70
|
thing whether the column sits on the fact table or across a join.
|
|
71
71
|
|
|
72
|
+
**One row per series, enforced.** "Once per series" means unique on the
|
|
73
|
+
join keys, and a sidecar that is not would multiply every observation it
|
|
74
|
+
matches — silently, in every count and aggregate over its columns.
|
|
75
|
+
`SchemaInspector.plan` refuses one with `SidecarNotUniqueError` (#412), so
|
|
76
|
+
a `Sidecar` that exists has been checked.
|
|
77
|
+
|
|
72
78
|
Attributes:
|
|
73
79
|
table: Fully-qualified sidecar table.
|
|
74
80
|
join_keys: Columns common to both sides — derived by
|
|
75
81
|
`SchemaInspector.plan` from the tables themselves, not declared.
|
|
82
|
+
Which is why uniqueness is checked against *these*: a sidecar unique
|
|
83
|
+
on its own key can still fan out if the two tables share fewer
|
|
84
|
+
columns than that key.
|
|
76
85
|
columns: Every column the sidecar has, for deciding which side a
|
|
77
86
|
component lives on.
|
|
78
87
|
"""
|
|
@@ -316,8 +325,17 @@ class ObsQuery(Query[ObsRow]):
|
|
|
316
325
|
here rather than raising.
|
|
317
326
|
"""
|
|
318
327
|
|
|
319
|
-
def col(self, component_id: str) ->
|
|
320
|
-
"""The column for one SDMX component. See `Observations.__getitem__`.
|
|
328
|
+
def col(self, component_id: str) -> Untyped[ObsRow]:
|
|
329
|
+
"""The column for one SDMX component. See `Observations.__getitem__`.
|
|
330
|
+
|
|
331
|
+
Declared `Untyped`, which is what it always returned (#413). It said
|
|
332
|
+
`Col` — the lie #347 corrected on `__getitem__` and left here — so the
|
|
333
|
+
operators `Untyped` really offers were invisible to every checker and
|
|
334
|
+
every reader: the comparisons, and `.str`'s `contains`, `starts_with`,
|
|
335
|
+
`ends_with` and `caseless` rung. Those four are SDMX-REST's `co`, `nc`
|
|
336
|
+
(negated), `sw` and `ew`, with the `LIKE` escaping done by
|
|
337
|
+
`_label_match` rather than by each consumer.
|
|
338
|
+
"""
|
|
321
339
|
root = self.plan.root
|
|
322
340
|
if not isinstance(root, Observations): # pragma: no cover - built one way
|
|
323
341
|
msg = f"this query is not rooted at observations, but at {type(root).__name__}"
|
|
@@ -1676,7 +1676,10 @@ class Query[R: RowKind, A = object]:
|
|
|
1676
1676
|
the grouped result, which can also order by an aggregate.
|
|
1677
1677
|
"""
|
|
1678
1678
|
if not keys:
|
|
1679
|
-
msg =
|
|
1679
|
+
msg = (
|
|
1680
|
+
"group_by() needs at least one key. To aggregate over every row at once, use "
|
|
1681
|
+
".agg(...) — several aggregates, one row — or .count() for the row count alone."
|
|
1682
|
+
)
|
|
1680
1683
|
raise TypeError(msg)
|
|
1681
1684
|
for key in keys:
|
|
1682
1685
|
_require_column("group_by", key)
|
|
@@ -1690,6 +1693,58 @@ class Query[R: RowKind, A = object]:
|
|
|
1690
1693
|
raise SliceError(msg)
|
|
1691
1694
|
return Grouped(query=self, keys=tuple(keys))
|
|
1692
1695
|
|
|
1696
|
+
def agg(self, **named: Agg[R]) -> Frame[R, A]:
|
|
1697
|
+
"""Several aggregates over every row of the query, in one scan (#414).
|
|
1698
|
+
|
|
1699
|
+
One row out, one column per aggregate — `group_by(...).agg(...)` with
|
|
1700
|
+
no keys, which is why it is not spelled that way: `group_by()` refuses
|
|
1701
|
+
zero keys, and a row count alone is `count()`. The case this serves is
|
|
1702
|
+
*several* answers from one pass, e.g. the distinct values each
|
|
1703
|
+
dimension takes under a filter, plus the row count — what SDMX-REST's
|
|
1704
|
+
`/availability` returns:
|
|
1705
|
+
|
|
1706
|
+
obs.agg(
|
|
1707
|
+
SEX=obs.col("SEX").unique(),
|
|
1708
|
+
TIME_PERIOD=obs.col("TIME_PERIOD").unique(),
|
|
1709
|
+
n=sl.len(),
|
|
1710
|
+
).collect()
|
|
1711
|
+
|
|
1712
|
+
Not always faster than one `select(col).unique()` per column. On
|
|
1713
|
+
cenprof-2021 (519M rows, six dimensions) one scan lost 2.8x unfiltered
|
|
1714
|
+
and won 1.1-4.8x under a filter, so it is an option to take when a
|
|
1715
|
+
filter narrows the scan, not a replacement.
|
|
1716
|
+
|
|
1717
|
+
An `Agg` here is a plain SQL aggregate over the query's rows — the same
|
|
1718
|
+
expression in `select()` is a correlated subquery over a fan-out.
|
|
1719
|
+
|
|
1720
|
+
Raises:
|
|
1721
|
+
TypeError: If no aggregates are given, or one is a plain column —
|
|
1722
|
+
with no group keys there is no single value a column could
|
|
1723
|
+
take. Project it with `select()`, or `group_by()` it.
|
|
1724
|
+
SliceError: If the query is ordered or sliced. An order means
|
|
1725
|
+
nothing once every row folds into one, and a limit applied
|
|
1726
|
+
first would silently aggregate a truncated set.
|
|
1727
|
+
"""
|
|
1728
|
+
if not named:
|
|
1729
|
+
msg = "agg() needs at least one aggregate. For the row count alone, use .count()."
|
|
1730
|
+
raise TypeError(msg)
|
|
1731
|
+
for name, item in named.items():
|
|
1732
|
+
if not isinstance(item, Agg):
|
|
1733
|
+
msg = (
|
|
1734
|
+
f"{name!r} is a column, not an aggregate — agg() folds every row into one, so a "
|
|
1735
|
+
f"column has no single value to give. Aggregate it (.unique(), .count(), .max(), …), "
|
|
1736
|
+
f"or keep it per row with select(), or per group with group_by()."
|
|
1737
|
+
)
|
|
1738
|
+
raise TypeError(msg)
|
|
1739
|
+
if self.order_keys or self._sliced():
|
|
1740
|
+
msg = (
|
|
1741
|
+
"cannot agg() a query that is ordered or sliced — an order means nothing once "
|
|
1742
|
+
"every row folds into one, and a limit before aggregating aggregates a truncated "
|
|
1743
|
+
"set. Aggregate the whole query, then order or slice something else."
|
|
1744
|
+
)
|
|
1745
|
+
raise SliceError(msg)
|
|
1746
|
+
return Frame(query=self, items=tuple(named.items()), aggregated=True)
|
|
1747
|
+
|
|
1693
1748
|
# ── no implicit execution ─────────────────────────────────────────────
|
|
1694
1749
|
|
|
1695
1750
|
def __iter__(self) -> None:
|
|
@@ -1757,7 +1812,7 @@ class Grouped[R: RowKind, A = object]:
|
|
|
1757
1812
|
# the binding's layout.
|
|
1758
1813
|
items: list[tuple[str, AnyCol | AnyAgg]] = [(key.projected_name, key) for key in self.keys]
|
|
1759
1814
|
items.extend(named.items())
|
|
1760
|
-
return Frame(query=self._query, items=tuple(items), group_keys=self.keys)
|
|
1815
|
+
return Frame(query=self._query, items=tuple(items), group_keys=self.keys, aggregated=True)
|
|
1761
1816
|
|
|
1762
1817
|
def len(self) -> Frame[R, A]:
|
|
1763
1818
|
"""Count rows per group, in a column named `len` — as polars names it.
|
|
@@ -1771,7 +1826,7 @@ class Grouped[R: RowKind, A = object]:
|
|
|
1771
1826
|
# the binding's layout.
|
|
1772
1827
|
items: list[tuple[str, AnyCol | AnyAgg]] = [(key.projected_name, key) for key in self.keys]
|
|
1773
1828
|
items.append(("len", Agg[R]("count", ())))
|
|
1774
|
-
return Frame(query=self._query, items=tuple(items), group_keys=self.keys)
|
|
1829
|
+
return Frame(query=self._query, items=tuple(items), group_keys=self.keys, aggregated=True)
|
|
1775
1830
|
|
|
1776
1831
|
@override
|
|
1777
1832
|
def __repr__(self) -> str:
|
|
@@ -1853,6 +1908,11 @@ class Frame[R: RowKind, A = object]:
|
|
|
1853
1908
|
#: aggregates are plain `GROUP BY` aggregates rather than correlated
|
|
1854
1909
|
#: subqueries.
|
|
1855
1910
|
group_keys: tuple[Col[R], ...] = ()
|
|
1911
|
+
#: True when every item is a plain aggregate over the query's rows — a
|
|
1912
|
+
#: `group_by().agg()` (with keys) or a root `Query.agg()` (without, #414).
|
|
1913
|
+
#: Separate from `group_keys` because zero keys is a real case: one row
|
|
1914
|
+
#: summarising the whole filtered query.
|
|
1915
|
+
aggregated: bool = False
|
|
1856
1916
|
#: `HAVING` terms — predicates over the *groups*, not over the rows.
|
|
1857
1917
|
having_terms: tuple[AggPred, ...] = ()
|
|
1858
1918
|
|
|
@@ -2076,7 +2136,7 @@ class Frame[R: RowKind, A = object]:
|
|
|
2076
2136
|
# impossible — the default order made it refuse, and `order_by()` with
|
|
2077
2137
|
# no arguments clears the caller's keys and falls back to that same
|
|
2078
2138
|
# default, so there was no spelling that worked.
|
|
2079
|
-
row_order = self._query.order_keys if (self.
|
|
2139
|
+
row_order = self._query.order_keys if (self.aggregated or self.distinct) else self._query._order
|
|
2080
2140
|
if self._query.unordered:
|
|
2081
2141
|
# Drops the grouped and `unique()` tiebreaks along with the row
|
|
2082
2142
|
# order, which is the point: those exist so groups come back in a
|
|
@@ -2092,6 +2152,7 @@ class Frame[R: RowKind, A = object]:
|
|
|
2092
2152
|
distinct=self.distinct,
|
|
2093
2153
|
qualify=self._query.qualifiers,
|
|
2094
2154
|
group_keys=self.group_keys,
|
|
2155
|
+
aggregated=self.aggregated,
|
|
2095
2156
|
having=self.having_terms,
|
|
2096
2157
|
positional_order=self.order,
|
|
2097
2158
|
fetch_scope=self._query._fetch_scope,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|