sdmxlib 0.64.0__tar.gz → 0.65.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/PKG-INFO +1 -1
  2. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/pyproject.toml +1 -1
  3. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/pyproject.toml.orig +1 -1
  4. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/__init__.py +24 -1
  5. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/writer.py +6 -5
  6. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_data_common.py +6 -2
  7. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/sql.py +69 -2
  8. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/dataset.py +14 -2
  9. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/polars.py +4 -0
  10. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_compile.py +32 -5
  11. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_expr.py +12 -6
  12. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/observations.py +21 -3
  13. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/query.py +65 -4
  14. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/README.md +0 -0
  15. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_duckdb.py +0 -0
  16. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_singleflight.py +0 -0
  17. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/_time.py +0 -0
  18. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/__init__.py +0 -0
  19. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/_freshness.py +0 -0
  20. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/client.py +0 -0
  21. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/federated.py +0 -0
  22. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/filters.py +0 -0
  23. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/policy.py +0 -0
  24. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/providers.py +0 -0
  25. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/query.py +0 -0
  26. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/registry.py +0 -0
  27. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/api/session.py +0 -0
  28. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/__init__.py +0 -0
  29. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/contentconstraint.py +0 -0
  30. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/builders/metadata_annotation.py +0 -0
  31. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/data_store.py +0 -0
  32. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/__init__.py +0 -0
  33. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/_coerce.py +0 -0
  34. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/_index.py +0 -0
  35. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/data_format.py +0 -0
  36. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/gaps.py +0 -0
  37. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/__init__.py +0 -0
  38. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/reader.py +0 -0
  39. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/__init__.py +0 -0
  40. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_dto.py +0 -0
  41. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_resolve.py +0 -0
  42. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_structure_dto.py +0 -0
  43. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_structure_ingest.py +0 -0
  44. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v1.py +0 -0
  45. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v2.py +0 -0
  46. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/metadata.py +0 -0
  47. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/reader.py +0 -0
  48. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/structure_values.py +0 -0
  49. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/writer.py +0 -0
  50. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json_1_0/__init__.py +0 -0
  51. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json_1_0/writer.py +0 -0
  52. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/__init__.py +0 -0
  53. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_common.py +0 -0
  54. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_v3_reader.py +0 -0
  55. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml/_v3_writer.py +0 -0
  56. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/__init__.py +0 -0
  57. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/_synthetic.py +0 -0
  58. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/namespaces.py +0 -0
  59. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/reader.py +0 -0
  60. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml21/writer.py +0 -0
  61. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/__init__.py +0 -0
  62. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/metadata.py +0 -0
  63. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/namespaces.py +0 -0
  64. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/reader.py +0 -0
  65. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml30/writer.py +0 -0
  66. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/__init__.py +0 -0
  67. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/namespaces.py +0 -0
  68. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/reader.py +0 -0
  69. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_ml31/writer.py +0 -0
  70. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/__init__.py +0 -0
  71. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/data_store.py +0 -0
  72. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/local/registry.py +0 -0
  73. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/__init__.py +0 -0
  74. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_interning.py +0 -0
  75. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_items.py +0 -0
  76. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_label_match.py +0 -0
  77. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/_refshape.py +0 -0
  78. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/annotations.py +0 -0
  79. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/base.py +0 -0
  80. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/binding.py +0 -0
  81. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/category.py +0 -0
  82. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/codelist.py +0 -0
  83. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/collections.py +0 -0
  84. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/concept.py +0 -0
  85. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/constraint.py +0 -0
  86. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/convert.py +0 -0
  87. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/dataflow.py +0 -0
  88. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/datastructure.py +0 -0
  89. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/errors.py +0 -0
  90. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/expr.py +0 -0
  91. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/hierarchy.py +0 -0
  92. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/in_memory_registry.py +0 -0
  93. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/istring.py +0 -0
  94. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/mapping.py +0 -0
  95. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/message.py +0 -0
  96. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadata_provision.py +0 -0
  97. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadataflow.py +0 -0
  98. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadataset.py +0 -0
  99. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/metadatastructure.py +0 -0
  100. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/organisation.py +0 -0
  101. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/provision.py +0 -0
  102. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/ref.py +0 -0
  103. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/registry.py +0 -0
  104. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/representation.py +0 -0
  105. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/rest_api.py +0 -0
  106. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/rollup.py +0 -0
  107. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/urn.py +0 -0
  108. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/validation.py +0 -0
  109. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/model/version.py +0 -0
  110. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/py.typed +0 -0
  111. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/__init__.py +0 -0
  112. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_closure.py +0 -0
  113. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_common.py +0 -0
  114. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_edges.py +0 -0
  115. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_execute.py +0 -0
  116. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_joins.py +0 -0
  117. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_plan.py +0 -0
  118. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_pushdown.py +0 -0
  119. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_rebase.py +0 -0
  120. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/_rest.py +0 -0
  121. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/accessors.py +0 -0
  122. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/raw.py +0 -0
  123. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/query/schema.py +0 -0
  124. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/rest.py +0 -0
  125. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/session.py +0 -0
  126. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/sql.py +0 -0
  127. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/__init__.py +0 -0
  128. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/_kinds.py +0 -0
  129. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/_projections.py +0 -0
  130. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/lazy.py +0 -0
  131. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/readers.py +0 -0
  132. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/resolve.py +0 -0
  133. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/schema.py +0 -0
  134. {sdmxlib-0.64.0 → sdmxlib-0.65.0}/src/sdmxlib/storage/writers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: sdmxlib
3
- Version: 0.64.0
3
+ Version: 0.65.0
4
4
  Summary: SDMX structural metadata library for Python
5
5
  Keywords: sdmx,statistics,metadata,datastructure
6
6
  Author: gabrielgellner
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sdmxlib"
3
- version = "0.64.0"
3
+ version = "0.65.0"
4
4
  description = "SDMX structural metadata library for Python"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.13"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sdmxlib"
3
- version = "0.64.0"
3
+ version = "0.65.0"
4
4
  description = "SDMX structural metadata library for Python"
5
5
  readme = "README.md"
6
6
  license = { text = "Apache-2.0" }
@@ -19,7 +19,7 @@ from sdmxlib.session import Session, SessionHook, ValueSet
19
19
  from sdmxlib.query import RawSql, RawSqlError, schema
20
20
  from sdmxlib.query.accessors import ArtefactRow, CodeRow, HCodeRow, ReportedAttributeRow, q
21
21
  from sdmxlib.query._compile import CompiledQuery
22
- from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Win, WinFn, WinPred
22
+ from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Untyped, Win, WinFn, WinPred
23
23
  from sdmxlib.query.observations import ObsQuery, ObsRow
24
24
  from sdmxlib.query.query import Frame, Grouped, Page, Query, union
25
25
  from sdmxlib.sql import SqlRelation
@@ -136,6 +136,27 @@ from sdmxlib.model import (
136
136
  artefact_urn,
137
137
  )
138
138
 
139
+
140
+ def len[R: RowKind]() -> Agg[R]: # noqa: A001 - polars' `pl.len()`; nothing in this module calls the builtin
141
+ """The number of rows, as an aggregate — `count(*)`, named for polars' `pl.len()`.
142
+
143
+ For `Query.agg`, which folds a whole query into one row, so the row count
144
+ can come out of the same scan as the other aggregates (#414):
145
+
146
+ obs.agg(SEX=obs.col("SEX").unique(), n=sl.len())
147
+
148
+ A function rather than a column method because it counts *rows*, not the
149
+ values of any column — the same reason `group_by(...).len()` is a method on
150
+ the grouping rather than on a column. The row kind is taken from where it is
151
+ used, so it fits any query's `agg`.
152
+
153
+ Shadows the builtin only as `sl.len`; this module never calls `len()`
154
+ itself, which is why it is defined here rather than re-exported from a
155
+ module that would need a second suppression.
156
+ """
157
+ return Agg[R]("count", ())
158
+
159
+
139
160
  __all__ = [
140
161
  "Agency",
141
162
  "AgencyScheme",
@@ -278,6 +299,7 @@ __all__ = [
278
299
  "SupportsUrn",
279
300
  "TimeDimension",
280
301
  "UnresolvedRef",
302
+ "Untyped",
281
303
  "ValidationError",
282
304
  "ValidationIssue",
283
305
  "ValueMap",
@@ -290,6 +312,7 @@ __all__ = [
290
312
  "dim",
291
313
  "from_dict",
292
314
  "item",
315
+ "len",
293
316
  "parse",
294
317
  "parse_components",
295
318
  "q",
@@ -23,7 +23,7 @@ column maps in their dict-insertion order, which matches the order in
23
23
  which they were declared on the structure.
24
24
 
25
25
  Performance: rows are serialised via Polars' Rust CSV writer
26
- (``pl.from_arrow(batch).write_csv``) rather than Python's `csv`,
26
+ (``pl.DataFrame(batch).write_csv``) rather than Python's `csv`,
27
27
  which makes the writer fast enough that bulk SDMX-CSV downloads are
28
28
  network-bound rather than writer-bound (#61).
29
29
 
@@ -43,7 +43,7 @@ Example:
43
43
  from __future__ import annotations
44
44
 
45
45
  import io
46
- from typing import TYPE_CHECKING, cast
46
+ from typing import TYPE_CHECKING
47
47
 
48
48
  import polars as pl
49
49
 
@@ -143,10 +143,11 @@ def write_sdmx_csv(
143
143
  )
144
144
  raise KeyError(msg)
145
145
 
146
- # pl.from_arrow returns DataFrame | Series; a RecordBatch always
147
- # gives back a DataFrame, but the union is sticky in the stubs.
146
+ # The constructor rather than `pl.from_arrow`, which is typed
147
+ # `DataFrame | Series` and since polars 2.0 really does return a Series
148
+ # for stream-only inputs — a `cast` here would have hidden that.
148
149
  df = (
149
- cast("pl.DataFrame", pl.from_arrow(batch))
150
+ pl.DataFrame(batch)
150
151
  .select(data_cols)
151
152
  .with_columns(
152
153
  pl.lit(structure).alias("STRUCTURE"),
@@ -14,7 +14,7 @@ from __future__ import annotations
14
14
  import calendar
15
15
  import re
16
16
  from datetime import datetime, timedelta
17
- from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict, cast
17
+ from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict
18
18
 
19
19
  import msgspec
20
20
  import polars as pl
@@ -640,7 +640,11 @@ def encode_batch(
640
640
  )
641
641
  raise KeyError(msg)
642
642
 
643
- df = cast("pl.DataFrame", pl.from_arrow(batch))
643
+ # The constructor, not `pl.from_arrow`: the same frame for a RecordBatch, but
644
+ # typed `DataFrame` rather than `DataFrame | Series` — and polars 2.0 made
645
+ # `from_arrow` return a Series for stream-only inputs, which a `cast` would
646
+ # have passed through unchecked.
647
+ df = pl.DataFrame(batch)
644
648
  n = df.height
645
649
  if n == 0:
646
650
  return b""
@@ -59,6 +59,20 @@ if TYPE_CHECKING:
59
59
  from sdmxlib.model.binding import DataflowBinding
60
60
 
61
61
 
62
+ class SidecarNotUniqueError(ValueError):
63
+ """A series-attributes sidecar holds more than one row per join key (#412).
64
+
65
+ The sidecar is joined as `LEFT JOIN … USING (<keys>)`, so every extra row
66
+ per key multiplies each observation it matches. Nothing errors downstream:
67
+ `count()`, `collect()` and every aggregate over a sidecar column come back
68
+ inflated by the fan-out factor, and every row looks valid.
69
+
70
+ A `ValueError` because it is the same kind of refusal `plan` already makes
71
+ for a sidecar sharing no columns — the binding names a table that cannot
72
+ serve as its series attributes — so a caller catching that catches this.
73
+ """
74
+
75
+
62
76
  @frozen(kw_only=True)
63
77
  class QueryPlan:
64
78
  """The resolved ``FROM`` side of a `DataflowBinding`'s query.
@@ -109,6 +123,10 @@ class SchemaInspector:
109
123
  def __init__(self, conn: duckdb.DuckDBPyConnection) -> None:
110
124
  self._conn = conn
111
125
  self._columns_cache: dict[str, tuple[str, ...]] = {}
126
+ #: (table, keys) -> (rows, distinct keys), for `key_counts`. Shares
127
+ #: `_columns_cache`'s lock and lifecycle: both describe a table as it
128
+ #: stands, and `invalidate` is the one signal that it changed.
129
+ self._key_counts_cache: dict[tuple[str, tuple[str, ...]], tuple[int, int]] = {}
112
130
  self._lock = threading.Lock()
113
131
 
114
132
  def columns(self, table: str) -> tuple[str, ...]:
@@ -139,15 +157,50 @@ class SchemaInspector:
139
157
  cols_b = set(self.columns(table_b))
140
158
  return tuple(c for c in self.columns(table_a) if c in cols_b)
141
159
 
160
+ def key_counts(self, table: str, keys: tuple[str, ...]) -> tuple[int, int]:
161
+ """``(rows, distinct key combinations)`` of ``table`` over ``keys`` (cached).
162
+
163
+ Rows with a NULL in any key are left out of both counts: `USING` never
164
+ matches NULL, so repeated NULL-keyed rows cannot multiply a join, and
165
+ refusing a table for them would be refusing a harmless one.
166
+
167
+ A scan of the table — ~5 ms on cenprof-2021-csip's 619-row geography
168
+ sidecar, ~37 ms cold — so memoised single-flight beside `columns`, and
169
+ dropped by `invalidate` with it.
170
+
171
+ Args:
172
+ table: The table to count, as `columns` takes it.
173
+ keys: The columns whose combinations are counted.
174
+ """
175
+ cache_key = (table, keys)
176
+ cached = self._key_counts_cache.get(cache_key)
177
+ if cached is not None:
178
+ return cached
179
+ with self._lock:
180
+ cached = self._key_counts_cache.get(cache_key)
181
+ if cached is not None:
182
+ return cached
183
+ tuple_sql = ", ".join(keys)
184
+ not_null = " AND ".join(f"{k} IS NOT NULL" for k in keys)
185
+ row = self._conn.execute(
186
+ f"SELECT count(*), count(DISTINCT ({tuple_sql})) FROM {table} WHERE {not_null}"
187
+ ).fetchone()
188
+ counts = (int(row[0]), int(row[1])) if row else (0, 0)
189
+ self._key_counts_cache[cache_key] = counts
190
+ return counts
191
+
142
192
  def invalidate(self, table: str) -> None:
143
- """Drop the cached column list for ``table`` (e.g. after a rewrite)."""
193
+ """Drop everything cached about ``table`` (e.g. after a rewrite)."""
144
194
  with self._lock:
145
195
  self._columns_cache.pop(table, None)
196
+ for cache_key in [k for k in self._key_counts_cache if k[0] == table]:
197
+ del self._key_counts_cache[cache_key]
146
198
 
147
199
  def invalidate_all(self) -> None:
148
- """Drop all cached column lists."""
200
+ """Drop everything cached about every table."""
149
201
  with self._lock:
150
202
  self._columns_cache.clear()
203
+ self._key_counts_cache.clear()
151
204
 
152
205
  def plan(self, binding: DataflowBinding) -> QueryPlan:
153
206
  """Resolve ``binding``'s main/sidecar tables into a `QueryPlan`.
@@ -163,6 +216,9 @@ class SchemaInspector:
163
216
  Raises:
164
217
  ValueError: If the binding names a sidecar that shares no columns
165
218
  with the main table (no derivable join keys).
219
+ SidecarNotUniqueError: If the sidecar holds more than one row for
220
+ some combination of those keys, which would multiply every
221
+ observation the join matches (#412).
166
222
  """
167
223
  main = f"{binding.database_role}.{binding.table_location}"
168
224
  sidecar_loc = binding.series_attributes_location
@@ -176,4 +232,15 @@ class SchemaInspector:
176
232
  f"shares no columns with {main!r}; cannot derive JOIN keys"
177
233
  )
178
234
  raise ValueError(msg)
235
+ rows, distinct = self.key_counts(sidecar, join_keys)
236
+ if rows != distinct:
237
+ keys = ", ".join(join_keys)
238
+ msg = (
239
+ f"series-attributes sidecar {sidecar!r} holds {rows:,} rows but only {distinct:,} distinct "
240
+ f"({keys}) — joined on those keys, every observation of a repeated series would be counted "
241
+ f"once per sidecar row, so counts and aggregates over its columns come back inflated (#412). "
242
+ f"The keys are the columns the two tables share; make the sidecar unique on them, or drop the "
243
+ f"shared columns that are not part of the series key."
244
+ )
245
+ raise SidecarNotUniqueError(msg)
179
246
  return QueryPlan(main=main, sidecar=sidecar, join_keys=join_keys)
@@ -156,8 +156,16 @@ class Dataset:
156
156
  if hint_cl is not None:
157
157
  on_columns = hint_cl.code_ids()
158
158
  else:
159
- # Codelist not resolved — discover column values from the data
160
- on_columns = self.data.select(pivot_id).unique().collect()[pivot_id].cast(pl.String).to_list()
159
+ # Codelist not resolved — discover column values from the data.
160
+ # Sorted, because `unique` promises no order and the wide
161
+ # frame's columns would otherwise change from run to run.
162
+ on_columns = (
163
+ self.data.select(pl.col(pivot_id).cast(pl.String))
164
+ .unique()
165
+ .sort(pivot_id)
166
+ .collect()[pivot_id]
167
+ .to_list()
168
+ )
161
169
  index = [d.id for d in self.structure.dimensions if d.id != pivot_id]
162
170
 
163
171
  else:
@@ -187,11 +195,15 @@ class Dataset:
187
195
  index.append(self.structure.time_dimension.id)
188
196
  measures = self.measures
189
197
  values = measures[0] if len(measures) == 1 else measures
198
+ # `maintain_order`: rows in the order their series first appear, which
199
+ # is `DataFrame.pivot`'s contract. The lazy pivot defaults to no order,
200
+ # and at a few thousand series that order differs run to run.
190
201
  return self.data.pivot(
191
202
  on=pivot_id,
192
203
  on_columns=pl.Series(on_columns),
193
204
  index=index,
194
205
  values=values,
206
+ maintain_order=True,
195
207
  )
196
208
 
197
209
  # ── SDMX-aware expressions ────────────────────────────────────────────────
@@ -208,6 +208,10 @@ def validation_report(
208
208
  .rename({comp.id: "invalid_value"})
209
209
  .with_columns(pl.lit(comp.id).alias("column"))
210
210
  .select("column", "invalid_value", "count")
211
+ # `group_by` promises no order, and at a few thousand values the
212
+ # order it gives differs run to run — on polars 1.x and 2.0 alike.
213
+ # Components stay in DSD order (the loop); values within one sort.
214
+ .sort("invalid_value")
211
215
  .collect()
212
216
  )
213
217
  if not bad.is_empty():
@@ -843,12 +843,28 @@ class _Scope:
843
843
  for windows, applied a third time.
844
844
  """
845
845
  if isinstance(item, AnyAgg):
846
- refs = [self._flat_reference(column, context="in a grouped aggregate") for column in item.columns]
847
- body = self._agg_body_from(item, refs)
846
+ body = self._grouped_agg_body(item, context="in a grouped aggregate")
848
847
  return Sql(f"{body.text} AS {name}", body.params)
849
848
  ref = self._flat_reference(item, context="as a group key")
850
849
  return Sql(f"{ref.text} AS {name}", ref.params)
851
850
 
851
+ def _grouped_agg_body(self, agg: AnyAgg, *, context: str) -> Sql:
852
+ """A plain aggregate over the group's rows, honouring the column's `.sort()`.
853
+
854
+ The order was dropped here until #414: `col.sort().unique()` compiled to
855
+ `list(DISTINCT col ORDER BY col)` through a fan-out but to an unordered
856
+ `list(DISTINCT col)` in a `group_by().agg()` — the caller asked for an
857
+ order and silently did not get one. Read off the column exactly as
858
+ `_agg_subquery` reads it, so the two readings of an `Agg` agree.
859
+ """
860
+ refs = [self._flat_reference(column, context=context) for column in agg.columns]
861
+ order = agg.columns[0]._order if agg.columns else None
862
+ order_sql: Sql | None = None
863
+ if order is not None:
864
+ key_ref = self._flat_reference(order[0], context=context)
865
+ order_sql = Sql(f"{key_ref.text}{' DESC' if order[1] else ''}", key_ref.params)
866
+ return self._agg_body_from(agg, refs, order_sql)
867
+
852
868
  def grouped_having(self, pred: AggPred) -> Sql:
853
869
  """One `HAVING` term — an aggregate compared to a value.
854
870
 
@@ -861,8 +877,7 @@ class _Scope:
861
877
  means the correlated-subquery reading instead, which is why the two
862
878
  clauses compile it through different methods rather than sharing one.
863
879
  """
864
- refs = [self._flat_reference(column, context="in a HAVING clause") for column in pred.agg.columns]
865
- body = self._agg_body_from(pred.agg, refs)
880
+ body = self._grouped_agg_body(pred.agg, context="in a HAVING clause")
866
881
  return Sql(f"{body.text} {pred.op} ?", (*body.params, pred.value))
867
882
 
868
883
  def _aggregate(self, name: str, agg: AnyAgg) -> Sql:
@@ -1433,6 +1448,7 @@ def compile_projection(
1433
1448
  distinct: bool = False,
1434
1449
  qualify: Sequence[AnyWinPred] = (),
1435
1450
  group_keys: Sequence[AnyCol] = (),
1451
+ aggregated: bool = False,
1436
1452
  having: Sequence[AggPred] = (),
1437
1453
  positional_order: Sequence[tuple[int, bool]] = (),
1438
1454
  fetch_scope: str | None = None,
@@ -1454,7 +1470,13 @@ def compile_projection(
1454
1470
  # front of `items`, so `_projected_prefixes` has already joined them.
1455
1471
  # Measured — adding one was a redundant call no mutation could kill.
1456
1472
  scope = _Scope(table, predicates, _Namespace(), extra_paths=extra, schema=root.__schema__, joins=root.__joins__)
1457
- projected = [scope.grouped(name, item) if group_keys else scope.projection(name, item) for name, item in items]
1473
+ # `aggregated` rather than `group_keys` decides how an item compiles (#414):
1474
+ # a root-level `Query.agg` aggregates over every row with no keys at all,
1475
+ # and keying this on the keys sent it down the per-row projection path,
1476
+ # which correlates each aggregate over a fan-out and refuses at the root.
1477
+ # The keys still decide `GROUP BY` and the tiebreak below.
1478
+ grouped = aggregated or bool(group_keys)
1479
+ projected = [scope.grouped(name, item) if grouped else scope.projection(name, item) for name, item in items]
1458
1480
  constraint = _constraint_clause(root.__constraint__, scope.alias)
1459
1481
  where = _conjoin(
1460
1482
  [constraint, _scope_clause(fetch_scope, scope.alias, root.__schema__), *(scope.compile(p) for p in predicates)]
@@ -1503,6 +1525,11 @@ def compile_projection(
1503
1525
  # would name columns the projection may have dropped.
1504
1526
  keys = [f"{position} {'ASC' if ascending else 'DESC'} NULLS LAST" for position, ascending in positional_order]
1505
1527
  ordered = Sql("\nORDER BY " + ", ".join([*keys, *(tiebreak or [])]))
1528
+ elif aggregated and not group_keys:
1529
+ # One row, so nothing to order — and the root-key fallback below would
1530
+ # name row columns that are no longer in an aggregate's scope, which
1531
+ # DuckDB rejects with a GROUP BY binder error.
1532
+ ordered = Sql("")
1506
1533
  else:
1507
1534
  ordered = _order_sql(root, scope, order, tiebreak=tiebreak)
1508
1535
  # `from_sql` last: it emits only the root joins something above referenced.
@@ -522,29 +522,35 @@ class Agg[R: RowKind](AnyAgg):
522
522
  #
523
523
  # Approved suppression, on the grounds that it mirrors `Win.__eq__`
524
524
  # exactly — same override, same reason, same pair of checkers.
525
+ #
526
+ # Declared `AggCompare[R]`, which is what all six return (#414). They said
527
+ # `Pred[R]`, so `having(col.count() > 1)` — whose parameter is `AggPred` —
528
+ # was a type error for every typed caller; the existing `having` tests
529
+ # passed only because their fixture is untyped. `AggCompare` is both an
530
+ # `AggPred` and a `Pred[R]`, so `filter` still takes it unchanged.
525
531
  def __hash__(self) -> int:
526
532
  return id(self)
527
533
 
528
- def __eq__(self, other: object) -> Pred[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
534
+ def __eq__(self, other: object) -> AggCompare[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
529
535
  return AggCompare(self, "=", other)
530
536
 
531
- def __ne__(self, other: object) -> Pred[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
537
+ def __ne__(self, other: object) -> AggCompare[R]: # pyright: ignore[reportIncompatibleMethodOverride] # pyrefly: ignore[bad-override]
532
538
  return AggCompare(self, "!=", other)
533
539
 
534
540
  def __bool__(self) -> bool:
535
541
  msg = _ambiguous_truth("an aggregate")
536
542
  raise TypeError(msg)
537
543
 
538
- def __gt__(self, other: object) -> Pred[R]:
544
+ def __gt__(self, other: object) -> AggCompare[R]:
539
545
  return AggCompare(self, ">", other)
540
546
 
541
- def __ge__(self, other: object) -> Pred[R]:
547
+ def __ge__(self, other: object) -> AggCompare[R]:
542
548
  return AggCompare(self, ">=", other)
543
549
 
544
- def __lt__(self, other: object) -> Pred[R]:
550
+ def __lt__(self, other: object) -> AggCompare[R]:
545
551
  return AggCompare(self, "<", other)
546
552
 
547
- def __le__(self, other: object) -> Pred[R]:
553
+ def __le__(self, other: object) -> AggCompare[R]:
548
554
  return AggCompare(self, "<=", other)
549
555
 
550
556
  def over(self, *partition: Col[R]) -> Win[R]:
@@ -44,7 +44,7 @@ from typing import TYPE_CHECKING, Self
44
44
  import attrs
45
45
 
46
46
  from sdmxlib.model.datastructure import ComponentRole
47
- from sdmxlib.query._expr import Col, Cols, RowKind, Untyped
47
+ from sdmxlib.query._expr import Cols, RowKind, Untyped
48
48
  from sdmxlib.query._joins import RootJoin
49
49
  from sdmxlib.query.query import Query
50
50
 
@@ -69,10 +69,19 @@ class Sidecar:
69
69
  changes the *plan* and never the spelling: `obs["GEO_NAME"]` means the same
70
70
  thing whether the column sits on the fact table or across a join.
71
71
 
72
+ **One row per series, enforced.** "Once per series" means unique on the
73
+ join keys, and a sidecar that is not would multiply every observation it
74
+ matches — silently, in every count and aggregate over its columns.
75
+ `SchemaInspector.plan` refuses one with `SidecarNotUniqueError` (#412), so
76
+ a `Sidecar` that exists has been checked.
77
+
72
78
  Attributes:
73
79
  table: Fully-qualified sidecar table.
74
80
  join_keys: Columns common to both sides — derived by
75
81
  `SchemaInspector.plan` from the tables themselves, not declared.
82
+ Which is why uniqueness is checked against *these*: a sidecar unique
83
+ on its own key can still fan out if the two tables share fewer
84
+ columns than that key.
76
85
  columns: Every column the sidecar has, for deciding which side a
77
86
  component lives on.
78
87
  """
@@ -316,8 +325,17 @@ class ObsQuery(Query[ObsRow]):
316
325
  here rather than raising.
317
326
  """
318
327
 
319
- def col(self, component_id: str) -> Col[ObsRow]:
320
- """The column for one SDMX component. See `Observations.__getitem__`."""
328
+ def col(self, component_id: str) -> Untyped[ObsRow]:
329
+ """The column for one SDMX component. See `Observations.__getitem__`.
330
+
331
+ Declared `Untyped`, which is what it always returned (#413). It said
332
+ `Col` — the lie #347 corrected on `__getitem__` and left here — so the
333
+ operators `Untyped` really offers were invisible to every checker and
334
+ every reader: the comparisons, and `.str`'s `contains`, `starts_with`,
335
+ `ends_with` and `caseless` rung. Those four are SDMX-REST's `co`, `nc`
336
+ (negated), `sw` and `ew`, with the `LIKE` escaping done by
337
+ `_label_match` rather than by each consumer.
338
+ """
321
339
  root = self.plan.root
322
340
  if not isinstance(root, Observations): # pragma: no cover - built one way
323
341
  msg = f"this query is not rooted at observations, but at {type(root).__name__}"
@@ -1676,7 +1676,10 @@ class Query[R: RowKind, A = object]:
1676
1676
  the grouped result, which can also order by an aggregate.
1677
1677
  """
1678
1678
  if not keys:
1679
- msg = "group_by() needs at least one key — grouping by nothing is what count() already does."
1679
+ msg = (
1680
+ "group_by() needs at least one key. To aggregate over every row at once, use "
1681
+ ".agg(...) — several aggregates, one row — or .count() for the row count alone."
1682
+ )
1680
1683
  raise TypeError(msg)
1681
1684
  for key in keys:
1682
1685
  _require_column("group_by", key)
@@ -1690,6 +1693,58 @@ class Query[R: RowKind, A = object]:
1690
1693
  raise SliceError(msg)
1691
1694
  return Grouped(query=self, keys=tuple(keys))
1692
1695
 
1696
+ def agg(self, **named: Agg[R]) -> Frame[R, A]:
1697
+ """Several aggregates over every row of the query, in one scan (#414).
1698
+
1699
+ One row out, one column per aggregate — `group_by(...).agg(...)` with
1700
+ no keys, which is why it is not spelled that way: `group_by()` refuses
1701
+ zero keys, and a row count alone is `count()`. The case this serves is
1702
+ *several* answers from one pass, e.g. the distinct values each
1703
+ dimension takes under a filter, plus the row count — what SDMX-REST's
1704
+ `/availability` returns:
1705
+
1706
+ obs.agg(
1707
+ SEX=obs.col("SEX").unique(),
1708
+ TIME_PERIOD=obs.col("TIME_PERIOD").unique(),
1709
+ n=sl.len(),
1710
+ ).collect()
1711
+
1712
+ Not always faster than one `select(col).unique()` per column. On
1713
+ cenprof-2021 (519M rows, six dimensions) one scan lost 2.8x unfiltered
1714
+ and won 1.1-4.8x under a filter, so it is an option to take when a
1715
+ filter narrows the scan, not a replacement.
1716
+
1717
+ An `Agg` here is a plain SQL aggregate over the query's rows — the same
1718
+ expression in `select()` is a correlated subquery over a fan-out.
1719
+
1720
+ Raises:
1721
+ TypeError: If no aggregates are given, or one is a plain column —
1722
+ with no group keys there is no single value a column could
1723
+ take. Project it with `select()`, or `group_by()` it.
1724
+ SliceError: If the query is ordered or sliced. An order means
1725
+ nothing once every row folds into one, and a limit applied
1726
+ first would silently aggregate a truncated set.
1727
+ """
1728
+ if not named:
1729
+ msg = "agg() needs at least one aggregate. For the row count alone, use .count()."
1730
+ raise TypeError(msg)
1731
+ for name, item in named.items():
1732
+ if not isinstance(item, Agg):
1733
+ msg = (
1734
+ f"{name!r} is a column, not an aggregate — agg() folds every row into one, so a "
1735
+ f"column has no single value to give. Aggregate it (.unique(), .count(), .max(), …), "
1736
+ f"or keep it per row with select(), or per group with group_by()."
1737
+ )
1738
+ raise TypeError(msg)
1739
+ if self.order_keys or self._sliced():
1740
+ msg = (
1741
+ "cannot agg() a query that is ordered or sliced — an order means nothing once "
1742
+ "every row folds into one, and a limit before aggregating aggregates a truncated "
1743
+ "set. Aggregate the whole query, then order or slice something else."
1744
+ )
1745
+ raise SliceError(msg)
1746
+ return Frame(query=self, items=tuple(named.items()), aggregated=True)
1747
+
1693
1748
  # ── no implicit execution ─────────────────────────────────────────────
1694
1749
 
1695
1750
  def __iter__(self) -> None:
@@ -1757,7 +1812,7 @@ class Grouped[R: RowKind, A = object]:
1757
1812
  # the binding's layout.
1758
1813
  items: list[tuple[str, AnyCol | AnyAgg]] = [(key.projected_name, key) for key in self.keys]
1759
1814
  items.extend(named.items())
1760
- return Frame(query=self._query, items=tuple(items), group_keys=self.keys)
1815
+ return Frame(query=self._query, items=tuple(items), group_keys=self.keys, aggregated=True)
1761
1816
 
1762
1817
  def len(self) -> Frame[R, A]:
1763
1818
  """Count rows per group, in a column named `len` — as polars names it.
@@ -1771,7 +1826,7 @@ class Grouped[R: RowKind, A = object]:
1771
1826
  # the binding's layout.
1772
1827
  items: list[tuple[str, AnyCol | AnyAgg]] = [(key.projected_name, key) for key in self.keys]
1773
1828
  items.append(("len", Agg[R]("count", ())))
1774
- return Frame(query=self._query, items=tuple(items), group_keys=self.keys)
1829
+ return Frame(query=self._query, items=tuple(items), group_keys=self.keys, aggregated=True)
1775
1830
 
1776
1831
  @override
1777
1832
  def __repr__(self) -> str:
@@ -1853,6 +1908,11 @@ class Frame[R: RowKind, A = object]:
1853
1908
  #: aggregates are plain `GROUP BY` aggregates rather than correlated
1854
1909
  #: subqueries.
1855
1910
  group_keys: tuple[Col[R], ...] = ()
1911
+ #: True when every item is a plain aggregate over the query's rows — a
1912
+ #: `group_by().agg()` (with keys) or a root `Query.agg()` (without, #414).
1913
+ #: Separate from `group_keys` because zero keys is a real case: one row
1914
+ #: summarising the whole filtered query.
1915
+ aggregated: bool = False
1856
1916
  #: `HAVING` terms — predicates over the *groups*, not over the rows.
1857
1917
  having_terms: tuple[AggPred, ...] = ()
1858
1918
 
@@ -2076,7 +2136,7 @@ class Frame[R: RowKind, A = object]:
2076
2136
  # impossible — the default order made it refuse, and `order_by()` with
2077
2137
  # no arguments clears the caller's keys and falls back to that same
2078
2138
  # default, so there was no spelling that worked.
2079
- row_order = self._query.order_keys if (self.group_keys or self.distinct) else self._query._order
2139
+ row_order = self._query.order_keys if (self.aggregated or self.distinct) else self._query._order
2080
2140
  if self._query.unordered:
2081
2141
  # Drops the grouped and `unique()` tiebreaks along with the row
2082
2142
  # order, which is the point: those exist so groups come back in a
@@ -2092,6 +2152,7 @@ class Frame[R: RowKind, A = object]:
2092
2152
  distinct=self.distinct,
2093
2153
  qualify=self._query.qualifiers,
2094
2154
  group_keys=self.group_keys,
2155
+ aggregated=self.aggregated,
2095
2156
  having=self.having_terms,
2096
2157
  positional_order=self.order,
2097
2158
  fetch_scope=self._query._fetch_scope,
File without changes
File without changes
File without changes
File without changes
File without changes