graphique 2.0.1__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {graphique-2.0.1 → graphique-2.0.2}/PKG-INFO +4 -5
- {graphique-2.0.1 → graphique-2.0.2}/README.md +1 -1
- {graphique-2.0.1 → graphique-2.0.2}/graphique/core.py +16 -16
- {graphique-2.0.1 → graphique-2.0.2}/graphique/inputs.py +53 -52
- {graphique-2.0.1 → graphique-2.0.2}/graphique/interface.py +24 -25
- {graphique-2.0.1 → graphique-2.0.2}/graphique/middleware.py +23 -20
- {graphique-2.0.1 → graphique-2.0.2}/graphique/models.py +23 -23
- {graphique-2.0.1 → graphique-2.0.2}/graphique/scalars.py +16 -18
- {graphique-2.0.1 → graphique-2.0.2}/graphique/service.py +8 -8
- {graphique-2.0.1 → graphique-2.0.2}/graphique/shell.py +14 -14
- {graphique-2.0.1 → graphique-2.0.2}/graphique.egg-info/PKG-INFO +4 -5
- {graphique-2.0.1 → graphique-2.0.2}/graphique.egg-info/requires.txt +1 -1
- {graphique-2.0.1 → graphique-2.0.2}/pyproject.toml +13 -18
- graphique-2.0.2/tests/test_core.py +44 -0
- {graphique-2.0.1 → graphique-2.0.2}/tests/test_dataset.py +64 -64
- graphique-2.0.2/tests/test_models.py +186 -0
- {graphique-2.0.1 → graphique-2.0.2}/tests/test_service.py +144 -144
- graphique-2.0.1/tests/test_core.py +0 -44
- graphique-2.0.1/tests/test_models.py +0 -186
- {graphique-2.0.1 → graphique-2.0.2}/LICENSE.txt +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/graphique/__init__.py +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/graphique/py.typed +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/graphique.egg-info/SOURCES.txt +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/graphique.egg-info/dependency_links.txt +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/graphique.egg-info/top_level.txt +0 -0
- {graphique-2.0.1 → graphique-2.0.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: graphique
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.2
|
|
4
4
|
Summary: GraphQL service for python dataframes and parquet datasets.
|
|
5
5
|
Author-email: Aric Coady <aric.coady@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -14,7 +14,6 @@ Classifier: Intended Audience :: Developers
|
|
|
14
14
|
Classifier: Intended Audience :: Science/Research
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
@@ -24,11 +23,11 @@ Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
|
|
|
24
23
|
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
25
24
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
25
|
Classifier: Typing :: Typed
|
|
27
|
-
Requires-Python: >=3.
|
|
26
|
+
Requires-Python: >=3.11
|
|
28
27
|
Description-Content-Type: text/markdown
|
|
29
28
|
License-File: LICENSE.txt
|
|
30
29
|
Requires-Dist: pyarrow>=20
|
|
31
|
-
Requires-Dist: strawberry-graphql[asgi,cli]>=0.
|
|
30
|
+
Requires-Dist: strawberry-graphql[asgi,cli]>=0.288
|
|
32
31
|
Requires-Dist: ibis-framework>=10.7
|
|
33
32
|
Requires-Dist: pyarrow-hotfix
|
|
34
33
|
Requires-Dist: isodate>=0.7
|
|
@@ -47,7 +46,7 @@ Dynamic: license-file
|
|
|
47
46
|
[](https://codecov.io/gh/coady/graphique/)
|
|
48
47
|
[](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
|
|
49
48
|
[](https://github.com/astral-sh/ruff)
|
|
50
|
-
[](https://github.com/astral-sh/ty)
|
|
51
50
|
|
|
52
51
|
[GraphQL](https://graphql.org) service for [ibis](https://ibis-project.org) dataframes, [arrow](https://arrow.apache.org) tables, and [parquet](https://parquet.apache.org) datasets. The schema for a query API is derived automatically.
|
|
53
52
|
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
[](https://codecov.io/gh/coady/graphique/)
|
|
7
7
|
[](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
|
|
8
8
|
[](https://github.com/astral-sh/ruff)
|
|
9
|
-
[](https://github.com/astral-sh/ty)
|
|
10
10
|
|
|
11
11
|
[GraphQL](https://graphql.org) service for [ibis](https://ibis-project.org) dataframes, [arrow](https://arrow.apache.org) tables, and [parquet](https://parquet.apache.org) datasets. The schema for a query API is derived automatically.
|
|
12
12
|
|
|
@@ -22,9 +22,9 @@ def getitems(obj, *keys):
|
|
|
22
22
|
return obj
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
def order_key(name: str) -> ibis.
|
|
25
|
+
def order_key(name: str) -> ibis.Value:
|
|
26
26
|
"""Parse sort order."""
|
|
27
|
-
return (ibis.desc if name.startswith(
|
|
27
|
+
return (ibis.desc if name.startswith("-") else ibis.asc)(ibis._[name.lstrip("-")])
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class Parquet(ds.Dataset):
|
|
@@ -32,24 +32,24 @@ class Parquet(ds.Dataset):
|
|
|
32
32
|
|
|
33
33
|
def schema(self) -> pa.Schema:
|
|
34
34
|
"""partition schema"""
|
|
35
|
-
return self.partitioning.schema if hasattr(self,
|
|
35
|
+
return self.partitioning.schema if hasattr(self, "partitioning") else pa.schema([])
|
|
36
36
|
|
|
37
37
|
def keys(self, *names) -> list:
|
|
38
38
|
"""Return prefix of matching partition keys."""
|
|
39
39
|
keys = set(Parquet.schema(self).names)
|
|
40
|
-
return list(itertools.takewhile(lambda name: name.lstrip(
|
|
40
|
+
return list(itertools.takewhile(lambda name: name.lstrip("-") in keys, names))
|
|
41
41
|
|
|
42
|
-
def fragments(self, counts: str =
|
|
42
|
+
def fragments(self, counts: str = "") -> ibis.Table:
|
|
43
43
|
"""Return partition fragments as a table."""
|
|
44
44
|
parts = []
|
|
45
|
-
for frag in self._get_fragments(self._scan_options.get(
|
|
45
|
+
for frag in self._get_fragments(self._scan_options.get("filter")):
|
|
46
46
|
parts.append(ds.get_partition_keys(frag.partition_expression))
|
|
47
|
-
parts[-1][
|
|
47
|
+
parts[-1]["__path__"] = frag.path
|
|
48
48
|
if counts:
|
|
49
49
|
parts[-1][counts] = frag.count_rows()
|
|
50
50
|
return ibis.memtable(pa.Table.from_pylist(parts))
|
|
51
51
|
|
|
52
|
-
def group(self, *names, counts: str =
|
|
52
|
+
def group(self, *names, counts: str = "") -> ibis.Table:
|
|
53
53
|
"""Return grouped partitions as a table."""
|
|
54
54
|
table = Parquet.fragments(self, counts)
|
|
55
55
|
agg = {counts: table[counts].sum()} if counts else {}
|
|
@@ -65,19 +65,19 @@ class Parquet(ds.Dataset):
|
|
|
65
65
|
|
|
66
66
|
def to_table(self) -> ibis.Table:
|
|
67
67
|
"""Return ibis `Table` from filtered dataset."""
|
|
68
|
-
paths = [frag.path for frag in self._get_fragments(self._scan_options.get(
|
|
68
|
+
paths = [frag.path for frag in self._get_fragments(self._scan_options.get("filter"))]
|
|
69
69
|
hive = isinstance(self.partitioning, ds.HivePartitioning)
|
|
70
70
|
return ibis.read_parquet(paths, hive_partitioning=hive)
|
|
71
71
|
|
|
72
72
|
def rank(self, limit: int, *names: str, dense: bool = False) -> ibis.Table:
|
|
73
73
|
"""Return ordered limited partitions as a table."""
|
|
74
|
-
keys = {name.strip(
|
|
75
|
-
table = Parquet.fragments(self, counts=
|
|
76
|
-
groups = table.aggregate(count=ibis._.count(), total=table[
|
|
74
|
+
keys = {name.strip("-"): order_key(name) for name in names}
|
|
75
|
+
table = Parquet.fragments(self, counts="_").order_by(*keys.values()).cache()
|
|
76
|
+
groups = table.aggregate(count=ibis._.count(), total=table["_"].sum(), by=list(keys))
|
|
77
77
|
groups = groups.order_by(*keys.values()).cache()
|
|
78
78
|
if not dense:
|
|
79
|
-
totals = itertools.accumulate(groups[
|
|
80
|
-
limit = next((index for index, total in enumerate(totals, 1) if total >= limit), None)
|
|
81
|
-
limit = groups[:limit][
|
|
79
|
+
totals = itertools.accumulate(groups["total"].to_list())
|
|
80
|
+
limit = next((index for index, total in enumerate(totals, 1) if total >= limit), None)
|
|
81
|
+
limit = groups[:limit]["count"].sum().to_pyarrow().as_py()
|
|
82
82
|
hive = isinstance(self.partitioning, ds.HivePartitioning)
|
|
83
|
-
return ibis.read_parquet(table[:limit][
|
|
83
|
+
return ibis.read_parquet(table[:limit]["__path__"].to_list(), hive_partitioning=hive)
|
|
@@ -7,8 +7,8 @@ from __future__ import annotations
|
|
|
7
7
|
import functools
|
|
8
8
|
import inspect
|
|
9
9
|
import operator
|
|
10
|
-
from collections.abc import Callable, Iterable
|
|
11
|
-
from datetime import date, datetime, time
|
|
10
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
11
|
+
from datetime import date, datetime, time
|
|
12
12
|
from decimal import Decimal
|
|
13
13
|
from typing import Generic, TypeVar
|
|
14
14
|
|
|
@@ -25,8 +25,9 @@ from strawberry.types.arguments import StrawberryArgument
|
|
|
25
25
|
from strawberry.types.field import StrawberryField
|
|
26
26
|
|
|
27
27
|
from .core import getitems, links, order_key
|
|
28
|
+
from .scalars import Duration
|
|
28
29
|
|
|
29
|
-
T = TypeVar(
|
|
30
|
+
T = TypeVar("T")
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
def use_doc(decorator: Callable, **kwargs):
|
|
@@ -53,9 +54,9 @@ def default_field(
|
|
|
53
54
|
) -> StrawberryField:
|
|
54
55
|
"""Use dataclass `default_factory` for `UNSET` or mutables."""
|
|
55
56
|
if func is not None:
|
|
56
|
-
kwargs[
|
|
57
|
+
kwargs["description"] = inspect.getdoc(func).splitlines()[0]
|
|
57
58
|
if not nullable and default is UNSET:
|
|
58
|
-
kwargs.setdefault(
|
|
59
|
+
kwargs.setdefault("directives", []).append(optional())
|
|
59
60
|
return strawberry.field(default_factory=type(default), **kwargs)
|
|
60
61
|
|
|
61
62
|
|
|
@@ -74,7 +75,7 @@ class Filter(Generic[T]):
|
|
|
74
75
|
gt: T | None = default_field(description=r"\>")
|
|
75
76
|
ge: T | None = default_field(description=r"\>=")
|
|
76
77
|
|
|
77
|
-
def __iter__(self) ->
|
|
78
|
+
def __iter__(self) -> Iterator[tuple]:
|
|
78
79
|
for name, value in self.__dict__.items():
|
|
79
80
|
if value is not UNSET:
|
|
80
81
|
yield name, value
|
|
@@ -87,18 +88,18 @@ class Filter(Generic[T]):
|
|
|
87
88
|
if types[name] not in (list, dict):
|
|
88
89
|
yield StrawberryArgument(name, name, annotation, default={})
|
|
89
90
|
annotation = StrawberryAnnotation(Expression | None)
|
|
90
|
-
yield StrawberryArgument(
|
|
91
|
+
yield StrawberryArgument("where", None, annotation, default=None)
|
|
91
92
|
|
|
92
93
|
@staticmethod
|
|
93
94
|
def to_exprs(**queries: Filter) -> Iterable[ibis.Deferred]:
|
|
94
95
|
"""Transform query syntax into ibis expressions."""
|
|
95
96
|
for name, query in queries.items():
|
|
96
97
|
field = ibis._[name]
|
|
97
|
-
for op, value in query:
|
|
98
|
+
for op, value in query:
|
|
98
99
|
match value, op:
|
|
99
|
-
case list(),
|
|
100
|
+
case list(), "eq":
|
|
100
101
|
yield field.isin(value)
|
|
101
|
-
case list(),
|
|
102
|
+
case list(), "ne":
|
|
102
103
|
yield field.notin(value)
|
|
103
104
|
case _:
|
|
104
105
|
yield getattr(operator, op)(field, value)
|
|
@@ -109,10 +110,10 @@ class Filter(Generic[T]):
|
|
|
109
110
|
exprs = []
|
|
110
111
|
for name, query in queries.items():
|
|
111
112
|
field = pc.field(name)
|
|
112
|
-
for op, value in dict(query).items():
|
|
113
|
+
for op, value in dict(query).items():
|
|
113
114
|
if isinstance(value, list):
|
|
114
115
|
expr = pc.is_in(field, pa.array(value))
|
|
115
|
-
exprs.append(pc.invert(expr) if op ==
|
|
116
|
+
exprs.append(pc.invert(expr) if op == "ne" else expr)
|
|
116
117
|
else:
|
|
117
118
|
exprs.append(getattr(operator, op)(field, value))
|
|
118
119
|
return functools.reduce(operator.and_, exprs or [None])
|
|
@@ -121,11 +122,11 @@ class Filter(Generic[T]):
|
|
|
121
122
|
@strawberry.input(description="name and optional alias for aggregation")
|
|
122
123
|
class Aggregate:
|
|
123
124
|
name: str = strawberry.field(description="column name")
|
|
124
|
-
alias: str = strawberry.field(default=
|
|
125
|
+
alias: str = strawberry.field(default="", description="output column name")
|
|
125
126
|
where: Expression | None = None
|
|
126
127
|
|
|
127
128
|
def to_ibis(self, func: str, **options) -> tuple:
|
|
128
|
-
options[
|
|
129
|
+
options["where"] = self.where and self.where.to_ibis()
|
|
129
130
|
return (self.alias or self.name), getattr(ibis._[self.name], func)(**options)
|
|
130
131
|
|
|
131
132
|
|
|
@@ -133,8 +134,8 @@ class Aggregate:
|
|
|
133
134
|
class UniqueAggregate(Aggregate):
|
|
134
135
|
approx: bool = False
|
|
135
136
|
|
|
136
|
-
def to_ibis(self, func: str, **options) -> tuple:
|
|
137
|
-
return super().to_ibis(
|
|
137
|
+
def to_ibis(self, func: str, **options) -> tuple:
|
|
138
|
+
return super().to_ibis("approx_" + func if self.approx else func, **options)
|
|
138
139
|
|
|
139
140
|
|
|
140
141
|
@strawberry.input
|
|
@@ -152,9 +153,9 @@ class OrderAggregate(Aggregate):
|
|
|
152
153
|
|
|
153
154
|
@strawberry.input
|
|
154
155
|
class VarAggregate(Aggregate):
|
|
155
|
-
how: str =
|
|
156
|
+
how: str = "sample"
|
|
156
157
|
|
|
157
|
-
def to_ibis(self, func: str) -> tuple:
|
|
158
|
+
def to_ibis(self, func: str, **_) -> tuple:
|
|
158
159
|
return super().to_ibis(func, how=self.how)
|
|
159
160
|
|
|
160
161
|
|
|
@@ -162,7 +163,7 @@ class VarAggregate(Aggregate):
|
|
|
162
163
|
class QuantileAggregate(UniqueAggregate):
|
|
163
164
|
q: float = 0.5
|
|
164
165
|
|
|
165
|
-
def to_ibis(self, func: str) -> tuple:
|
|
166
|
+
def to_ibis(self, func: str, **_) -> tuple:
|
|
166
167
|
return super().to_ibis(func, quantile=self.q)
|
|
167
168
|
|
|
168
169
|
|
|
@@ -170,7 +171,7 @@ class QuantileAggregate(UniqueAggregate):
|
|
|
170
171
|
class CollectAggregate(OrderAggregate):
|
|
171
172
|
distinct: bool = False
|
|
172
173
|
|
|
173
|
-
def to_ibis(self, func: str) -> tuple:
|
|
174
|
+
def to_ibis(self, func: str, **_) -> tuple:
|
|
174
175
|
return super().to_ibis(func, distinct=self.distinct)
|
|
175
176
|
|
|
176
177
|
|
|
@@ -199,11 +200,11 @@ class Aggregates:
|
|
|
199
200
|
|
|
200
201
|
@strawberry.input(description="typed scalars")
|
|
201
202
|
class Scalars:
|
|
202
|
-
base64:
|
|
203
|
+
base64: strawberry.scalars.Base64 | None = default_field(description="binary scalar")
|
|
203
204
|
date: date | None = default_field(description="date scalar")
|
|
204
205
|
datetime: datetime | None = default_field(description="datetime scalar")
|
|
205
206
|
decimal: Decimal | None = default_field(description="decimal scalar")
|
|
206
|
-
duration:
|
|
207
|
+
duration: Duration | None = default_field(description="duration scalar")
|
|
207
208
|
time: time | None = default_field(description="time scalar")
|
|
208
209
|
|
|
209
210
|
def __iter__(self):
|
|
@@ -229,14 +230,14 @@ class Expression:
|
|
|
229
230
|
notin: list[Expression] = default_field([], func=ibis.Column.notin)
|
|
230
231
|
|
|
231
232
|
inv: Expression | None = default_field(description="~")
|
|
232
|
-
and_: list[Expression] = default_field([], name=
|
|
233
|
-
or_: list[Expression] = default_field([], name=
|
|
233
|
+
and_: list[Expression] = default_field([], name="and", description="&")
|
|
234
|
+
or_: list[Expression] = default_field([], name="or", description="|")
|
|
234
235
|
xor: list[Expression] = default_field([], description="^")
|
|
235
236
|
|
|
236
237
|
add: list[Expression] = default_field([], description="+")
|
|
237
238
|
sub: list[Expression] = default_field([], description="-")
|
|
238
239
|
mul: list[Expression] = default_field([], description="*")
|
|
239
|
-
truediv: list[Expression] = default_field([], name=
|
|
240
|
+
truediv: list[Expression] = default_field([], name="div", description="/")
|
|
240
241
|
|
|
241
242
|
coalesce: list[Expression] = default_field([], func=ibis.Column.coalesce)
|
|
242
243
|
cume_dist: Expression | None = default_field(func=ibis.Column.cume_dist)
|
|
@@ -271,18 +272,18 @@ class Expression:
|
|
|
271
272
|
yield ibis.row_number()
|
|
272
273
|
for name, (expr, *args) in self.items():
|
|
273
274
|
match name:
|
|
274
|
-
case
|
|
275
|
+
case "eq" | "ne" | "lt" | "le" | "gt" | "ge" | "inv" | "and_" | "or_" | "xor":
|
|
275
276
|
yield getattr(operator, name)(expr, *args)
|
|
276
|
-
case
|
|
277
|
+
case "add" | "sub" | "mul" | "truediv":
|
|
277
278
|
yield getattr(operator, name)(expr, *args)
|
|
278
279
|
case _:
|
|
279
280
|
yield getattr(expr, name)(*args)
|
|
280
281
|
for field in (self.array, self.numeric, self.string, self.temporal, self.window):
|
|
281
282
|
if field:
|
|
282
|
-
yield from field
|
|
283
|
+
yield from field
|
|
283
284
|
|
|
284
285
|
def to_ibis(self) -> ibis.Deferred | None:
|
|
285
|
-
fields = list(self) or [None]
|
|
286
|
+
fields = list(self) or [None]
|
|
286
287
|
if len(fields) == 1:
|
|
287
288
|
return fields[0]
|
|
288
289
|
raise ValueError(f"conflicting inputs: {', '.join(map(str, fields))}")
|
|
@@ -290,10 +291,10 @@ class Expression:
|
|
|
290
291
|
|
|
291
292
|
@strawberry.input(description="an `Expression` with an optional alias")
|
|
292
293
|
class Projection(Expression):
|
|
293
|
-
alias: str = strawberry.field(default=
|
|
294
|
+
alias: str = strawberry.field(default="", description="name of projected column")
|
|
294
295
|
|
|
295
|
-
def to_ibis(self) -> tuple:
|
|
296
|
-
name = self.alias or
|
|
296
|
+
def to_ibis(self) -> tuple: # type: ignore
|
|
297
|
+
name = self.alias or ".".join(self.name)
|
|
297
298
|
if name:
|
|
298
299
|
return name, super().to_ibis()
|
|
299
300
|
raise ValueError(f"projected fields require a name or alias: {self}")
|
|
@@ -319,12 +320,12 @@ class Arrays:
|
|
|
319
320
|
offset: int = 0
|
|
320
321
|
limit: int | None = None
|
|
321
322
|
|
|
322
|
-
def __iter__(self) ->
|
|
323
|
-
for name, (expr, *args) in Expression.items(self):
|
|
323
|
+
def __iter__(self) -> Iterator[ibis.Deferred]:
|
|
324
|
+
for name, (expr, *args) in Expression.items(self):
|
|
324
325
|
match name:
|
|
325
|
-
case
|
|
326
|
+
case "slice":
|
|
326
327
|
yield expr[self.offset :][: self.limit]
|
|
327
|
-
case
|
|
328
|
+
case "value":
|
|
328
329
|
yield expr[self.offset]
|
|
329
330
|
case _:
|
|
330
331
|
yield getattr(expr, name)(*args)
|
|
@@ -356,15 +357,15 @@ class Numeric:
|
|
|
356
357
|
cumsum: Expression | None = default_field(func=ibis.expr.types.NumericColumn.cumsum)
|
|
357
358
|
|
|
358
359
|
buckets: list[JSON] = default_field([])
|
|
359
|
-
closed: str =
|
|
360
|
+
closed: str = "left"
|
|
360
361
|
close_extreme: bool = True
|
|
361
362
|
include_under: bool = False
|
|
362
363
|
include_over: bool = False
|
|
363
364
|
|
|
364
|
-
def __iter__(self) ->
|
|
365
|
-
for name, (expr, *args) in Expression.items(self):
|
|
365
|
+
def __iter__(self) -> Iterator[ibis.Deferred]:
|
|
366
|
+
for name, (expr, *args) in Expression.items(self):
|
|
366
367
|
match name:
|
|
367
|
-
case
|
|
368
|
+
case "bucket":
|
|
368
369
|
yield expr.bucket(
|
|
369
370
|
self.buckets,
|
|
370
371
|
closed=self.closed,
|
|
@@ -398,8 +399,8 @@ class Strings:
|
|
|
398
399
|
strip: list[Expression] = default_field([], func=ibis.expr.types.StringColumn.strip)
|
|
399
400
|
upper: Expression | None = default_field(func=ibis.expr.types.StringColumn.upper)
|
|
400
401
|
|
|
401
|
-
def __iter__(self) ->
|
|
402
|
-
for name, (expr, *args) in Expression.items(self):
|
|
402
|
+
def __iter__(self) -> Iterator[ibis.Deferred]:
|
|
403
|
+
for name, (expr, *args) in Expression.items(self):
|
|
403
404
|
yield getattr(expr, name)(*args)
|
|
404
405
|
|
|
405
406
|
|
|
@@ -427,17 +428,17 @@ class Temporal:
|
|
|
427
428
|
)
|
|
428
429
|
year: Expression | None = default_field(func=ibis.expr.types.TimestampColumn.year)
|
|
429
430
|
|
|
430
|
-
format_str: str =
|
|
431
|
-
unit: str =
|
|
431
|
+
format_str: str = ""
|
|
432
|
+
unit: str = ""
|
|
432
433
|
|
|
433
|
-
def __iter__(self) ->
|
|
434
|
-
for name, (expr, *args) in Expression.items(self):
|
|
434
|
+
def __iter__(self) -> Iterator[ibis.Deferred]:
|
|
435
|
+
for name, (expr, *args) in Expression.items(self):
|
|
435
436
|
match name:
|
|
436
|
-
case
|
|
437
|
+
case "delta":
|
|
437
438
|
yield expr.delta(*args, unit=self.unit)
|
|
438
|
-
case
|
|
439
|
+
case "truncate":
|
|
439
440
|
yield expr.truncate(self.unit)
|
|
440
|
-
case
|
|
441
|
+
case "strftime":
|
|
441
442
|
yield expr.strftime(self.format_str)
|
|
442
443
|
case _:
|
|
443
444
|
yield getattr(expr, name)(*args)
|
|
@@ -462,11 +463,11 @@ class Window:
|
|
|
462
463
|
default: JSON | None = default_field(None, description="default JSON scalar")
|
|
463
464
|
scalar: Scalars | None = default_field(description="default typed scalar")
|
|
464
465
|
|
|
465
|
-
def __iter__(self) ->
|
|
466
|
+
def __iter__(self) -> Iterator[ibis.Deferred]:
|
|
466
467
|
(default,) = self.scalar or [self.default]
|
|
467
|
-
for name, (expr,) in Expression.items(self):
|
|
468
|
+
for name, (expr,) in Expression.items(self):
|
|
468
469
|
match name:
|
|
469
|
-
case
|
|
470
|
+
case "lag" | "lead":
|
|
470
471
|
yield getattr(expr, name)(self.offset, default)
|
|
471
472
|
case _:
|
|
472
473
|
yield getattr(operator, name)(expr, expr.lag(self.offset)).fill_null(default)
|
|
@@ -7,14 +7,13 @@ Doesn't require knowledge of the schema.
|
|
|
7
7
|
import itertools
|
|
8
8
|
import operator
|
|
9
9
|
from collections.abc import Iterable, Iterator, Mapping
|
|
10
|
-
from typing import TypeAlias, no_type_check
|
|
10
|
+
from typing import Self, TypeAlias, no_type_check
|
|
11
11
|
|
|
12
12
|
import ibis
|
|
13
13
|
import pyarrow.dataset as ds
|
|
14
14
|
import strawberry
|
|
15
15
|
from strawberry import UNSET, Info
|
|
16
16
|
from strawberry.scalars import JSON
|
|
17
|
-
from typing_extensions import Self
|
|
18
17
|
|
|
19
18
|
from .core import Parquet, getitems, order_key
|
|
20
19
|
from .inputs import Aggregates, Expression, Field, Filter, Projection, Scalars, provisional
|
|
@@ -27,7 +26,7 @@ Source: TypeAlias = ibis.Table | ds.Dataset
|
|
|
27
26
|
def references(field) -> Iterator:
|
|
28
27
|
"""Generate every possible column reference from strawberry `SelectedField`."""
|
|
29
28
|
if isinstance(field, str):
|
|
30
|
-
yield field.lstrip(
|
|
29
|
+
yield field.lstrip("-")
|
|
31
30
|
elif isinstance(field, Iterable):
|
|
32
31
|
for value in field:
|
|
33
32
|
yield from references(value)
|
|
@@ -35,7 +34,7 @@ def references(field) -> Iterator:
|
|
|
35
34
|
for value in field.values():
|
|
36
35
|
yield from references(value)
|
|
37
36
|
else:
|
|
38
|
-
for name in (
|
|
37
|
+
for name in ("name", "arguments", "selections"):
|
|
39
38
|
yield from references(getattr(field, name, []))
|
|
40
39
|
|
|
41
40
|
|
|
@@ -63,7 +62,7 @@ class Dataset:
|
|
|
63
62
|
def resolve(self, info: Info, source: ibis.Table) -> Self:
|
|
64
63
|
"""Cache the table if it will be reused."""
|
|
65
64
|
counts = selections(*info.selected_fields)
|
|
66
|
-
counts[
|
|
65
|
+
counts["type"] = counts["schema"] = 0
|
|
67
66
|
if counts.total() > 1 and isinstance(source, ibis.Table):
|
|
68
67
|
if names := self.select(info, source):
|
|
69
68
|
source = source.select(*names).cache()
|
|
@@ -109,7 +108,7 @@ class Dataset:
|
|
|
109
108
|
|
|
110
109
|
Schema derived fields provide syntax for simple queries; `where` supports complex queries.
|
|
111
110
|
"""
|
|
112
|
-
exprs: list = [] if where is None else list(where)
|
|
111
|
+
exprs: list = [] if where is None else list(where)
|
|
113
112
|
source = Parquet.filter(self.source, Filter.to_arrow(**queries))
|
|
114
113
|
if source is None:
|
|
115
114
|
exprs += Filter.to_exprs(**queries)
|
|
@@ -156,7 +155,7 @@ class Dataset:
|
|
|
156
155
|
return self.source.count_rows()
|
|
157
156
|
|
|
158
157
|
@doc_field
|
|
159
|
-
def any(self, limit: BigInt = 1) -> bool:
|
|
158
|
+
def any(self, limit: BigInt = BigInt(1)) -> bool:
|
|
160
159
|
"""Whether there are at least `limit` rows.
|
|
161
160
|
|
|
162
161
|
May be significantly faster than `count` for out-of-core data.
|
|
@@ -168,7 +167,7 @@ class Dataset:
|
|
|
168
167
|
cast=f"cast expression to indicated {links.types}",
|
|
169
168
|
try_="return null if cast fails",
|
|
170
169
|
)
|
|
171
|
-
def column(self, name: list[str], cast: str =
|
|
170
|
+
def column(self, name: list[str], cast: str = "", try_: bool = False) -> Column | None:
|
|
172
171
|
"""Column of any type by name.
|
|
173
172
|
|
|
174
173
|
If the column is in the schema, `columns` can be used instead.
|
|
@@ -182,7 +181,7 @@ class Dataset:
|
|
|
182
181
|
offset="number of rows to skip; negative value skips from the end",
|
|
183
182
|
limit="maximum number of rows to return",
|
|
184
183
|
)
|
|
185
|
-
def slice(self, info: Info, offset: BigInt = 0, limit: BigInt | None = None) -> Self:
|
|
184
|
+
def slice(self, info: Info, offset: BigInt = BigInt(0), limit: BigInt | None = None) -> Self:
|
|
186
185
|
"""[Limit](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.limit) row selection."""
|
|
187
186
|
return self.resolve(info, self.table[offset:][:limit])
|
|
188
187
|
|
|
@@ -196,9 +195,9 @@ class Dataset:
|
|
|
196
195
|
self,
|
|
197
196
|
info: Info,
|
|
198
197
|
on: list[str] | None = None,
|
|
199
|
-
keep: str | None =
|
|
200
|
-
counts: str =
|
|
201
|
-
order: str =
|
|
198
|
+
keep: str | None = "first",
|
|
199
|
+
counts: str = "",
|
|
200
|
+
order: str = "",
|
|
202
201
|
) -> Self:
|
|
203
202
|
"""[Remove duplicate](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.distinct) rows from table.
|
|
204
203
|
|
|
@@ -228,8 +227,8 @@ class Dataset:
|
|
|
228
227
|
self,
|
|
229
228
|
info: Info,
|
|
230
229
|
by: list[str] = [],
|
|
231
|
-
counts: str =
|
|
232
|
-
order: str =
|
|
230
|
+
counts: str = "",
|
|
231
|
+
order: str = "",
|
|
233
232
|
aggregate: Aggregates = {}, # type: ignore
|
|
234
233
|
) -> Self:
|
|
235
234
|
"""[Group](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.group_by) table by columns."""
|
|
@@ -262,8 +261,8 @@ class Dataset:
|
|
|
262
261
|
else:
|
|
263
262
|
table = self.table
|
|
264
263
|
if dense and limit is not None:
|
|
265
|
-
groups = table.aggregate(_=ibis._.count(), by=[name.lstrip(
|
|
266
|
-
limit = groups.order_by(*map(order_key, by))[:limit][
|
|
264
|
+
groups = table.aggregate(_=ibis._.count(), by=[name.lstrip("-") for name in by])
|
|
265
|
+
limit = groups.order_by(*map(order_key, by))[:limit]["_"].sum().to_pyarrow().as_py()
|
|
267
266
|
return self.resolve(info, table.order_by(*map(order_key, by))[:limit])
|
|
268
267
|
|
|
269
268
|
@doc_field(
|
|
@@ -276,9 +275,9 @@ class Dataset:
|
|
|
276
275
|
self,
|
|
277
276
|
info: Info,
|
|
278
277
|
name: str,
|
|
279
|
-
offset: str =
|
|
278
|
+
offset: str = "",
|
|
280
279
|
keep_empty: bool = False,
|
|
281
|
-
row_number: str =
|
|
280
|
+
row_number: str = "",
|
|
282
281
|
) -> Self:
|
|
283
282
|
"""[Unnest](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.unnest) an array column from a table."""
|
|
284
283
|
table = self.table
|
|
@@ -300,9 +299,9 @@ class Dataset:
|
|
|
300
299
|
right: str,
|
|
301
300
|
keys: list[str],
|
|
302
301
|
rkeys: list[str] = [],
|
|
303
|
-
how: str =
|
|
304
|
-
lname: str =
|
|
305
|
-
rname: str =
|
|
302
|
+
how: str = "inner",
|
|
303
|
+
lname: str = "",
|
|
304
|
+
rname: str = "{name}_right",
|
|
306
305
|
) -> Self:
|
|
307
306
|
"""[Join](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.join) two tables."""
|
|
308
307
|
left = self.table
|
|
@@ -325,7 +324,7 @@ class Dataset:
|
|
|
325
324
|
return type(self)(ibis.memtable(table))
|
|
326
325
|
|
|
327
326
|
@doc_field(subset="columns names; defaults to all", how="remove if `any` or `all` are null")
|
|
328
|
-
def drop_null(self, info: Info, subset: list[str] | None = None, how: str =
|
|
327
|
+
def drop_null(self, info: Info, subset: list[str] | None = None, how: str = "any") -> Self:
|
|
329
328
|
"""[Drop](https://ibis-project.org/reference/expression-tables#ibis.expr.types.relations.Table.drop_null) rows with null values."""
|
|
330
329
|
return self.resolve(info, self.table.drop_null(subset, how=how))
|
|
331
330
|
|
|
@@ -338,7 +337,7 @@ class Dataset:
|
|
|
338
337
|
scalar: Scalars = {}, # type: ignore
|
|
339
338
|
) -> Self:
|
|
340
339
|
"""[Fill null](https://ibis-project.org/reference/expression-tables.html#ibis.expr.types.relations.Table.fill_null) values."""
|
|
341
|
-
(value,) = itertools.chain(scalar, [] if value is UNSET else [value])
|
|
340
|
+
(value,) = itertools.chain(scalar, [] if value is UNSET else [value])
|
|
342
341
|
replacements = dict.fromkeys(ibis_schema(self.source) if name is None else name, value)
|
|
343
342
|
return self.resolve(info, self.table.fill_null(replacements))
|
|
344
343
|
|
|
@@ -363,8 +362,8 @@ class Dataset:
|
|
|
363
362
|
info: Info,
|
|
364
363
|
by: list[str] = [],
|
|
365
364
|
split: list[Projection] = [],
|
|
366
|
-
counts: str =
|
|
367
|
-
alias: str =
|
|
365
|
+
counts: str = "",
|
|
366
|
+
alias: str = "{}_index",
|
|
368
367
|
aggregate: Aggregates = {}, # type: ignore
|
|
369
368
|
) -> Self:
|
|
370
369
|
"""Provisionally group table by adjacent values in columns."""
|