graphique 1.5__tar.gz → 1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {graphique-1.5 → graphique-1.7}/PKG-INFO +6 -5
  2. {graphique-1.5 → graphique-1.7}/README.md +3 -2
  3. {graphique-1.5 → graphique-1.7}/graphique/__init__.py +0 -2
  4. {graphique-1.5 → graphique-1.7}/graphique/core.py +32 -26
  5. {graphique-1.5 → graphique-1.7}/graphique/inputs.py +4 -4
  6. {graphique-1.5 → graphique-1.7}/graphique/interface.py +3 -3
  7. {graphique-1.5 → graphique-1.7}/graphique/middleware.py +1 -2
  8. {graphique-1.5 → graphique-1.7}/graphique/models.py +4 -4
  9. graphique-1.7/graphique/scalars.py +111 -0
  10. {graphique-1.5 → graphique-1.7}/graphique/shell.py +25 -25
  11. {graphique-1.5 → graphique-1.7}/graphique.egg-info/PKG-INFO +6 -5
  12. {graphique-1.5 → graphique-1.7}/graphique.egg-info/SOURCES.txt +1 -0
  13. graphique-1.7/graphique.egg-info/requires.txt +8 -0
  14. {graphique-1.5 → graphique-1.7}/pyproject.toml +4 -11
  15. graphique-1.7/requirements.in +2 -0
  16. graphique-1.7/tests/test_bench.py +21 -0
  17. {graphique-1.5 → graphique-1.7}/tests/test_core.py +29 -1
  18. {graphique-1.5 → graphique-1.7}/tests/test_dataset.py +12 -6
  19. {graphique-1.5 → graphique-1.7}/tests/test_models.py +39 -24
  20. {graphique-1.5 → graphique-1.7}/tests/test_service.py +58 -29
  21. graphique-1.5/graphique/scalars.py +0 -65
  22. graphique-1.5/graphique.egg-info/requires.txt +0 -8
  23. graphique-1.5/requirements.in +0 -2
  24. {graphique-1.5 → graphique-1.7}/LICENSE.txt +0 -0
  25. {graphique-1.5 → graphique-1.7}/graphique/py.typed +0 -0
  26. {graphique-1.5 → graphique-1.7}/graphique/service.py +0 -0
  27. {graphique-1.5 → graphique-1.7}/graphique.egg-info/dependency_links.txt +0 -0
  28. {graphique-1.5 → graphique-1.7}/graphique.egg-info/top_level.txt +0 -0
  29. {graphique-1.5 → graphique-1.7}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: graphique
3
- Version: 1.5
3
+ Version: 1.7
4
4
  Summary: GraphQL service for arrow tables and parquet data sets.
5
5
  Author-email: Aric Coady <aric.coady@gmail.com>
6
6
  License: Copyright 2022 Aric Coady
@@ -40,8 +40,8 @@ Classifier: Typing :: Typed
40
40
  Requires-Python: >=3.9
41
41
  Description-Content-Type: text/markdown
42
42
  License-File: LICENSE.txt
43
- Requires-Dist: pyarrow>=15
44
- Requires-Dist: strawberry-graphql[asgi,cli]>=0.198
43
+ Requires-Dist: pyarrow>=17
44
+ Requires-Dist: strawberry-graphql[asgi,cli]>=0.236
45
45
  Provides-Extra: server
46
46
  Requires-Dist: uvicorn[standard]; extra == "server"
47
47
  Provides-Extra: cli
@@ -51,9 +51,10 @@ Requires-Dist: tqdm; extra == "cli"
51
51
  ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
52
52
  [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
53
53
  ![image](https://img.shields.io/pypi/status/graphique.svg)
54
- [![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
54
+ [![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
55
55
  [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
56
- [![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
56
+ [![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
57
+ [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
57
58
  [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
58
59
  [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
59
60
 
@@ -2,9 +2,10 @@
2
2
  ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
3
3
  [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
4
4
  ![image](https://img.shields.io/pypi/status/graphique.svg)
5
- [![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
5
+ [![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
6
6
  [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
7
- [![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
7
+ [![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
8
+ [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
8
9
  [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
9
10
  [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
10
11
 
@@ -1,3 +1 @@
1
1
  from .middleware import GraphQL # noqa: F401
2
-
3
- __version__ = '1.5'
@@ -12,10 +12,10 @@ import inspect
12
12
  import itertools
13
13
  import operator
14
14
  import json
15
- from collections.abc import Callable, Iterable, Iterator, Sequence
15
+ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
16
16
  from dataclasses import dataclass
17
17
  from typing import Optional, Union, get_type_hints
18
- import numpy as np # type: ignore
18
+ import numpy as np
19
19
  import pyarrow as pa
20
20
  import pyarrow.acero as ac
21
21
  import pyarrow.compute as pc
@@ -402,10 +402,10 @@ class Table(pa.Table):
402
402
  if the predicate has args, it will be called on the differences
403
403
  """
404
404
  offsets = pa.chunked_array(
405
- Column.run_offsets(self[name], *predicates.get(name, (pc.not_equal,)))
405
+ Column.run_offsets(self[name], *predicates.get(name, ()))
406
406
  for name in names + tuple(predicates)
407
- ).unique()
408
- offsets = offsets.take(pc.sort_indices(offsets))
407
+ )
408
+ offsets = offsets.unique().sort()
409
409
  scalars = self.select(names).take(offsets[:-1])
410
410
  lists = self.select(set(self.schema.names) - set(names))
411
411
  table = Table.union(scalars, Table.from_offsets(lists, offsets))
@@ -419,13 +419,16 @@ class Table(pa.Table):
419
419
  counts: alias for optional row counts
420
420
  **funcs: aggregate funcs with columns options
421
421
  """
422
- if isinstance(self, pa.Table):
423
- self = self.unify_dictionaries()
424
422
  prefix = 'hash_' if names else ''
425
423
  aggs = [agg.astuple(prefix + func) for func in funcs for agg in funcs[func]]
424
+ columns = list(names) + [agg[0] for agg in aggs]
426
425
  if counts:
427
426
  aggs.append(([], 'hash_count_all', None, counts))
428
- return Declarations(self).aggregate(aggs, names).to_table()
427
+ if isinstance(self, pa.Table):
428
+ decl = Declaration('table_source', self.unify_dictionaries())
429
+ else:
430
+ decl = Declaration.scan(self, columns=columns)
431
+ return decl.aggregate(aggs, names).to_table(use_threads=False)
429
432
 
430
433
  def aggregate(self, counts: str = '', **funcs: Sequence[Agg]) -> dict:
431
434
  """Return aggregated scalars as a row of data."""
@@ -468,7 +471,7 @@ class Table(pa.Table):
468
471
  func = functools.partial(pc.sort_indices, null_placement=null_placement)
469
472
  if length is not None and length < len(self):
470
473
  func = functools.partial(pc.select_k_unstable, k=length)
471
- keys = dict(map(sort_key, names)) # type: ignore
474
+ keys = dict(map(sort_key, names))
472
475
  table = pa.table({name: Column.sort_values(self[name]) for name in keys})
473
476
  return func(table, sort_keys=keys.items()) if table else pa.array([], 'int64')
474
477
 
@@ -560,7 +563,7 @@ class Table(pa.Table):
560
563
  dense: use dense rank; false indicates sorting
561
564
  """
562
565
  schema = set(Table.fragment_keys(self))
563
- keys = dict(itertools.takewhile(lambda key: key[0] in schema, map(sort_key, names))) # type: ignore
566
+ keys = dict(itertools.takewhile(lambda key: key[0] in schema, map(sort_key, names)))
564
567
  if not keys:
565
568
  return None, names
566
569
  parts = []
@@ -629,8 +632,8 @@ class Table(pa.Table):
629
632
  return f'{size:n} {prefix}B'
630
633
 
631
634
 
632
- class Declarations(list):
633
- """[Acero](https://arrow.apache.org/docs/python/api/acero.html) engine declarations."""
635
+ class Declaration(ac.Declaration):
636
+ """[Acero](https://arrow.apache.org/docs/python/api/acero.html) engine declaration."""
634
637
 
635
638
  option_map = {
636
639
  'table_source': ac.TableSourceNodeOptions,
@@ -642,21 +645,24 @@ class Declarations(list):
642
645
  'hashjoin': ac.HashJoinNodeOptions,
643
646
  }
644
647
 
645
- def __init__(self, source: Union[pa.Table, ds.Dataset]):
646
- if isinstance(source, ds.Dataset):
647
- expr = source._scan_options.get('filter')
648
- self.scan(source, filter=expr)
649
- if expr is not None:
650
- self.filter(expr)
651
- else:
652
- self.table_source(source)
653
-
654
- def apply(self, name: str, *args, **kwargs) -> Self:
655
- self.append(ac.Declaration(name, self.option_map[name](*args, **kwargs)))
648
+ def __init__(self, name, *args, inputs=None, **options):
649
+ super().__init__(name, self.option_map[name](*args, **options), inputs)
650
+
651
+ @classmethod
652
+ def scan(cls, dataset: ds.Dataset, columns=None) -> Self:
653
+ """Return source node from a dataset."""
654
+ expr = dataset._scan_options.get('filter')
655
+ self = cls('scan', dataset, filter=expr, columns=columns)
656
+ if expr is not None:
657
+ self = self.filter(expr)
658
+ if isinstance(columns, Mapping):
659
+ self = self.project(columns.values(), columns)
660
+ elif columns is not None:
661
+ self = self.project(map(pc.field, columns))
656
662
  return self
657
663
 
664
+ def apply(self, name: str, *args, **options) -> Self:
665
+ return type(self)(name, *args, inputs=[self], **options)
666
+
658
667
  def __getattr__(self, name: str) -> Callable:
659
668
  return functools.partial(self.apply, name)
660
-
661
- def to_table(self, use_threads: bool = False) -> pa.Table:
662
- return ac.Declaration.from_sequence(self).to_table(use_threads)
@@ -15,9 +15,9 @@ import pyarrow.dataset as ds
15
15
  import strawberry
16
16
  from strawberry import UNSET
17
17
  from strawberry.annotation import StrawberryAnnotation
18
- from strawberry.arguments import StrawberryArgument
18
+ from strawberry.types.arguments import StrawberryArgument
19
19
  from strawberry.schema_directive import Location
20
- from strawberry.field import StrawberryField
20
+ from strawberry.types.field import StrawberryField
21
21
  from strawberry.scalars import JSON
22
22
  from typing_extensions import Self
23
23
  from .core import Agg
@@ -377,11 +377,11 @@ class Expression:
377
377
  fields.append(field)
378
378
  for group in operator.attrgetter(*self.groups)(self):
379
379
  if group is not UNSET:
380
- fields += group.to_fields() # type: ignore
380
+ fields += group.to_fields()
381
381
  for op in self.unaries:
382
382
  expr = getattr(self, op)
383
383
  if expr is not UNSET:
384
- fields.append(self.getfunc(op)(expr.to_arrow())) # type: ignore
384
+ fields.append(self.getfunc(op)(expr.to_arrow()))
385
385
  if not fields:
386
386
  return None
387
387
  if len(fields) > 1:
@@ -15,8 +15,8 @@ import pyarrow as pa
15
15
  import pyarrow.compute as pc
16
16
  import pyarrow.dataset as ds
17
17
  import strawberry.asgi
18
+ from strawberry import Info
18
19
  from strawberry.extensions.utils import get_path_from_info
19
- from strawberry.types import Info
20
20
  from typing_extensions import Self
21
21
  from .core import Batch, Column as C, ListChunk, Table as T
22
22
  from .inputs import CountAggregate, Cumulative, Diff, Expression, Field, Filter
@@ -91,11 +91,11 @@ class Dataset:
91
91
 
92
92
  @classmethod
93
93
  @no_type_check
94
- def resolve_reference(cls, info, **keys) -> Self:
94
+ def resolve_reference(cls, info: Info, **keys) -> Self:
95
95
  """Return table from federated keys."""
96
96
  self = getattr(info.root_value, cls.field)
97
97
  queries = {name: Filter(eq=[keys[name]]) for name in keys}
98
- return self.filter(Info(info, None), **queries)
98
+ return self.filter(info, **queries)
99
99
 
100
100
  def columns(self, info: Info) -> dict:
101
101
  """fields for each column"""
@@ -9,10 +9,9 @@ from keyword import iskeyword
9
9
  from typing import Optional
10
10
  import pyarrow.dataset as ds
11
11
  import strawberry.asgi
12
- from strawberry import UNSET
12
+ from strawberry import Info, UNSET
13
13
  from strawberry.extensions import tracing
14
14
  from strawberry.utils.str_converters import to_camel_case
15
- from strawberry.types import Info
16
15
  from .core import Column as C
17
16
  from .inputs import Filter
18
17
  from .interface import Dataset, Root
@@ -13,11 +13,11 @@ from typing import Annotated, Generic, Optional, TypeVar, TYPE_CHECKING, get_arg
13
13
  import pyarrow as pa
14
14
  import pyarrow.compute as pc
15
15
  import strawberry
16
- from strawberry.field import StrawberryField
17
- from strawberry.types import Info
16
+ from strawberry import Info
17
+ from strawberry.types.field import StrawberryField
18
18
  from .core import Column as C
19
19
  from .inputs import links
20
- from .scalars import Interval, Long, scalar_map, type_map
20
+ from .scalars import Long, scalar_map, type_map
21
21
 
22
22
  if TYPE_CHECKING: # pragma: no cover
23
23
  from .interface import Dataset
@@ -120,7 +120,7 @@ class Set(Generic[T]):
120
120
  return self.array.to_pylist()
121
121
 
122
122
 
123
- @Column.register(timedelta, Interval)
123
+ @Column.register(timedelta, pa.MonthDayNano)
124
124
  @strawberry.type(name='Column', description="column of elapsed times")
125
125
  class NominalColumn(Generic[T], Column):
126
126
  values = doc_field(Set.values)
@@ -0,0 +1,111 @@
1
+ """
2
+ GraphQL scalars.
3
+ """
4
+
5
+ import functools
6
+ import re
7
+ from datetime import date, datetime, time, timedelta
8
+ from decimal import Decimal
9
+ from typing import Union, no_type_check
10
+ import pyarrow as pa
11
+ import strawberry
12
+
13
+
14
+ def parse_long(value) -> int:
15
+ if isinstance(value, int):
16
+ return value
17
+ raise TypeError(f"Long cannot represent value: {value}")
18
+
19
+
20
+ @no_type_check
21
+ def parse_duration(value: str):
22
+ months = days = seconds = 0
23
+ d_val, _, t_val = value.partition('T')
24
+ parts = re.split(r'(-?\d+\.?\d*)', d_val.lower() + t_val)
25
+ if parts.pop(0) != 'p':
26
+ raise ValueError("Duration format must start with `P`")
27
+ multipliers = {'y': 12, 'w': 7, 'H': 3600, 'M': 60}
28
+ for num, key in zip(parts[::2], parts[1::2]):
29
+ value = (float if '.' in num else int)(num) * multipliers.get(key, 1)
30
+ if key in 'ym':
31
+ months += value
32
+ elif key in 'wd':
33
+ days += value
34
+ elif key in 'HMS':
35
+ seconds += value
36
+ else:
37
+ raise ValueError(f"Invalid duration field: {key.upper()}")
38
+ if set(d_val).isdisjoint('YM'):
39
+ return timedelta(days, seconds)
40
+ return pa.MonthDayNano([months, days, int(seconds * 1_000_000_000)])
41
+
42
+
43
+ @functools.singledispatch
44
+ def duration_isoformat(months: int, days: int, seconds: int, fraction: str = '.') -> str:
45
+ minutes, seconds = divmod(seconds, 60)
46
+ items = zip('YMDHM', divmod(months, 12) + (days,) + divmod(minutes, 60))
47
+ year, month, day, hour, minute = (f'{value}{key}' if value else '' for key, value in items)
48
+ fraction = fraction.rstrip('0').rstrip('.')
49
+ return f'P{year}{month}{day}T{hour}{minute}{seconds}{fraction}S'
50
+
51
+
52
+ @duration_isoformat.register
53
+ def _(td: timedelta) -> str: # type: ignore
54
+ return duration_isoformat(0, td.days, td.seconds, f'.{td.microseconds:06}')
55
+
56
+
57
+ @duration_isoformat.register
58
+ def _(mdn: pa.MonthDayNano) -> str:
59
+ seconds, nanoseconds = divmod(mdn.nanoseconds, 1_000_000_000)
60
+ value = duration_isoformat(mdn.months, mdn.days, seconds, f'.{nanoseconds:09}')
61
+ return value if mdn.months else value.replace('P', 'P0M')
62
+
63
+
64
+ Long = strawberry.scalar(int, name='Long', description="64-bit int", parse_value=parse_long)
65
+ Duration = strawberry.scalar(
66
+ Union[timedelta, pa.MonthDayNano],
67
+ name='Duration',
68
+ description="Duration (isoformat)",
69
+ specified_by_url="https://en.wikipedia.org/wiki/ISO_8601#Durations",
70
+ serialize=duration_isoformat,
71
+ parse_value=parse_duration,
72
+ )
73
+ scalar_map = {
74
+ bytes: strawberry.scalars.Base64,
75
+ dict: strawberry.scalars.JSON,
76
+ timedelta: Duration,
77
+ pa.MonthDayNano: Duration,
78
+ }
79
+
80
+ type_map = {
81
+ pa.lib.Type_BOOL: bool,
82
+ pa.lib.Type_UINT8: int,
83
+ pa.lib.Type_INT8: int,
84
+ pa.lib.Type_UINT16: int,
85
+ pa.lib.Type_INT16: int,
86
+ pa.lib.Type_UINT32: Long,
87
+ pa.lib.Type_INT32: int,
88
+ pa.lib.Type_UINT64: Long,
89
+ pa.lib.Type_INT64: Long,
90
+ pa.lib.Type_HALF_FLOAT: float,
91
+ pa.lib.Type_FLOAT: float,
92
+ pa.lib.Type_DOUBLE: float,
93
+ pa.lib.Type_DECIMAL128: Decimal,
94
+ pa.lib.Type_DECIMAL256: Decimal,
95
+ pa.lib.Type_DATE32: date,
96
+ pa.lib.Type_DATE64: date,
97
+ pa.lib.Type_TIMESTAMP: datetime,
98
+ pa.lib.Type_TIME32: time,
99
+ pa.lib.Type_TIME64: time,
100
+ pa.lib.Type_DURATION: timedelta,
101
+ pa.lib.Type_INTERVAL_MONTH_DAY_NANO: pa.MonthDayNano,
102
+ pa.lib.Type_BINARY: bytes,
103
+ pa.lib.Type_FIXED_SIZE_BINARY: bytes,
104
+ pa.lib.Type_LARGE_BINARY: bytes,
105
+ pa.lib.Type_STRING: str,
106
+ pa.lib.Type_LARGE_STRING: str,
107
+ pa.lib.Type_LIST: list,
108
+ pa.lib.Type_FIXED_SIZE_LIST: list,
109
+ pa.lib.Type_LARGE_LIST: list,
110
+ pa.lib.Type_STRUCT: dict,
111
+ }
@@ -9,11 +9,13 @@ sufficient once partitioned, but there is a `fragments` option to optimize for m
9
9
  progress on the second pass.
10
10
  """
11
11
 
12
- import argparse
13
12
  import shutil
14
- from collections.abc import Iterable
15
13
  from pathlib import Path
14
+ from typing import Annotated
15
+ import numpy as np
16
+ import pyarrow.compute as pc
16
17
  import pyarrow.dataset as ds
18
+ import typer # type: ignore
17
19
  from tqdm import tqdm # type: ignore
18
20
 
19
21
 
@@ -22,12 +24,19 @@ def sort_key(name: str) -> tuple:
22
24
  return name.lstrip('-'), ('descending' if name.startswith('-') else 'ascending')
23
25
 
24
26
 
25
- def write_batches(scanner: ds.Scanner, base_dir: str, *partitioning: str, **options):
26
- """Partition dataset by batches."""
27
+ def write_batches(
28
+ scanner: ds.Scanner, base_dir: str, *partitioning: str, indices: str = '', **options
29
+ ):
30
+ """Partition dataset by batches.
31
+
32
+ Optionally include original indices.
33
+ """
27
34
  options.update(format='parquet', partitioning=partitioning)
28
35
  options.update(partitioning_flavor='hive', existing_data_behavior='overwrite_or_ignore')
29
36
  with tqdm(total=scanner.count_rows(), desc="Batches") as pbar:
30
37
  for index, batch in enumerate(scanner.to_batches()):
38
+ if indices:
39
+ batch = batch.append_column(indices, pc.add(np.arange(len(batch)), pbar.n))
31
40
  options['basename_template'] = f'part-{index}-{{i}}.parquet'
32
41
  ds.write_dataset(batch, base_dir, **options)
33
42
  pbar.update(len(batch))
@@ -45,34 +54,25 @@ def write_fragments(dataset: ds.Dataset, base_dir: str, sorting=(), **options):
45
54
 
46
55
 
47
56
  def partition(
48
- scanner: ds.Scanner,
49
- base_dir: str,
50
- *partitioning: str,
51
- fragments: bool = False,
52
- sort: Iterable[str] = (),
53
- **options,
57
+ src: Annotated[str, typer.Argument(help="source path")],
58
+ dest: Annotated[str, typer.Argument(help="destination path")],
59
+ partitioning: Annotated[list[str], typer.Argument(help="partition keys")],
60
+ fragments: Annotated[bool, typer.Option(help="iterate over fragments")] = False,
61
+ sort: Annotated[list[str], typer.Option(help="sort keys; will load fragments")] = [],
54
62
  ):
55
63
  """Partition dataset by keys."""
56
- temp = Path(base_dir) / 'temp'
57
- write_batches(scanner, str(temp), *partitioning)
64
+ temp = Path(dest) / 'temp'
65
+ write_batches(ds.dataset(src, partitioning='hive'), str(temp), *partitioning)
58
66
  dataset = ds.dataset(temp, partitioning='hive')
67
+ options = dict(partitioning_flavor='hive', existing_data_behavior='overwrite_or_ignore')
59
68
  if fragments or sort:
60
- write_fragments(dataset, base_dir, tuple(map(sort_key, sort)), **options)
69
+ write_fragments(dataset, dest, tuple(map(sort_key, sort)))
61
70
  else:
62
- options.update(partitioning_flavor='hive', existing_data_behavior='overwrite_or_ignore')
63
71
  with tqdm(desc="Partitions"):
64
- ds.write_dataset(dataset, base_dir, partitioning=partitioning, **options)
72
+ ds.write_dataset(dataset, dest, partitioning=partitioning, **options)
65
73
  shutil.rmtree(temp)
66
74
 
67
75
 
68
- parser = argparse.ArgumentParser(description=__doc__)
69
- parser.add_argument('src', help="source path")
70
- parser.add_argument('dest', help="destination path")
71
- parser.add_argument('partitioning', nargs='+', help="partition keys")
72
- parser.add_argument('--fragments', action='store_true', help="iterate over fragments")
73
- parser.add_argument('--sort', nargs='*', default=(), help="sort keys; will load fragments")
74
-
75
76
  if __name__ == '__main__':
76
- args = parser.parse_args()
77
- dataset = ds.dataset(args.src, partitioning='hive')
78
- partition(dataset, args.dest, *args.partitioning, fragments=args.fragments, sort=args.sort)
77
+ partition.__doc__ = __doc__
78
+ typer.run(partition)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: graphique
3
- Version: 1.5
3
+ Version: 1.7
4
4
  Summary: GraphQL service for arrow tables and parquet data sets.
5
5
  Author-email: Aric Coady <aric.coady@gmail.com>
6
6
  License: Copyright 2022 Aric Coady
@@ -40,8 +40,8 @@ Classifier: Typing :: Typed
40
40
  Requires-Python: >=3.9
41
41
  Description-Content-Type: text/markdown
42
42
  License-File: LICENSE.txt
43
- Requires-Dist: pyarrow>=15
44
- Requires-Dist: strawberry-graphql[asgi,cli]>=0.198
43
+ Requires-Dist: pyarrow>=17
44
+ Requires-Dist: strawberry-graphql[asgi,cli]>=0.236
45
45
  Provides-Extra: server
46
46
  Requires-Dist: uvicorn[standard]; extra == "server"
47
47
  Provides-Extra: cli
@@ -51,9 +51,10 @@ Requires-Dist: tqdm; extra == "cli"
51
51
  ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
52
52
  [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
53
53
  ![image](https://img.shields.io/pypi/status/graphique.svg)
54
- [![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
54
+ [![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
55
55
  [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
56
- [![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
56
+ [![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
57
+ [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
57
58
  [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
58
59
  [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
59
60
 
@@ -17,6 +17,7 @@ graphique.egg-info/SOURCES.txt
17
17
  graphique.egg-info/dependency_links.txt
18
18
  graphique.egg-info/requires.txt
19
19
  graphique.egg-info/top_level.txt
20
+ tests/test_bench.py
20
21
  tests/test_core.py
21
22
  tests/test_dataset.py
22
23
  tests/test_models.py
@@ -0,0 +1,8 @@
1
+ pyarrow>=17
2
+ strawberry-graphql[asgi,cli]>=0.236
3
+
4
+ [cli]
5
+ tqdm
6
+
7
+ [server]
8
+ uvicorn[standard]
@@ -1,10 +1,11 @@
1
1
  [build-system]
2
- requires = ["setuptools>=61", "wheel"]
2
+ requires = ["setuptools>=61"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "graphique"
7
- dynamic = ["version", "dependencies"]
7
+ version = "1.7"
8
+ dynamic = ["dependencies"]
8
9
  description = "GraphQL service for arrow tables and parquet data sets."
9
10
  readme = "README.md"
10
11
  requires-python = ">=3.9"
@@ -39,26 +40,18 @@ Issues = "https://github.com/coady/graphique/issues"
39
40
  server = ["uvicorn[standard]"]
40
41
  cli = ["tqdm"]
41
42
 
42
- [tool.setuptools]
43
- packages = ["graphique"]
44
-
45
43
  [tool.setuptools.dynamic]
46
- version = {attr = "graphique.__version__"}
47
44
  dependencies = {file = "requirements.in"}
48
45
 
49
- [tool.setuptools.package-data]
50
- graphique = ["py.typed"]
51
-
52
46
  [tool.ruff]
53
47
  line-length = 100
54
48
  extend-include = ["*.ipynb"]
55
49
 
56
50
  [tool.ruff.format]
57
- preview = true
58
51
  quote-style = "preserve"
59
52
 
60
53
  [[tool.mypy.overrides]]
61
- module = ["pyarrow.*", "strawberry.*", "starlette.*"]
54
+ module = ["numpy.*", "pyarrow.*", "strawberry.*", "starlette.*"]
62
55
  ignore_missing_imports = true
63
56
 
64
57
  [tool.coverage.run]
@@ -0,0 +1,2 @@
1
+ pyarrow>=17
2
+ strawberry-graphql[asgi,cli]>=0.236
@@ -0,0 +1,21 @@
1
+ import pytest
2
+ from graphique.core import Table as T
3
+
4
+
5
+ @pytest.mark.benchmark
6
+ def test_group(table):
7
+ T.group(table, 'state', 'county', 'city')
8
+ T.runs(table, 'state', 'county', 'city')
9
+
10
+
11
+ @pytest.mark.benchmark
12
+ def test_rank(table):
13
+ T.ranked(table, 1, 'state', 'county', 'city')
14
+ T.ranked(table, 10, 'state', 'county', 'city')
15
+
16
+
17
+ @pytest.mark.benchmark
18
+ def test_sort(table):
19
+ T.sort(table, 'state', 'county', 'city', length=1)
20
+ T.sort(table, 'state', 'county', 'city', length=10)
21
+ T.sort(table, 'state', 'county', 'city')
@@ -1,7 +1,28 @@
1
1
  import pyarrow as pa
2
2
  import pyarrow.compute as pc
3
+ import pyarrow.dataset as ds
3
4
  import pytest
4
- from graphique.core import Agg, ListChunk, Column as C, Table as T
5
+ from graphique.core import Agg, Declaration, ListChunk, Column as C, Table as T
6
+ from graphique.scalars import parse_duration, duration_isoformat
7
+
8
+
9
+ def test_duration():
10
+ assert duration_isoformat(parse_duration('P1Y1M1DT1H1M1.1S')) == 'P1Y1M1DT1H1M1.1S'
11
+ assert duration_isoformat(parse_duration('P1M1DT1H1M1.1S')) == 'P1M1DT1H1M1.1S'
12
+ assert duration_isoformat(parse_duration('P1DT1H1M1.1S')) == 'P1DT1H1M1.1S'
13
+ assert duration_isoformat(parse_duration('PT1H1M1.1S')) == 'PT1H1M1.1S'
14
+ assert duration_isoformat(parse_duration('PT1M1.1S')) == 'PT1M1.1S'
15
+ assert duration_isoformat(parse_duration('PT1.1S')) == 'PT1.1S'
16
+ assert duration_isoformat(parse_duration('PT1S')) == 'PT1S'
17
+ assert duration_isoformat(parse_duration('P0D')) == 'PT0S'
18
+ assert duration_isoformat(parse_duration('PT0S')) == 'PT0S'
19
+ assert duration_isoformat(parse_duration('P-1DT-1H')) == 'P-2DT23H0S'
20
+ assert duration_isoformat(parse_duration('P0MT')) == 'P0MT0S'
21
+ assert duration_isoformat(parse_duration('P0YT')) == 'P0MT0S'
22
+ with pytest.raises(ValueError):
23
+ duration_isoformat(parse_duration('T1H'))
24
+ with pytest.raises(ValueError):
25
+ duration_isoformat(parse_duration('P1H'))
5
26
 
6
27
 
7
28
  def test_dictionary(table):
@@ -69,6 +90,13 @@ def test_membership():
69
90
  assert C.index(array, 1, start=2) == -1
70
91
 
71
92
 
93
+ def test_declaration(table):
94
+ dataset = ds.dataset(table).filter(pc.field('state') == 'CA')
95
+ assert Declaration.scan(dataset).to_table()['state'].unique().to_pylist() == ['CA']
96
+ (column,) = Declaration.scan(dataset, columns={'_': pc.field('state')}).to_table()
97
+ assert column.unique().to_pylist() == ['CA']
98
+
99
+
72
100
  def test_group(table):
73
101
  groups = T.group(table, 'state', list=[Agg('county'), Agg('city')])
74
102
  assert len(groups) == 52
@@ -85,11 +85,13 @@ def test_group(dsclient):
85
85
  assert data['group']['row']['county']
86
86
  data = dsclient.execute(
87
87
  """{ group(by: ["state"], aggregate: {mean: {name: "zipcode"}}) { slice(length: 1) {
88
- column(name: "zipcode") { ... on FloatColumn { values } } } } }""")
88
+ column(name: "zipcode") { ... on FloatColumn { values } } } } }"""
89
+ )
89
90
  assert data == {'group': {'slice': {'column': {'values': [pytest.approx(12614.62721)]}}}}
90
91
  data = dsclient.execute(
91
92
  """{ group(by: ["state"], aggregate: {list: {name: "zipcode"}}) { aggregate(mean: {name: "zipcode"}) {
92
- slice(length: 1) { column(name: "zipcode") { ... on FloatColumn { values } } } } } }""")
93
+ slice(length: 1) { column(name: "zipcode") { ... on FloatColumn { values } } } } } }"""
94
+ )
93
95
  assert data == {
94
96
  'group': {'aggregate': {'slice': {'column': {'values': [pytest.approx(12614.62721)]}}}}
95
97
  }
@@ -101,7 +103,8 @@ def test_group(dsclient):
101
103
  def test_list(partclient):
102
104
  data = partclient.execute(
103
105
  """{ group(by: "state", aggregate: {distinct: {alias: "counties", name: "county"}}) {
104
- tables { row { state } column(name: "counties") { length } } } } """)
106
+ tables { row { state } column(name: "counties") { length } } } } """
107
+ )
105
108
  (table,) = [table for table in data['group']['tables'] if table['row']['state'] == 'PR']
106
109
  assert table == {'row': {'state': 'PR'}, 'column': {'length': 78}}
107
110
  data = partclient.execute("""{ group(by: "north", aggregate: {distinct: {name: "west"}}) {
@@ -133,7 +136,8 @@ def test_fragments(partclient):
133
136
  assert data == {'group': {'length': 1, 'row': {'state': 'AK'}}}
134
137
  data = partclient.execute(
135
138
  """{ group(by: ["north", "west"], aggregate: {distinct: {name: "city"}, mean: {name: "zipcode"}}) {
136
- length column(name: "city") { type } } }""")
139
+ length column(name: "city") { type } } }"""
140
+ )
137
141
  assert data == {'group': {'length': 4, 'column': {'type': 'large_list<item: string>'}}}
138
142
  data = partclient.execute("""{ group(by: "north", aggregate: {countDistinct: {name: "west"}}) {
139
143
  column(name: "west") { ... on LongColumn { values } } } }""")
@@ -240,14 +244,16 @@ def test_federation(fedclient):
240
244
  assert set(table['schema']['names']) > {'zipcode', 'timezone', 'latitude'}
241
245
  data = fedclient.execute(
242
246
  """{ zipcodes { scan(columns: {alias: "zip", name: "zipcode", cast: "int64"}) {
243
- join(right: "zip_db", keys: "zip", joinType: "right outer") { length schema { names } } } } }""")
247
+ join(right: "zip_db", keys: "zip", joinType: "right outer") { length schema { names } } } } }"""
248
+ )
244
249
  table = data['zipcodes']['scan']['join']
245
250
  assert table['length'] == 42724
246
251
  assert set(table['schema']['names']) > {'zip', 'timezone', 'latitude'}
247
252
 
248
253
  data = fedclient.execute(
249
254
  """{ _entities(representations: {__typename: "ZipcodesTable", zipcode: 90001}) {
250
- ... on ZipcodesTable { length row { state } schema { names } } } }""")
255
+ ... on ZipcodesTable { length row { state } schema { names } } } }"""
256
+ )
251
257
  assert data == {
252
258
  '_entities': [{'length': 1, 'row': {'state': 'CA'}, 'schema': {'names': ['state']}}]
253
259
  }
@@ -1,4 +1,3 @@
1
- import sys
2
1
  import pytest
3
2
 
4
3
 
@@ -102,7 +101,8 @@ def test_boolean(executor):
102
101
  assert data == {'scan': {'length': 1}}
103
102
  data = executor(
104
103
  """{ scan(columns: {alias: "bool", andNot: [{inv: {name: "bool"}}, {name: "bool"}], kleene: true})
105
- { columns { bool { values } } } }""")
104
+ { columns { bool { values } } } }"""
105
+ )
106
106
  assert data == {'scan': {'columns': {'bool': {'values': [True, None]}}}}
107
107
 
108
108
 
@@ -153,27 +153,30 @@ def test_numeric(executor):
153
153
  assert data == {'scan': {'columns': {'int32': {'values': [0, None]}}}}
154
154
  data = executor(
155
155
  """{ scan(columns: {alias: "float", coalesce: [{name: "float"}, {name: "int32"}]}) {
156
- columns { float { values } } } }""")
156
+ columns { float { values } } } }"""
157
+ )
157
158
  assert data == {'scan': {'columns': {'float': {'values': [0.0, None]}}}}
158
159
  data = executor("""{ scan(columns: {bitWise: {not: {name: "int32"}}, alias: "int32"}) {
159
160
  columns { int32 { values } } } }""")
160
161
  assert data == {'scan': {'columns': {'int32': {'values': [-1, None]}}}}
161
162
  data = executor(
162
163
  """{ scan(columns: {bitWise: {or: [{name: "int32"}, {name: "int64"}]}, alias: "int64"}) {
163
- columns { int64 { values } } } }""")
164
+ columns { int64 { values } } } }"""
165
+ )
164
166
  assert data == {'scan': {'columns': {'int64': {'values': [0, None]}}}}
165
167
 
166
168
 
167
- @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or pytz")
168
169
  def test_datetime(executor):
169
170
  for name in ('timestamp', 'date32'):
170
171
  data = executor(
171
172
  f"""{{ scan(columns: {{alias: "year", temporal: {{year: {{name: "{name}"}}}}}})
172
- {{ column(name: "year") {{ ... on LongColumn {{ values }} }} }} }}""")
173
+ {{ column(name: "year") {{ ... on LongColumn {{ values }} }} }} }}"""
174
+ )
173
175
  assert data == {'scan': {'column': {'values': [1970, None]}}}
174
176
  data = executor(
175
177
  f"""{{ scan(columns: {{alias: "quarter", temporal: {{quarter: {{name: "{name}"}}}}}})
176
- {{ column(name: "quarter") {{ ... on LongColumn {{ values }} }} }} }}""")
178
+ {{ column(name: "quarter") {{ ... on LongColumn {{ values }} }} }} }}"""
179
+ )
177
180
  assert data == {'scan': {'column': {'values': [1, None]}}}
178
181
  data = executor(f"""{{ scan(columns: {{alias: "{name}",
179
182
  temporal: {{yearsBetween: [{{name: "{name}"}}, {{name: "{name}"}}]}}}})
@@ -181,16 +184,19 @@ def test_datetime(executor):
181
184
  assert data == {'scan': {'column': {'values': [0, None]}}}
182
185
  data = executor(
183
186
  """{ scan(columns: {alias: "timestamp", temporal: {strftime: {name: "timestamp"}}}) {
184
- column(name: "timestamp") { type } } }""")
187
+ column(name: "timestamp") { type } } }"""
188
+ )
185
189
  assert data == {'scan': {'column': {'type': 'string'}}}
186
190
  for name in ('timestamp', 'time32'):
187
191
  data = executor(
188
192
  f"""{{ scan(columns: {{alias: "hour", temporal: {{hour: {{name: "{name}"}}}}}})
189
- {{ column(name: "hour") {{ ... on LongColumn {{ values }} }} }} }}""")
193
+ {{ column(name: "hour") {{ ... on LongColumn {{ values }} }} }} }}"""
194
+ )
190
195
  assert data == {'scan': {'column': {'values': [0, None]}}}
191
196
  data = executor(
192
197
  f"""{{ scan(columns: {{alias: "subsecond", temporal: {{subsecond: {{name: "{name}"}}}}}})
193
- {{ column(name: "subsecond") {{ ... on FloatColumn {{ values }} }} }} }}""")
198
+ {{ column(name: "subsecond") {{ ... on FloatColumn {{ values }} }} }} }}"""
199
+ )
194
200
  assert data == {'scan': {'column': {'values': [0.0, None]}}}
195
201
  data = executor(f"""{{ scan(columns: {{alias: "hours",
196
202
  temporal: {{hoursBetween: [{{name: "{name}"}}, {{name: "{name}"}}]}}}})
@@ -200,27 +206,29 @@ def test_datetime(executor):
200
206
  executor('{ columns { time64 { between(unit: "hours") { values } } } }')
201
207
  data = executor(
202
208
  """{ scan(columns: {alias: "timestamp", temporal: {assumeTimezone: {name: "timestamp"}, timezone: "UTC"}}) {
203
- columns { timestamp { values } } } }""")
209
+ columns { timestamp { values } } } }"""
210
+ )
204
211
  dates = data['scan']['columns']['timestamp']['values']
205
212
  assert dates == ['1970-01-01T00:00:00+00:00', None]
206
213
  data = executor(
207
214
  """{ scan(columns: {alias: "time32", temporal: {round: {name: "time32"}, unit: "hour"}}) {
208
- columns { time32 { values } } } }""")
215
+ columns { time32 { values } } } }"""
216
+ )
209
217
  assert data == {'scan': {'columns': {'time32': {'values': ['00:00:00', None]}}}}
210
218
 
211
219
 
212
220
  def test_duration(executor):
213
221
  data = executor(
214
222
  """{ scan(columns: {alias: "diff", checked: true, subtract: [{name: "timestamp"}, {name: "timestamp"}]})
215
- { column(name: "diff") { ... on DurationColumn { unique { values } } } } }""")
216
- assert data == {'scan': {'column': {'unique': {'values': [0.0, None]}}}}
223
+ { column(name: "diff") { ... on DurationColumn { unique { values } } } } }"""
224
+ )
225
+ assert data == {'scan': {'column': {'unique': {'values': ['PT0S', None]}}}}
217
226
  data = executor('{ runs(split: [{name: "timestamp", gt: 0.0}]) { length } }')
218
227
  assert data == {'runs': {'length': 1}}
219
228
  data = executor("""{ scan(columns: {alias: "diff", temporal:
220
229
  {monthDayNanoIntervalBetween: [{name: "timestamp"}, {name: "timestamp"}]}})
221
- { column(name: "diff") { ... on IntervalColumn { values { months days nanoseconds } } } } }""")
222
- value = {'months': 0, 'days': 0, 'nanoseconds': 0}
223
- assert data == {'scan': {'column': {'values': [value, None]}}}
230
+ { column(name: "diff") { ... on DurationColumn { values } } } }""")
231
+ assert data == {'scan': {'column': {'values': ['P0MT0S', None]}}}
224
232
 
225
233
 
226
234
  def test_list(executor):
@@ -250,7 +258,8 @@ def test_list(executor):
250
258
  assert data == {'apply': {'column': {'values': [1, -1]}}}
251
259
  data = executor(
252
260
  """{ scan(columns: {list: {element: [{name: "list"}, {value: 1}]}, alias: "value"}) {
253
- column(name: "value") { ... on IntColumn { values } } } }""")
261
+ column(name: "value") { ... on IntColumn { values } } } }"""
262
+ )
254
263
  assert data == {'scan': {'column': {'values': [1, None]}}}
255
264
  data = executor("""{ scan(columns: {list: {slice: {name: "list"}, stop: 1}, alias: "value"}) {
256
265
  column(name: "value") { ... on ListColumn { flatten { ... on IntColumn { values } } } } } }""")
@@ -267,12 +276,14 @@ def test_list(executor):
267
276
  data = executor(
268
277
  """{ aggregate(stddev: {name: "list"}, variance: {name: "list", alias: "var", ddof: 1}) {
269
278
  column(name: "list") { ... on FloatColumn { values } }
270
- var: column(name: "var") { ... on FloatColumn { values } } } }""")
279
+ var: column(name: "var") { ... on FloatColumn { values } } } }"""
280
+ )
271
281
  assert data['aggregate']['column']['values'] == [pytest.approx((2 / 3) ** 0.5), None]
272
282
  assert data['aggregate']['var']['values'] == [1, None]
273
283
  data = executor(
274
284
  """{ runs(by: "int32") { scan(columns: {binary: {join: [{name: "binary"}, {base64: ""}]}, alias: "binary"}) {
275
- column(name: "binary") { ... on Base64Column { values } } } } }""")
285
+ column(name: "binary") { ... on Base64Column { values } } } } }"""
286
+ )
276
287
  assert data == {'runs': {'scan': {'column': {'values': [None]}}}}
277
288
  data = executor('{ columns { list { value { type } } } }')
278
289
  assert data == {'columns': {'list': {'value': {'type': 'int32'}}}}
@@ -318,7 +329,8 @@ def test_dictionary(executor):
318
329
  assert data == {'group': {'column': {'values': [1, 0]}}}
319
330
  data = executor(
320
331
  """{ scan(columns: {alias: "string", coalesce: [{name: "string"}, {value: ""}]}) {
321
- columns { string { values } } } }""")
332
+ columns { string { values } } } }"""
333
+ )
322
334
  assert data == {'scan': {'columns': {'string': {'values': ['', '']}}}}
323
335
 
324
336
 
@@ -334,7 +346,8 @@ def test_selections(executor):
334
346
  def test_conditions(executor):
335
347
  data = executor(
336
348
  """{ scan(columns: {alias: "bool", ifElse: [{name: "bool"}, {name: "int32"}, {name: "float"}]}) {
337
- column(name: "bool") { type } } }""")
349
+ column(name: "bool") { type } } }"""
350
+ )
338
351
  assert data == {'scan': {'column': {'type': 'float'}}}
339
352
  with pytest.raises(ValueError, match="no kernel"):
340
353
  executor("""{ scan(columns: {alias: "bool",
@@ -352,7 +365,8 @@ def test_base64(executor):
352
365
  assert data == {'scan': {'column': {'values': [0, None]}}}
353
366
  data = executor(
354
367
  """{ scan(columns: {alias: "binary", binary: {repeat: [{name: "binary"}, {value: 2}]}}) {
355
- columns { binary { values } } } }""")
368
+ columns { binary { values } } } }"""
369
+ )
356
370
  assert data == {'scan': {'columns': {'binary': {'values': ['', None]}}}}
357
371
  data = executor(
358
372
  '{ apply(fillNullForward: {name: "binary"}) { columns { binary { values } } } }'
@@ -360,7 +374,8 @@ def test_base64(executor):
360
374
  assert data == {'apply': {'columns': {'binary': {'values': ['', '']}}}}
361
375
  data = executor(
362
376
  """{ scan(columns: {alias: "binary", coalesce: [{name: "binary"}, {base64: "Xw=="}]}) {
363
- columns { binary { values } } } }""")
377
+ columns { binary { values } } } }"""
378
+ )
364
379
  assert data == {'scan': {'columns': {'binary': {'values': ['', 'Xw==']}}}}
365
380
  data = executor("""{ scan(columns: {alias: "binary", binary: {joinElementWise: [
366
381
  {name: "binary"}, {name: "binary"}, {base64: "Xw=="}], nullHandling: "replace"}}) {
@@ -48,11 +48,13 @@ def test_floats(client):
48
48
  assert quantile == pytest.approx(39.12054)
49
49
  data = client.execute(
50
50
  """{ scan(columns: {alias: "latitude", elementWise: {min: [{name: "latitude"}, {name: "longitude"}]}}) {
51
- columns { latitude { min } } } }""")
51
+ columns { latitude { min } } } }"""
52
+ )
52
53
  assert data == {'scan': {'columns': {'latitude': {'min': pytest.approx(-174.213333)}}}}
53
54
  data = client.execute(
54
55
  """{scan(columns: {alias: "l", setLookup: {digitize: [{name: "latitude"}, {value: [40]}]}}) {
55
- column(name: "l") { ... on LongColumn { unique { values counts } } } } }""")
56
+ column(name: "l") { ... on LongColumn { unique { values counts } } } } }"""
57
+ )
56
58
  assert data == {"scan": {"column": {"unique": {"values": [1, 0], "counts": [17955, 23745]}}}}
57
59
  data = client.execute(
58
60
  '{ scan(columns: {alias: "latitude", log: {logb: [{name: "latitude"}, {value: 3}]}}) { row { latitude } } }'
@@ -106,33 +108,39 @@ def test_strings(client):
106
108
  assert data == {'scan': {'length': 41700}}
107
109
  data = client.execute(
108
110
  """{ scan(columns: {alias: "city", substring: {match: {name: "city"}, pattern: "Mountain"}})
109
- { scan(filter: {name: "city"}) { length } } }""")
111
+ { scan(filter: {name: "city"}) { length } } }"""
112
+ )
110
113
  assert data == {'scan': {'scan': {'length': 88}}}
111
114
  data = client.execute(
112
115
  """{ scan(filter: {substring: {match: {name: "city"}, pattern: "mountain", ignoreCase: true}})
113
- { length } }""")
116
+ { length } }"""
117
+ )
114
118
  assert data == {'scan': {'length': 88}}
115
119
  data = client.execute(
116
120
  """{ scan(filter: {substring: {match: {name: "city"}, pattern: "^Mountain", regex: true}})
117
- { length } }""")
121
+ { length } }"""
122
+ )
118
123
  assert data == {'scan': {'length': 42}}
119
124
  data = client.execute(
120
125
  """{ scan(columns: {alias: "idx", setLookup: {indexIn: [{name: "state"}, {value: ["CA", "OR"]}]}})
121
- { column(name: "idx") { ... on IntColumn { unique { values } } } } }""")
126
+ { column(name: "idx") { ... on IntColumn { unique { values } } } } }"""
127
+ )
122
128
  assert data == {'scan': {'column': {'unique': {'values': [None, 0, 1]}}}}
123
129
 
124
130
 
125
131
  def test_string_methods(client):
126
132
  data = client.execute(
127
133
  """{ scan(columns: {alias: "split", substring: {split: {name: "city"}, pattern: "-", maxSplits: 1}}) {
128
- column(name: "split") { type } } }""")
134
+ column(name: "split") { type } } }"""
135
+ )
129
136
  assert data == {'scan': {'column': {'type': 'list<item: string>'}}}
130
137
  data = client.execute("""{ scan(columns: {alias: "split", substring: {split: {name: "city"}}}) {
131
138
  column(name: "split") { type } } }""")
132
139
  assert data == {'scan': {'column': {'type': 'list<item: string>'}}}
133
140
  data = client.execute(
134
141
  """{ scan(columns: {alias: "state", utf8: {trim: {name: "state"}, characters: "C"}}) {
135
- columns { state { values } } } }""")
142
+ columns { state { values } } } }"""
143
+ )
136
144
  states = data['scan']['columns']['state']['values']
137
145
  assert 'CA' not in states and 'A' in states
138
146
  data = client.execute(
@@ -141,11 +149,13 @@ def test_string_methods(client):
141
149
  assert data == {'scan': {'length': 41700}}
142
150
  data = client.execute(
143
151
  """{ scan(columns: {alias: "state", utf8: {center: {name: "state"}, width: 4, padding: "_"}})
144
- { row { state } } }""")
152
+ { row { state } } }"""
153
+ )
145
154
  assert data == {'scan': {'row': {'state': '_NY_'}}}
146
155
  data = client.execute(
147
156
  """{ scan(columns: {alias: "state", utf8: {replaceSlice: {name: "state"}, start: 0, stop: 2, replacement: ""}})
148
- { columns { state { unique { values } } } } }""")
157
+ { columns { state { unique { values } } } } }"""
158
+ )
149
159
  assert data == {'scan': {'columns': {'state': {'unique': {'values': ['']}}}}}
150
160
  data = client.execute(
151
161
  '{ scan(columns: {alias: "state", utf8: {sliceCodeunits: {name: "state"}, start: 0, stop: 1}}) { row { state } } }'
@@ -153,7 +163,8 @@ def test_string_methods(client):
153
163
  assert data == {'scan': {'row': {'state': 'N'}}}
154
164
  data = client.execute(
155
165
  """{ scan(columns: {alias: "state", substring: {replace: {name: "state"}, pattern: "C", replacement: "A"}})
156
- { columns { state { values } } } }""")
166
+ { columns { state { values } } } }"""
167
+ )
157
168
  assert 'AA' in data['scan']['columns']['state']['values']
158
169
 
159
170
 
@@ -213,15 +224,18 @@ def test_scan(client):
213
224
  assert data['scan']['length'] == 2805
214
225
  data = client.execute(
215
226
  """{ scan(columns: {alias: "zipcode", add: [{name: "zipcode"}, {name: "zipcode"}]})
216
- { columns { zipcode { min } } } }""")
227
+ { columns { zipcode { min } } } }"""
228
+ )
217
229
  assert data['scan']['columns']['zipcode']['min'] == 1002
218
230
  data = client.execute(
219
231
  """{ scan(columns: {alias: "zipcode", subtract: [{name: "zipcode"}, {name: "zipcode"}]})
220
- { columns { zipcode { unique { values } } } } }""")
232
+ { columns { zipcode { unique { values } } } } }"""
233
+ )
221
234
  assert data['scan']['columns']['zipcode']['unique']['values'] == [0]
222
235
  data = client.execute(
223
236
  """{ scan(columns: {alias: "product", multiply: [{name: "latitude"}, {name: "longitude"}]})
224
- { scan(filter: {gt: [{name: "product"}, {value: 0}]}) { length } } }""")
237
+ { scan(filter: {gt: [{name: "product"}, {value: 0}]}) { length } } }"""
238
+ )
225
239
  assert data['scan']['scan']['length'] == 0
226
240
  data = client.execute(
227
241
  '{ scan(columns: {name: "zipcode", cast: "float"}) { column(name: "zipcode") { type } } }'
@@ -237,26 +251,31 @@ def test_scan(client):
237
251
  assert data == {'scan': {'column': {'type': 'int32'}}}
238
252
  data = client.execute(
239
253
  """{ scan(columns: {alias: "longitude", elementWise: {max: [{name: "longitude"}, {name: "latitude"}]}})
240
- { columns { longitude { min } } } }""")
254
+ { columns { longitude { min } } } }"""
255
+ )
241
256
  assert data['scan']['columns']['longitude']['min'] == pytest.approx(17.963333)
242
257
  data = client.execute(
243
258
  """{ scan(columns: {alias: "latitude", elementWise: {min: [{name: "longitude"}, {name: "latitude"}]}})
244
- { columns { latitude { max } } } }""")
259
+ { columns { latitude { max } } } }"""
260
+ )
245
261
  assert data['scan']['columns']['latitude']['max'] == pytest.approx(-65.301389)
246
262
  data = client.execute(
247
263
  """{ scan(columns: {alias: "state", elementWise: {min: [{name: "state"}, {name: "county"}], skipNulls: false}})
248
- { columns { state { values } } } }""")
264
+ { columns { state { values } } } }"""
265
+ )
249
266
  assert data['scan']['columns']['state']['values'][0] == 'NY'
250
267
 
251
268
 
252
269
  def test_apply(client):
253
270
  data = client.execute(
254
271
  """{ scan(columns: {alias: "city", substring: {find: {name: "city"}, pattern: "mountain"}})
255
- { column(name: "city") { ... on IntColumn { unique { values } } } } }""")
272
+ { column(name: "city") { ... on IntColumn { unique { values } } } } }"""
273
+ )
256
274
  assert data['scan']['column']['unique']['values'] == [-1]
257
275
  data = client.execute(
258
276
  """{ scan(columns: {alias: "city", substring: {count: {name: "city"}, pattern: "mountain", ignoreCase: true}})
259
- { column(name: "city") { ... on IntColumn { unique { values } } } } }""")
277
+ { column(name: "city") { ... on IntColumn { unique { values } } } } }"""
278
+ )
260
279
  assert data['scan']['column']['unique']['values'] == [0, 1]
261
280
  data = client.execute("""{ scan(columns: {alias: "state", binary: {joinElementWise: [
262
281
  {name: "state"}, {name: "county"}, {value: " "}]}}) { columns { state { values } } } }""")
@@ -275,7 +294,8 @@ def test_apply(client):
275
294
  assert data == {'apply': {'row': {'zipcode': 1}}}
276
295
  data = client.execute(
277
296
  """{ apply(rank: {name: "zipcode", sortKeys: "descending", nullPlacement: "at_start", tiebreaker: "dense"})
278
- { row { zipcode } } }""")
297
+ { row { zipcode } } }"""
298
+ )
279
299
  assert data == {'apply': {'row': {'zipcode': 41700}}}
280
300
 
281
301
 
@@ -300,15 +320,18 @@ def test_sort(client):
300
320
  assert data['sort']['columns']['state']['values'] == ['AK', 'AK']
301
321
  data = client.execute(
302
322
  """{ group(by: ["state"], aggregate: {list: {name: "county"}}) { apply(list: {sort: {by: ["county"]}})
303
- { aggregate(first: [{name: "county"}]) { row { state county } } } } }""")
323
+ { aggregate(first: [{name: "county"}]) { row { state county } } } } }"""
324
+ )
304
325
  assert data['group']['apply']['aggregate']['row'] == {'state': 'NY', 'county': 'Albany'}
305
326
  data = client.execute(
306
327
  """{ group(by: ["state"], aggregate: {list: {name: "county"}}) { apply(list: {sort: {by: ["-county"], length: 1}})
307
- { aggregate(first: [{name: "county"}]) { row { state county } } } } }""")
328
+ { aggregate(first: [{name: "county"}]) { row { state county } } } } }"""
329
+ )
308
330
  assert data['group']['apply']['aggregate']['row'] == {'state': 'NY', 'county': 'Yates'}
309
331
  data = client.execute(
310
332
  """{ group(by: ["state"], aggregate: {list: {name: "county"}}) { apply(list: {sort: {by: "county", length: 2}})
311
- { row { state } column(name: "county") { ... on ListColumn { value { length } } } } } }""")
333
+ { row { state } column(name: "county") { ... on ListColumn { value { length } } } } } }"""
334
+ )
312
335
  assert data['group']['apply'] == {'row': {'state': 'NY'}, 'column': {'value': {'length': 2}}}
313
336
 
314
337
 
@@ -321,7 +344,8 @@ def test_group(client):
321
344
  """{ group(by: ["state"], aggregate: {list: {name: "county"}}) { length tables { length
322
345
  columns { state { values } county { min max } } }
323
346
  scan(columns: {list: {valueLength: {name: "county"}}, alias: "c"}) {
324
- column(name: "c") { ... on IntColumn { values } } } } }""")
347
+ column(name: "c") { ... on IntColumn { values } } } } }"""
348
+ )
325
349
  assert len(data['group']['tables']) == data['group']['length'] == 52
326
350
  table = data['group']['tables'][0]
327
351
  assert table['length'] == data['group']['scan']['column']['values'][0] == 2205
@@ -332,7 +356,8 @@ def test_group(client):
332
356
  scan(filter: {gt: [{name: "counts"}, {value: 200}]}) {
333
357
  aggregate(min: [{name: "city", alias: "min"}], max: [{name: "city", alias: "max"}]) {
334
358
  min: column(name: "min") { ... on StringColumn { values } }
335
- max: column(name: "max") { ... on StringColumn { values } } } } } }""")
359
+ max: column(name: "max") { ... on StringColumn { values } } } } } }"""
360
+ )
336
361
  agg = data['group']['scan']['aggregate']
337
362
  assert agg['min']['values'] == ['Naval Anacost Annex', 'Alsip', 'Alief', 'Acton']
338
363
  assert agg['max']['values'] == ['Washington Navy Yard', 'Worth', 'Webster', 'Woodland Hills']
@@ -340,7 +365,8 @@ def test_group(client):
340
365
  """{ group(by: ["state", "county"], counts: "c", aggregate: {list: [{name: "zipcode"}, {name: "latitude"}, {name: "longitude"}]}) {
341
366
  sort(by: ["-c"], length: 4) { aggregate(sum: [{name: "latitude"}], mean: [{name: "longitude"}]) {
342
367
  columns { latitude { values } longitude { values } }
343
- column(name: "zipcode") { type } } } } }""")
368
+ column(name: "zipcode") { type } } } } }"""
369
+ )
344
370
  agg = data['group']['sort']['aggregate']
345
371
  assert agg['column']['type'] == 'list<item: int32>'
346
372
  assert all(latitude > 1000 for latitude in agg['columns']['latitude']['values'])
@@ -368,7 +394,8 @@ def test_flatten(client):
368
394
  assert data == {'group': {'flatten': {'columns': {'city': {'type': 'string'}}}}}
369
395
  data = client.execute(
370
396
  """{ group(by: "state", aggregate: {list: {name: "city"}}) { flatten(indices: "idx") { columns { city { type } }
371
- column(name: "idx") { ... on LongColumn { unique { values counts } } } } } }""")
397
+ column(name: "idx") { ... on LongColumn { unique { values counts } } } } } }"""
398
+ )
372
399
  idx = data['group']['flatten']['column']['unique']
373
400
  assert idx['values'] == list(range(52))
374
401
  assert sum(idx['counts']) == 41700
@@ -381,7 +408,8 @@ def test_aggregate(client):
381
408
  countDistinct: [{name: "city", alias: "cd"}]}) { slice(length: 3) {
382
409
  c: column(name: "c") { ... on LongColumn { values } }
383
410
  cd: column(name: "cd") { ... on LongColumn { values } }
384
- columns { state { values } county { values } } } } }""")
411
+ columns { state { values } county { values } } } } }"""
412
+ )
385
413
  assert data['group']['slice'] == {
386
414
  'c': {'values': [2205, 176, 703]},
387
415
  'cd': {'values': [1612, 99, 511]},
@@ -394,7 +422,8 @@ def test_aggregate(client):
394
422
  """{ group(by: ["state", "county"], aggregate: {list: {name: "city"}, min: {name: "city", alias: "first"}}) {
395
423
  aggregate(max: {name: "city", alias: "last"}) { slice(length: 3) {
396
424
  first: column(name: "first") { ... on StringColumn { values } }
397
- last: column(name: "last") { ... on StringColumn { values } } } } } }""")
425
+ last: column(name: "last") { ... on StringColumn { values } } } } } }"""
426
+ )
398
427
  assert data['group']['aggregate']['slice'] == {
399
428
  'first': {'values': ['Amagansett', 'Adjuntas', 'Aguada']},
400
429
  'last': {'values': ['Yaphank', 'Adjuntas', 'Aguada']},
@@ -1,65 +0,0 @@
1
- """
2
- GraphQL scalars.
3
- """
4
-
5
- from datetime import date, datetime, time, timedelta
6
- from decimal import Decimal
7
- import pyarrow as pa
8
- import strawberry
9
-
10
-
11
- def parse_long(value) -> int:
12
- if isinstance(value, int):
13
- return value
14
- raise TypeError(f"Long cannot represent value: {value}")
15
-
16
-
17
- @strawberry.type(description="month day nano interval")
18
- class Interval:
19
- months: int
20
- days: int
21
- nanoseconds: 'Long' # type: ignore
22
-
23
-
24
- Long = strawberry.scalar(int, name='Long', description="64-bit int", parse_value=parse_long)
25
- Duration = strawberry.scalar( # pragma: no branch
26
- timedelta,
27
- name='Duration',
28
- description="duration float (in seconds)",
29
- serialize=timedelta.total_seconds,
30
- parse_value=lambda s: timedelta(seconds=s),
31
- )
32
- scalar_map = {bytes: strawberry.scalars.Base64, dict: strawberry.scalars.JSON, timedelta: Duration}
33
-
34
- type_map = {
35
- pa.lib.Type_BOOL: bool,
36
- pa.lib.Type_UINT8: int,
37
- pa.lib.Type_INT8: int,
38
- pa.lib.Type_UINT16: int,
39
- pa.lib.Type_INT16: int,
40
- pa.lib.Type_UINT32: Long,
41
- pa.lib.Type_INT32: int,
42
- pa.lib.Type_UINT64: Long,
43
- pa.lib.Type_INT64: Long,
44
- pa.lib.Type_HALF_FLOAT: float,
45
- pa.lib.Type_FLOAT: float,
46
- pa.lib.Type_DOUBLE: float,
47
- pa.lib.Type_DECIMAL128: Decimal,
48
- pa.lib.Type_DECIMAL256: Decimal,
49
- pa.lib.Type_DATE32: date,
50
- pa.lib.Type_DATE64: date,
51
- pa.lib.Type_TIMESTAMP: datetime,
52
- pa.lib.Type_TIME32: time,
53
- pa.lib.Type_TIME64: time,
54
- pa.lib.Type_DURATION: timedelta,
55
- pa.lib.Type_INTERVAL_MONTH_DAY_NANO: Interval,
56
- pa.lib.Type_BINARY: bytes,
57
- pa.lib.Type_FIXED_SIZE_BINARY: bytes,
58
- pa.lib.Type_LARGE_BINARY: bytes,
59
- pa.lib.Type_STRING: str,
60
- pa.lib.Type_LARGE_STRING: str,
61
- pa.lib.Type_LIST: list,
62
- pa.lib.Type_FIXED_SIZE_LIST: list,
63
- pa.lib.Type_LARGE_LIST: list,
64
- pa.lib.Type_STRUCT: dict,
65
- }
@@ -1,8 +0,0 @@
1
- pyarrow>=15
2
- strawberry-graphql[asgi,cli]>=0.198
3
-
4
- [cli]
5
- tqdm
6
-
7
- [server]
8
- uvicorn[standard]
@@ -1,2 +0,0 @@
1
- pyarrow>=15
2
- strawberry-graphql[asgi,cli]>=0.198
File without changes
File without changes
File without changes
File without changes