graphique 1.6__tar.gz → 1.7__tar.gz

{graphique-1.6 → graphique-1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: graphique
-Version: 1.6
+Version: 1.7
 Summary: GraphQL service for arrow tables and parquet data sets.
 Author-email: Aric Coady <aric.coady@gmail.com>
 License: Copyright 2022 Aric Coady
@@ -40,8 +40,8 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: pyarrow>=16
-Requires-Dist: strawberry-graphql[asgi,cli]>=0.221
+Requires-Dist: pyarrow>=17
+Requires-Dist: strawberry-graphql[asgi,cli]>=0.236
 Provides-Extra: server
 Requires-Dist: uvicorn[standard]; extra == "server"
 Provides-Extra: cli
@@ -51,9 +51,10 @@ Requires-Dist: tqdm; extra == "cli"
 ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
 [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
 ![image](https://img.shields.io/pypi/status/graphique.svg)
-[![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
+[![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
 [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
-[![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
+[![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
 [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
 

{graphique-1.6 → graphique-1.7}/README.md

@@ -2,9 +2,10 @@
 ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
 [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
 ![image](https://img.shields.io/pypi/status/graphique.svg)
-[![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
+[![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
 [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
-[![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
+[![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
 [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
 

{graphique-1.6 → graphique-1.7}/graphique/__init__.py

@@ -1,3 +1 @@
 from .middleware import GraphQL  # noqa: F401
-
-__version__ = '1.6'

{graphique-1.6 → graphique-1.7}/graphique/core.py

@@ -12,7 +12,7 @@ import inspect
 import itertools
 import operator
 import json
-from collections.abc import Callable, Iterable, Iterator, Sequence
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Optional, Union, get_type_hints
 import numpy as np
@@ -419,14 +419,16 @@ class Table(pa.Table):
             counts: alias for optional row counts
             **funcs: aggregate funcs with columns options
         """
-        if isinstance(self, pa.Table):
-            self = self.unify_dictionaries()
         prefix = 'hash_' if names else ''
         aggs = [agg.astuple(prefix + func) for func in funcs for agg in funcs[func]]
         columns = list(names) + [agg[0] for agg in aggs]
         if counts:
             aggs.append(([], 'hash_count_all', None, counts))
-        return Declarations(self, columns).aggregate(aggs, names).to_table()
+        if isinstance(self, pa.Table):
+            decl = Declaration('table_source', self.unify_dictionaries())
+        else:
+            decl = Declaration.scan(self, columns=columns)
+        return decl.aggregate(aggs, names).to_table(use_threads=False)
 
     def aggregate(self, counts: str = '', **funcs: Sequence[Agg]) -> dict:
         """Return aggregated scalars as a row of data."""
@@ -630,8 +632,8 @@ class Table(pa.Table):
         return f'{size:n} {prefix}B'
 
 
-class Declarations(list):
-    """[Acero](https://arrow.apache.org/docs/python/api/acero.html) engine declarations."""
+class Declaration(ac.Declaration):
+    """[Acero](https://arrow.apache.org/docs/python/api/acero.html) engine declaration."""
 
     option_map = {
         'table_source': ac.TableSourceNodeOptions,
@@ -643,21 +645,24 @@ class Declarations(list):
         'hashjoin': ac.HashJoinNodeOptions,
     }
 
-    def __init__(self, source: Union[pa.Table, ds.Dataset], columns=None):
-        if isinstance(source, ds.Dataset):
-            expr = source._scan_options.get('filter')
-            self.scan(source, filter=expr, columns=columns)
-            if expr is not None:
-                self.filter(expr)
-        else:
-            self.table_source(source)
-
-    def apply(self, name: str, *args, **kwargs) -> Self:
-        self.append(ac.Declaration(name, self.option_map[name](*args, **kwargs)))
+    def __init__(self, name, *args, inputs=None, **options):
+        super().__init__(name, self.option_map[name](*args, **options), inputs)
+
+    @classmethod
+    def scan(cls, dataset: ds.Dataset, columns=None) -> Self:
+        """Return source node from a dataset."""
+        expr = dataset._scan_options.get('filter')
+        self = cls('scan', dataset, filter=expr, columns=columns)
+        if expr is not None:
+            self = self.filter(expr)
+        if isinstance(columns, Mapping):
+            self = self.project(columns.values(), columns)
+        elif columns is not None:
+            self = self.project(map(pc.field, columns))
         return self
 
+    def apply(self, name: str, *args, **options) -> Self:
+        return type(self)(name, *args, inputs=[self], **options)
+
     def __getattr__(self, name: str) -> Callable:
         return functools.partial(self.apply, name)
-
-    def to_table(self, use_threads: bool = False) -> pa.Table:
-        return ac.Declaration.from_sequence(self).to_table(use_threads)

{graphique-1.6 → graphique-1.7}/graphique/inputs.py

@@ -15,9 +15,9 @@ import pyarrow.dataset as ds
 import strawberry
 from strawberry import UNSET
 from strawberry.annotation import StrawberryAnnotation
-from strawberry.arguments import StrawberryArgument
+from strawberry.types.arguments import StrawberryArgument
 from strawberry.schema_directive import Location
-from strawberry.field import StrawberryField
+from strawberry.types.field import StrawberryField
 from strawberry.scalars import JSON
 from typing_extensions import Self
 from .core import Agg

{graphique-1.6 → graphique-1.7}/graphique/interface.py

@@ -91,11 +91,11 @@ class Dataset:
 
     @classmethod
     @no_type_check
-    def resolve_reference(cls, info, **keys) -> Self:
+    def resolve_reference(cls, info: Info, **keys) -> Self:
         """Return table from federated keys."""
         self = getattr(info.root_value, cls.field)
         queries = {name: Filter(eq=[keys[name]]) for name in keys}
-        return self.filter(Info(info, None), **queries)
+        return self.filter(info, **queries)
 
     def columns(self, info: Info) -> dict:
         """fields for each column"""

{graphique-1.6 → graphique-1.7}/graphique/models.py

@@ -14,7 +14,7 @@ import pyarrow as pa
 import pyarrow.compute as pc
 import strawberry
 from strawberry import Info
-from strawberry.field import StrawberryField
+from strawberry.types.field import StrawberryField
 from .core import Column as C
 from .inputs import links
 from .scalars import Long, scalar_map, type_map

{graphique-1.6 → graphique-1.7}/graphique/scalars.py

@@ -24,35 +24,41 @@ def parse_duration(value: str):
     parts = re.split(r'(-?\d+\.?\d*)', d_val.lower() + t_val)
     if parts.pop(0) != 'p':
         raise ValueError("Duration format must start with `P`")
+    multipliers = {'y': 12, 'w': 7, 'H': 3600, 'M': 60}
     for num, key in zip(parts[::2], parts[1::2]):
-        if n := float(num) if '.' in num else int(num):
-            if key in 'ym':
-                months += n * 12 if key == 'y' else n
-            elif key in 'wd':
-                days += n * 7 if key == 'w' else n
-            elif key in 'HMS':
-                seconds += n * {'H': 3600, 'M': 60, 'S': 1}[key]
-            else:
-                raise ValueError(f"Invalid duration field: {key.upper()}")
-    if months:
-        return pa.MonthDayNano([months, days, int(seconds * 1_000_000_000)])
-    return timedelta(days, seconds)
+        value = (float if '.' in num else int)(num) * multipliers.get(key, 1)
+        if key in 'ym':
+            months += value
+        elif key in 'wd':
+            days += value
+        elif key in 'HMS':
+            seconds += value
+        else:
+            raise ValueError(f"Invalid duration field: {key.upper()}")
+    if set(d_val).isdisjoint('YM'):
+        return timedelta(days, seconds)
+    return pa.MonthDayNano([months, days, int(seconds * 1_000_000_000)])
 
 
 @functools.singledispatch
-def duration_isoformat(td: timedelta) -> str:
-    days = f'{td.days}D' if td.days else ''
-    fraction = f'.{td.microseconds:06}' if td.microseconds else ''
-    return f'P{days}T{td.seconds}{fraction}S'
+def duration_isoformat(months: int, days: int, seconds: int, fraction: str = '.') -> str:
+    minutes, seconds = divmod(seconds, 60)
+    items = zip('YMDHM', divmod(months, 12) + (days,) + divmod(minutes, 60))
+    year, month, day, hour, minute = (f'{value}{key}' if value else '' for key, value in items)
+    fraction = fraction.rstrip('0').rstrip('.')
+    return f'P{year}{month}{day}T{hour}{minute}{seconds}{fraction}S'
+
+
+@duration_isoformat.register
+def _(td: timedelta) -> str:  # type: ignore
+    return duration_isoformat(0, td.days, td.seconds, f'.{td.microseconds:06}')
 
 
 @duration_isoformat.register
 def _(mdn: pa.MonthDayNano) -> str:
-    months = f'{mdn.months}M' if mdn.months else ''
-    days = f'{mdn.days}D' if mdn.days else ''
     seconds, nanoseconds = divmod(mdn.nanoseconds, 1_000_000_000)
-    fraction = f'.{nanoseconds:09}' if nanoseconds else ''
-    return f'P{months}{days}T{seconds}{fraction}S'
+    value = duration_isoformat(mdn.months, mdn.days, seconds, f'.{nanoseconds:09}')
+    return value if mdn.months else value.replace('P', 'P0M')
 
 
 Long = strawberry.scalar(int, name='Long', description="64-bit int", parse_value=parse_long)

{graphique-1.6 → graphique-1.7}/graphique/shell.py

@@ -12,6 +12,8 @@ progress on the second pass.
 import shutil
 from pathlib import Path
 from typing import Annotated
+import numpy as np
+import pyarrow.compute as pc
 import pyarrow.dataset as ds
 import typer  # type: ignore
 from tqdm import tqdm  # type: ignore
@@ -22,12 +24,19 @@ def sort_key(name: str) -> tuple:
     return name.lstrip('-'), ('descending' if name.startswith('-') else 'ascending')
 
 
-def write_batches(scanner: ds.Scanner, base_dir: str, *partitioning: str, **options):
-    """Partition dataset by batches."""
+def write_batches(
+    scanner: ds.Scanner, base_dir: str, *partitioning: str, indices: str = '', **options
+):
+    """Partition dataset by batches.
+
+    Optionally include original indices.
+    """
     options.update(format='parquet', partitioning=partitioning)
     options.update(partitioning_flavor='hive', existing_data_behavior='overwrite_or_ignore')
     with tqdm(total=scanner.count_rows(), desc="Batches") as pbar:
         for index, batch in enumerate(scanner.to_batches()):
+            if indices:
+                batch = batch.append_column(indices, pc.add(np.arange(len(batch)), pbar.n))
             options['basename_template'] = f'part-{index}-{{i}}.parquet'
             ds.write_dataset(batch, base_dir, **options)
             pbar.update(len(batch))

{graphique-1.6 → graphique-1.7}/graphique.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: graphique
-Version: 1.6
+Version: 1.7
 Summary: GraphQL service for arrow tables and parquet data sets.
 Author-email: Aric Coady <aric.coady@gmail.com>
 License: Copyright 2022 Aric Coady
@@ -40,8 +40,8 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: pyarrow>=16
-Requires-Dist: strawberry-graphql[asgi,cli]>=0.221
+Requires-Dist: pyarrow>=17
+Requires-Dist: strawberry-graphql[asgi,cli]>=0.236
 Provides-Extra: server
 Requires-Dist: uvicorn[standard]; extra == "server"
 Provides-Extra: cli
@@ -51,9 +51,10 @@ Requires-Dist: tqdm; extra == "cli"
 ![image](https://img.shields.io/pypi/pyversions/graphique.svg)
 [![image](https://pepy.tech/badge/graphique)](https://pepy.tech/project/graphique)
 ![image](https://img.shields.io/pypi/status/graphique.svg)
-[![image](https://github.com/coady/graphique/workflows/build/badge.svg)](https://github.com/coady/graphique/actions)
+[![build](https://github.com/coady/graphique/actions/workflows/build.yml/badge.svg)](https://github.com/coady/graphique/actions/workflows/build.yml)
 [![image](https://codecov.io/gh/coady/graphique/branch/main/graph/badge.svg)](https://codecov.io/gh/coady/graphique/)
-[![image](https://github.com/coady/graphique/workflows/codeql/badge.svg)](https://github.com/coady/graphique/security/code-scanning)
+[![CodeQL](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/coady/graphique/actions/workflows/github-code-scanning/codeql)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/coady/graphique)
 [![image](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![image](https://mypy-lang.org/static/mypy_badge.svg)](https://mypy-lang.org/)
 

{graphique-1.6 → graphique-1.7}/graphique.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ graphique.egg-info/SOURCES.txt
 graphique.egg-info/dependency_links.txt
 graphique.egg-info/requires.txt
 graphique.egg-info/top_level.txt
+tests/test_bench.py
 tests/test_core.py
 tests/test_dataset.py
 tests/test_models.py

graphique-1.7/graphique.egg-info/requires.txt

@@ -0,0 +1,8 @@
+pyarrow>=17
+strawberry-graphql[asgi,cli]>=0.236
+
+[cli]
+tqdm
+
+[server]
+uvicorn[standard]

{graphique-1.6 → graphique-1.7}/pyproject.toml

@@ -1,10 +1,11 @@
 [build-system]
-requires = ["setuptools>=61", "wheel"]
+requires = ["setuptools>=61"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "graphique"
-dynamic = ["version", "dependencies"]
+version = "1.7"
+dynamic = ["dependencies"]
 description = "GraphQL service for arrow tables and parquet data sets."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -39,22 +40,14 @@ Issues = "https://github.com/coady/graphique/issues"
 server = ["uvicorn[standard]"]
 cli = ["tqdm"]
 
-[tool.setuptools]
-packages = ["graphique"]
-
 [tool.setuptools.dynamic]
-version = {attr = "graphique.__version__"}
 dependencies = {file = "requirements.in"}
 
-[tool.setuptools.package-data]
-graphique = ["py.typed"]
-
 [tool.ruff]
 line-length = 100
 extend-include = ["*.ipynb"]
 
 [tool.ruff.format]
-preview = true
 quote-style = "preserve"
 
 [[tool.mypy.overrides]]

graphique-1.7/requirements.in

@@ -0,0 +1,2 @@
+pyarrow>=17
+strawberry-graphql[asgi,cli]>=0.236

graphique-1.7/tests/test_bench.py

@@ -0,0 +1,21 @@
+import pytest
+from graphique.core import Table as T
+
+
+@pytest.mark.benchmark
+def test_group(table):
+    T.group(table, 'state', 'county', 'city')
+    T.runs(table, 'state', 'county', 'city')
+
+
+@pytest.mark.benchmark
+def test_rank(table):
+    T.ranked(table, 1, 'state', 'county', 'city')
+    T.ranked(table, 10, 'state', 'county', 'city')
+
+
+@pytest.mark.benchmark
+def test_sort(table):
+    T.sort(table, 'state', 'county', 'city', length=1)
+    T.sort(table, 'state', 'county', 'city', length=10)
+    T.sort(table, 'state', 'county', 'city')

{graphique-1.6 → graphique-1.7}/tests/test_core.py

@@ -1,21 +1,24 @@
 import pyarrow as pa
 import pyarrow.compute as pc
+import pyarrow.dataset as ds
 import pytest
-from graphique.core import Agg, ListChunk, Column as C, Table as T
+from graphique.core import Agg, Declaration, ListChunk, Column as C, Table as T
 from graphique.scalars import parse_duration, duration_isoformat
 
 
 def test_duration():
-    assert duration_isoformat(parse_duration('P1Y1M1DT1H1M1.1S')) == 'P13M1DT3661.100000000S'
-    assert duration_isoformat(parse_duration('P1M1DT1H1M1.1S')) == 'P1M1DT3661.100000000S'
-    assert duration_isoformat(parse_duration('P1DT1H1M1.1S')) == 'P1DT3661.100000S'
-    assert duration_isoformat(parse_duration('PT1H1M1.1S')) == 'PT3661.100000S'
-    assert duration_isoformat(parse_duration('PT1M1.1S')) == 'PT61.100000S'
-    assert duration_isoformat(parse_duration('PT1.1S')) == 'PT1.100000S'
+    assert duration_isoformat(parse_duration('P1Y1M1DT1H1M1.1S')) == 'P1Y1M1DT1H1M1.1S'
+    assert duration_isoformat(parse_duration('P1M1DT1H1M1.1S')) == 'P1M1DT1H1M1.1S'
+    assert duration_isoformat(parse_duration('P1DT1H1M1.1S')) == 'P1DT1H1M1.1S'
+    assert duration_isoformat(parse_duration('PT1H1M1.1S')) == 'PT1H1M1.1S'
+    assert duration_isoformat(parse_duration('PT1M1.1S')) == 'PT1M1.1S'
+    assert duration_isoformat(parse_duration('PT1.1S')) == 'PT1.1S'
     assert duration_isoformat(parse_duration('PT1S')) == 'PT1S'
     assert duration_isoformat(parse_duration('P0D')) == 'PT0S'
     assert duration_isoformat(parse_duration('PT0S')) == 'PT0S'
-    assert duration_isoformat(parse_duration('P-1DT-1H')) == 'P-2DT82800S'
+    assert duration_isoformat(parse_duration('P-1DT-1H')) == 'P-2DT23H0S'
+    assert duration_isoformat(parse_duration('P0MT')) == 'P0MT0S'
+    assert duration_isoformat(parse_duration('P0YT')) == 'P0MT0S'
     with pytest.raises(ValueError):
         duration_isoformat(parse_duration('T1H'))
     with pytest.raises(ValueError):
@@ -87,6 +90,13 @@ def test_membership():
     assert C.index(array, 1, start=2) == -1
 
 
+def test_declaration(table):
+    dataset = ds.dataset(table).filter(pc.field('state') == 'CA')
+    assert Declaration.scan(dataset).to_table()['state'].unique().to_pylist() == ['CA']
+    (column,) = Declaration.scan(dataset, columns={'_': pc.field('state')}).to_table()
+    assert column.unique().to_pylist() == ['CA']
+
+
 def test_group(table):
     groups = T.group(table, 'state', list=[Agg('county'), Agg('city')])
     assert len(groups) == 52

{graphique-1.6 → graphique-1.7}/tests/test_models.py

@@ -1,4 +1,3 @@
-import sys
 import pytest
 
 
@@ -167,7 +166,6 @@ def test_numeric(executor):
     assert data == {'scan': {'columns': {'int64': {'values': [0, None]}}}}
 
 
-@pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or pytz")
 def test_datetime(executor):
     for name in ('timestamp', 'date32'):
         data = executor(
@@ -230,7 +228,7 @@ def test_duration(executor):
     data = executor("""{ scan(columns: {alias: "diff", temporal:
         {monthDayNanoIntervalBetween: [{name: "timestamp"}, {name: "timestamp"}]}})
         { column(name: "diff") { ... on DurationColumn { values } } } }""")
-    assert data == {'scan': {'column': {'values': ['PT0S', None]}}}
+    assert data == {'scan': {'column': {'values': ['P0MT0S', None]}}}
 
 
 def test_list(executor):

graphique-1.6/graphique.egg-info/requires.txt

@@ -1,8 +0,0 @@
-pyarrow>=16
-strawberry-graphql[asgi,cli]>=0.221
-
-[cli]
-tqdm
-
-[server]
-uvicorn[standard]

graphique-1.6/requirements.in

@@ -1,2 +0,0 @@
-pyarrow>=16
-strawberry-graphql[asgi,cli]>=0.221