pyaggregation 0.1.0__py3-none-any.whl

pyaggregation/compilers.py

@@ -0,0 +1,273 @@
+"""Compile a report from JSON: `{"aggregates": [{"name", "fold", "of", "where"}], "group_by"}`.
+
+pyaggregation does not know the expression languages inside `of`, `where` and
+`group_by`. They are delegated to injected compilers, e.g. pyformula's
+`FormulaCompiler` for values and pyspecification's `PredicateCompiler` for conditions.
+"""
+
+from collections.abc import Callable, Hashable, Mapping
+from typing import Any, NamedTuple, NotRequired, Protocol, TypedDict, cast
+
+from pyvocabulary import index_path, key_path, located, validate_name
+
+from .aggregates import (
+    average,
+    collect,
+    count,
+    count_distinct,
+    every,
+    first,
+    join,
+    last,
+    maximum,
+    median,
+    minimum,
+    mode,
+    percentile,
+    product,
+    share,
+    some,
+    stdev,
+    total,
+    variance,
+)
+from .exceptions import (
+    DuplicateResultNameError,
+    InvalidAggregateError,
+    InvalidReportDefinitionError,
+    InvalidResultNameError,
+)
+from .report import Condition, GroupedReport, Report, Value
+
+type Build = Callable[[str, Any, Any, dict[str, Any]], Report[Any]]
+
+
+class FoldSpec(NamedTuple):
+    """The JSON fields of one fold, besides `name` and `fold`, and how to build it."""
+
+    required: frozenset[str]
+    optional: frozenset[str]
+    build: Build
+
+
+def _of_first(build: Callable[..., Report[Any]]) -> Build:
+    return lambda name, of, where, options: build(of, name=name, where=where, **options)
+
+
+_OF = frozenset({"of"})
+_WHERE = frozenset({"where"})
+
+
+def _simple(build: Callable[..., Report[Any]]) -> FoldSpec:
+    return FoldSpec(_OF, _WHERE, _of_first(build))
+
+
+def _count(name: str, _of: Any, where: Any, _options: dict[str, Any]) -> Report[Any]:
+    return count(name=name, where=where)
+
+
+def _share(name: str, of: Any, where: Any, _options: dict[str, Any]) -> Report[Any]:
+    return share(name=name, where=where, of=of)
+
+
+FOLDS: Mapping[str, FoldSpec] = {
+    "average": _simple(average),
+    "collect": _simple(collect),
+    "count": FoldSpec(frozenset(), _WHERE, _count),
+    "count_distinct": _simple(count_distinct),
+    "every": _simple(every),
+    "first": _simple(first),
+    "join": FoldSpec(_OF, _WHERE | {"separator"}, _of_first(join)),
+    "last": _simple(last),
+    "maximum": _simple(maximum),
+    "median": _simple(median),
+    "minimum": _simple(minimum),
+    "mode": _simple(mode),
+    "percentile": FoldSpec(_OF | {"at"}, _WHERE, _of_first(percentile)),
+    "product": _simple(product),
+    "share": FoldSpec(_WHERE, _OF, _share),
+    "some": _simple(some),
+    "stdev": FoldSpec(_OF, _WHERE | {"sample"}, _of_first(stdev)),
+    "total": _simple(total),
+    "variance": FoldSpec(_OF, _WHERE | {"sample"}, _of_first(variance)),
+}
+
+# Plain JSON values passed to a built-in as they are; `of` and `where` are compiled.
+_OPTIONS = frozenset({"at", "sample", "separator"})
+
+_REPORT_KEYS = frozenset({"aggregates", "group_by"})
+
+
+class AggregateDict(TypedDict):
+    name: str
+    fold: str
+    of: NotRequired[Any]
+    where: NotRequired[Any]
+
+
+class ReportDict(TypedDict):
+    aggregates: list[AggregateDict]
+    group_by: NotRequired[Any]
+
+
+class Compiler[F](Protocol):
+    """Anything with a `compile(definition) -> F` method."""
+
+    def compile(self, definition: Any, /) -> F: ...
+
+
+class ReportCompiler[T]:
+    """Compile report definitions, delegating `of`, `where` and `group_by` to injected compilers.
+
+    Every problem is reported here, before any item is folded. Errors from the
+    injected compilers keep their type and get a note with the full JSON path.
+
+    Args:
+        values: Compiles each `of`.
+        conditions: Compiles each `where`.
+        keys: Compiles `group_by`. Without it, grouped definitions are rejected.
+        start: The `start` of every `total`, e.g. `Decimal(0)` in a money app,
+            since JSON has no `Decimal`.
+
+    Example:
+    ```python
+    >>> class Field:
+    ...     def compile(self, name):
+    ...         return lambda item: item[name]
+    >>> compiler = ReportCompiler(Field(), Field(), Field())
+    >>> report = compiler.compile({
+    ...     "aggregates": [
+    ...         {"name": "stock", "fold": "total", "of": "quantity"},
+    ...         {"name": "on_sale", "fold": "count", "where": "sale"},
+    ...     ],
+    ...     "group_by": "shop",
+    ... })
+    >>> report
+    Report(stock, on_sale).group_by(<lambda>)
+    >>> report([
+    ...     {"shop": "north", "quantity": 3, "sale": True},
+    ...     {"shop": "north", "quantity": 4, "sale": False},
+    ... ])
+    {'north': {'stock': 7, 'on_sale': 1}}
+
+    ```
+
+    """
+
+    __slots__ = ("_conditions", "_keys", "_start", "_values")
+
+    def __init__(
+        self,
+        values: Compiler[Value[T]],
+        conditions: Compiler[Condition[T]],
+        keys: Compiler[Callable[[T], Hashable]] | None = None,
+        /,
+        *,
+        start: Any = 0,
+    ) -> None:
+        self._values = values
+        self._conditions = conditions
+        self._keys = keys
+        self._start = start
+
+    def compile(
+        self, definition: ReportDict | Mapping[str, Any], /
+    ) -> Report[T] | GroupedReport[T, Hashable]:
+        """Compile `definition` into a `Report`, or a `GroupedReport` with `group_by`.
+
+        Raises:
+            InvalidReportDefinitionError: If the definition has the wrong shape.
+
+        """
+        _check_keys(definition, required={"aggregates"}, allowed=_REPORT_KEYS, path="$")
+
+        path = key_path("$", "aggregates")
+        items = definition["aggregates"]
+
+        if not isinstance(items, list) or not items:
+            msg = f"Invalid report at {path}: expected a non-empty list, received {items!r}"
+            raise InvalidReportDefinitionError(msg)
+
+        compiled = [
+            self._compile_aggregate(item, index_path(path, index))
+            for index, item in enumerate(cast("list[Any]", items))
+        ]
+
+        with located(path, DuplicateResultNameError):
+            report = Report(tuple(aggregate for each in compiled for aggregate in each.aggregates))
+
+        if "group_by" not in definition:
+            return report
+
+        path = key_path("$", "group_by")
+
+        if self._keys is None:
+            msg = f"Invalid report at {path}: grouping needs a key compiler, and none was given"
+            raise InvalidReportDefinitionError(msg)
+
+        with located(path, Exception):
+            key = self._keys.compile(definition["group_by"])
+
+        return report.group_by(key)
+
+    def _compile_aggregate(self, item: Any, path: str) -> Report[T]:
+        _check_keys(item, required=set(), allowed=None, path=path)
+
+        fold = item.get("fold")
+
+        if fold not in FOLDS:
+            msg = (
+                f"Invalid report at {key_path(path, 'fold')}: unknown fold {fold!r}. "
+                f"Available folds: {', '.join(FOLDS)}"
+            )
+            raise InvalidReportDefinitionError(msg)
+
+        required, optional, build = FOLDS[fold]
+        _check_keys(
+            item,
+            required={"name", "fold", *required},
+            allowed={"name", "fold", *required, *optional},
+            path=path,
+        )
+
+        with located(key_path(path, "name"), InvalidResultNameError):
+            name = validate_name(item["name"], error=InvalidResultNameError)
+
+        of = where = None
+
+        if "of" in item:
+            with located(key_path(path, "of"), Exception):
+                of = self._values.compile(item["of"])
+
+        if "where" in item:
+            with located(key_path(path, "where"), Exception):
+                where = self._conditions.compile(item["where"])
+
+        options = {key: item[key] for key in _OPTIONS & item.keys()}
+
+        if fold == "total":
+            options["start"] = self._start
+
+        with located(path, InvalidAggregateError):
+            return build(name, of, where, options)
+
+
+def _check_keys(
+    item: Any, *, required: set[str], allowed: set[str] | frozenset[str] | None, path: str
+) -> None:
+    if not isinstance(item, Mapping):
+        msg = f"Invalid report at {path}: expected an object, received {item!r}"
+        raise InvalidReportDefinitionError(msg)
+
+    keys = set(cast("Mapping[str, Any]", item))
+
+    if missing := sorted(required - keys):
+        msg = f"Invalid report at {path}: missing {', '.join(map(repr, missing))}"
+        raise InvalidReportDefinitionError(msg)
+
+    if allowed is not None and (unknown := sorted(keys - allowed)):
+        msg = (
+            f"Invalid report at {path}: unknown {', '.join(map(repr, unknown))}. "
+            f"Allowed keys: {', '.join(sorted(allowed))}"
+        )
+        raise InvalidReportDefinitionError(msg)

pyaggregation/exceptions.py

@@ -0,0 +1,51 @@
+"""Exceptions raised by pyaggregation.
+
+Errors raised while a report runs (by `of`, `where`, a key or a fold) keep their
+own type; pyaggregation only adds a note naming the aggregate, e.g.
+`in aggregate 'payroll'`.
+"""
+
+from collections.abc import Iterable
+
+from pyvocabulary import InvalidNameError
+
+
+class PyaggregationError(Exception):
+    """Base class for every exception raised by pyaggregation."""
+
+
+class InvalidResultNameError(PyaggregationError, InvalidNameError):
+    """Raised when a result name is not a valid ASCII Python identifier."""
+
+    def __init__(self, name: object, /) -> None:
+        super().__init__(name, kind="result")
+
+
+class DuplicateResultNameError(PyaggregationError, ValueError):
+    """Raised when two aggregates of one report share a result name.
+
+    Example:
+    ```python
+    >>> str(DuplicateResultNameError("payroll", ["payroll", "headcount"]))
+    "Result name 'payroll' is used more than once. Result names: payroll, headcount"
+
+    ```
+
+    """
+
+    def __init__(self, name: str, names: Iterable[str], /) -> None:
+        super().__init__(
+            f"Result name '{name}' is used more than once. Result names: {', '.join(names)}"
+        )
+
+
+class InvalidAggregateError(PyaggregationError, TypeError):
+    """Raised when an aggregate or report is built from parts that cannot work."""
+
+
+class InvalidGroupKeyError(PyaggregationError, TypeError):
+    """Raised when `group_by` is given a key that is not callable."""
+
+
+class InvalidReportDefinitionError(PyaggregationError, ValueError):
+    """Raised when a JSON report definition does not have the expected shape."""

pyaggregation/json_schema.py

@@ -0,0 +1,99 @@
+"""The JSON Schema of a report definition, as accepted by `ReportCompiler.compile`.
+
+pyaggregation does not know the languages inside `of`, `where` and `group_by`,
+so their schemas are passed in, e.g. pyspecification's `get_expression_json_schema(rules)`.
+"""
+
+from collections.abc import Mapping
+from typing import Any
+
+from pyvocabulary import JSON_SCHEMA_DIALECT
+
+from .compilers import FOLDS
+
+VALUE_SCHEMA_ID = "urn:pyaggregation:value"
+CONDITION_SCHEMA_ID = "urn:pyaggregation:condition"
+KEY_SCHEMA_ID = "urn:pyaggregation:key"
+
+_NAME_SCHEMA = {"type": "string", "pattern": "^[a-zA-Z_][a-zA-Z0-9_]*$"}
+_FIELD_SCHEMAS = {
+    "of": {"$ref": VALUE_SCHEMA_ID},
+    "where": {"$ref": CONDITION_SCHEMA_ID},
+    "at": {"type": "number", "minimum": 0, "maximum": 1},
+    "sample": {"type": "boolean"},
+    "separator": {"type": "string"},
+}
+
+
+def get_report_json_schema(
+    value_schema: Mapping[str, Any],
+    condition_schema: Mapping[str, Any],
+    key_schema: Mapping[str, Any] | None = None,
+    /,
+) -> dict[str, Any]:
+    """Return the JSON Schema of `{"aggregates": [{"name", "fold", "of", "where"}], "group_by"}`.
+
+    Each fold has its own variant, so `of` and `where` are required or forbidden
+    exactly as the compiler requires or forbids them. Each sub-schema is embedded
+    under its own `$id`, so the `#/$defs/...` refs it uses keep pointing into itself.
+
+    Args:
+        value_schema: The schema of one `of`, e.g. a pyformula expression.
+        condition_schema: The schema of one `where`, e.g. a pyspecification expression.
+        key_schema: The schema of `group_by`. Without it, `group_by` is not allowed,
+            matching a `ReportCompiler` without a key compiler.
+
+    Example:
+    ```python
+    >>> schema = get_report_json_schema({"type": "string"}, {"type": "string"})
+    >>> list(schema["properties"]), schema["$defs"]["count"]["required"]
+    (['aggregates'], ['name', 'fold'])
+    >>> schema["$defs"]["share"]["required"]
+    ['name', 'fold', 'where']
+
+    ```
+
+    """
+    defs: dict[str, Any] = {
+        fold: _aggregate_schema(fold, spec.required, spec.optional) for fold, spec in FOLDS.items()
+    }
+    defs["value"] = {**value_schema, "$id": VALUE_SCHEMA_ID}
+    defs["condition"] = {**condition_schema, "$id": CONDITION_SCHEMA_ID}
+
+    properties: dict[str, Any] = {
+        "aggregates": {
+            "type": "array",
+            "minItems": 1,
+            "items": {"oneOf": [{"$ref": f"#/$defs/{fold}"} for fold in FOLDS]},
+        },
+    }
+
+    if key_schema is not None:
+        properties["group_by"] = {"$ref": KEY_SCHEMA_ID}
+        defs["key"] = {**key_schema, "$id": KEY_SCHEMA_ID}
+
+    return {
+        "$schema": JSON_SCHEMA_DIALECT,
+        "type": "object",
+        "properties": properties,
+        "required": ["aggregates"],
+        "additionalProperties": False,
+        "$defs": defs,
+    }
+
+
+def _aggregate_schema(
+    fold: str, required: frozenset[str], optional: frozenset[str]
+) -> dict[str, Any]:
+    fields = sorted(required | optional)
+
+    return {
+        "type": "object",
+        "properties": {
+            "name": _NAME_SCHEMA,
+            "fold": {"const": fold},
+            **{field: _FIELD_SCHEMAS[field] for field in fields},
+        },
+        "required": ["name", "fold", *sorted(required)],
+        "additionalProperties": False,
+    }

pyaggregation/report.py

@@ -0,0 +1,232 @@
+"""Reports: several folds run together over one iterable, in one pass.
+
+```python
+report = total(salary, name="payroll") & count(name="managers", where=is_manager)
+report(employees)                        # {"payroll": ..., "managers": 2}
+report.group_by(department)(employees)   # {"Sales": {...}, "IT": {...}}
+```
+
+The input is iterated exactly once, so generators and database cursors work and
+only the accumulators are held in memory.
+"""
+
+from collections.abc import Callable, Hashable, Iterable
+from dataclasses import dataclass
+from functools import partial
+from typing import Any, cast
+
+from pyvocabulary import validate_name
+
+from .exceptions import (
+    DuplicateResultNameError,
+    InvalidAggregateError,
+    InvalidGroupKeyError,
+    InvalidResultNameError,
+)
+
+type Value[T] = Callable[[T], Any]
+type Condition[T] = Callable[[T], Any]
+
+
+def identity[V](value: V, /) -> V:
+    """Return `value` unchanged: the default `of` of an aggregate."""
+    return value
+
+
+@dataclass(frozen=True, slots=True)
+class Fold[V, A, R]:
+    """Three pure functions that reduce values to one result.
+
+    `initial` is a function, not a value, so every run starts from a fresh
+    accumulator, even a mutable one.
+
+    Example:
+    ```python
+    >>> longest = Fold(lambda: "", lambda acc, word: max(acc, word, key=len), str.upper)
+    >>> aggregate(longest, name="longest")(["fold", "report", "sum"])
+    {'longest': 'REPORT'}
+
+    ```
+
+    """
+
+    initial: Callable[[], A]
+    step: Callable[[A, V], A]
+    finish: Callable[[A], R]
+
+
+@dataclass(frozen=True, slots=True)
+class Aggregate[T]:
+    """One named result: `fold` over `of(item)` for every item that passes `where`."""
+
+    name: str
+    fold: Fold[Any, Any, Any]
+    of: Value[T] = identity
+    where: Condition[T] | None = None
+
+    def __post_init__(self) -> None:
+        validate_name(self.name, error=InvalidResultNameError)
+
+        # The annotation is a promise to type checkers; callers may still pass anything.
+        if not isinstance(cast("object", self.fold), Fold):
+            msg = f"Aggregate '{self.name}' must have a Fold, received {self.fold!r}"
+            raise InvalidAggregateError(msg)
+
+        check_callable(self.of, role="of", name=self.name)
+
+        if self.where is not None:
+            check_callable(self.where, role="where", name=self.name)
+
+
+@dataclass(frozen=True, slots=True)
+class Report[T]:
+    """Aggregates in result order. Calling it folds an iterable into one dict.
+
+    Build reports with the built-ins (`total`, `count`, ...) or `aggregate`, then
+    join them with `&`. A repeated result name is rejected when the report is built.
+
+    Example:
+    ```python
+    >>> from pyaggregation import count, total
+    >>> report = total(len, name="letters") & count(name="words", where=str.istitle)
+    >>> report
+    Report(letters, words)
+    >>> report(iter(["Fold", "every", "Word"]))
+    {'letters': 13, 'words': 2}
+    >>> report.group_by(len)(["Fold", "every", "Word"])
+    {4: {'letters': 8, 'words': 2}, 5: {'letters': 5, 'words': 0}}
+
+    ```
+
+    """
+
+    aggregates: tuple[Aggregate[T], ...]
+
+    def __post_init__(self) -> None:
+        if not self.aggregates:
+            msg = "A report needs at least one aggregate, received none"
+            raise InvalidAggregateError(msg)
+
+        names = [aggregate.name for aggregate in self.aggregates]
+
+        if duplicate := next((name for name in names if names.count(name) > 1), None):
+            raise DuplicateResultNameError(duplicate, dict.fromkeys(names))
+
+    def __call__(self, items: Iterable[T], /) -> dict[str, Any]:
+        accumulators = _start(self.aggregates)
+
+        for item in items:
+            _step(self.aggregates, accumulators, item)
+
+        return _finish(self.aggregates, accumulators)
+
+    def __and__(self, other: "Report[T]", /) -> "Report[T]":
+        match other:
+            case Report(aggregates=aggregates):
+                return Report((*self.aggregates, *aggregates))
+            case _:
+                return NotImplemented
+
+    def group_by[K: Hashable](self, key: Callable[[T], K], /) -> "GroupedReport[T, K]":
+        """Return a report that folds each group of items with the same `key` separately."""
+        return GroupedReport(self, key)
+
+    def __repr__(self) -> str:
+        return f"Report({', '.join(aggregate.name for aggregate in self.aggregates)})"
+
+
+@dataclass(frozen=True, slots=True)
+class GroupedReport[T, K: Hashable]:
+    """A report per key, still in one pass. Groups come in first-seen order."""
+
+    report: Report[T]
+    key: Callable[[T], K]
+
+    def __post_init__(self) -> None:
+        if not callable(self.key):
+            msg = f"The group_by key must be callable, received {self.key!r}"
+            raise InvalidGroupKeyError(msg)
+
+    def __call__(self, items: Iterable[T], /) -> dict[K, dict[str, Any]]:
+        aggregates = self.report.aggregates
+        groups: dict[K, list[Any]] = {}
+
+        for item in items:
+            key = self.key(item)
+
+            if (accumulators := groups.get(key)) is None:
+                accumulators = groups[key] = _start(aggregates)
+
+            _step(aggregates, accumulators, item)
+
+        return {key: _finish(aggregates, accumulators) for key, accumulators in groups.items()}
+
+    def __repr__(self) -> str:
+        return f"{self.report!r}.group_by({getattr(self.key, '__name__', self.key)})"
+
+
+def aggregate[T](
+    fold: Fold[Any, Any, Any],
+    /,
+    *,
+    name: str,
+    of: Value[T] | None = None,
+    where: Condition[T] | None = None,
+) -> Report[T]:
+    """Return a one-result report from any fold: the way to add folds beyond the built-ins.
+
+    Args:
+        fold: Reduces the values to the result.
+        name: The result's key in the report's dict.
+        of: Computes the value folded for each item. Defaults to the item itself.
+        where: Keeps only the items it is true for. Runs before `of`.
+
+    Example:
+    ```python
+    >>> distinct = Fold(frozenset, lambda seen, value: seen | {value}, len)
+    >>> aggregate(distinct, name="letters", of=str.lower)("AbBa")
+    {'letters': 2}
+
+    ```
+
+    """
+    return Report((Aggregate(name, fold, identity if of is None else of, where),))
+
+
+def check_callable(value: object, /, *, role: str, name: str) -> None:
+    """Raise `InvalidAggregateError` unless `value` is callable."""
+    if not callable(value):
+        msg = f"The {role} of aggregate '{name}' must be callable, received {value!r}"
+        raise InvalidAggregateError(msg)
+
+
+# --- one pass
+
+
+def _start[T](aggregates: tuple[Aggregate[T], ...]) -> list[Any]:
+    return [_named(aggregate, aggregate.fold.initial) for aggregate in aggregates]
+
+
+def _step[T](aggregates: tuple[Aggregate[T], ...], accumulators: list[Any], item: T) -> None:
+    for index, aggregate in enumerate(aggregates):
+        try:
+            if aggregate.where is None or aggregate.where(item):
+                accumulators[index] = aggregate.fold.step(accumulators[index], aggregate.of(item))
+        except Exception as error:
+            error.add_note(f"in aggregate '{aggregate.name}'")
+            raise
+
+
+def _finish[T](aggregates: tuple[Aggregate[T], ...], accumulators: list[Any]) -> dict[str, Any]:
+    return {
+        aggregate.name: _named(aggregate, partial(aggregate.fold.finish, accumulator))
+        for aggregate, accumulator in zip(aggregates, accumulators, strict=True)
+    }
+
+
+def _named[R](aggregate: Aggregate[Any], call: Callable[[], R]) -> R:
+    try:
+        return call()
+    except Exception as error:
+        error.add_note(f"in aggregate '{aggregate.name}'")
+        raise