truegrain 0.1.0__py3-none-any.whl

truegrain/__init__.py

@@ -0,0 +1,63 @@
+"""Python client for the semantic engine.
+
+An Apache Ossie semantic model compiled to governed, dialect-correct SQL. This
+package talks to it over HTTP. There is no method that sends SQL, because there
+is no endpoint that accepts it: the only expressible request is a semantic one.
+
+Quick start:
+
+    >>> from truegrain import Client, filters
+    >>> client = Client.from_env()          # SEMANTIC_URL, SEMANTIC_TOKEN
+    >>> for metric in client.metrics(search="revenue"):
+    ...     print(metric.name, "-", metric.description)
+    >>> result = client.query(
+    ...     metrics=["sales.order_revenue"],
+    ...     dimensions=["sales.customers.region"],
+    ...     filters=[filters.in_("sales.orders.status", "shipped", "delivered")],
+    ...     order_by=[{"field": "order_revenue", "desc": True}],
+    ... )
+    >>> result.to_dataframe()
+
+Refusals are answers, not failures. Catch :class:`Refused` and branch on
+``retry``:
+
+    >>> try:
+    ...     client.query(metrics=["sales.order_revenue"],
+    ...                  dimensions=["sales.products.category"])
+    ... except Refused as refusal:
+    ...     if refusal.should_modify():
+    ...         print("try instead:", refusal.hint)
+
+The engine has no required dependencies here: this package uses only the
+standard library. pandas is optional and needed only for
+:meth:`Result.to_dataframe`.
+"""
+
+from __future__ import annotations
+
+from . import filters, tools
+from .client import OPERATIONS, Client
+from .errors import Refused, Retry, SemanticError, TransportError, Unauthorized
+from .models import Compiled, Dimension, Health, Job, Metric, Namespace, Result
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Client",
+    "Compiled",
+    "Dimension",
+    "Health",
+    "Job",
+    "Metric",
+    "Namespace",
+    "OPERATIONS",
+    "Refused",
+    "Result",
+    "Retry",
+    "SemanticError",
+    "TransportError",
+    "Unauthorized",
+    "__version__",
+    "filters",
+    "tools",
+]

truegrain/client.py

@@ -0,0 +1,445 @@
+"""HTTP client for the semantic engine.
+
+Deliberately dependency-free. It uses only the standard library so that
+installing it into an agent's environment cannot conflict with anything already
+there. pandas is optional and used only by :meth:`Result.to_dataframe`.
+
+There is no method that sends SQL, because there is no endpoint that accepts it.
+The only expressible request is a semantic one.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Iterable, Mapping
+
+from .errors import Refused, SemanticError, TransportError, Unauthorized
+from .models import Compiled, Dimension, Health, Job, Metric, Namespace, Result
+
+__all__ = ["Client"]
+
+#: Maps each OpenAPI operationId to the method that implements it.
+#:
+#: tests/test_covers_spec.py asserts this covers every operation in
+#: api/openapi.yaml, so an endpoint added to the engine cannot quietly go
+#: unsupported here.
+OPERATIONS: dict[str, str] = {
+    "getHealth": "health",
+    "getModelVersion": "model_version",
+    "listNamespaces": "namespaces",
+    "listMetrics": "metrics",
+    "describeMetric": "metric",
+    "listDimensions": "dimensions",
+    "query": "query",
+    "compile": "compile",
+    "submitJob": "submit",
+    "getJob": "job",
+    "cancelJob": "cancel_job",
+}
+
+DEFAULT_TIMEOUT = 60.0
+
+
+class Client:
+    """A connection to one semantic engine.
+
+    Args:
+        base_url: Where the engine is served, for example
+            ``http://127.0.0.1:8080``.
+        token: Bearer token identifying the workload. Read it from the
+            environment; never hard-code one. Omit it only against a local
+            engine running without authentication.
+        timeout: Seconds to wait for a response.
+        user_agent: Overrides the identifying header, which is useful when you
+            want the audit log to distinguish one agent from another.
+
+    Example:
+        >>> from truegrain import Client, filters
+        >>> c = Client("http://127.0.0.1:8080", token=os.environ["SEMANTIC_TOKEN"])
+        >>> result = c.query(
+        ...     metrics=["sales.order_revenue"],
+        ...     dimensions=["sales.customers.region"],
+        ...     filters=[filters.eq("sales.orders.status", "shipped")],
+        ... )
+        >>> result.to_dataframe()
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        token: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        user_agent: str = "truegrain-python/1.0",
+    ) -> None:
+        if not base_url:
+            raise ValueError("base_url is required")
+        self.base_url = base_url.rstrip("/")
+        self._token = token
+        self.timeout = timeout
+        self.user_agent = user_agent
+
+    @classmethod
+    def from_env(
+        cls,
+        url_var: str = "SEMANTIC_URL",
+        token_var: str = "SEMANTIC_TOKEN",
+        **kwargs: Any,
+    ) -> "Client":
+        """Build a client from environment variables.
+
+        Keeps the token out of source and out of the process list, which is
+        where a token passed as a command-line flag ends up.
+        """
+        url = os.environ.get(url_var)
+        if not url:
+            raise SemanticError(f"{url_var} is not set")
+        return cls(url, token=os.environ.get(token_var), **kwargs)
+
+    # ---------- metadata ----------
+
+    def health(self) -> Health:
+        """What this deployment enforces, including what it does not.
+
+        Worth reading before trusting the layer with anything sensitive:
+        ``health().enforcement_notes`` states the gaps in plain language.
+        """
+        return Health.parse(self._get("/v1/health"))
+
+    def model_version(self) -> str:
+        """The workspace digest currently served.
+
+        Every query result carries this too. Two results with the same digest
+        were produced by exactly the same definitions.
+        """
+        return str(self._get("/v1/model/version").get("model_version", ""))
+
+    def namespaces(self) -> list[Namespace]:
+        """The namespaces in the workspace, their owners and availability."""
+        payload = self._get("/v1/namespaces")
+        return [Namespace.parse(n) for n in payload.get("namespaces", [])]
+
+    def metrics(self, search: str | None = None) -> list[Metric]:
+        """Every metric this identity may read.
+
+        Args:
+            search: Case-insensitive substring over name and description.
+        """
+        params = {"search": search} if search else None
+        payload = self._get("/v1/metrics", params)
+        return [Metric.parse(m) for m in payload.get("metrics", [])]
+
+    def metric(self, name: str) -> Metric:
+        """One metric's definition and the dimensions legal for it.
+
+        Args:
+            name: Qualified as ``namespace.metric``, or bare when unambiguous
+                across the workspace.
+        """
+        if not name:
+            raise ValueError("metric name is required")
+        return Metric.parse(self._get(f"/v1/metrics/{urllib.parse.quote(name, safe='')}"))
+
+    def dimensions(self, metric: str | None = None) -> list[Dimension]:
+        """Dimensions available for grouping and filtering.
+
+        Args:
+            metric: Restrict to dimensions valid for this metric, which is
+                almost always what you want.
+        """
+        params = {"metric": metric} if metric else None
+        payload = self._get("/v1/dimensions", params)
+        return [Dimension.parse(d) for d in payload.get("dimensions", [])]
+
+    # ---------- query ----------
+
+    def query(
+        self,
+        metrics: Iterable[str],
+        dimensions: Iterable[str] | None = None,
+        filters: Iterable[Mapping[str, Any]] | None = None,
+        grain: str | None = None,
+        limit: int | None = None,
+        order_by: Iterable[Mapping[str, Any]] | None = None,
+    ) -> Result:
+        """Answer a question and return rows.
+
+        Args:
+            metrics: Metric names. Metrics at different grains, or in different
+                namespaces, are aggregated separately and joined on the shared
+                dimensions.
+            dimensions: ``dataset.field`` or ``namespace.dataset.field``.
+            filters: Structured filters. Use :mod:`truegrain.filters` to
+                build them.
+            grain: Time bucket for the selected time dimension.
+            limit: Maximum rows. Defaults to the server's limit.
+            order_by: ``[{"field": "order_revenue", "desc": True}]``.
+
+        Raises:
+            Refused: The engine declined. Check ``err.retry`` before retrying.
+        """
+        return Result.parse(
+            self._post("/v1/query", self._request_body(metrics, dimensions, filters, grain, limit, order_by))
+        )
+
+    def compile(
+        self,
+        metrics: Iterable[str],
+        dimensions: Iterable[str] | None = None,
+        filters: Iterable[Mapping[str, Any]] | None = None,
+        grain: str | None = None,
+        limit: int | None = None,
+        order_by: Iterable[Mapping[str, Any]] | None = None,
+    ) -> Compiled:
+        """Return the SQL a request compiles to, without running it.
+
+        A dry run is still governed: a request you may not run returns a refusal
+        and no SQL, and the inspection is audited. It is a way to see what a
+        query would do, not a way around the gate.
+        """
+        return Compiled.parse(
+            self._post("/v1/compile", self._request_body(metrics, dimensions, filters, grain, limit, order_by))
+        )
+
+    # ---------- asynchronous execution ----------
+
+    def run(
+        self,
+        metrics: Iterable[str],
+        dimensions: Iterable[str] | None = None,
+        filters: Iterable[Mapping[str, Any]] | None = None,
+        grain: str | None = None,
+        limit: int | None = None,
+        order_by: Iterable[Mapping[str, Any]] | None = None,
+        max_wait: float = 900.0,
+        page_size: int | None = None,
+    ) -> Result:
+        """Run a query that may take longer than an HTTP request should.
+
+        Submits the query, polls until it finishes, collects every page, and
+        returns one :class:`Result`. Use this instead of :meth:`query` whenever
+        the warehouse might be slow: a large BigQuery scan routinely outlives
+        the default timeout of an agent framework, and a synchronous call that
+        times out leaves the query running and billable with nobody reading it.
+
+        The governance gate runs during submission, so a request this identity
+        may not make raises :class:`Refused` immediately rather than after a
+        wait.
+
+        Args:
+            max_wait: Seconds to wait before giving up. On expiry the job is
+                cancelled rather than left running, so an abandoned wait does
+                not keep spending warehouse time.
+            page_size: Rows per page while collecting. The default suits most
+                results; lower it when rows are wide.
+
+        Raises:
+            Refused: The engine declined, or the query failed. Check
+                ``err.retry`` before retrying.
+            SemanticError: The wait expired, or the job was cancelled.
+        """
+        job = self.submit(metrics, dimensions, filters, grain, limit, order_by)
+        job = self.wait(job.job_id, max_wait=max_wait, page_size=page_size)
+        job.raise_for_state()
+
+        # Walk the remaining pages. row_count is the size of the whole result,
+        # so it bounds the walk without the loop having to trust the cursor to
+        # eventually come back empty.
+        rows = list(job.rows)
+        cursor = job.next_cursor
+        total = job.row_count or None
+        while cursor and (total is None or len(rows) < total):
+            page = self.job(job.job_id, cursor=cursor, page_size=page_size)
+            if not page.rows:
+                break
+            rows.extend(page.rows)
+            cursor = page.next_cursor
+        return job.as_result(tuple(rows))
+
+    def submit(
+        self,
+        metrics: Iterable[str],
+        dimensions: Iterable[str] | None = None,
+        filters: Iterable[Mapping[str, Any]] | None = None,
+        grain: str | None = None,
+        limit: int | None = None,
+        order_by: Iterable[Mapping[str, Any]] | None = None,
+    ) -> Job:
+        """Start a query in the background and return immediately.
+
+        The returned job already carries ``compiled_sql``, because compilation
+        happened synchronously. A job coming back at all means the query was
+        authorized, not merely accepted.
+
+        Prefer :meth:`run` unless you genuinely need to do something else while
+        the query runs.
+        """
+        return Job.parse(
+            self._post(
+                "/v1/jobs",
+                self._request_body(metrics, dimensions, filters, grain, limit, order_by),
+            )
+        )
+
+    def job(self, job_id: str, cursor: str | None = None, page_size: int | None = None) -> Job:
+        """Poll a job and read one page of its rows.
+
+        Args:
+            cursor: From a previous page's ``next_cursor``. Omit for the first.
+            page_size: Rows in this page.
+        """
+        if not job_id:
+            raise ValueError("job_id is required")
+        params: dict[str, str] = {}
+        if cursor:
+            params["cursor"] = cursor
+        if page_size is not None:
+            params["page_size"] = str(page_size)
+        return Job.parse(
+            self._get(f"/v1/jobs/{urllib.parse.quote(job_id, safe='')}", params or None)
+        )
+
+    def cancel_job(self, job_id: str) -> Job:
+        """Stop a running query. Idempotent."""
+        if not job_id:
+            raise ValueError("job_id is required")
+        return Job.parse(
+            self._delete(f"/v1/jobs/{urllib.parse.quote(job_id, safe='')}")
+        )
+
+    def wait(
+        self,
+        job_id: str,
+        max_wait: float = 900.0,
+        page_size: int | None = None,
+    ) -> Job:
+        """Poll until a job reaches a terminal state.
+
+        Backs off from a fast first poll to a two second ceiling, so a query
+        that finishes quickly is not delayed and a slow one is not hammered.
+
+        On expiry the job is cancelled before raising, so giving up here also
+        stops the warehouse work.
+        """
+        deadline = time.monotonic() + max_wait
+        interval = 0.25
+        while True:
+            job = self.job(job_id, page_size=page_size)
+            if job.is_done:
+                return job
+            if time.monotonic() >= deadline:
+                # Leaving it running would keep spending warehouse time for an
+                # answer this caller has already stopped waiting for.
+                try:
+                    self.cancel_job(job_id)
+                except SemanticError:
+                    pass
+                raise SemanticError(
+                    f"job {job_id} did not finish within {max_wait:g}s and was cancelled"
+                )
+            time.sleep(min(interval, max(0.0, deadline - time.monotonic())))
+            interval = min(interval * 1.5, 2.0)
+
+    # ---------- internals ----------
+
+    @staticmethod
+    def _request_body(
+        metrics: Iterable[str],
+        dimensions: Iterable[str] | None,
+        filters: Iterable[Mapping[str, Any]] | None,
+        grain: str | None,
+        limit: int | None,
+        order_by: Iterable[Mapping[str, Any]] | None,
+    ) -> dict[str, Any]:
+        names = list(metrics)
+        if not names:
+            raise ValueError("at least one metric is required")
+        body: dict[str, Any] = {"metrics": names}
+        # Omitted rather than sent empty: the server rejects unknown and
+        # malformed fields, and an empty list is not the same as absent.
+        if dimensions:
+            body["dimensions"] = list(dimensions)
+        if filters:
+            body["filters"] = [dict(f) for f in filters]
+        if grain:
+            body["grain"] = grain
+        if limit is not None:
+            body["limit"] = limit
+        if order_by:
+            body["order_by"] = [dict(o) for o in order_by]
+        return body
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"Accept": "application/json", "User-Agent": self.user_agent}
+        if self._token:
+            headers["Authorization"] = f"Bearer {self._token}"
+        return headers
+
+    def _get(self, path: str, params: Mapping[str, str] | None = None) -> dict[str, Any]:
+        url = self.base_url + path
+        if params:
+            url += "?" + urllib.parse.urlencode({k: v for k, v in params.items() if v})
+        return self._send(urllib.request.Request(url, headers=self._headers(), method="GET"))
+
+    def _post(self, path: str, body: Mapping[str, Any]) -> dict[str, Any]:
+        headers = self._headers()
+        headers["Content-Type"] = "application/json"
+        payload = json.dumps(body).encode("utf-8")
+        request = urllib.request.Request(
+            self.base_url + path, data=payload, headers=headers, method="POST"
+        )
+        return self._send(request)
+
+    def _delete(self, path: str) -> dict[str, Any]:
+        return self._send(
+            urllib.request.Request(self.base_url + path, headers=self._headers(), method="DELETE")
+        )
+
+    def _send(self, request: urllib.request.Request) -> dict[str, Any]:
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                return self._decode(response.read(), response.status)
+        except urllib.error.HTTPError as exc:
+            # A refusal arrives as an error status with a JSON body. It is an
+            # answer, not a transport failure, so it is raised as Refused with
+            # the code and retry class intact.
+            raw = exc.read()
+            if exc.code == 401:
+                raise Unauthorized(self._message(raw, "credentials were not accepted")) from None
+            try:
+                payload = json.loads(raw.decode("utf-8"))
+            except (ValueError, UnicodeDecodeError):
+                raise TransportError(
+                    f"HTTP {exc.code} from {request.full_url}: {self._message(raw, exc.reason)}"
+                ) from None
+            if isinstance(payload, dict) and "code" in payload:
+                raise Refused.from_payload(payload, exc.code) from None
+            raise TransportError(f"HTTP {exc.code} from {request.full_url}: {payload}") from None
+        except urllib.error.URLError as exc:
+            raise TransportError(f"cannot reach {request.full_url}: {exc.reason}") from None
+        except TimeoutError:
+            raise TransportError(
+                f"{request.full_url} did not respond within {self.timeout:g}s"
+            ) from None
+
+    @staticmethod
+    def _decode(raw: bytes, status: int) -> dict[str, Any]:
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except (ValueError, UnicodeDecodeError) as exc:
+            raise TransportError(f"HTTP {status} body was not JSON: {exc}") from None
+        if not isinstance(payload, dict):
+            raise TransportError(f"HTTP {status} body was not a JSON object")
+        return payload
+
+    @staticmethod
+    def _message(raw: bytes, fallback: Any) -> str:
+        text = raw.decode("utf-8", errors="replace").strip()
+        return text or str(fallback)
+
+    def __repr__(self) -> str:
+        return f"Client(base_url={self.base_url!r}, authenticated={bool(self._token)})"

truegrain/errors.py

@@ -0,0 +1,99 @@
+"""Errors raised by the client.
+
+A refusal is not an exception in the usual sense. The engine understood the
+request and declined it, and it told you what to do about that. The distinction
+matters most to an agent: the difference between "change the arguments" and
+"stop" is the difference between a self-correcting loop and an infinite one.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+Retry = Literal["modify", "later", "never"]
+
+
+class SemanticError(Exception):
+    """Base class, so a caller can catch everything this library raises."""
+
+
+class TransportError(SemanticError):
+    """The engine could not be reached, or answered with something unparseable.
+
+    This is a network or deployment problem, never a statement about the
+    request. Retrying the same request is reasonable.
+    """
+
+
+class Unauthorized(SemanticError):
+    """Credentials are missing or not recognised."""
+
+
+class Refused(SemanticError):
+    """The engine declined the request and said why.
+
+    Attributes:
+        code: Machine-readable reason. Branch on this, never on the text.
+        reason: One sentence stating what was wrong.
+        hint: What to do instead. For a fan-out refusal this names the metrics
+            defined at the grain where the question is well defined.
+        retry: One of "modify", "later" or "never". See :meth:`should_modify`,
+            :meth:`should_wait` and :meth:`is_final`.
+        status: The HTTP status that carried the refusal.
+    """
+
+    def __init__(
+        self,
+        code: str,
+        reason: str,
+        hint: str = "",
+        retry: Retry = "never",
+        status: int = 0,
+    ) -> None:
+        self.code = code
+        self.reason = reason
+        self.hint = hint
+        self.retry: Retry = retry
+        self.status = status
+        super().__init__(self._render())
+
+    def _render(self) -> str:
+        text = f"{self.code}: {self.reason}"
+        if self.hint:
+            text += f"\n  hint: {self.hint}"
+        text += f"\n  retry: {self.retry}"
+        return text
+
+    @classmethod
+    def from_payload(cls, payload: dict[str, Any], status: int) -> "Refused":
+        return cls(
+            code=str(payload.get("code", "unknown")),
+            reason=str(payload.get("reason", "")),
+            hint=str(payload.get("hint", "")),
+            retry=payload.get("retry", "never"),
+            status=status,
+        )
+
+    def should_modify(self) -> bool:
+        """The request is answerable, but not as written.
+
+        Change the arguments and try again. Repeating it unchanged will not
+        work. The hint usually names what to change.
+        """
+        return self.retry == "modify"
+
+    def should_wait(self) -> bool:
+        """Nothing about the request is wrong.
+
+        Something outside it failed, such as the policy source being
+        unreachable. The same request may succeed shortly.
+        """
+        return self.retry == "later"
+
+    def is_final(self) -> bool:
+        """No version of this request from this caller will succeed.
+
+        Usually a denial. Say so rather than substituting a different metric
+        that answers a different question.
+        """
+        return self.retry == "never"

truegrain/filters.py

@@ -0,0 +1,85 @@
+"""Constructors for structured filters.
+
+A filter is never a SQL fragment. That is what keeps an ungoverned predicate
+inexpressible, and it is the reason these are small functions rather than a
+string builder.
+
+Using them instead of hand-written dictionaries catches a misspelled operator at
+the call site rather than as a refusal from the server.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+Filter = dict[str, Any]
+
+
+def _filter(dimension: str, op: str, *values: Any) -> Filter:
+    out: Filter = {"dimension": dimension, "op": op}
+    if values:
+        out["values"] = list(values)
+    return out
+
+
+def eq(dimension: str, value: Any) -> Filter:
+    """`dimension = value`."""
+    return _filter(dimension, "eq", value)
+
+
+def ne(dimension: str, value: Any) -> Filter:
+    """`dimension <> value`."""
+    return _filter(dimension, "ne", value)
+
+
+def in_(dimension: str, *values: Any) -> Filter:
+    """`dimension IN (...)`. Named with a trailing underscore; `in` is a keyword."""
+    if not values:
+        raise ValueError("in_ needs at least one value")
+    return _filter(dimension, "in", *values)
+
+
+def not_in(dimension: str, *values: Any) -> Filter:
+    """`dimension NOT IN (...)`."""
+    if not values:
+        raise ValueError("not_in needs at least one value")
+    return _filter(dimension, "not_in", *values)
+
+
+def gt(dimension: str, value: Any) -> Filter:
+    """`dimension > value`."""
+    return _filter(dimension, "gt", value)
+
+
+def gte(dimension: str, value: Any) -> Filter:
+    """`dimension >= value`."""
+    return _filter(dimension, "gte", value)
+
+
+def lt(dimension: str, value: Any) -> Filter:
+    """`dimension < value`."""
+    return _filter(dimension, "lt", value)
+
+
+def lte(dimension: str, value: Any) -> Filter:
+    """`dimension <= value`."""
+    return _filter(dimension, "lte", value)
+
+
+def between(dimension: str, low: Any, high: Any) -> Filter:
+    """`dimension BETWEEN low AND high`.
+
+    The engine refuses a reversed range rather than matching no rows, because an
+    empty result that looks like a real answer is worse than an error.
+    """
+    return _filter(dimension, "between", low, high)
+
+
+def is_null(dimension: str) -> Filter:
+    """`dimension IS NULL`."""
+    return _filter(dimension, "is_null")
+
+
+def is_not_null(dimension: str) -> Filter:
+    """`dimension IS NOT NULL`."""
+    return _filter(dimension, "is_not_null")