turbine-api 1.0.0__py3-none-any.whl

turbine/api/__init__.py

@@ -0,0 +1 @@
+"""Data Product API Extension: serve the published Datasets over HTTP."""

turbine/api/app.py

@@ -0,0 +1,658 @@
+"""Build the Litestar application that serves the Data Product API.
+
+One route exists for each published Dataset::
+
+    project --> products --> API servers --> datasets --> inspection
+            --> PublishedDataset --> route handler --> JSON page
+"""
+
+from __future__ import annotations
+
+import hashlib
+import inspect
+import logging
+from dataclasses import replace
+from http import HTTPStatus
+from types import GenericAlias
+from typing import Annotated, Any, Literal
+
+import msgspec
+import pyarrow as pa
+from litestar import Litestar, Request, Response, get
+from litestar.exceptions import HTTPException
+from litestar.openapi.config import OpenAPIConfig
+from litestar.openapi.datastructures import ResponseSpec
+from litestar.openapi.plugins import (
+    JsonRenderPlugin,
+    RapidocRenderPlugin,
+    RedocRenderPlugin,
+    ScalarRenderPlugin,
+    StoplightRenderPlugin,
+    SwaggerRenderPlugin,
+    YamlRenderPlugin,
+)
+from litestar.openapi.spec import Server as OpenAPIServer
+from litestar.params import QueryParameter
+from litestar.plugins.problem_details import (
+    ProblemDetailsConfig,
+    ProblemDetailsException,
+    ProblemDetailsPlugin,
+)
+from litestar.types import Scope
+from sqlglot import exp
+from turbine.api.cursor import encode_cursor
+from turbine.api.extension import MOUNT_PATH
+from turbine.api.lint import (
+    published_columns_have_types,
+    published_dataset_has_primary_key,
+    publishes_api,
+)
+from turbine.api.pool import PooledConnectionOpener
+from turbine.api.publication import PublishedColumn, PublishedDataset, dataset_path
+from turbine.api.query import META_PARAMETERS, PageQuery, filter_parameter
+
+from turbine.core.sdk import (
+    Finding,
+    ProductIdentity,
+    ServeContext,
+    ServeDatasetInspection,
+    ServeDatasource,
+    ServerNotSelectedError,
+    Severity,
+    TextLocation,
+    TurbineError,
+)
+from turbine.core.sdk.datasource import DriverFault
+from turbine.core.sdk.lint import RuleOutcome
+from turbine.core.sdk.nodes import (
+    Column,
+    ContributionRole,
+    DataProduct,
+    Dataset,
+    Server,
+    SlaPropertyKind,
+)
+
+DEFAULT_PAGE_SIZE = 100
+MAX_PAGE_SIZE = 100_000
+ENCODE_CHUNK_ROWS = 65_536
+"""Rows encoded per Arrow batch, so one page never holds every row as Python objects."""
+PUBLICATION_RULES = (published_columns_have_types, published_dataset_has_primary_key)
+"""The lint rules that a Dataset must pass before the Data API publishes it.
+
+`turbine lint` and the editor show the same findings before the server starts.
+"""
+
+
+class QueryProblem(msgspec.Struct):
+    """Body of a 400 response, as problem details."""
+
+    status: Literal[400]
+    title: str
+    detail: str
+
+
+class PageLinks(msgspec.Struct):
+    """Links section of a page response."""
+
+    next: str | None
+
+
+def _problem_details_from_http_exception(error: HTTPException) -> ProblemDetailsException:
+    """Convert an HTTP error into problem details.
+
+    The conversion renames the internal key 'page_size' to 'page[size]', so
+    the response speaks the names of the public query parameters.
+    """
+    extra = error.extra
+    if isinstance(extra, list):
+        extra = [
+            {**item, "key": "page[size]"}
+            if isinstance(item, dict) and item.get("key") == "page_size"
+            else item
+            for item in extra
+        ]
+    return ProblemDetailsException(
+        status_code=error.status_code,
+        title=HTTPStatus(error.status_code).phrase,
+        detail=str(error.detail),
+        extra=extra,
+        headers=error.headers,
+    )
+
+
+class ProductApiScalarRenderPlugin(ScalarRenderPlugin):
+    """Scalar documentation plugin that knows the mount path."""
+
+    @staticmethod
+    def get_openapi_json_route(request: Request[Any, Any, Any]) -> str:
+        """Give the URL of the OpenAPI document below the mount path."""
+        del request
+        return f"{MOUNT_PATH}/docs/openapi.json"
+
+
+async def _log_request_exception(error: Exception, _scope: Scope) -> None:
+    """Log a server error. Client errors make no log record."""
+    if isinstance(error, HTTPException) and error.status_code < HTTPStatus.INTERNAL_SERVER_ERROR:
+        return
+    logging.getLogger(__name__).debug(
+        "Data API request failed", exc_info=(type(error), error, error.__traceback__)
+    )
+
+
+def create_data_api(context: ServeContext) -> Litestar:
+    """Build the Data API application for the project.
+
+    Parameters
+    ----------
+    context : ServeContext
+        The project, the datasource bindings, and the output channels.
+
+    Returns
+    -------
+    Litestar
+        An application with one route for each published Dataset.
+
+    Raises
+    ------
+    TurbineError
+        If two Data Products use the same API path.
+    """
+    pool = PooledConnectionOpener(context.connection_opener)
+    resources = _published_datasets(context)
+    resources_by_path: dict[str, PublishedDataset] = {}
+    for resource in resources:
+        previous = resources_by_path.get(resource.path)
+        if previous is not None:
+            raise TurbineError(
+                f"Data Products {previous.product.key} and {resource.product.key} use "
+                f"the same API path {resource.path!r}. Give the products different major versions. "
+                "Or disable publication for one version. Remove its API Server from the Data "
+                "Contract, or set api: false at the top level of the standalone Quality Spec."
+            )
+        resources_by_path[resource.path] = resource
+    handlers = [_route(resource, context, pool) for resource in resources]
+    app = Litestar(
+        route_handlers=handlers,
+        logging_config=None,
+        openapi_config=OpenAPIConfig(
+            title="Turbine Data Product API",
+            version="v1",
+            path="/docs",
+            servers=[OpenAPIServer(url=MOUNT_PATH)],
+            render_plugins=(
+                ProductApiScalarRenderPlugin(),
+                StoplightRenderPlugin(),
+                SwaggerRenderPlugin(),
+                RedocRenderPlugin(),
+                RapidocRenderPlugin(),
+                JsonRenderPlugin(),
+                YamlRenderPlugin(),
+            ),
+        ),
+        plugins=[
+            ProblemDetailsPlugin(
+                ProblemDetailsConfig(
+                    exception_to_problem_detail_map={HTTPException: _problem_details_from_http_exception}
+                )
+            )
+        ],
+        after_exception=[_log_request_exception],
+        on_shutdown=[pool.close_all],
+    )
+    base_url = f"{context.base_url}{MOUNT_PATH}"
+    context.announce(f"\nOpen the Data API documentation in a browser:\n  {base_url}/docs\n")
+    for resource in resources:
+        fields = ", ".join(sorted(resource.queryable)) or "none"
+        context.announce(f"Dataset URL: {base_url}{resource.path}\n  Filterable fields: {fields}")
+    if not resources:
+        context.announce(
+            "The Data API has no published Datasets.\n"
+            "Read the publication warnings. Run turbine serve --help for configuration instructions."
+        )
+    return app
+
+
+def _published_datasets(context: ServeContext) -> tuple[PublishedDataset, ...]:
+    """Collect the published datasets of all Data Products."""
+    return tuple(
+        resource
+        for product in context.project.data_products
+        for resource in _published_product(product, context)
+    )
+
+
+def _published_product(product: DataProduct, context: ServeContext) -> tuple[PublishedDataset, ...]:
+    """Collect the published datasets of one Data Product.
+
+    A product that cannot connect makes a warning and no route. The other
+    products stay available.
+    """
+    api_servers = _api_servers(product, context.environment)
+    datasets = _api_datasets(product, api_servers)
+    if not datasets:
+        return ()
+    product = replace(product, datasets=datasets)
+    try:
+        inspection = context.inspect(product, api_servers)
+        return tuple(
+            resource
+            for dataset in inspection.datasets
+            if (resource := _validated_resource(product, dataset, inspection.datasource, context))
+            is not None
+        )
+    except ServerNotSelectedError as error:
+        # No connection was opened, so the advice about connections does not apply.
+        _report(
+            context,
+            product,
+            "turbine-api.serve.server-not-selected",
+            Severity.WARNING,
+            f"Data Product '{product.key.id}' is not published. Cause: {error}",
+        )
+        return ()
+    except Exception as error:  # noqa: BLE001 - one failed product must not stop siblings.
+        _report(
+            context,
+            product,
+            "turbine-api.serve.connection-failed",
+            Severity.WARNING,
+            f"Data Product '{product.key.id}' is not published. Cause: {error}\n"
+            "Run turbine status. Check the Server bindings and Datasource connections.",
+            # A Turbine error tells the cause. Turbine did not expect any other error.
+            report_bug=not isinstance(error, TurbineError),
+        )
+        return ()
+
+
+def _api_datasets(product: DataProduct, api_servers: tuple[Server, ...]) -> tuple[Dataset, ...]:
+    """Give the Datasets that the product offers to the API."""
+    if product.role is ContributionRole.STANDALONE_QUALITY_SPEC:
+        return product.datasets if product.api_enabled is True else ()
+    if product.role is not ContributionRole.DATA_CONTRACT or not api_servers:
+        return ()
+    return product.datasets
+
+
+def _api_servers(product: DataProduct, environment: str | None) -> tuple[Server, ...]:
+    """Give the API Servers of the product for an environment."""
+    return tuple(server for server in product.metadata.servers_in(environment) if publishes_api(server))
+
+
+def _validated_resource(
+    product: DataProduct,
+    inspection: ServeDatasetInspection,
+    datasource: ServeDatasource,
+    context: ServeContext,
+) -> PublishedDataset | None:
+    """Make a published dataset, or report why publication is not possible.
+
+    A Dataset is published only when it has a name, a database name, public
+    types, a primary key, and a schema that matches the live table.
+
+    Returns
+    -------
+    PublishedDataset or None
+        The resource, or None when the Dataset is not published.
+    """
+    dataset = inspection.dataset
+    if not isinstance(dataset.name, str) or dataset.database_name is None:
+        return None
+    for rule in PUBLICATION_RULES:
+        outcome = rule(dataset, product)
+        if isinstance(outcome, RuleOutcome):
+            _report(
+                context, product, rule.rule_id, Severity.WARNING, outcome.message, source=dataset.source
+            )
+            return None
+    errors = tuple(
+        finding for finding in inspection.validation.findings if finding.severity is Severity.ERROR
+    )
+    if errors:
+        _report(
+            context,
+            product,
+            "turbine-api.serve.schema-incompatible",
+            Severity.WARNING,
+            f"Dataset '{dataset.name}' is not published. "
+            + ". ".join(item.headline.rstrip(".") for item in errors)
+            + ".\nMake the declared columns and the live table columns the same.",
+        )
+        return None
+    columns = tuple(
+        _published_column(column, inspection)
+        for column in dataset.columns
+        if isinstance(column.name, str)
+    )
+    return PublishedDataset(product, dataset, datasource, inspection.table, columns)
+
+
+def _published_column(column: Column, inspection: ServeDatasetInspection) -> PublishedColumn:
+    """Make a published column with the facts that only the live column gives."""
+    live_name = (column.database_name or str(column.name)).casefold()
+    return PublishedColumn.from_column(
+        column,
+        approximate=live_name in inspection.validation.approximate_columns,
+        geometry=live_name in inspection.validation.geometry_columns,
+    )
+
+
+def _route(resource: PublishedDataset, context: ServeContext, pool: PooledConnectionOpener) -> Any:
+    """Build the GET route handler of one published dataset.
+
+    Notes
+    -----
+    The handler signature is made at runtime, because each dataset has its
+    own filter parameters and its own response schema::
+
+        request --> PageQuery --> SQL --> Arrow table --> JSON page
+                                                     --> ETag, Link headers
+    """
+    summary = f"List {resource.dataset.name}"
+
+    def read_dataset(
+        request: Request[Any, Any, Any], page_size: int | None = None, **_query: str | None
+    ) -> Response[bytes]:
+        """Return one page of rows of the dataset."""
+        size = DEFAULT_PAGE_SIZE if page_size is None else page_size
+        query = PageQuery.build(resource, request.query_params, size)
+        table = _query_table(resource, pool, query)
+        has_next = table.num_rows > size
+        if has_next and all(
+            table[name][size - 1].as_py() == table[name][size].as_py() for name in query.sort_fields
+        ):
+            raise HTTPException(
+                status_code=409,
+                detail="The API cannot return the next page. The pagination key is not unique. "
+                "Ask the Data Product owner to check the primary key. "
+                "Ask the owner to check for duplicate rows.",
+            )
+        page = table.slice(0, size)
+        next_url = (
+            _next_page_url(request, encode_cursor(page, query.sort_fields, query.cursor_context))
+            if has_next
+            else None
+        )
+        body = _encode_page(page, resource, query.fields, next_url)
+        headers = _headers(resource, context, body, next_url=next_url)
+        etag = headers["ETag"]
+        if _etag_matches(request.headers.get("If-None-Match"), etag):
+            return Response(content=b"", status_code=304, headers=headers)
+        return Response(content=body, media_type="application/json", headers=headers)
+
+    schema_name = _schema_name(resource)
+    read_dataset.__name__ = f"read_{schema_name}"
+    return_annotation = _response_schema(resource)
+    parameters = _openapi_parameters(resource)
+    # Litestar reads the handler signature with inspect.signature, which uses `__signature__`.
+    # The function type does not declare that attribute, so write it to the function dict.
+    read_dataset.__dict__["__signature__"] = inspect.Signature(
+        parameters, return_annotation=return_annotation
+    )
+    read_dataset.__annotations__ = {parameter.name: parameter.annotation for parameter in parameters} | {
+        "return": return_annotation
+    }
+    return get(
+        resource.path,
+        summary=summary,
+        tags=[resource.product.metadata.domain or resource.product.key.id],
+        deprecated=resource.product.metadata.end_of_life,
+        responses={
+            304: ResponseSpec(data_container=None, description="Response has not changed."),
+            400: ResponseSpec(
+                data_container=QueryProblem,
+                media_type="application/problem+json",
+                description="Invalid query. Read title for the cause and correction.",
+                generate_examples=False,
+            ),
+        },
+        sync_to_thread=True,
+    )(read_dataset)
+
+
+def _query_table(resource: PublishedDataset, pool: PooledConnectionOpener, query: PageQuery) -> pa.Table:
+    """Run the page query and give the rows.
+
+    A driver error that the request caused becomes a 400 response. All
+    other errors stay as they are.
+    """
+    with pool.lease(resource.datasource.name) as connection:
+        try:
+            return connection.query(
+                connection.render_sql(query.select),
+                parameters=query.parameters,
+                timeout=resource.datasource.connection_timeout,
+            )
+        except Exception as error:
+            fault = connection.classify(error)
+            if query.select.find(exp.RegexpLike) is not None and fault is DriverFault.BAD_QUERY:
+                raise HTTPException(
+                    status_code=400,
+                    detail="The regular expression is invalid or unsupported. "
+                    "Correct the pattern after '*=' or use an equality filter.",
+                ) from error
+            if query.parameters and (fault is DriverFault.BAD_QUERY or isinstance(error, OverflowError)):
+                raise HTTPException(
+                    status_code=400,
+                    detail="A filter value cannot be used by this dataset. "
+                    "Use values that fit the field types in this dataset's API schema.",
+                ) from error
+            raise
+
+
+def _openapi_parameters(resource: PublishedDataset) -> list[inspect.Parameter]:
+    """Build the handler parameters that OpenAPI documents."""
+    request = inspect.Parameter(
+        "request", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request[Any, Any, Any]
+    )
+    descriptions = {
+        "fields": "Return only these comma-separated field names. Use 'none' for primary keys only.",
+        "sort": "Sort by comma-separated field names. Prefix a name with '-' for descending order.",
+        "page[after]": "Use links.next instead of constructing the next page request.",
+        "page[size]": (
+            f"Maximum rows per page: 1 through {MAX_PAGE_SIZE}. Default: {DEFAULT_PAGE_SIZE}."
+        ),
+    }
+    query_names = (
+        *sorted(META_PARAMETERS),
+        *(filter_parameter(name) for name in sorted(resource.queryable)),
+    )
+    query_parameters: list[inspect.Parameter] = []
+    for index, name in enumerate(query_names):
+        description = descriptions.get(
+            name,
+            "Filter by a value, comma-separated values, or a value prefixed with !=, >, >=, <, or <=. "
+            "Repeat the parameter to give more than one condition. "
+            "For string fields only, use '*=' followed by a regular expression.",
+        )
+        annotation: Any
+        if name == "page[size]":
+            annotation = Annotated[
+                int | None,
+                QueryParameter(
+                    name=name, description=description, required=False, ge=1, le=MAX_PAGE_SIZE
+                ),
+            ]
+        else:
+            annotation = Annotated[
+                str | None, QueryParameter(name=name, description=description, required=False)
+            ]
+        parameter_name = "page_size" if name == "page[size]" else f"query_{index}"
+        query_parameters.append(
+            inspect.Parameter(
+                parameter_name, inspect.Parameter.KEYWORD_ONLY, annotation=annotation, default=None
+            )
+        )
+    return [request, *query_parameters]
+
+
+def _response_schema(resource: PublishedDataset) -> type[msgspec.Struct]:
+    """Build the response type of one dataset.
+
+    Notes
+    -----
+    The result has this shape::
+
+        <name>_row_page
+          data  : list of <name>_row
+          links : PageLinks
+    """
+    fields: list[tuple[str, Any, msgspec.UnsetType]] = []
+    renames: dict[str, str] = {}
+    for index, column in enumerate(resource.columns):
+        internal_name = f"field_{index}"
+        fields.append((internal_name, column.wire_type, msgspec.UNSET))
+        renames[internal_name] = column.name
+    row = msgspec.defstruct(
+        f"{_schema_name(resource)}_row", fields, rename=renames, module="turbine.api.generated"
+    )
+    # msgspec accepts every type expression, but its stub asks for `type`. The value
+    # `list[row]` is a GenericAlias, not a `type`. Python has no static type for a
+    # type expression, thus the value is Any.
+    rows: Any = GenericAlias(list, row)
+    page_fields: list[str | tuple[str, type] | tuple[str, type, Any]] = [
+        ("data", rows),
+        ("links", PageLinks),
+    ]
+    return msgspec.defstruct(f"{row.__name__}_page", page_fields, module="turbine.api.generated")
+
+
+def _next_page_url(request: Request[Any, Any, Any], cursor: str) -> str:
+    """Build the URL of the next page."""
+    query = request.query_params.copy()
+    query["page[after]"] = cursor
+    return str(request.url.with_replacements(query=query))
+
+
+def _encode_page(
+    table: pa.Table, resource: PublishedDataset, selected: tuple[str, ...], next_url: str | None
+) -> bytes:
+    """Encode a page as a JSON body.
+
+    The rows are encoded in batches, so one page never holds all rows as
+    Python objects at the same time.
+    """
+    encoder = msgspec.json.Encoder()
+    prepared = pa.table(
+        [resource.column(name).to_wire(table[name]) for name in selected], names=selected
+    )
+    batches = prepared.to_batches(max_chunksize=ENCODE_CHUNK_ROWS)
+    object_names = tuple(name for name in selected if resource.column(name).python_type is dict)
+    chunks = [
+        encoder.encode(_decode_object_fields(batch.to_pylist(), object_names))[1:-1]
+        for batch in batches
+        if batch.num_rows
+    ]
+    return b'{"data":[' + b",".join(chunks) + b'],"links":{"next":' + encoder.encode(next_url) + b"}}"
+
+
+def _decode_object_fields(
+    rows: list[dict[str, Any]], object_names: tuple[str, ...]
+) -> list[dict[str, Any]]:
+    """Replace JSON text in object fields with the decoded object."""
+    for row in rows:
+        for name in object_names:
+            value = row[name]
+            if isinstance(value, str):
+                row[name] = msgspec.json.decode(value, type=dict[str, Any])
+    return rows
+
+
+def _etag_matches(header: str | None, etag: str) -> bool:
+    """Tell if an If-None-Match header holds the ETag."""
+    if header is None:
+        return False
+    expected = etag.removeprefix("W/")
+    return any(
+        candidate == "*" or candidate.removeprefix("W/") == expected
+        for candidate in (part.strip() for part in header.split(","))
+    )
+
+
+def _headers(
+    resource: PublishedDataset, context: ServeContext, body: bytes, *, next_url: str | None
+) -> dict[str, str]:
+    """Build the response headers of a page.
+
+    The headers carry the contract status, the cache instructions, the
+    ETag, and the links to the next page and to a successor version.
+    """
+    product = resource.product
+    status = product.metadata.status
+    headers: dict[str, str] = {
+        "X-Contract-Id": product.key.id,
+        "X-Contract-Version": product.metadata.version or "",
+        "Cache-Control": _property(product, "cacheControl") or "no-cache",
+    }
+    links = [f'<{next_url}>; rel="next"'] if next_url is not None else []
+    if status is not None:
+        headers["X-Contract-Status"] = status.value
+    if product.metadata.end_of_life:
+        headers["Deprecation"] = "true"
+    sunset = next(
+        (
+            str(item.value)
+            for item in product.metadata.sla_properties
+            if item.property is SlaPropertyKind.END_OF_LIFE and item.value is not None
+        ),
+        None,
+    ) or _property(product, "sunsetDate")
+    if sunset is not None:
+        headers["Sunset"] = sunset
+    successor = _property(product, "supersededBy")
+    if successor is not None:
+        replacement = context.project.product(ProductIdentity(id=successor))
+        if replacement is not None and (path := _successor_path(resource, replacement)) is not None:
+            links.append(f'<{path}>; rel="successor-version"')
+    if links:
+        headers["Link"] = ", ".join(links)
+    digest = hashlib.sha256(
+        product.key.id.encode() + (product.metadata.version or "").encode() + body
+    ).hexdigest()
+    headers["ETag"] = f'"{digest}"'
+    return headers
+
+
+def _successor_path(resource: PublishedDataset, replacement: DataProduct) -> str | None:
+    """Give the URL of the same Dataset in the successor product."""
+    dataset = replacement.declared_dataset(resource.dataset.name)
+    if dataset is None:
+        return None
+    return f"{MOUNT_PATH}{dataset_path(replacement, dataset)}"
+
+
+def _report(
+    context: ServeContext,
+    product: DataProduct,
+    finding_id: str,
+    severity: Severity,
+    message: str,
+    *,
+    source: TextLocation | None = None,
+    report_bug: bool = False,
+) -> None:
+    """Add a publication finding to the diagnostics of the context.
+
+    ``report_bug`` adds the last help line that tells where to open an issue.
+    """
+    finding = Finding(finding_id, severity, message)
+    if report_bug:
+        finding = finding.with_bug_report()
+    if source is not None:
+        finding = finding.at_location(source)
+    elif product.defining_uri is not None:
+        finding = finding.at(product.defining_uri)
+    context.diagnostics.append(finding)
+
+
+def _property(product: DataProduct, name: str) -> str | None:
+    """Read a turbine extension property of the product."""
+    return product.extension_properties.text("turbine", name)
+
+
+def _schema_name(resource: PublishedDataset) -> str:
+    """Build a unique name for the generated schema of a dataset."""
+    product = resource.product.key.id.replace("-", "_")
+    dataset = str(resource.dataset.name).replace("-", "_")
+    route_identity = hashlib.sha256(resource.path.encode()).hexdigest()
+    return f"{product}_{dataset}_{route_identity}"