structured-data-models-client 1.0.0__py3-none-any.whl

sdm_client/materializers/sources.py

@@ -0,0 +1,331 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""The engine's graph over tables that stay in their source.
+
+One place builds it, because two need it: `Schema.from_source` reads the
+source's metadata through it, and the relational materializer samples through
+it. Both must agree on which columns a table has and where it lives.
+"""
+
+from __future__ import annotations
+
+import warnings
+from collections.abc import Iterator, Mapping, Sequence
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+from sdm_client.core.warnings import SDMWarning, caller_stacklevel
+from sdm_client.errors import MissingExtraError, StructuredDataError
+
+if TYPE_CHECKING:
+    from sdm_relational_engine.rfm import Graph
+
+
+class EngineTables:
+    r"""Engine tables read once per source.
+
+    Each gather gets copies, because it writes keys, time columns and stypes
+    onto them. Time ranges are not kept, so a schema sees new rows.
+    """
+
+    def __init__(self) -> None:
+        self._tables: dict[str, Any] = {}
+
+    def keep(self, tables: Mapping[str, Any]) -> None:
+        r"""Keyed by the schema's source name; Databricks and Postgres qualify
+        the engine's.
+        """
+        for key, table in tables.items():
+            self._tables.setdefault(key, table)
+
+    def graph(
+        self, source: Any, tables: Mapping[str, tuple[str, Sequence[str]]]
+    ) -> Graph:
+        r"""A fresh graph over copies of ``tables`` in ``source``, without
+        edges: the schema decides those.
+
+        Args:
+            source: A ``Source``, or a ``SemanticModel`` kept in one.
+            tables: Schema name to (source name, columns kept). Only tables
+                not read yet are read.
+        """
+        from sdm_relational_engine.rfm import Graph
+
+        missing = [
+            source_name
+            for source_name, _ in tables.values()
+            if source_name not in self._tables
+        ]
+        if missing:
+            if is_semantic_model(source):
+                model = semantic_model_graph(source)
+                self.keep({name: model[name] for name in model.tables})
+            else:
+                read = engine_graph(
+                    source,
+                    {name: name for name in missing},
+                    infer_metadata=False,
+                )
+                self.keep({name: read[name] for name in missing})
+        return Graph(
+            tables=[
+                _copy_for_one_gather(self._tables[source_name], name, columns)
+                for name, (source_name, columns) in tables.items()
+            ],
+            edges=[],
+        )
+
+
+def _copy_for_one_gather(table: Any, name: str, columns: Sequence[str]) -> Any:
+    copied = table.copy(name)
+    copied.end_time_column = None
+    copied.time_column = None
+    copied.primary_key = None
+    for column in [c.name for c in copied.columns if c.name not in columns]:
+        copied.remove_column(column)
+    return copied
+
+
+# The engine's dtypes a scalar wire column can carry, as the wire names them.
+# Lists, binary and durations have no scalar wire form.
+_WIRE_DTYPE = {
+    'bool': 'bool',
+    'int': 'int64',
+    'byte': 'int32',
+    'int16': 'int32',
+    'int32': 'int32',
+    'int64': 'int64',
+    'float': 'float64',
+    'float32': 'float32',
+    'float64': 'float64',
+    'string': 'string',
+    'date': 'timestamp[us]',
+    'time': 'timestamp[us]',
+}
+
+
+def wire_dtype(engine_dtype: Any) -> str | None:
+    r"""The wire dtype for an engine column, or ``None`` if it has none."""
+    return _WIRE_DTYPE.get(str(engine_dtype))
+
+
+@contextmanager
+def _schema_says_time_columns() -> Iterator[None]:
+    r"""Keep the engine from warning about a time column it kept unchecked:
+    the schema says that, with its own override, in one notice.
+    """
+    from sdm_relational_engine.rfm import TimeColumnWarning
+
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', TimeColumnWarning)
+        yield
+
+
+def engine_graph(
+    source: Any, tables: Mapping[str, str] | None, *, infer_metadata: bool
+) -> Graph:
+    r"""The engine's graph over ``tables`` in ``source``, without edges: the
+    schema decides those.
+
+    Args:
+        source: A ``Source``.
+        tables: Schema name to source name; ``None`` reads every table.
+        infer_metadata: Let the engine infer keys, time columns and stypes.
+    """
+    try:
+        from sdm_relational_engine import (
+            MetricViewAsTableError,
+            NoTablesFoundError,
+            TableNotFoundError,
+        )
+    except ImportError as error:  # pragma: no cover - optional extra
+        raise MissingExtraError(
+            'relational', 'structured-data-models-relational-engine'
+        ) from error
+
+    specs = (
+        None
+        if tables is None
+        else [
+            {'name': name, 'source_name': source_name}
+            for name, source_name in tables.items()
+        ]
+    )
+    try:
+        with _schema_says_time_columns():
+            return _read(source, specs, infer_metadata)
+    except NoTablesFoundError as error:
+        # The engine's own advice names its API, which cannot fill a namespace.
+        raise StructuredDataError(
+            f'{source!r} holds no table to describe', code='INVALID_REQUEST'
+        ) from error
+    except TableNotFoundError as error:
+        raise StructuredDataError(str(error), code='INVALID_REQUEST') from error
+    except MetricViewAsTableError as error:
+        # The engine's advice names its own API; here a source reads the view.
+        raise StructuredDataError(
+            f'{error.name!r} is a metric view, not a table: its measures are '
+            f'aggregations with no value per row. Read it as a semantic model '
+            f'instead, with source.semantic_model({error.name!r}).',
+            code='INVALID_REQUEST',
+            details={'metric_view': error.name},
+        ) from error
+
+
+def _read(
+    source: Any, specs: list[dict[str, Any]] | None, infer_metadata: bool
+) -> Graph:
+    from sdm_relational_engine.rfm import Graph
+
+    backend = source.backend
+    namespace = source.namespace
+    connection = source.connection
+    if backend == 'databricks':
+        return Graph.from_databricks(
+            connection,
+            tables=specs,
+            catalog=namespace.get('catalog'),
+            schema=namespace.get('schema'),
+            edges=[],
+            infer_metadata=infer_metadata,
+            verbose=False,
+        )
+    if backend == 'snowflake':
+        return Graph.from_snowflake(
+            connection,
+            tables=specs,
+            database=namespace.get('database'),
+            schema=namespace.get('schema'),
+            edges=[],
+            infer_metadata=infer_metadata,
+            verbose=False,
+        )
+    if backend == 'postgres':
+        return Graph.from_postgres(
+            connection,
+            tables=specs,
+            schema=namespace.get('schema'),
+            edges=[],
+            infer_metadata=infer_metadata,
+            verbose=False,
+        )
+    if backend == 'duckdb':
+        return Graph.from_duckdb(
+            connection,
+            tables=specs,
+            edges=[],
+            infer_metadata=infer_metadata,
+            verbose=False,
+        )
+    if backend == 'sqlite':
+        return Graph.from_sqlite(
+            connection,
+            tables=specs,
+            edges=[],
+            infer_metadata=infer_metadata,
+            verbose=False,
+        )
+    raise StructuredDataError(  # pragma: no cover - Source refuses others
+        f'no engine backend samples a {backend!r} source',
+        code='INVALID_REQUEST',
+    )
+
+
+def is_semantic_model(source: Any) -> bool:
+    r"""Whether ``source`` is a semantic model rather than a namespace."""
+    try:
+        from sdm_connectors import SemanticModel
+    except ImportError:  # pragma: no cover - nothing could have built one
+        return False
+    return isinstance(source, SemanticModel)
+
+
+def semantic_model_graph(model: Any) -> Graph:
+    r"""The engine's graph of a semantic model: a Databricks metric view or a
+    Snowflake semantic view.
+
+    Named after the model's own tables, with its links, keys and time columns
+    as the model declares them. Discovery and gathering both read it here, so
+    they keep the same columns.
+
+    What the model has that a graph cannot hold is said in one
+    :class:`SDMWarning`, and nothing is said for a model that loses nothing:
+    how it was read is in the graph's ``conversion_notes``, which the schema
+    renders as text. A model the
+    engine refuses, for a filter on a joined table say, is refused with a
+    :class:`StructuredDataError` that carries the engine's reason.
+    """
+    try:
+        from sdm_relational_engine import (
+            TableNotFoundError,
+            ViewConversionError,
+        )
+        from sdm_relational_engine.rfm import Graph
+    except ImportError as error:  # pragma: no cover - optional extra
+        raise MissingExtraError(
+            'relational', 'structured-data-models-relational-engine'
+        ) from error
+
+    try:
+        # The engine is told not to warn, rather than its warnings filtered,
+        # since a filter holds for every thread: what it would say is said
+        # below, and by the schema, to the caller.
+        graph = _model_graph(Graph, model)
+    except (ViewConversionError, TableNotFoundError) as error:
+        # The engine's refusals of the model, each saying what to change in
+        # it, and a table it reads that is not there, as for a source's own
+        # tables. Any other failure is not the model's, and is not one.
+        raise StructuredDataError(
+            str(error),
+            code='INVALID_REQUEST',
+            details={'semantic_model': model.name},
+        ) from error
+    if graph.conversion_messages:
+        warnings.warn(
+            f'{model!r} has what a graph cannot hold, so it is read in '
+            f'part:\n'
+            + '\n'.join(
+                f'- {message}' for message in graph.conversion_messages
+            ),
+            category=SDMWarning,
+            stacklevel=caller_stacklevel(),
+        )
+    return graph
+
+
+def _model_graph(graph_type: type[Graph], model: Any) -> Graph:
+    namespace = model.namespace
+    if model.backend == 'databricks':
+        return graph_type.from_databricks_metric_view(
+            model.name,
+            model.connection,
+            catalog=namespace.get('catalog'),
+            schema=namespace.get('schema'),
+            verbose=False,
+            trust_expressions=model.trust_expressions,
+            warn=False,
+        )
+    if model.backend == 'snowflake':
+        # The engine resolves an unqualified name against the connection's
+        # current database and schema, so the source's own are written in.
+        parts = [
+            namespace[part]
+            for part in ('database', 'schema')
+            if part in namespace
+        ]
+        name = (
+            model.name if '.' in model.name else '.'.join([*parts, model.name])
+        )
+        return graph_type.from_snowflake_semantic_view(
+            name,
+            model.connection,
+            verbose=False,
+            trust_expressions=model.trust_expressions,
+            warn=False,
+        )
+    raise StructuredDataError(  # pragma: no cover - SemanticModel refuses
+        f'no semantic model is kept in a {model.backend!r} source',
+        code='INVALID_REQUEST',
+    )

sdm_client/materializers/tabular.py

@@ -0,0 +1,339 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+r"""Build single-table requests from rows supplied by the caller."""
+
+from __future__ import annotations
+
+import numbers
+import warnings
+from collections.abc import Hashable, Sequence
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+from predictive_query import Stype
+from predictive_query.validator.utils import col_name, table_name
+
+from sdm_client.core.warnings import SDMWarning, caller_stacklevel
+from sdm_client.errors import StructuredDataError
+from sdm_client.models import TABULAR_LIMITS
+from sdm_client.request import Request
+from sdm_client.schema import Schema, SourceTable, Table
+
+if TYPE_CHECKING:
+    from predictive_query import ValidatedPredictiveQuery
+
+_CONTEXT_SEED = 42
+_CONTEXT_ROWS_MAX = TABULAR_LIMITS['context_rows_max']
+
+
+TABULAR_TASK_KINDS = frozenset({'classification', 'regression'})
+
+
+def _is_whole(value: object) -> bool:
+    r"""Whether ``value`` is a number with nothing after the decimal point."""
+    if isinstance(value, bool):
+        return True
+    if isinstance(value, numbers.Integral):
+        return True
+    if isinstance(value, Decimal):
+        return value.is_finite() and value == value.to_integral_value()
+    if isinstance(value, numbers.Real):
+        try:
+            return float(value).is_integer()
+        except (OverflowError, TypeError, ValueError):
+            return False
+    return False
+
+
+def _derive_task_kind(
+    query: ValidatedPredictiveQuery, target: pd.Series
+) -> str:
+    r"""The task the target implies, when the caller named none."""
+    if query.target_stype == Stype.categorical:
+        return 'classification'
+    if _holds_two_whole_numbers(target):
+        warnings.warn(
+            f'{query.target_column} holds two whole numbers, which is how '
+            f'a binary label is usually written, so this is being asked as '
+            f'a classification. If they are quantities rather than labels, '
+            f'pass task="regression" to ask for one; pass '
+            f'task="classification" to say this was meant and silence '
+            f'this.',
+            category=SDMWarning,
+            stacklevel=caller_stacklevel(),
+        )
+        return 'classification'
+    return 'regression'
+
+
+def _holds_two_whole_numbers(target: pd.Series) -> bool:
+    r"""Whether ``target`` is two distinct whole numbers and nothing else.
+
+    The values decide, not their storage: a 0/1 column with a null arrives as
+    ``float64`` and is still a label. Anything wider or fractional is left to
+    ``task=``.
+    """
+    values = target.dropna().unique()
+    if len(values) != 2:
+        return False
+    return all(_is_whole(value) for value in values)
+
+
+def _asking(request: Request, query: Any) -> Request:
+    """Record the question a request was built from.
+
+    A session pins one question and only the rows travel on a scoring call, so
+    the question is what tells one request from another. The relational route
+    records it when it gathers; this route builds the request directly and did
+    not, which left a tabular session unable to tell that it had been handed
+    something else.
+    """
+    request.query = query.to_string()
+    return request
+
+
+class TabularMaterializer:
+    r"""Materialize resolved PQL against a single-table schema.
+
+    The ``TABULAR`` profile has already refused anything this cannot honour
+    (a filtered entity, an ``ASSUMING`` clause, a windowed or derived
+    target),
+    so the target here is a real column of the one table.
+    """
+
+    def __init__(self, schema: Schema) -> None:
+        self.schema = schema
+
+    def table(
+        self,
+        *,
+        query: ValidatedPredictiveQuery,
+        predict: pd.DataFrame,
+        task: str | None = None,
+    ) -> Request:
+        r"""Build a request from rows supplied separately by the caller.
+
+        The context is the table minus the rows being scored and minus the
+        rows with nothing to teach. The other route already left out both;
+        this one left out neither, so the same table asked the same question
+        two ways built two different contexts.
+        """
+        entity_table_name = table_name(query.entity_column)
+        entity_col = col_name(query.entity_column)
+        target_col = col_name(query.target_column)
+
+        context = _context_within_limit(
+            _rows_that_carry_an_answer(
+                _without_the_rows_being_scored(
+                    _rows(self.schema[entity_table_name]), predict, entity_col
+                ),
+                target_col,
+            ),
+            None,
+            entity_table_name,
+            'This route takes no num_context_samples, so choose the '
+            'context by building the schema from the rows you want.',
+        )
+
+        return _asking(
+            Request(
+                context=context,
+                predict=predict,
+                target=target_col,
+                task=self._task_kind(query, context[target_col], task),
+                schema=self.schema.snapshot(),
+                entity_table=entity_table_name,
+            ),
+            query,
+        )
+
+    def indices(
+        self,
+        query: ValidatedPredictiveQuery,
+        indices: Sequence[Any],
+        num_context_samples: int | None,
+        task: str | None = None,
+    ) -> Request:
+        r"""Build a single-table request for rows in the table."""
+        entity_table_name = table_name(query.entity_column)
+        entity_table = _rows(self.schema[entity_table_name])
+        entity_col = col_name(query.entity_column)
+        target_col = col_name(query.target_column)
+
+        indices = list(indices)
+        positions: dict[Hashable, list[int]] = {}
+        for position, value in enumerate(entity_table[entity_col]):
+            positions.setdefault(value, []).append(position)
+
+        missing = [value for value in indices if value not in positions]
+        if missing:
+            shown = missing[:10]
+            len_missing = len(missing)
+            tail = '' if len_missing == len(shown) else ', ...'
+            raise StructuredDataError(
+                f'{len_missing} of the {len(indices)} indices are not in '
+                f'{query.entity_column}: {shown}{tail}. Pass predict=<frame> '
+                f'to score rows not present in the table',
+                code='INVALID_REQUEST',
+            )
+
+        scored = [
+            position for value in indices for position in positions[value]
+        ]
+        predict = entity_table.iloc[scored].drop(columns=[target_col])
+        context_mask = entity_table[target_col].notna()
+        context_mask.iloc[scored] = False
+        candidates = entity_table[context_mask]
+
+        context = _context_within_limit(
+            candidates,
+            num_context_samples,
+            entity_table_name,
+            'Pass num_context_samples to choose how many, or select the rows '
+            'yourself.',
+        )
+
+        return _asking(
+            Request(
+                context=context,
+                predict=predict,
+                target=target_col,
+                task=self._task_kind(query, context[target_col], task),
+                schema=self.schema.snapshot(),
+                entity_table=entity_table_name,
+            ),
+            query,
+        )
+
+    @staticmethod
+    def _task_kind(
+        query: ValidatedPredictiveQuery,
+        target: pd.Series,
+        stated: str | None,
+    ) -> str:
+        r"""What is being asked of the target, and who decided it.
+
+        ``target`` is the context that will be sent, not the column it came
+        from; the two differ, and the narrower one is what the model learns
+        from.
+        """
+        if query.target_stype not in {Stype.categorical, Stype.numerical}:
+            raise StructuredDataError(
+                f'tabular materialization requires a categorical or '
+                f'numerical target; {query.target_column!r} has semantic '
+                f'type {query.target_stype.value!r}',
+                code='UNSUPPORTED_REQUEST',
+            )
+        if stated is not None and (
+            not isinstance(stated, str) or stated not in TABULAR_TASK_KINDS
+        ):
+            raise StructuredDataError(
+                f'the tabular model does not serve the task kind '
+                f'{stated!r}; it serves {sorted(TABULAR_TASK_KINDS)}',
+                code='UNSUPPORTED_REQUEST',
+                details={
+                    'task': stated,
+                    'supported': sorted(TABULAR_TASK_KINDS),
+                },
+            )
+
+        return (
+            stated if stated is not None else _derive_task_kind(query, target)
+        )
+
+
+def _context_within_limit(
+    candidates: pd.DataFrame,
+    num_context_samples: int | None,
+    table: str,
+    remedy: str,
+) -> pd.DataFrame:
+    r"""The context rows, never more than the endpoint will take.
+
+    The endpoint refuses a context above its limit, and refuses it *after* the
+    whole thing has been encoded into JSON and transmitted. A table of
+    hundreds of thousands of rows sent in full to be told the limit is ten
+    thousand. The relational side has declared its limits and refused locally
+    all along; this is the same thing on the other model.
+
+    A number the caller named above the limit is an error, because they asked
+    for something that cannot happen and quietly giving them less would answer
+    a different question. Saying nothing is a choice about how much data the
+    model sees, so it is made here and said out loud.
+    """
+    if num_context_samples is not None:
+        if num_context_samples > _CONTEXT_ROWS_MAX:
+            raise StructuredDataError(
+                f'num_context_samples is {num_context_samples:,}, and the '
+                f'endpoint accepts at most {_CONTEXT_ROWS_MAX:,} context '
+                f'rows',
+                code='UNSUPPORTED_REQUEST',
+            )
+        wanted = num_context_samples
+    else:
+        wanted = _CONTEXT_ROWS_MAX
+    if len(candidates) <= wanted:
+        return candidates
+    if num_context_samples is None:
+        warnings.warn(
+            f'{table} has {len(candidates):,} rows that could be context and '
+            f'the endpoint accepts {_CONTEXT_ROWS_MAX:,}, so {wanted:,} were '
+            f'sampled. {remedy}',
+            category=SDMWarning,
+            stacklevel=caller_stacklevel(),
+        )
+    return candidates.sample(n=wanted, random_state=_CONTEXT_SEED).sort_index()
+
+
+def _rows_that_carry_an_answer(
+    frame: pd.DataFrame, target: str
+) -> pd.DataFrame:
+    r"""The rows that can teach the model something.
+
+    A row whose target is null is not an example of anything. It takes up
+    room in a context the endpoint caps, and says nothing in exchange.
+    """
+    if target not in frame:
+        return frame
+    return frame[frame[target].notna()]
+
+
+def _without_the_rows_being_scored(
+    table: pd.DataFrame, predict: pd.DataFrame, entity_col: str
+) -> pd.DataFrame:
+    r"""The table, minus any row the caller is about to ask about.
+
+    The `indices=` route does this and this one did not, so the two
+    single-table routes disagreed about whether a row may be both scored and
+    shown. Rows supplied separately are often rows the table already holds,
+    which is how anyone checks a model against answers they have. Leaving
+    them in the context hands the model those answers, labelled, and returns
+    what reads like accuracy.
+
+    Rows the table does not hold are what the route is for, and they simply
+    match nothing here.
+    """
+    if entity_col not in table.columns or entity_col not in predict.columns:
+        return table
+    scored = set(predict[entity_col])
+    if not scored:
+        return table
+    return table[~table[entity_col].isin(scored)]
+
+
+def _rows(table: Table | SourceTable) -> pd.DataFrame:
+    r"""The rows of a table the tabular model reads, which it holds in memory.
+
+    `PredictiveQuery.materialize` sends a schema over tables in their source to
+    the relational model, so only a hand-built route reaches here with one.
+    """
+    if isinstance(table, SourceTable):
+        raise StructuredDataError(
+            f'{table.name!r} stays in its source, and the tabular model reads '
+            f'rows in memory; build the schema with Schema.from_data',
+            code='INVALID_REQUEST',
+        )
+    return table.data