kumoai 2.13.0.dev202511131731__cp310-cp310-macosx_11_0_arm64.whl → 2.14.0.dev202512271732__cp310-cp310-macosx_11_0_arm64.whl

kumoai/experimental/rfm/sagemaker.py

@@ -0,0 +1,138 @@
+import base64
+import json
+from typing import Any
+
+import requests
+
+from kumoai.client import KumoClient
+from kumoai.client.endpoints import Endpoint, HTTPMethod
+from kumoai.exceptions import HTTPException
+
+try:
+    # isort: off
+    from mypy_boto3_sagemaker_runtime.client import SageMakerRuntimeClient
+    from mypy_boto3_sagemaker_runtime.type_defs import (
+        InvokeEndpointOutputTypeDef, )
+    # isort: on
+except ImportError:
+    SageMakerRuntimeClient = Any
+    InvokeEndpointOutputTypeDef = Any
+
+
+class SageMakerResponseAdapter(requests.Response):
+    def __init__(self, sm_response: InvokeEndpointOutputTypeDef):
+        super().__init__()
+        # Read the body bytes
+        self._content = sm_response['Body'].read()
+        self.status_code = 200
+        self.headers['Content-Type'] = sm_response.get('ContentType',
+                                                       'application/json')
+        # Optionally, you can store original sm_response for debugging
+        self.sm_response = sm_response
+
+    @property
+    def text(self) -> str:
+        assert isinstance(self._content, bytes)
+        return self._content.decode('utf-8')
+
+    def json(self, **kwargs) -> dict[str, Any]:  # type: ignore
+        return json.loads(self.text, **kwargs)
+
+
+class KumoClient_SageMakerAdapter(KumoClient):
+    def __init__(self, region: str, endpoint_name: str):
+        import boto3
+        self._client: SageMakerRuntimeClient = boto3.client(
+            service_name="sagemaker-runtime", region_name=region)
+        self._endpoint_name = endpoint_name
+
+        # Recording buffers.
+        self._recording_active = False
+        self._recorded_reqs: list[dict[str, Any]] = []
+        self._recorded_resps: list[dict[str, Any]] = []
+
+    def authenticate(self) -> None:
+        # TODO(siyang): call /ping to verify?
+        pass
+
+    def _request(self, endpoint: Endpoint, **kwargs: Any) -> requests.Response:
+        assert endpoint.method == HTTPMethod.POST
+        if 'json' in kwargs:
+            payload = json.dumps(kwargs.pop('json'))
+        elif 'data' in kwargs:
+            raw_payload = kwargs.pop('data')
+            assert isinstance(raw_payload, bytes)
+            payload = base64.b64encode(raw_payload).decode()
+        else:
+            raise HTTPException(400, 'Unable to send data to KumoRFM.')
+
+        request = {
+            'method': endpoint.get_path().rsplit('/')[-1],
+            'payload': payload,
+        }
+        response: InvokeEndpointOutputTypeDef = self._client.invoke_endpoint(
+            EndpointName=self._endpoint_name,
+            ContentType="application/json",
+            Body=json.dumps(request),
+        )
+
+        adapted_response = SageMakerResponseAdapter(response)
+
+        # If validation is active, store input/output
+        if self._recording_active:
+            self._recorded_reqs.append(request)
+            self._recorded_resps.append(adapted_response.json())
+
+        return adapted_response
+
+    def start_recording(self) -> None:
+        """Start recording requests/responses to/from sagemaker endpoint."""
+        assert not self._recording_active
+        self._recording_active = True
+        self._recorded_reqs.clear()
+        self._recorded_resps.clear()
+
+    def end_recording(self) -> list[tuple[dict[str, Any], dict[str, Any]]]:
+        """Stop recording and return recorded requests/responses."""
+        assert self._recording_active
+        self._recording_active = False
+        recorded = list(zip(self._recorded_reqs, self._recorded_resps))
+        self._recorded_reqs.clear()
+        self._recorded_resps.clear()
+        return recorded
+
+
+class KumoClient_SageMakerProxy_Local(KumoClient):
+    def __init__(self, url: str):
+        self._client = KumoClient(url, api_key=None)
+        self._client._api_url = self._client._url
+        self._endpoint = Endpoint('/invocations', HTTPMethod.POST)
+
+    def authenticate(self) -> None:
+        try:
+            self._client._session.get(
+                self._url + '/ping',
+                verify=self._verify_ssl).raise_for_status()
+        except Exception:
+            raise ValueError(
+                "Client authentication failed. Please check if you "
+                "have a valid API key/credentials.")
+
+    def _request(self, endpoint: Endpoint, **kwargs: Any) -> requests.Response:
+        assert endpoint.method == HTTPMethod.POST
+        if 'json' in kwargs:
+            payload = json.dumps(kwargs.pop('json'))
+        elif 'data' in kwargs:
+            raw_payload = kwargs.pop('data')
+            assert isinstance(raw_payload, bytes)
+            payload = base64.b64encode(raw_payload).decode()
+        else:
+            raise HTTPException(400, 'Unable to send data to KumoRFM.')
+        return self._client._request(
+            self._endpoint,
+            json={
+                'method': endpoint.get_path().rsplit('/')[-1],
+                'payload': payload,
+            },
+            **kwargs,
+        )

kumoai/pquery/predictive_query.py

@@ -370,9 +370,11 @@ class PredictiveQuery:
         train_table_job_api = global_state.client.generate_train_table_job_api
         job_id: GenerateTrainTableJobID = train_table_job_api.create(
             GenerateTrainTableRequest(
-                dict(custom_tags), pq_id, plan,
-                graph_snapshot_id=self.graph.snapshot(
-                    non_blocking=non_blocking)))
+                dict(custom_tags),
+                pq_id,
+                plan,
+                None,
+            ))
 
         self._train_table = TrainingTableJob(job_id=job_id)
         if non_blocking:
@@ -451,9 +453,11 @@ class PredictiveQuery:
         bp_table_api = global_state.client.generate_prediction_table_job_api
         job_id: GeneratePredictionTableJobID = bp_table_api.create(
             GeneratePredictionTableRequest(
-                dict(custom_tags), pq_id, plan,
-                graph_snapshot_id=self.graph.snapshot(
-                    non_blocking=non_blocking)))
+                dict(custom_tags),
+                pq_id,
+                plan,
+                None,
+            ))
 
         self._prediction_table = PredictionTableJob(job_id=job_id)
         if non_blocking:

kumoai/spcs.py

@@ -54,9 +54,7 @@ def _refresh_spcs_token() -> None:
         api_key=global_state._api_key,
         spcs_token=spcs_token,
     )
-    if not client.authenticate():
-        raise ValueError("Client authentication failed. Please check if you "
-                         "have a valid API key.")
+    client.authenticate()
 
     # Update state:
     global_state.set_spcs_token(spcs_token)

kumoai/testing/decorators.py

@@ -25,7 +25,7 @@ def onlyFullTest(func: Callable) -> Callable:
 def has_package(package: str) -> bool:
     r"""Returns ``True`` in case ``package`` is installed."""
     req = Requirement(package)
-    if importlib.util.find_spec(req.name) is None:
+    if importlib.util.find_spec(req.name) is None:  # type: ignore
         return False
 
     try:

kumoai/testing/snow.py

@@ -0,0 +1,50 @@
+import json
+import os
+
+from kumoai.experimental.rfm.backend.snow import Connection
+from kumoai.experimental.rfm.backend.snow import connect as _connect
+
+
+def connect(
+    region: str,
+    id: str,
+    account: str,
+    user: str,
+    warehouse: str,
+    database: str | None = None,
+    schema: str | None = None,
+) -> Connection:
+
+    kwargs = dict(password=os.getenv('SNOWFLAKE_PASSWORD'))
+    if kwargs['password'] is None:
+        import boto3
+        from cryptography.hazmat.primitives import serialization
+
+        client = boto3.client(
+            service_name='secretsmanager',
+            region_name=region,
+        )
+        secret_id = (f'arn:aws:secretsmanager:{region}:{id}:secret:'
+                     f'{account}.snowflakecomputing.com')
+        response = client.get_secret_value(SecretId=secret_id)['SecretString']
+        secret = json.loads(response)
+
+        private_key = serialization.load_pem_private_key(
+            secret['kumo_user_secretkey'].encode(),
+            password=None,
+        )
+        kwargs['private_key'] = private_key.private_bytes(
+            encoding=serialization.Encoding.DER,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+    return _connect(
+        account=account,
+        user=user,
+        warehouse='WH_XS',
+        database='KUMO',
+        schema=schema,
+        session_parameters=dict(CLIENT_TELEMETRY_ENABLED=False),
+        **kwargs,
+    )

kumoai/trainer/distilled_trainer.py

@@ -0,0 +1,175 @@
+import logging
+from typing import Literal, Mapping, Optional, Union, overload
+
+from kumoapi.distilled_model_plan import DistilledModelPlan
+from kumoapi.jobs import DistillationJobRequest, DistillationJobResource
+
+from kumoai import global_state
+from kumoai.client.jobs import TrainingJobID
+from kumoai.graph import Graph
+from kumoai.pquery.training_table import TrainingTable, TrainingTableJob
+from kumoai.trainer.job import TrainingJob, TrainingJobResult
+
+logger = logging.getLogger(__name__)
+
+
+class DistillationTrainer:
+    r"""A trainer supports creating a Kumo machine learning model
+    for use in an online serving endpoint. The distllation process involes
+    training a shallow model on a :class:`~kumoai.pquery.PredictiveQuery` using
+    the embeddings generated by a base model :args:`base_training_job_id`.
+
+    Args:
+        model_plan: The distilled model plan to use for the distillation process.
+        base_training_job_id: The ID of the base training job to use for the distillation process.
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        model_plan: DistilledModelPlan,
+        base_training_job_id: TrainingJobID,
+    ) -> None:
+        self.model_plan: DistilledModelPlan = model_plan
+        self.base_training_job_id: TrainingJobID = base_training_job_id
+
+        # Cached from backend:
+        self._training_job_id: Optional[TrainingJobID] = None
+
+    # Metadata ################################################################
+
+    @property
+    def is_trained(self) -> bool:
+        r"""Returns ``True`` if this trainer instance has successfully been
+        trained (and is therefore ready for prediction); ``False`` otherwise.
+        """
+        raise NotImplementedError(
+            "Checking if a distilled trainer is trained is not "
+            "implemented yet.")
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+    ) -> TrainingJobResult:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: Literal[False],
+    ) -> TrainingJobResult:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: Literal[True],
+    ) -> TrainingJob:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: bool,
+    ) -> Union[TrainingJob, TrainingJobResult]:
+        pass
+
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: bool = False,
+        custom_tags: Mapping[str, str] = {},
+    ) -> Union[TrainingJob, TrainingJobResult]:
+        r"""Fits a model to the specified graph and training table, with the
+        strategy defined by :class:`DistilledTrainer`'s :obj:`model_plan`.
+
+        Args:
+            graph: The :class:`~kumoai.graph.Graph` object that represents the
+                tables and relationships that Kumo will learn from.
+            train_table: The :class:`~kumoai.pquery.TrainingTable`, or
+                in-progress :class:`~kumoai.pquery.TrainingTableJob`, that
+                represents the training data produced by a
+                :class:`~kumoai.pquery.PredictiveQuery` on :obj:`graph`.
+            non_blocking: Whether this operation should return immediately
+                after launching the training job, or await completion of the
+                training job.
+            custom_tags: Additional, customer defined k-v tags to be associated
+                with the job to be launched. Job tags are useful for grouping
+                and searching jobs.
+
+        Returns:
+            Union[TrainingJobResult, TrainingJob]:
+                If ``non_blocking=False``, returns a training job object. If
+                ``non_blocking=True``, returns a training job future object.
+        """
+        # TODO(manan, siyang): remove soon:
+        job_id = train_table.job_id
+        assert job_id is not None
+
+        train_table_job_api = global_state.client.generate_train_table_job_api
+        pq_id = train_table_job_api.get(job_id).config.pquery_id
+        assert pq_id is not None
+
+        custom_table = None
+        if isinstance(train_table, TrainingTable):
+            custom_table = train_table._custom_train_table
+
+        # NOTE the backend implementation currently handles sequentialization
+        # between a training table future and a training job; that is, if the
+        # training table future is still executing, the backend will wait on
+        # the job ID completion before executing a training job. This preserves
+        # semantics for both futures, ensures that Kumo works as expected if
+        # used only via REST API, and allows us to avoid chaining calllbacks
+        # in an ugly way here:
+        api = global_state.client.distillation_job_api
+        self._training_job_id = api.create(
+            DistillationJobRequest(
+                dict(custom_tags),
+                pquery_id=pq_id,
+                base_training_job_id=self.base_training_job_id,
+                distilled_model_plan=self.model_plan,
+                graph_snapshot_id=graph.snapshot(non_blocking=non_blocking),
+                train_table_job_id=job_id,
+                custom_train_table=custom_table,
+            ))
+
+        out = TrainingJob(job_id=self._training_job_id)
+        if non_blocking:
+            return out
+        return out.attach()
+
+    @classmethod
+    def _load_from_job(
+        cls,
+        job: DistillationJobResource,
+    ) -> 'DistillationTrainer':
+        trainer = cls(job.config.distilled_model_plan,
+                      job.config.base_training_job_id)
+        trainer._training_job_id = job.job_id
+        return trainer
+
+    @classmethod
+    def load(cls, job_id: TrainingJobID) -> 'DistillationTrainer':
+        r"""Creates a :class:`~kumoai.trainer.Trainer` instance from a training
+        job ID.
+        """
+        raise NotImplementedError(
+            "Loading a distilled trainer from a job ID is not implemented yet."
+        )
+
+    @classmethod
+    def load_from_tags(cls, tags: Mapping[str, str]) -> 'DistillationTrainer':
+        raise NotImplementedError(
+            "Loading a distilled trainer from tags is not implemented yet.")

kumoai/utils/__init__.py

@@ -1,10 +1,11 @@
-from .progress_logger import ProgressLogger, InteractiveProgressLogger
+from .sql import quote_ident
+from .progress_logger import ProgressLogger
 from .forecasting import ForecastVisualizer
 from .datasets import from_relbench
 
 __all__ = [
+    'quote_ident',
     'ProgressLogger',
-    'InteractiveProgressLogger',
     'ForecastVisualizer',
     'from_relbench',
 ]

kumoai/utils/progress_logger.py

@@ -1,6 +1,7 @@
+import re
 import sys
 import time
-from typing import Any, List, Optional, Union
+from typing import Any
 
 from rich.console import Console, ConsoleOptions, RenderResult
 from rich.live import Live
@@ -20,12 +21,22 @@ from typing_extensions import Self
 
 
 class ProgressLogger:
-    def __init__(self, msg: str) -> None:
+    def __init__(self, msg: str, verbose: bool = True) -> None:
         self.msg = msg
-        self.logs: List[str] = []
+        self.verbose = verbose
+
+        self.logs: list[str] = []
+
+        self.start_time: float | None = None
+        self.end_time: float | None = None
+
+    @classmethod
+    def default(cls, msg: str, verbose: bool = True) -> 'ProgressLogger':
+        from kumoai import in_snowflake_notebook
 
-        self.start_time: Optional[float] = None
-        self.end_time: Optional[float] = None
+        if in_snowflake_notebook():
+            return StreamlitProgressLogger(msg, verbose)
+        return RichProgressLogger(msg, verbose)
 
     @property
     def duration(self) -> float:
@@ -37,6 +48,12 @@ class ProgressLogger:
     def log(self, msg: str) -> None:
         self.logs.append(msg)
 
+    def init_progress(self, total: int, description: str) -> None:
+        pass
+
+    def step(self) -> None:
+        pass
+
     def __enter__(self) -> Self:
         self.start_time = time.perf_counter()
         return self
@@ -66,22 +83,21 @@ class ColoredTimeRemainingColumn(TimeRemainingColumn):
         return Text(str(super().render(task)), style=self.style)
 
 
-class InteractiveProgressLogger(ProgressLogger):
+class RichProgressLogger(ProgressLogger):
     def __init__(
         self,
         msg: str,
         verbose: bool = True,
         refresh_per_second: int = 10,
     ) -> None:
-        super().__init__(msg=msg)
+        super().__init__(msg=msg, verbose=verbose)
 
-        self.verbose = verbose
         self.refresh_per_second = refresh_per_second
 
-        self._progress: Optional[Progress] = None
-        self._task: Optional[int] = None
+        self._progress: Progress | None = None
+        self._task: int | None = None
 
-        self._live: Optional[Live] = None
+        self._live: Live | None = None
         self._exception: bool = False
 
     def init_progress(self, total: int, description: str) -> None:
@@ -151,7 +167,7 @@ class InteractiveProgressLogger(ProgressLogger):
 
         table = Table.grid(padding=(0, 1))
 
-        icon: Union[Text, Padding]
+        icon: Text | Padding
         if self._exception:
             style = 'red'
             icon = Text('❌', style=style)
@@ -175,3 +191,153 @@ class InteractiveProgressLogger(ProgressLogger):
 
         if self.verbose and self._progress is not None:
             yield self._progress.get_renderable()
+
+
+class StreamlitProgressLogger(ProgressLogger):
+    def __init__(
+        self,
+        msg: str,
+        verbose: bool = True,
+    ) -> None:
+        super().__init__(msg=msg, verbose=verbose)
+
+        self._status: Any = None
+
+        self._total = 0
+        self._current = 0
+        self._description: str = ''
+        self._progress: Any = None
+
+    def __enter__(self) -> Self:
+        super().__enter__()
+
+        import streamlit as st
+
+        # Adjust layout for prettier output:
+        st.markdown(STREAMLIT_CSS, unsafe_allow_html=True)
+
+        if self.verbose:
+            self._status = st.status(
+                f':blue[{self._sanitize_text(self.msg)}]',
+                expanded=True,
+            )
+
+        return self
+
+    def log(self, msg: str) -> None:
+        super().log(msg)
+        if self.verbose and self._status is not None:
+            self._status.write(self._sanitize_text(msg))
+
+    def init_progress(self, total: int, description: str) -> None:
+        if self.verbose and self._status is not None:
+            self._total = total
+            self._current = 0
+            self._description = self._sanitize_text(description)
+            percent = min(self._current / self._total, 1.0)
+            self._progress = self._status.progress(
+                value=percent,
+                text=f'{self._description} [{self._current}/{self._total}]',
+            )
+
+    def step(self) -> None:
+        self._current += 1
+
+        if self.verbose and self._progress is not None:
+            percent = min(self._current / self._total, 1.0)
+            self._progress.progress(
+                value=percent,
+                text=f'{self._description} [{self._current}/{self._total}]',
+            )
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        super().__exit__(exc_type, exc_val, exc_tb)
+
+        if not self.verbose or self._status is None:
+            return
+
+        label = f'{self._sanitize_text(self.msg)} ({self.duration:.2f}s)'
+
+        if exc_type is not None:
+            self._status.update(
+                label=f':red[{label}]',
+                state='error',
+                expanded=True,
+            )
+        else:
+            self._status.update(
+                label=f':green[{label}]',
+                state='complete',
+                expanded=True,
+            )
+
+    @staticmethod
+    def _sanitize_text(msg: str) -> str:
+        return re.sub(r'\[/?bold\]', '**', msg)
+
+
+STREAMLIT_CSS = """
+<style>
+/* Fix horizontal scrollbar */
+.stExpander summary {
+    width: auto;
+}
+
+/* Fix paddings/margins */
+.stExpander summary {
+    padding: 0.75rem 1rem 0.5rem;
+}
+.stExpander p {
+    margin: 0px 0px 0.2rem;
+}
+.stExpander [data-testid="stExpanderDetails"] {
+    padding-bottom: 1.45rem;
+}
+.stExpander .stProgress div:first-child {
+    padding-bottom: 4px;
+}
+
+/* Fix expand icon position */
+.stExpander summary svg {
+    height: 1.5rem;
+}
+
+/* Fix summary icons */
+.stExpander summary [data-testid="stExpanderIconCheck"] {
+    font-size: 1.8rem;
+    margin-top: -3px;
+    color: rgb(21, 130, 55);
+}
+.stExpander summary [data-testid="stExpanderIconError"] {
+    font-size: 1.8rem;
+    margin-top: -3px;
+    color: rgb(255, 43, 43);
+}
+.stExpander summary span:first-child span:first-child {
+    width: 1.6rem;
+}
+
+/* Add border between title and content */
+.stExpander [data-testid="stExpanderDetails"] {
+    border-top: 1px solid rgba(30, 37, 47, 0.2);
+    padding-top: 0.5rem;
+}
+
+/* Fix title font size */
+.stExpander summary p {
+    font-size: 1rem;
+}
+
+/* Gray out content */
+.stExpander [data-testid="stExpanderDetails"] {
+    color: rgba(30, 37, 47, 0.5);
+}
+
+/* Fix progress bar font size */
+.stExpander .stProgress p {
+    line-height: 1.6;
+    font-size: 1rem;
+    color: rgba(30, 37, 47, 0.5);
+}
+</style>
+"""

kumoai/utils/sql.py

@@ -0,0 +1,3 @@
+def quote_ident(name: str) -> str:
+    r"""Quotes a SQL identifier."""
+    return '"' + name.replace('"', '""') + '"'