kumoai 2.13.0.dev202512031731__cp312-cp312-macosx_11_0_arm64.whl → 2.14.0.dev202512301731__cp312-cp312-macosx_11_0_arm64.whl

kumoai/experimental/rfm/sagemaker.py

@@ -1,6 +1,6 @@
 import base64
 import json
-from typing import Any, Dict, List, Tuple
+from typing import Any
 
 import requests
 
@@ -48,8 +48,8 @@ class KumoClient_SageMakerAdapter(KumoClient):
 
         # Recording buffers.
         self._recording_active = False
-        self._recorded_reqs: List[Dict[str, Any]] = []
-        self._recorded_resps: List[Dict[str, Any]] = []
+        self._recorded_reqs: list[dict[str, Any]] = []
+        self._recorded_resps: list[dict[str, Any]] = []
 
     def authenticate(self) -> None:
         # TODO(siyang): call /ping to verify?
@@ -92,7 +92,7 @@ class KumoClient_SageMakerAdapter(KumoClient):
         self._recorded_reqs.clear()
         self._recorded_resps.clear()
 
-    def end_recording(self) -> List[Tuple[Dict[str, Any], Dict[str, Any]]]:
+    def end_recording(self) -> list[tuple[dict[str, Any], dict[str, Any]]]:
         """Stop recording and return recorded requests/responses."""
         assert self._recording_active
         self._recording_active = False

kumoai/pquery/predictive_query.py

@@ -370,9 +370,11 @@ class PredictiveQuery:
         train_table_job_api = global_state.client.generate_train_table_job_api
         job_id: GenerateTrainTableJobID = train_table_job_api.create(
             GenerateTrainTableRequest(
-                dict(custom_tags), pq_id, plan,
-                graph_snapshot_id=self.graph.snapshot(
-                    non_blocking=non_blocking)))
+                dict(custom_tags),
+                pq_id,
+                plan,
+                None,
+            ))
 
         self._train_table = TrainingTableJob(job_id=job_id)
         if non_blocking:
@@ -451,9 +453,11 @@ class PredictiveQuery:
         bp_table_api = global_state.client.generate_prediction_table_job_api
         job_id: GeneratePredictionTableJobID = bp_table_api.create(
             GeneratePredictionTableRequest(
-                dict(custom_tags), pq_id, plan,
-                graph_snapshot_id=self.graph.snapshot(
-                    non_blocking=non_blocking)))
+                dict(custom_tags),
+                pq_id,
+                plan,
+                None,
+            ))
 
         self._prediction_table = PredictionTableJob(job_id=job_id)
         if non_blocking:

kumoai/testing/snow.py

@@ -0,0 +1,50 @@
+import json
+import os
+
+from kumoai.experimental.rfm.backend.snow import Connection
+from kumoai.experimental.rfm.backend.snow import connect as _connect
+
+
+def connect(
+    region: str,
+    id: str,
+    account: str,
+    user: str,
+    warehouse: str,
+    database: str | None = None,
+    schema: str | None = None,
+) -> Connection:
+
+    kwargs = dict(password=os.getenv('SNOWFLAKE_PASSWORD'))
+    if kwargs['password'] is None:
+        import boto3
+        from cryptography.hazmat.primitives import serialization
+
+        client = boto3.client(
+            service_name='secretsmanager',
+            region_name=region,
+        )
+        secret_id = (f'arn:aws:secretsmanager:{region}:{id}:secret:'
+                     f'{account}.snowflakecomputing.com')
+        response = client.get_secret_value(SecretId=secret_id)['SecretString']
+        secret = json.loads(response)
+
+        private_key = serialization.load_pem_private_key(
+            secret['kumo_user_secretkey'].encode(),
+            password=None,
+        )
+        kwargs['private_key'] = private_key.private_bytes(
+            encoding=serialization.Encoding.DER,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+    return _connect(
+        account=account,
+        user=user,
+        warehouse='WH_XS',
+        database='KUMO',
+        schema=schema,
+        session_parameters=dict(CLIENT_TELEMETRY_ENABLED=False),
+        **kwargs,
+    )

kumoai/trainer/distilled_trainer.py

@@ -0,0 +1,175 @@
+import logging
+from typing import Literal, Mapping, Optional, Union, overload
+
+from kumoapi.distilled_model_plan import DistilledModelPlan
+from kumoapi.jobs import DistillationJobRequest, DistillationJobResource
+
+from kumoai import global_state
+from kumoai.client.jobs import TrainingJobID
+from kumoai.graph import Graph
+from kumoai.pquery.training_table import TrainingTable, TrainingTableJob
+from kumoai.trainer.job import TrainingJob, TrainingJobResult
+
+logger = logging.getLogger(__name__)
+
+
+class DistillationTrainer:
+    r"""A trainer supports creating a Kumo machine learning model
+    for use in an online serving endpoint. The distllation process involes
+    training a shallow model on a :class:`~kumoai.pquery.PredictiveQuery` using
+    the embeddings generated by a base model :args:`base_training_job_id`.
+
+    Args:
+        model_plan: The distilled model plan to use for the distillation process.
+        base_training_job_id: The ID of the base training job to use for the distillation process.
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        model_plan: DistilledModelPlan,
+        base_training_job_id: TrainingJobID,
+    ) -> None:
+        self.model_plan: DistilledModelPlan = model_plan
+        self.base_training_job_id: TrainingJobID = base_training_job_id
+
+        # Cached from backend:
+        self._training_job_id: Optional[TrainingJobID] = None
+
+    # Metadata ################################################################
+
+    @property
+    def is_trained(self) -> bool:
+        r"""Returns ``True`` if this trainer instance has successfully been
+        trained (and is therefore ready for prediction); ``False`` otherwise.
+        """
+        raise NotImplementedError(
+            "Checking if a distilled trainer is trained is not "
+            "implemented yet.")
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+    ) -> TrainingJobResult:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: Literal[False],
+    ) -> TrainingJobResult:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: Literal[True],
+    ) -> TrainingJob:
+        pass
+
+    @overload
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: bool,
+    ) -> Union[TrainingJob, TrainingJobResult]:
+        pass
+
+    def fit(
+        self,
+        graph: Graph,
+        train_table: Union[TrainingTable, TrainingTableJob],
+        *,
+        non_blocking: bool = False,
+        custom_tags: Mapping[str, str] = {},
+    ) -> Union[TrainingJob, TrainingJobResult]:
+        r"""Fits a model to the specified graph and training table, with the
+        strategy defined by :class:`DistilledTrainer`'s :obj:`model_plan`.
+
+        Args:
+            graph: The :class:`~kumoai.graph.Graph` object that represents the
+                tables and relationships that Kumo will learn from.
+            train_table: The :class:`~kumoai.pquery.TrainingTable`, or
+                in-progress :class:`~kumoai.pquery.TrainingTableJob`, that
+                represents the training data produced by a
+                :class:`~kumoai.pquery.PredictiveQuery` on :obj:`graph`.
+            non_blocking: Whether this operation should return immediately
+                after launching the training job, or await completion of the
+                training job.
+            custom_tags: Additional, customer defined k-v tags to be associated
+                with the job to be launched. Job tags are useful for grouping
+                and searching jobs.
+
+        Returns:
+            Union[TrainingJobResult, TrainingJob]:
+                If ``non_blocking=False``, returns a training job object. If
+                ``non_blocking=True``, returns a training job future object.
+        """
+        # TODO(manan, siyang): remove soon:
+        job_id = train_table.job_id
+        assert job_id is not None
+
+        train_table_job_api = global_state.client.generate_train_table_job_api
+        pq_id = train_table_job_api.get(job_id).config.pquery_id
+        assert pq_id is not None
+
+        custom_table = None
+        if isinstance(train_table, TrainingTable):
+            custom_table = train_table._custom_train_table
+
+        # NOTE the backend implementation currently handles sequentialization
+        # between a training table future and a training job; that is, if the
+        # training table future is still executing, the backend will wait on
+        # the job ID completion before executing a training job. This preserves
+        # semantics for both futures, ensures that Kumo works as expected if
+        # used only via REST API, and allows us to avoid chaining calllbacks
+        # in an ugly way here:
+        api = global_state.client.distillation_job_api
+        self._training_job_id = api.create(
+            DistillationJobRequest(
+                dict(custom_tags),
+                pquery_id=pq_id,
+                base_training_job_id=self.base_training_job_id,
+                distilled_model_plan=self.model_plan,
+                graph_snapshot_id=graph.snapshot(non_blocking=non_blocking),
+                train_table_job_id=job_id,
+                custom_train_table=custom_table,
+            ))
+
+        out = TrainingJob(job_id=self._training_job_id)
+        if non_blocking:
+            return out
+        return out.attach()
+
+    @classmethod
+    def _load_from_job(
+        cls,
+        job: DistillationJobResource,
+    ) -> 'DistillationTrainer':
+        trainer = cls(job.config.distilled_model_plan,
+                      job.config.base_training_job_id)
+        trainer._training_job_id = job.job_id
+        return trainer
+
+    @classmethod
+    def load(cls, job_id: TrainingJobID) -> 'DistillationTrainer':
+        r"""Creates a :class:`~kumoai.trainer.Trainer` instance from a training
+        job ID.
+        """
+        raise NotImplementedError(
+            "Loading a distilled trainer from a job ID is not implemented yet."
+        )
+
+    @classmethod
+    def load_from_tags(cls, tags: Mapping[str, str]) -> 'DistillationTrainer':
+        raise NotImplementedError(
+            "Loading a distilled trainer from tags is not implemented yet.")

kumoai/utils/__init__.py

@@ -1,10 +1,11 @@
-from .progress_logger import ProgressLogger, InteractiveProgressLogger
+from .sql import quote_ident
+from .progress_logger import ProgressLogger
 from .forecasting import ForecastVisualizer
 from .datasets import from_relbench
 
 __all__ = [
+    'quote_ident',
     'ProgressLogger',
-    'InteractiveProgressLogger',
     'ForecastVisualizer',
     'from_relbench',
 ]

kumoai/utils/display.py

@@ -0,0 +1,51 @@
+from collections.abc import Sequence
+
+import pandas as pd
+
+from kumoai import in_notebook, in_snowflake_notebook
+
+
+def message(msg: str) -> None:
+    msg = msg.replace("`", "'") if not in_notebook() else msg
+
+    if in_snowflake_notebook():
+        import streamlit as st
+        st.markdown(msg)
+    elif in_notebook():
+        from IPython.display import Markdown, display
+        display(Markdown(msg))
+    else:
+        print(msg)
+
+
+def title(msg: str) -> None:
+    message(f"### {msg}" if in_notebook() else f"{msg}:")
+
+
+def italic(msg: str) -> None:
+    message(f"*{msg}*" if in_notebook() else msg)
+
+
+def unordered_list(items: Sequence[str]) -> None:
+    if in_notebook():
+        msg = '\n'.join([f"- {item}" for item in items])
+    else:
+        msg = '\n'.join([f"• {item.replace('`', '')}" for item in items])
+    message(msg)
+
+
+def dataframe(df: pd.DataFrame) -> None:
+    if in_snowflake_notebook():
+        import streamlit as st
+        st.dataframe(df, hide_index=True)
+    elif in_notebook():
+        from IPython.display import display
+        try:
+            if hasattr(df.style, 'hide'):
+                display(df.style.hide(axis='index'))  # pandas=2
+            else:
+                display(df.style.hide_index())  # pandas<1.3
+        except ImportError:
+            print(df.to_string(index=False))  # missing jinja2
+    else:
+        print(df.to_string(index=False))

kumoai/utils/progress_logger.py

@@ -1,6 +1,7 @@
+import re
 import sys
 import time
-from typing import Any, List, Optional, Union
+from typing import Any
 
 from rich.console import Console, ConsoleOptions, RenderResult
 from rich.live import Live
@@ -20,12 +21,22 @@ from typing_extensions import Self
 
 
 class ProgressLogger:
-    def __init__(self, msg: str) -> None:
+    def __init__(self, msg: str, verbose: bool = True) -> None:
         self.msg = msg
-        self.logs: List[str] = []
+        self.verbose = verbose
+
+        self.logs: list[str] = []
+
+        self.start_time: float | None = None
+        self.end_time: float | None = None
+
+    @classmethod
+    def default(cls, msg: str, verbose: bool = True) -> 'ProgressLogger':
+        from kumoai import in_snowflake_notebook
 
-        self.start_time: Optional[float] = None
-        self.end_time: Optional[float] = None
+        if in_snowflake_notebook():
+            return StreamlitProgressLogger(msg, verbose)
+        return RichProgressLogger(msg, verbose)
 
     @property
     def duration(self) -> float:
@@ -37,6 +48,12 @@ class ProgressLogger:
     def log(self, msg: str) -> None:
         self.logs.append(msg)
 
+    def init_progress(self, total: int, description: str) -> None:
+        pass
+
+    def step(self) -> None:
+        pass
+
     def __enter__(self) -> Self:
         self.start_time = time.perf_counter()
         return self
@@ -66,22 +83,21 @@ class ColoredTimeRemainingColumn(TimeRemainingColumn):
         return Text(str(super().render(task)), style=self.style)
 
 
-class InteractiveProgressLogger(ProgressLogger):
+class RichProgressLogger(ProgressLogger):
     def __init__(
         self,
         msg: str,
         verbose: bool = True,
         refresh_per_second: int = 10,
     ) -> None:
-        super().__init__(msg=msg)
+        super().__init__(msg=msg, verbose=verbose)
 
-        self.verbose = verbose
         self.refresh_per_second = refresh_per_second
 
-        self._progress: Optional[Progress] = None
-        self._task: Optional[int] = None
+        self._progress: Progress | None = None
+        self._task: int | None = None
 
-        self._live: Optional[Live] = None
+        self._live: Live | None = None
         self._exception: bool = False
 
     def init_progress(self, total: int, description: str) -> None:
@@ -151,7 +167,7 @@ class InteractiveProgressLogger(ProgressLogger):
 
         table = Table.grid(padding=(0, 1))
 
-        icon: Union[Text, Padding]
+        icon: Text | Padding
         if self._exception:
             style = 'red'
             icon = Text('❌', style=style)
@@ -175,3 +191,153 @@ class InteractiveProgressLogger(ProgressLogger):
 
         if self.verbose and self._progress is not None:
             yield self._progress.get_renderable()
+
+
+class StreamlitProgressLogger(ProgressLogger):
+    def __init__(
+        self,
+        msg: str,
+        verbose: bool = True,
+    ) -> None:
+        super().__init__(msg=msg, verbose=verbose)
+
+        self._status: Any = None
+
+        self._total = 0
+        self._current = 0
+        self._description: str = ''
+        self._progress: Any = None
+
+    def __enter__(self) -> Self:
+        super().__enter__()
+
+        import streamlit as st
+
+        # Adjust layout for prettier output:
+        st.markdown(STREAMLIT_CSS, unsafe_allow_html=True)
+
+        if self.verbose:
+            self._status = st.status(
+                f':blue[{self._sanitize_text(self.msg)}]',
+                expanded=True,
+            )
+
+        return self
+
+    def log(self, msg: str) -> None:
+        super().log(msg)
+        if self.verbose and self._status is not None:
+            self._status.write(self._sanitize_text(msg))
+
+    def init_progress(self, total: int, description: str) -> None:
+        if self.verbose and self._status is not None:
+            self._total = total
+            self._current = 0
+            self._description = self._sanitize_text(description)
+            percent = min(self._current / self._total, 1.0)
+            self._progress = self._status.progress(
+                value=percent,
+                text=f'{self._description} [{self._current}/{self._total}]',
+            )
+
+    def step(self) -> None:
+        self._current += 1
+
+        if self.verbose and self._progress is not None:
+            percent = min(self._current / self._total, 1.0)
+            self._progress.progress(
+                value=percent,
+                text=f'{self._description} [{self._current}/{self._total}]',
+            )
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        super().__exit__(exc_type, exc_val, exc_tb)
+
+        if not self.verbose or self._status is None:
+            return
+
+        label = f'{self._sanitize_text(self.msg)} ({self.duration:.2f}s)'
+
+        if exc_type is not None:
+            self._status.update(
+                label=f':red[{label}]',
+                state='error',
+                expanded=True,
+            )
+        else:
+            self._status.update(
+                label=f':green[{label}]',
+                state='complete',
+                expanded=True,
+            )
+
+    @staticmethod
+    def _sanitize_text(msg: str) -> str:
+        return re.sub(r'\[/?bold\]', '**', msg)
+
+
+STREAMLIT_CSS = """
+<style>
+/* Fix horizontal scrollbar */
+.stExpander summary {
+    width: auto;
+}
+
+/* Fix paddings/margins */
+.stExpander summary {
+    padding: 0.75rem 1rem 0.5rem;
+}
+.stExpander p {
+    margin: 0px 0px 0.2rem;
+}
+.stExpander [data-testid="stExpanderDetails"] {
+    padding-bottom: 1.45rem;
+}
+.stExpander .stProgress div:first-child {
+    padding-bottom: 4px;
+}
+
+/* Fix expand icon position */
+.stExpander summary svg {
+    height: 1.5rem;
+}
+
+/* Fix summary icons */
+.stExpander summary [data-testid="stExpanderIconCheck"] {
+    font-size: 1.8rem;
+    margin-top: -3px;
+    color: rgb(21, 130, 55);
+}
+.stExpander summary [data-testid="stExpanderIconError"] {
+    font-size: 1.8rem;
+    margin-top: -3px;
+    color: rgb(255, 43, 43);
+}
+.stExpander summary span:first-child span:first-child {
+    width: 1.6rem;
+}
+
+/* Add border between title and content */
+.stExpander [data-testid="stExpanderDetails"] {
+    border-top: 1px solid rgba(30, 37, 47, 0.2);
+    padding-top: 0.5rem;
+}
+
+/* Fix title font size */
+.stExpander summary p {
+    font-size: 1rem;
+}
+
+/* Gray out content */
+.stExpander [data-testid="stExpanderDetails"] {
+    color: rgba(30, 37, 47, 0.5);
+}
+
+/* Fix progress bar font size */
+.stExpander .stProgress p {
+    line-height: 1.6;
+    font-size: 1rem;
+    color: rgba(30, 37, 47, 0.5);
+}
+</style>
+"""

kumoai/utils/sql.py

@@ -0,0 +1,3 @@
+def quote_ident(name: str) -> str:
+    r"""Quotes a SQL identifier."""
+    return '"' + name.replace('"', '""') + '"'

{kumoai-2.13.0.dev202512031731.dist-info → kumoai-2.14.0.dev202512301731.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kumoai
-Version: 2.13.0.dev202512031731
+Version: 2.14.0.dev202512301731
 Summary: AI on the Modern Data Stack
 Author-email: "Kumo.AI" <hello@kumo.ai>
 License-Expression: MIT
@@ -23,7 +23,7 @@ Requires-Dist: requests>=2.28.2
 Requires-Dist: urllib3
 Requires-Dist: plotly
 Requires-Dist: typing_extensions>=4.5.0
-Requires-Dist: kumo-api==0.48.0
+Requires-Dist: kumo-api==0.49.0
 Requires-Dist: tqdm>=4.66.0
 Requires-Dist: aiohttp>=3.10.0
 Requires-Dist: pydantic>=1.10.21
@@ -41,7 +41,9 @@ Requires-Dist: requests-mock; extra == "test"
 Provides-Extra: sqlite
 Requires-Dist: adbc_driver_sqlite; extra == "sqlite"
 Provides-Extra: snowflake
+Requires-Dist: numpy<2.0; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: pyyaml; extra == "snowflake"
 Provides-Extra: sagemaker
 Requires-Dist: boto3<2.0,>=1.30.0; extra == "sagemaker"
 Requires-Dist: mypy-boto3-sagemaker-runtime<2.0,>=1.34.0; extra == "sagemaker"