kumoai 2.13.0.dev202511261731__cp310-cp310-win_amd64.whl → 2.13.0.dev202512021731__cp310-cp310-win_amd64.whl

kumoai/experimental/rfm/{local_graph.py → graph.py}

@@ -3,7 +3,7 @@ import io
 import warnings
 from collections import defaultdict
 from importlib.util import find_spec
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
 
 import pandas as pd
 from kumoapi.graph import ColumnKey, ColumnKeyGroup, GraphDefinition
@@ -12,19 +12,19 @@ from kumoapi.typing import Stype
 from typing_extensions import Self
 
 from kumoai import in_notebook
-from kumoai.experimental.rfm import LocalTable
+from kumoai.experimental.rfm import Table
 from kumoai.graph import Edge
 
 if TYPE_CHECKING:
     import graphviz
 
 
-class LocalGraph:
-    r"""A graph of :class:`LocalTable` objects, akin to relationships between
+class Graph:
+    r"""A graph of :class:`Table` objects, akin to relationships between
     tables in a relational database.
 
     Creating a graph is the final step of data definition; after a
-    :class:`LocalGraph` is created, you can use it to initialize the
+    :class:`Graph` is created, you can use it to initialize the
     Kumo Relational Foundation Model (:class:`KumoRFM`).
 
     .. code-block:: python
@@ -44,7 +44,7 @@ class LocalGraph:
         >>> table3 = rfm.LocalTable(name="table3", data=df3)
 
         >>> # Create a graph from a dictionary of tables:
-        >>> graph = rfm.LocalGraph({
+        >>> graph = rfm.Graph({
         ...     "table1": table1,
         ...     "table2": table2,
         ...     "table3": table3,
@@ -75,33 +75,44 @@ class LocalGraph:
 
     def __init__(
         self,
-        tables: List[LocalTable],
-        edges: Optional[List[Edge]] = None,
+        tables: Sequence[Table],
+        edges: Optional[Sequence[Edge]] = None,
     ) -> None:
 
-        self._tables: Dict[str, LocalTable] = {}
+        self._tables: Dict[str, Table] = {}
         self._edges: List[Edge] = []
 
         for table in tables:
             self.add_table(table)
 
+        for table in tables:
+            for fkey, dst_table, pkey in table._get_source_foreign_keys():
+                if self[dst_table].primary_key is None:
+                    self[dst_table].primary_key = pkey
+                elif self[dst_table]._primary_key != pkey:
+                    raise ValueError(f"Found duplicate primary key definition "
+                                     f"'{self[dst_table]._primary_key}' and "
+                                     f"'{pkey}' in table '{dst_table}'.")
+                self.link(table.name, fkey, dst_table)
+
         for edge in (edges or []):
             _edge = Edge._cast(edge)
             assert _edge is not None
-            self.link(*_edge)
+            if _edge not in self._edges:
+                self.link(*_edge)
 
     @classmethod
     def from_data(
         cls,
         df_dict: Dict[str, pd.DataFrame],
-        edges: Optional[List[Edge]] = None,
+        edges: Optional[Sequence[Edge]] = None,
         infer_metadata: bool = True,
         verbose: bool = True,
     ) -> Self:
-        r"""Creates a :class:`LocalGraph` from a dictionary of
+        r"""Creates a :class:`Graph` from a dictionary of
         :class:`pandas.DataFrame` objects.
 
-        Automatically infers table metadata and links.
+        Automatically infers table metadata and links by default.
 
         .. code-block:: python
 
@@ -115,7 +126,7 @@ class LocalGraph:
             >>> df3 = pd.DataFrame(...)
 
             >>> # Create a graph from a dictionary of data frames:
-            >>> graph = rfm.LocalGraph.from_data({
+            >>> graph = rfm.Graph.from_data({
             ...     "table1": df1,
             ...     "table2": df2,
             ...     "table3": df3,
@@ -131,39 +142,103 @@ class LocalGraph:
         Args:
             df_dict: A dictionary of data frames, where the keys are the names
                 of the tables and the values hold table data.
-            infer_metadata: Whether to infer metadata for all tables in the
-                graph.
             edges: An optional list of :class:`~kumoai.graph.Edge` objects to
                 add to the graph. If not provided, edges will be automatically
-                inferred from the data.
+                inferred from the data in case ``infer_metadata=True``.
+            infer_metadata: Whether to infer metadata for all tables in the
+                graph.
             verbose: Whether to print verbose output.
+        """
+        from kumoai.experimental.rfm.backend.local import LocalTable
+        tables = [LocalTable(df, name) for name, df in df_dict.items()]
 
-        Note:
-            This method will automatically infer metadata and links for the
-            graph.
+        graph = cls(tables, edges=edges or [])
+
+        if infer_metadata:
+            graph.infer_metadata(False)
+
+            if edges is None:
+                graph.infer_links(False)
+
+        if verbose:
+            graph.print_metadata()
+            graph.print_links()
+
+        return graph
+
+    @classmethod
+    def from_sqlite(
+        cls,
+        uri: Any,
+        table_names: Optional[Sequence[str]] = None,
+        edges: Optional[Sequence[Edge]] = None,
+        infer_metadata: bool = True,
+        verbose: bool = True,
+        conn_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Self:
+        r"""Creates a :class:`Graph` from a :class:`sqlite` database.
+
+        Automatically infers table metadata and links by default.
+
+        .. code-block:: python
 
-        Example:
             >>> # doctest: +SKIP
             >>> import kumoai.experimental.rfm as rfm
-            >>> df1 = pd.DataFrame(...)
-            >>> df2 = pd.DataFrame(...)
-            >>> df3 = pd.DataFrame(...)
-            >>> graph = rfm.LocalGraph.from_data(data={
-            ...     "table1": df1,
-            ...     "table2": df2,
-            ...     "table3": df3,
-            ... })
-            >>> graph.validate()
+
+            >>> # Create a graph from a SQLite database:
+            >>> graph = rfm.Graph.from_sqlite('data.db')
+
+            >>> # Inspect table metadata:
+            >>> for table in graph.tables.values():
+            ...     table.print_metadata()
+
+            >>> # Visualize graph (if graphviz is installed):
+            >>> graph.visualize()
+
+        Args:
+            uri: The path to the database file or an open connection obtained
+                from :meth:`~kumoai.experimental.rfm.backend.sqlite.connect`.
+            table_names: Set of table names to include. If ``None``, will add
+                all tables present in the database.
+            edges: An optional list of :class:`~kumoai.graph.Edge` objects to
+                add to the graph. If not provided, edges will be automatically
+                inferred from the data in case ``infer_metadata=True``.
+            infer_metadata: Whether to infer metadata for all tables in the
+                graph.
+            verbose: Whether to print verbose output.
+            conn_kwargs: Additional connection arguments, following the
+                :class:`adbc_driver_sqlite` protocol.
         """
-        tables = [LocalTable(df, name) for name, df in df_dict.items()]
+        from kumoai.experimental.rfm.backend.sqlite import (
+            Connection,
+            SQLiteTable,
+            connect,
+        )
+
+        if not isinstance(uri, Connection):
+            connection = connect(uri, **(conn_kwargs or {}))
+        else:
+            connection = uri
+
+        if table_names is None:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT name FROM sqlite_master "
+                               "WHERE type='table'")
+                table_names = [row[0] for row in cursor.fetchall()]
+
+        tables = [SQLiteTable(connection, name) for name in table_names]
 
         graph = cls(tables, edges=edges or [])
 
         if infer_metadata:
-            graph.infer_metadata(verbose)
+            graph.infer_metadata(False)
 
             if edges is None:
-                graph.infer_links(verbose)
+                graph.infer_links(False)
+
+        if verbose:
+            graph.print_metadata()
+            graph.print_links()
 
         return graph
 
@@ -175,7 +250,7 @@ class LocalGraph:
         """
         return name in self.tables
 
-    def table(self, name: str) -> LocalTable:
+    def table(self, name: str) -> Table:
         r"""Returns the table with name ``name`` in the graph.
 
         Raises:
@@ -186,11 +261,11 @@ class LocalGraph:
         return self.tables[name]
 
     @property
-    def tables(self) -> Dict[str, LocalTable]:
+    def tables(self) -> Dict[str, Table]:
         r"""Returns the dictionary of table objects."""
         return self._tables
 
-    def add_table(self, table: LocalTable) -> Self:
+    def add_table(self, table: Table) -> Self:
         r"""Adds a table to the graph.
 
         Args:
@@ -199,11 +274,21 @@ class LocalGraph:
         Raises:
             KeyError: If a table with the same name already exists in the
                 graph.
+            ValueError: If the table belongs to a different backend than the
+                rest of the tables in the graph.
         """
         if table.name in self._tables:
             raise KeyError(f"Cannot add table with name '{table.name}' to "
                            f"this graph; table names must be globally unique.")
 
+        if len(self._tables) > 0:
+            cls = next(iter(self._tables.values())).__class__
+            if table.__class__ != cls:
+                raise ValueError(f"Cannot register a "
+                                 f"'{table.__class__.__name__}' to this "
+                                 f"graph since other tables are of type "
+                                 f"'{cls.__name__}'.")
+
         self._tables[table.name] = table
 
         return self
@@ -241,7 +326,7 @@ class LocalGraph:
         Example:
             >>> # doctest: +SKIP
             >>> import kumoai.experimental.rfm as rfm
-            >>> graph = rfm.LocalGraph(tables=...).infer_metadata()
+            >>> graph = rfm.Graph(tables=...).infer_metadata()
             >>> graph.metadata  # doctest: +SKIP
                 name   primary_key  time_column  end_time_column
             0   users      user_id            -                -
@@ -263,7 +348,7 @@ class LocalGraph:
         })
 
     def print_metadata(self) -> None:
-        r"""Prints the :meth:`~LocalGraph.metadata` of the graph."""
+        r"""Prints the :meth:`~Graph.metadata` of the graph."""
         if in_notebook():
             from IPython.display import Markdown, display
             display(Markdown('### 🗂️ Graph Metadata'))
@@ -287,7 +372,7 @@ class LocalGraph:
 
         Note:
             For more information, please see
-            :meth:`kumoai.experimental.rfm.LocalTable.infer_metadata`.
+            :meth:`kumoai.experimental.rfm.Table.infer_metadata`.
         """
         for table in self.tables.values():
             table.infer_metadata(verbose=False)
@@ -305,7 +390,7 @@ class LocalGraph:
         return self._edges
 
     def print_links(self) -> None:
-        r"""Prints the :meth:`~LocalGraph.edges` of the graph."""
+        r"""Prints the :meth:`~Graph.edges` of the graph."""
         edges = [(edge.dst_table, self[edge.dst_table]._primary_key,
                   edge.src_table, edge.fkey) for edge in self.edges]
         edges = sorted(edges)
@@ -333,9 +418,9 @@ class LocalGraph:
 
     def link(
         self,
-        src_table: Union[str, LocalTable],
+        src_table: Union[str, Table],
         fkey: str,
-        dst_table: Union[str, LocalTable],
+        dst_table: Union[str, Table],
     ) -> Self:
         r"""Links two tables (``src_table`` and ``dst_table``) from the foreign
         key ``fkey`` in the source table to the primary key in the destination
@@ -358,11 +443,11 @@ class LocalGraph:
                 table does not exist in the graph, if the source key does not
                 exist in the source table.
         """
-        if isinstance(src_table, LocalTable):
+        if isinstance(src_table, Table):
             src_table = src_table.name
         assert isinstance(src_table, str)
 
-        if isinstance(dst_table, LocalTable):
+        if isinstance(dst_table, Table):
             dst_table = dst_table.name
         assert isinstance(dst_table, str)
 
@@ -396,9 +481,9 @@ class LocalGraph:
 
     def unlink(
         self,
-        src_table: Union[str, LocalTable],
+        src_table: Union[str, Table],
         fkey: str,
-        dst_table: Union[str, LocalTable],
+        dst_table: Union[str, Table],
     ) -> Self:
         r"""Removes an :class:`~kumoai.graph.Edge` from the graph.
 
@@ -410,11 +495,11 @@ class LocalGraph:
         Raises:
             ValueError: if the edge is not present in the graph.
         """
-        if isinstance(src_table, LocalTable):
+        if isinstance(src_table, Table):
             src_table = src_table.name
         assert isinstance(src_table, str)
 
-        if isinstance(dst_table, LocalTable):
+        if isinstance(dst_table, Table):
             dst_table = dst_table.name
         assert isinstance(dst_table, str)
 
@@ -428,17 +513,13 @@ class LocalGraph:
         return self
 
     def infer_links(self, verbose: bool = True) -> Self:
-        r"""Infers links for the tables and adds them as edges to the graph.
+        r"""Infers missing links for the tables and adds them as edges to the
+        graph.
 
         Args:
             verbose: Whether to print verbose output.
-
-        Note:
-            This function expects graph edges to be undefined upfront.
         """
-        if len(self.edges) > 0:
-            warnings.warn("Cannot infer links if graph edges already exist")
-            return self
+        known_edges = {(edge.src_table, edge.fkey) for edge in self.edges}
 
         # A list of primary key candidates (+score) for every column:
         candidate_dict: dict[
@@ -463,6 +544,9 @@ class LocalGraph:
                 src_table_name = src_table.name.lower()
 
                 for src_key in src_table.columns:
+                    if (src_table.name, src_key.name) in known_edges:
+                        continue
+
                     if src_key == src_table.primary_key:
                         continue  # Cannot link to primary key.
 
@@ -528,7 +612,10 @@ class LocalGraph:
                         score += 1.0
 
                     # Cardinality ratio:
-                    if len(src_table._data) > len(dst_table._data):
+                    src_num_rows = src_table._num_rows()
+                    dst_num_rows = dst_table._num_rows()
+                    if (src_num_rows is not None and dst_num_rows is not None
+                            and src_num_rows > dst_num_rows):
                         score += 1.0
 
                     if score < 5.0:
@@ -790,7 +877,7 @@ class LocalGraph:
     def __contains__(self, name: str) -> bool:
         return self.has_table(name)
 
-    def __getitem__(self, name: str) -> LocalTable:
+    def __getitem__(self, name: str) -> Table:
         return self.table(name)
 
     def __delitem__(self, name: str) -> None:

kumoai/experimental/rfm/infer/__init__.py

@@ -2,10 +2,12 @@ from .id import contains_id
 from .timestamp import contains_timestamp
 from .categorical import contains_categorical
 from .multicategorical import contains_multicategorical
+from .stype import infer_stype
 
 __all__ = [
     'contains_id',
     'contains_timestamp',
     'contains_categorical',
     'contains_multicategorical',
+    'infer_stype',
 ]

kumoai/experimental/rfm/infer/stype.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from kumoapi.typing import Dtype, Stype
+
+from kumoai.experimental.rfm.infer import (
+    contains_categorical,
+    contains_id,
+    contains_multicategorical,
+    contains_timestamp,
+)
+
+
+def infer_stype(ser: pd.Series, column_name: str, dtype: Dtype) -> Stype:
+    r"""Infers the semantic type of a column.
+
+    Args:
+        ser: A :class:`pandas.Series` to analyze.
+        column_name: The name of the column (used for pattern matching).
+        dtype: The data type.
+
+    Returns:
+        The semantic type.
+    """
+    if contains_id(ser, column_name, dtype):
+        return Stype.ID
+
+    if contains_timestamp(ser, column_name, dtype):
+        return Stype.timestamp
+
+    if contains_multicategorical(ser, column_name, dtype):
+        return Stype.multicategorical
+
+    if contains_categorical(ser, column_name, dtype):
+        return Stype.categorical
+
+    return dtype.default_stype

kumoai/experimental/rfm/local_graph_store.py

@@ -6,7 +6,7 @@ import pandas as pd
 from kumoapi.rfm.context import Subgraph
 from kumoapi.typing import Stype
 
-from kumoai.experimental.rfm import LocalGraph
+from kumoai.experimental.rfm import Graph, LocalTable
 from kumoai.experimental.rfm.utils import normalize_text
 from kumoai.utils import InteractiveProgressLogger, ProgressLogger
 
@@ -20,7 +20,7 @@ except ImportError:
 class LocalGraphStore:
     def __init__(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
@@ -105,7 +105,7 @@ class LocalGraphStore:
 
     def sanitize(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
     ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, np.ndarray]]:
         r"""Sanitizes raw data according to table schema definition:
@@ -120,10 +120,11 @@ class LocalGraphStore:
         data for faster model processing. In particular, it:
         * tokenizes any text column that is not a foreign key
         """
-        df_dict: Dict[str, pd.DataFrame] = {
-            table_name: table._data.copy(deep=False).reset_index(drop=True)
-            for table_name, table in graph.tables.items()
-        }
+        df_dict: Dict[str, pd.DataFrame] = {}
+        for table_name, table in graph.tables.items():
+            assert isinstance(table, LocalTable)
+            df = table._data
+            df_dict[table_name] = df.copy(deep=False).reset_index(drop=True)
 
         foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
 
@@ -165,7 +166,7 @@ class LocalGraphStore:
 
         return df_dict, mask_dict
 
-    def get_stype_dict(self, graph: LocalGraph) -> Dict[str, Dict[str, Stype]]:
+    def get_stype_dict(self, graph: Graph) -> Dict[str, Dict[str, Stype]]:
         stype_dict: Dict[str, Dict[str, Stype]] = {}
         foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
         for table in graph.tables.values():
@@ -180,7 +181,7 @@ class LocalGraphStore:
 
     def get_pkey_data(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[str, str],
             Dict[str, pd.DataFrame],
@@ -218,7 +219,7 @@ class LocalGraphStore:
 
     def get_time_data(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[str, str],
             Dict[str, str],
@@ -259,7 +260,7 @@ class LocalGraphStore:
 
     def get_csc(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[Tuple[str, str, str], np.ndarray],
             Dict[Tuple[str, str, str], np.ndarray],

kumoai/experimental/rfm/rfm.py

@@ -32,7 +32,7 @@ from kumoapi.task import TaskType
 
 from kumoai.client.rfm import RFMAPI
 from kumoai.exceptions import HTTPException
-from kumoai.experimental.rfm import LocalGraph
+from kumoai.experimental.rfm import Graph
 from kumoai.experimental.rfm.local_graph_sampler import LocalGraphSampler
 from kumoai.experimental.rfm.local_graph_store import LocalGraphStore
 from kumoai.experimental.rfm.local_pquery_driver import (
@@ -123,17 +123,17 @@ class KumoRFM:
     :class:`KumoRFM` is a foundation model to generate predictions for any
     relational dataset without training.
     The model is pre-trained and the class provides an interface to query the
-    model from a :class:`LocalGraph` object.
+    model from a :class:`Graph` object.
 
     .. code-block:: python
 
-        from kumoai.experimental.rfm import LocalGraph, KumoRFM
+        from kumoai.experimental.rfm import Graph, KumoRFM
 
         df_users = pd.DataFrame(...)
         df_items = pd.DataFrame(...)
         df_orders = pd.DataFrame(...)
 
-        graph = LocalGraph.from_data({
+        graph = Graph.from_data({
             'users': df_users,
             'items': df_items,
             'orders': df_orders,
@@ -163,7 +163,7 @@ class KumoRFM:
     """
     def __init__(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:

kumoai/experimental/rfm/sagemaker.py

@@ -2,15 +2,22 @@ import base64
 import json
 from typing import Any, Dict, List, Tuple
 
-import boto3
 import requests
-from mypy_boto3_sagemaker_runtime.client import SageMakerRuntimeClient
-from mypy_boto3_sagemaker_runtime.type_defs import InvokeEndpointOutputTypeDef
 
 from kumoai.client import KumoClient
 from kumoai.client.endpoints import Endpoint, HTTPMethod
 from kumoai.exceptions import HTTPException
 
+try:
+    # isort: off
+    from mypy_boto3_sagemaker_runtime.client import SageMakerRuntimeClient
+    from mypy_boto3_sagemaker_runtime.type_defs import (
+        InvokeEndpointOutputTypeDef, )
+    # isort: on
+except ImportError:
+    SageMakerRuntimeClient = Any
+    InvokeEndpointOutputTypeDef = Any
+
 
 class SageMakerResponseAdapter(requests.Response):
     def __init__(self, sm_response: InvokeEndpointOutputTypeDef):
@@ -34,6 +41,7 @@ class SageMakerResponseAdapter(requests.Response):
 
 class KumoClient_SageMakerAdapter(KumoClient):
     def __init__(self, region: str, endpoint_name: str):
+        import boto3
         self._client: SageMakerRuntimeClient = boto3.client(
             service_name="sagemaker-runtime", region_name=region)
         self._endpoint_name = endpoint_name