kumoai 2.13.0.dev202511231731__cp312-cp312-macosx_11_0_arm64.whl → 2.13.0.dev202512011731__cp312-cp312-macosx_11_0_arm64.whl

kumoai/experimental/rfm/{local_table.py → base/table.py}

@@ -1,5 +1,5 @@
-from dataclasses import dataclass
-from typing import Any, Dict, List, Optional
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, Sequence
 
 import pandas as pd
 from kumoapi.source_table import UnavailableSourceTable
@@ -9,107 +9,17 @@ from kumoapi.typing import Dtype, Stype
 from typing_extensions import Self
 
 from kumoai import in_notebook
-from kumoai.experimental.rfm import utils
+from kumoai.experimental.rfm.base import Column
 
 
-@dataclass(init=False, repr=False, eq=False)
-class Column:
-    stype: Stype
-
-    def __init__(
-        self,
-        name: str,
-        dtype: Dtype,
-        stype: Stype,
-        is_primary_key: bool = False,
-        is_time_column: bool = False,
-        is_end_time_column: bool = False,
-    ) -> None:
-        self._name = name
-        self._dtype = Dtype(dtype)
-        self._is_primary_key = is_primary_key
-        self._is_time_column = is_time_column
-        self._is_end_time_column = is_end_time_column
-        self.stype = Stype(stype)
-
-    @property
-    def name(self) -> str:
-        return self._name
-
-    @property
-    def dtype(self) -> Dtype:
-        return self._dtype
-
-    def __setattr__(self, key: str, val: Any) -> None:
-        if key == 'stype':
-            if isinstance(val, str):
-                val = Stype(val)
-            assert isinstance(val, Stype)
-            if not val.supports_dtype(self.dtype):
-                raise ValueError(f"Column '{self.name}' received an "
-                                 f"incompatible semantic type (got "
-                                 f"dtype='{self.dtype}' and stype='{val}')")
-            if self._is_primary_key and val != Stype.ID:
-                raise ValueError(f"Primary key '{self.name}' must have 'ID' "
-                                 f"semantic type (got '{val}')")
-            if self._is_time_column and val != Stype.timestamp:
-                raise ValueError(f"Time column '{self.name}' must have "
-                                 f"'timestamp' semantic type (got '{val}')")
-            if self._is_end_time_column and val != Stype.timestamp:
-                raise ValueError(f"End time column '{self.name}' must have "
-                                 f"'timestamp' semantic type (got '{val}')")
-
-        super().__setattr__(key, val)
-
-    def __hash__(self) -> int:
-        return hash((self.name, self.stype, self.dtype))
-
-    def __eq__(self, other: Any) -> bool:
-        if not isinstance(other, Column):
-            return False
-        return hash(self) == hash(other)
-
-    def __repr__(self) -> str:
-        return (f'{self.__class__.__name__}(name={self.name}, '
-                f'stype={self.stype}, dtype={self.dtype})')
-
-
-class LocalTable:
-    r"""A table backed by a :class:`pandas.DataFrame`.
-
-    A :class:`LocalTable` fully specifies the relevant metadata, *i.e.*
-    selected columns, column semantic types, primary keys and time columns.
-    :class:`LocalTable` is used to create a :class:`LocalGraph`.
-
-    .. code-block:: python
-
-        import pandas as pd
-        import kumoai.experimental.rfm as rfm
-
-        # Load data from a CSV file:
-        df = pd.read_csv("data.csv")
-
-        # Create a table from a `pandas.DataFrame` and infer its metadata ...
-        table = rfm.LocalTable(df, name="my_table").infer_metadata()
-
-        # ... or create a table explicitly:
-        table = rfm.LocalTable(
-            df=df,
-            name="my_table",
-            primary_key="id",
-            time_column="time",
-            end_time_column=None,
-        )
-
-        # Verify metadata:
-        table.print_metadata()
-
-        # Change the semantic type of a column:
-        table[column].stype = "text"
+class Table(ABC):
+    r"""A :class:`Table` fully specifies the relevant metadata of a single
+    table, *i.e.* its selected columns, data types, semantic types, primary
+    keys and time columns.
 
     Args:
-        df: The data frame to create the table from.
-        name: The name of the table.
+        name: The name of this table.
+        columns: The selected columns of this table.
         primary_key: The name of the primary key of this table, if it exists.
         time_column: The name of the time column of this table, if it exists.
         end_time_column: The name of the end time column of this table, if it
@@ -117,46 +27,40 @@ class LocalTable:
     """
     def __init__(
         self,
-        df: pd.DataFrame,
         name: str,
+        columns: Optional[Sequence[str]] = None,
         primary_key: Optional[str] = None,
         time_column: Optional[str] = None,
         end_time_column: Optional[str] = None,
     ) -> None:
 
-        if df.empty:
-            raise ValueError("Data frame must have at least one row")
-        if isinstance(df.columns, pd.MultiIndex):
-            raise ValueError("Data frame must not have a multi-index")
-        if not df.columns.is_unique:
-            raise ValueError("Data frame must have unique column names")
-        if any(col == '' for col in df.columns):
-            raise ValueError("Data frame must have non-empty column names")
-
-        df = df.copy(deep=False)
-
-        self._data = df
         self._name = name
         self._primary_key: Optional[str] = None
         self._time_column: Optional[str] = None
         self._end_time_column: Optional[str] = None
 
         self._columns: Dict[str, Column] = {}
-        for column_name in df.columns:
+        for column_name in columns or []:
             self.add_column(column_name)
 
         if primary_key is not None:
+            if primary_key not in self:
+                self.add_column(primary_key)
             self.primary_key = primary_key
 
         if time_column is not None:
+            if time_column not in self:
+                self.add_column(time_column)
             self.time_column = time_column
 
         if end_time_column is not None:
+            if end_time_column not in self:
+                self.add_column(end_time_column)
             self.end_time_column = end_time_column
 
     @property
     def name(self) -> str:
-        r"""The name of the table."""
+        r"""The name of this table."""
         return self._name
 
     # Data column #############################################################
@@ -200,24 +104,25 @@ class LocalTable:
             raise KeyError(f"Column '{name}' already exists in table "
                            f"'{self.name}'")
 
-        if name not in self._data.columns:
-            raise KeyError(f"Column '{name}' does not exist in the underyling "
-                           f"data frame")
+        if not self._has_source_column(name):
+            raise KeyError(f"Column '{name}' does not exist in the underlying "
+                           f"source table")
 
         try:
-            dtype = utils.to_dtype(self._data[name])
+            dtype = self._get_source_dtype(name)
         except Exception as e:
-            raise RuntimeError(f"Data type inference for column '{name}' in "
-                               f"table '{self.name}' failed. Consider "
-                               f"changing the data type of the column or "
-                               f"removing it from the table.") from e
+            raise RuntimeError(f"Could not obtain data type for column "
+                               f"'{name}' in table '{self.name}'. Change "
+                               f"the data type of the column in the source "
+                               f"table or remove it from the table.") from e
+
         try:
-            stype = utils.infer_stype(self._data[name], name, dtype)
+            stype = self._get_source_stype(name, dtype)
         except Exception as e:
-            raise RuntimeError(f"Semantic type inference for column '{name}' "
-                               f"in table '{self.name}' failed. Consider "
-                               f"changing the data type of the column or "
-                               f"removing it from the table.") from e
+            raise RuntimeError(f"Could not obtain semantic type for column "
+                               f"'{name}' in table '{self.name}'. Change "
+                               f"the data type of the column in the source "
+                               f"table or remove it from the table.") from e
 
         self._columns[name] = Column(
             name=name,
@@ -432,12 +337,14 @@ class LocalTable:
         })
 
     def print_metadata(self) -> None:
-        r"""Prints the :meth:`~LocalTable.metadata` of the table."""
+        r"""Prints the :meth:`~metadata` of this table."""
+        num_rows = self._num_rows()
+        num_rows_repr = ' ({num_rows:,} rows)' if num_rows is not None else ''
+
         if in_notebook():
             from IPython.display import Markdown, display
-            display(
-                Markdown(f"### 🏷️ Metadata of Table `{self.name}` "
-                         f"({len(self._data):,} rows)"))
+            md_repr = f"### 🏷️ Metadata of Table `{self.name}`{num_rows_repr}"
+            display(Markdown(md_repr))
             df = self.metadata
             try:
                 if hasattr(df.style, 'hide'):
@@ -447,8 +354,7 @@ class LocalTable:
             except ImportError:
                 print(df.to_string(index=False))  # missing jinja2
         else:
-            print(f"🏷️ Metadata of Table '{self.name}' "
-                  f"({len(self._data):,} rows):")
+            print(f"🏷️ Metadata of Table '{self.name}'{num_rows_repr}")
             print(self.metadata.to_string(index=False))
 
     def infer_metadata(self, verbose: bool = True) -> Self:
@@ -478,11 +384,7 @@ class LocalTable:
                 column.name for column in self.columns if is_candidate(column)
             ]
 
-            if primary_key := utils.detect_primary_key(
-                    table_name=self.name,
-                    df=self._data,
-                    candidates=candidates,
-            ):
+            if primary_key := self._infer_primary_key(candidates):
                 self.primary_key = primary_key
                 logs.append(f"primary key '{primary_key}'")
 
@@ -493,7 +395,7 @@ class LocalTable:
                 if column.stype == Stype.timestamp
                 and column.name != self._end_time_column
             ]
-            if time_column := utils.detect_time_column(self._data, candidates):
+            if time_column := self._infer_time_column(candidates):
                 self.time_column = time_column
                 logs.append(f"time column '{time_column}'")
 
@@ -543,3 +445,29 @@ class LocalTable:
                 f'  time_column={self._time_column},\n'
                 f'  end_time_column={self._end_time_column},\n'
                 f')')
+
+    # Abstract method #########################################################
+
+    @abstractmethod
+    def _has_source_column(self, name: str) -> bool:
+        pass
+
+    @abstractmethod
+    def _get_source_dtype(self, name: str) -> Dtype:
+        pass
+
+    @abstractmethod
+    def _get_source_stype(self, name: str, dtype: Dtype) -> Stype:
+        pass
+
+    @abstractmethod
+    def _infer_primary_key(self, candidates: List[str]) -> Optional[str]:
+        pass
+
+    @abstractmethod
+    def _infer_time_column(self, candidates: List[str]) -> Optional[str]:
+        pass
+
+    @abstractmethod
+    def _num_rows(self) -> Optional[int]:
+        pass

kumoai/experimental/rfm/{local_graph.py → graph.py}

@@ -3,7 +3,7 @@ import io
 import warnings
 from collections import defaultdict
 from importlib.util import find_spec
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union
 
 import pandas as pd
 from kumoapi.graph import ColumnKey, ColumnKeyGroup, GraphDefinition
@@ -12,19 +12,19 @@ from kumoapi.typing import Stype
 from typing_extensions import Self
 
 from kumoai import in_notebook
-from kumoai.experimental.rfm import LocalTable
+from kumoai.experimental.rfm import Table
 from kumoai.graph import Edge
 
 if TYPE_CHECKING:
     import graphviz
 
 
-class LocalGraph:
-    r"""A graph of :class:`LocalTable` objects, akin to relationships between
+class Graph:
+    r"""A graph of :class:`Table` objects, akin to relationships between
     tables in a relational database.
 
     Creating a graph is the final step of data definition; after a
-    :class:`LocalGraph` is created, you can use it to initialize the
+    :class:`Graph` is created, you can use it to initialize the
     Kumo Relational Foundation Model (:class:`KumoRFM`).
 
     .. code-block:: python
@@ -44,7 +44,7 @@ class LocalGraph:
         >>> table3 = rfm.LocalTable(name="table3", data=df3)
 
         >>> # Create a graph from a dictionary of tables:
-        >>> graph = rfm.LocalGraph({
+        >>> graph = rfm.Graph({
         ...     "table1": table1,
         ...     "table2": table2,
         ...     "table3": table3,
@@ -75,11 +75,11 @@ class LocalGraph:
 
     def __init__(
         self,
-        tables: List[LocalTable],
-        edges: Optional[List[Edge]] = None,
+        tables: Sequence[Table],
+        edges: Optional[Sequence[Edge]] = None,
     ) -> None:
 
-        self._tables: Dict[str, LocalTable] = {}
+        self._tables: Dict[str, Table] = {}
         self._edges: List[Edge] = []
 
         for table in tables:
@@ -94,11 +94,11 @@ class LocalGraph:
     def from_data(
         cls,
         df_dict: Dict[str, pd.DataFrame],
-        edges: Optional[List[Edge]] = None,
+        edges: Optional[Sequence[Edge]] = None,
         infer_metadata: bool = True,
         verbose: bool = True,
     ) -> Self:
-        r"""Creates a :class:`LocalGraph` from a dictionary of
+        r"""Creates a :class:`Graph` from a dictionary of
         :class:`pandas.DataFrame` objects.
 
         Automatically infers table metadata and links.
@@ -115,7 +115,7 @@ class LocalGraph:
             >>> df3 = pd.DataFrame(...)
 
             >>> # Create a graph from a dictionary of data frames:
-            >>> graph = rfm.LocalGraph.from_data({
+            >>> graph = rfm.Graph.from_data({
             ...     "table1": df1,
             ...     "table2": df2,
             ...     "table3": df3,
@@ -148,13 +148,14 @@ class LocalGraph:
             >>> df1 = pd.DataFrame(...)
             >>> df2 = pd.DataFrame(...)
             >>> df3 = pd.DataFrame(...)
-            >>> graph = rfm.LocalGraph.from_data(data={
+            >>> graph = rfm.Graph.from_data(data={
             ...     "table1": df1,
             ...     "table2": df2,
             ...     "table3": df3,
             ... })
             >>> graph.validate()
         """
+        from kumoai.experimental.rfm import LocalTable
         tables = [LocalTable(df, name) for name, df in df_dict.items()]
 
         graph = cls(tables, edges=edges or [])
@@ -175,7 +176,7 @@ class LocalGraph:
         """
         return name in self.tables
 
-    def table(self, name: str) -> LocalTable:
+    def table(self, name: str) -> Table:
         r"""Returns the table with name ``name`` in the graph.
 
         Raises:
@@ -186,11 +187,11 @@ class LocalGraph:
         return self.tables[name]
 
     @property
-    def tables(self) -> Dict[str, LocalTable]:
+    def tables(self) -> Dict[str, Table]:
         r"""Returns the dictionary of table objects."""
         return self._tables
 
-    def add_table(self, table: LocalTable) -> Self:
+    def add_table(self, table: Table) -> Self:
         r"""Adds a table to the graph.
 
         Args:
@@ -199,11 +200,21 @@ class LocalGraph:
         Raises:
             KeyError: If a table with the same name already exists in the
                 graph.
+            ValueError: If the table belongs to a different backend than the
+                rest of the tables in the graph.
         """
         if table.name in self._tables:
             raise KeyError(f"Cannot add table with name '{table.name}' to "
                            f"this graph; table names must be globally unique.")
 
+        if len(self._tables) > 0:
+            cls = next(iter(self._tables.values())).__class__
+            if table.__class__ != cls:
+                raise ValueError(f"Cannot register a "
+                                 f"'{table.__class__.__name__}' to this "
+                                 f"graph since other tables are of type "
+                                 f"'{cls.__name__}'.")
+
         self._tables[table.name] = table
 
         return self
@@ -241,7 +252,7 @@ class LocalGraph:
         Example:
             >>> # doctest: +SKIP
             >>> import kumoai.experimental.rfm as rfm
-            >>> graph = rfm.LocalGraph(tables=...).infer_metadata()
+            >>> graph = rfm.Graph(tables=...).infer_metadata()
             >>> graph.metadata  # doctest: +SKIP
                 name   primary_key  time_column  end_time_column
             0   users      user_id            -                -
@@ -263,7 +274,7 @@ class LocalGraph:
         })
 
     def print_metadata(self) -> None:
-        r"""Prints the :meth:`~LocalGraph.metadata` of the graph."""
+        r"""Prints the :meth:`~Graph.metadata` of the graph."""
         if in_notebook():
             from IPython.display import Markdown, display
             display(Markdown('### 🗂️ Graph Metadata'))
@@ -287,7 +298,7 @@ class LocalGraph:
 
         Note:
             For more information, please see
-            :meth:`kumoai.experimental.rfm.LocalTable.infer_metadata`.
+            :meth:`kumoai.experimental.rfm.Table.infer_metadata`.
         """
         for table in self.tables.values():
             table.infer_metadata(verbose=False)
@@ -305,7 +316,7 @@ class LocalGraph:
         return self._edges
 
     def print_links(self) -> None:
-        r"""Prints the :meth:`~LocalGraph.edges` of the graph."""
+        r"""Prints the :meth:`~Graph.edges` of the graph."""
         edges = [(edge.dst_table, self[edge.dst_table]._primary_key,
                   edge.src_table, edge.fkey) for edge in self.edges]
         edges = sorted(edges)
@@ -333,9 +344,9 @@ class LocalGraph:
 
     def link(
         self,
-        src_table: Union[str, LocalTable],
+        src_table: Union[str, Table],
         fkey: str,
-        dst_table: Union[str, LocalTable],
+        dst_table: Union[str, Table],
     ) -> Self:
         r"""Links two tables (``src_table`` and ``dst_table``) from the foreign
         key ``fkey`` in the source table to the primary key in the destination
@@ -358,11 +369,11 @@ class LocalGraph:
                 table does not exist in the graph, if the source key does not
                 exist in the source table.
         """
-        if isinstance(src_table, LocalTable):
+        if isinstance(src_table, Table):
             src_table = src_table.name
         assert isinstance(src_table, str)
 
-        if isinstance(dst_table, LocalTable):
+        if isinstance(dst_table, Table):
             dst_table = dst_table.name
         assert isinstance(dst_table, str)
 
@@ -396,9 +407,9 @@ class LocalGraph:
 
     def unlink(
         self,
-        src_table: Union[str, LocalTable],
+        src_table: Union[str, Table],
         fkey: str,
-        dst_table: Union[str, LocalTable],
+        dst_table: Union[str, Table],
     ) -> Self:
         r"""Removes an :class:`~kumoai.graph.Edge` from the graph.
 
@@ -410,11 +421,11 @@ class LocalGraph:
         Raises:
             ValueError: if the edge is not present in the graph.
         """
-        if isinstance(src_table, LocalTable):
+        if isinstance(src_table, Table):
             src_table = src_table.name
         assert isinstance(src_table, str)
 
-        if isinstance(dst_table, LocalTable):
+        if isinstance(dst_table, Table):
             dst_table = dst_table.name
         assert isinstance(dst_table, str)
 
@@ -528,7 +539,10 @@ class LocalGraph:
                         score += 1.0
 
                     # Cardinality ratio:
-                    if len(src_table._data) > len(dst_table._data):
+                    src_num_rows = src_table._num_rows()
+                    dst_num_rows = dst_table._num_rows()
+                    if (src_num_rows is not None and dst_num_rows is not None
+                            and src_num_rows > dst_num_rows):
                         score += 1.0
 
                     if score < 5.0:
@@ -790,7 +804,7 @@ class LocalGraph:
     def __contains__(self, name: str) -> bool:
         return self.has_table(name)
 
-    def __getitem__(self, name: str) -> LocalTable:
+    def __getitem__(self, name: str) -> Table:
         return self.table(name)
 
     def __delitem__(self, name: str) -> None:

kumoai/experimental/rfm/local_graph_store.py

@@ -6,7 +6,7 @@ import pandas as pd
 from kumoapi.rfm.context import Subgraph
 from kumoapi.typing import Stype
 
-from kumoai.experimental.rfm import LocalGraph
+from kumoai.experimental.rfm import Graph, LocalTable
 from kumoai.experimental.rfm.utils import normalize_text
 from kumoai.utils import InteractiveProgressLogger, ProgressLogger
 
@@ -20,7 +20,7 @@ except ImportError:
 class LocalGraphStore:
     def __init__(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
@@ -105,7 +105,7 @@ class LocalGraphStore:
 
     def sanitize(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
     ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, np.ndarray]]:
         r"""Sanitizes raw data according to table schema definition:
@@ -120,10 +120,11 @@ class LocalGraphStore:
         data for faster model processing. In particular, it:
         * tokenizes any text column that is not a foreign key
         """
-        df_dict: Dict[str, pd.DataFrame] = {
-            table_name: table._data.copy(deep=False).reset_index(drop=True)
-            for table_name, table in graph.tables.items()
-        }
+        df_dict: Dict[str, pd.DataFrame] = {}
+        for table_name, table in graph.tables.items():
+            assert isinstance(table, LocalTable)
+            df = table._data
+            df_dict[table_name] = df.copy(deep=False).reset_index(drop=True)
 
         foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
 
@@ -165,7 +166,7 @@ class LocalGraphStore:
 
         return df_dict, mask_dict
 
-    def get_stype_dict(self, graph: LocalGraph) -> Dict[str, Dict[str, Stype]]:
+    def get_stype_dict(self, graph: Graph) -> Dict[str, Dict[str, Stype]]:
         stype_dict: Dict[str, Dict[str, Stype]] = {}
         foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
         for table in graph.tables.values():
@@ -180,7 +181,7 @@ class LocalGraphStore:
 
     def get_pkey_data(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[str, str],
             Dict[str, pd.DataFrame],
@@ -218,7 +219,7 @@ class LocalGraphStore:
 
     def get_time_data(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[str, str],
             Dict[str, str],
@@ -259,7 +260,7 @@ class LocalGraphStore:
 
     def get_csc(
         self,
-        graph: LocalGraph,
+        graph: Graph,
     ) -> Tuple[
             Dict[Tuple[str, str, str], np.ndarray],
             Dict[Tuple[str, str, str], np.ndarray],

kumoai/experimental/rfm/rfm.py

@@ -32,7 +32,7 @@ from kumoapi.task import TaskType
 
 from kumoai.client.rfm import RFMAPI
 from kumoai.exceptions import HTTPException
-from kumoai.experimental.rfm import LocalGraph
+from kumoai.experimental.rfm import Graph
 from kumoai.experimental.rfm.local_graph_sampler import LocalGraphSampler
 from kumoai.experimental.rfm.local_graph_store import LocalGraphStore
 from kumoai.experimental.rfm.local_pquery_driver import (
@@ -123,17 +123,17 @@ class KumoRFM:
     :class:`KumoRFM` is a foundation model to generate predictions for any
     relational dataset without training.
     The model is pre-trained and the class provides an interface to query the
-    model from a :class:`LocalGraph` object.
+    model from a :class:`Graph` object.
 
     .. code-block:: python
 
-        from kumoai.experimental.rfm import LocalGraph, KumoRFM
+        from kumoai.experimental.rfm import Graph, KumoRFM
 
         df_users = pd.DataFrame(...)
         df_items = pd.DataFrame(...)
         df_orders = pd.DataFrame(...)
 
-        graph = LocalGraph.from_data({
+        graph = Graph.from_data({
             'users': df_users,
             'items': df_items,
             'orders': df_orders,
@@ -163,7 +163,7 @@ class KumoRFM:
     """
     def __init__(
         self,
-        graph: LocalGraph,
+        graph: Graph,
         preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
@@ -172,10 +172,19 @@ class KumoRFM:
         self._graph_store = LocalGraphStore(graph, preprocess, verbose)
         self._graph_sampler = LocalGraphSampler(self._graph_store)
 
+        self._client: Optional[RFMAPI] = None
+
         self._batch_size: Optional[int | Literal['max']] = None
         self.num_retries: int = 0
+
+    @property
+    def _api_client(self) -> RFMAPI:
+        if self._client is not None:
+            return self._client
+
         from kumoai.experimental.rfm import global_state
-        self._api_client = RFMAPI(global_state.client)
+        self._client = RFMAPI(global_state.client)
+        return self._client
 
     def __repr__(self) -> str:
         return f'{self.__class__.__name__}()'