kumoai 2.13.0.dev202512011731__cp312-cp312-macosx_11_0_arm64.whl → 2.14.0.dev202512181731__cp312-cp312-macosx_11_0_arm64.whl

kumoai/experimental/rfm/base/source.py

@@ -0,0 +1,19 @@
+from dataclasses import dataclass
+
+from kumoapi.typing import Dtype
+
+
+@dataclass
+class SourceColumn:
+    name: str
+    dtype: Dtype
+    is_primary_key: bool
+    is_unique_key: bool
+    is_nullable: bool
+
+
+@dataclass
+class SourceForeignKey:
+    name: str
+    dst_table: str
+    primary_key: str

kumoai/experimental/rfm/base/sql_sampler.py

@@ -0,0 +1,84 @@
+from abc import abstractmethod
+from typing import TYPE_CHECKING, Literal
+
+import numpy as np
+import pandas as pd
+
+from kumoai.experimental.rfm.base import Sampler, SamplerOutput, SQLTable
+from kumoai.utils import ProgressLogger
+
+if TYPE_CHECKING:
+    from kumoai.experimental.rfm import Graph
+
+
+class SQLSampler(Sampler):
+    def __init__(
+        self,
+        graph: 'Graph',
+        verbose: bool | ProgressLogger = True,
+    ) -> None:
+        super().__init__(graph=graph, verbose=verbose)
+
+        self._fqn_dict: dict[str, str] = {}
+        for table in graph.tables.values():
+            assert isinstance(table, SQLTable)
+            self._connection = table._connection
+            self._fqn_dict[table.name] = table.fqn
+
+    @property
+    def fqn_dict(self) -> dict[str, str]:
+        r"""The fully-qualified quoted source name for all table names in the
+        graph.
+        """
+        return self._fqn_dict
+
+    def _sample_subgraph(
+        self,
+        entity_table_name: str,
+        entity_pkey: pd.Series,
+        anchor_time: pd.Series | Literal['entity'],
+        columns_dict: dict[str, set[str]],
+        num_neighbors: list[int],
+    ) -> SamplerOutput:
+
+        df, batch = self._by_pkey(
+            table_name=entity_table_name,
+            pkey=entity_pkey,
+            columns=columns_dict[entity_table_name],
+        )
+        if len(batch) != len(entity_pkey):
+            mask = np.ones(len(entity_pkey), dtype=bool)
+            mask[batch] = False
+            raise KeyError(f"The primary keys "
+                           f"{entity_pkey.iloc[mask].tolist()} do not exist "
+                           f"in the '{entity_table_name}' table")
+
+        perm = batch.argsort()
+        batch = batch[perm]
+        df = df.iloc[perm].reset_index(drop=True)
+
+        if not isinstance(anchor_time, pd.Series):
+            time_column = self.time_column_dict[entity_table_name]
+            anchor_time = df[time_column]
+
+        return SamplerOutput(
+            anchor_time=anchor_time.astype(int).to_numpy(),
+            df_dict={entity_table_name: df},
+            inverse_dict={},
+            batch_dict={entity_table_name: batch},
+            num_sampled_nodes_dict={entity_table_name: [len(batch)]},
+            row_dict={},
+            col_dict={},
+            num_sampled_edges_dict={},
+        )
+
+    # Abstract Methods ########################################################
+
+    @abstractmethod
+    def _by_pkey(
+        self,
+        table_name: str,
+        pkey: pd.Series,
+        columns: set[str],
+    ) -> tuple[pd.DataFrame, np.ndarray]:
+        pass

kumoai/experimental/rfm/base/sql_table.py

@@ -0,0 +1,113 @@
+from abc import abstractmethod
+from collections import defaultdict
+from collections.abc import Sequence
+from functools import cached_property
+from typing import Any
+
+from kumoapi.model_plan import MissingType
+
+from kumoai.experimental.rfm.base import (
+    ColumnExpressionType,
+    SourceForeignKey,
+    Table,
+)
+from kumoai.utils import quote_ident
+
+
+class SQLTable(Table):
+    r"""A :class:`SQLTable` specifies a :class:`Table` backed by a SQL
+    database.
+
+    Args:
+        name: The logical name of this table.
+        source_name: The physical name of this table in the database. If set to
+            ``None``, ``name`` is being used.
+        columns: The selected physical columns of this table.
+        column_expressions: The logical columns of this table.
+        primary_key: The name of the primary key of this table, if it exists.
+        time_column: The name of the time column of this table, if it exists.
+        end_time_column: The name of the end time column of this table, if it
+            exists.
+    """
+    def __init__(
+        self,
+        name: str,
+        source_name: str | None = None,
+        columns: Sequence[str] | None = None,
+        column_expressions: Sequence[ColumnExpressionType] | None = None,
+        primary_key: MissingType | str | None = MissingType.VALUE,
+        time_column: str | None = None,
+        end_time_column: str | None = None,
+    ) -> None:
+
+        self._connection: Any
+        self._source_name = source_name or name
+
+        super().__init__(
+            name=name,
+            columns=[],
+            primary_key=None,
+            time_column=None,
+            end_time_column=None,
+        )
+
+        if isinstance(primary_key, MissingType):
+            primary_key = self._source_primary_key
+
+        # Add column expressions with highest priority:
+        self._add_column_expressions(column_expressions or [])
+
+        if columns is None:
+            for column_name in self._source_column_dict.keys():
+                if column_name not in self:
+                    self.add_column(column_name)
+        else:
+            for column_name in columns:
+                self.add_column(column_name)
+
+        if primary_key is not None:
+            if primary_key not in self:
+                self.add_column(primary_key)
+            self.primary_key = primary_key
+
+        if time_column is not None:
+            if time_column not in self:
+                self.add_column(time_column)
+            self.time_column = time_column
+
+        if end_time_column is not None:
+            if end_time_column not in self:
+                self.add_column(end_time_column)
+            self.end_time_column = end_time_column
+
+    @property
+    def fqn(self) -> str:
+        r"""The fully-qualified quoted source table name."""
+        return quote_ident(self._source_name)
+
+    # Column ##################################################################
+
+    def _add_column_expressions(
+        self,
+        columns: Sequence[ColumnExpressionType],
+    ) -> None:
+        pass
+
+    # Abstract Methods ########################################################
+
+    @cached_property
+    def _source_foreign_key_dict(self) -> dict[str, SourceForeignKey]:
+        fkeys = self._get_source_foreign_keys()
+        # NOTE Drop all keys that link to multiple keys in the same table since
+        # we don't support composite keys yet:
+        table_pkeys: dict[str, set[str]] = defaultdict(set)
+        for fkey in fkeys:
+            table_pkeys[fkey.dst_table].add(fkey.primary_key)
+        return {
+            fkey.name: fkey
+            for fkey in fkeys if len(table_pkeys[fkey.dst_table]) == 1
+        }
+
+    @abstractmethod
+    def _get_source_foreign_keys(self) -> list[SourceForeignKey]:
+        pass

kumoai/experimental/rfm/base/table.py

@@ -1,15 +1,25 @@
 from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Sequence
+from collections.abc import Sequence
+from functools import cached_property
 
 import pandas as pd
+from kumoapi.model_plan import MissingType
 from kumoapi.source_table import UnavailableSourceTable
 from kumoapi.table import Column as ColumnDefinition
 from kumoapi.table import TableDefinition
-from kumoapi.typing import Dtype, Stype
+from kumoapi.typing import Stype
 from typing_extensions import Self
 
-from kumoai import in_notebook
-from kumoai.experimental.rfm.base import Column
+from kumoai import in_notebook, in_snowflake_notebook
+from kumoai.experimental.rfm.base import Column, DataBackend, SourceColumn
+from kumoai.experimental.rfm.infer import (
+    contains_categorical,
+    contains_id,
+    contains_multicategorical,
+    contains_timestamp,
+    infer_primary_key,
+    infer_time_column,
+)
 
 
 class Table(ABC):
@@ -28,19 +38,29 @@ class Table(ABC):
     def __init__(
         self,
         name: str,
-        columns: Optional[Sequence[str]] = None,
-        primary_key: Optional[str] = None,
-        time_column: Optional[str] = None,
-        end_time_column: Optional[str] = None,
+        columns: Sequence[str] | None = None,
+        primary_key: MissingType | str | None = MissingType.VALUE,
+        time_column: str | None = None,
+        end_time_column: str | None = None,
     ) -> None:
 
         self._name = name
-        self._primary_key: Optional[str] = None
-        self._time_column: Optional[str] = None
-        self._end_time_column: Optional[str] = None
+        self._primary_key: str | None = None
+        self._time_column: str | None = None
+        self._end_time_column: str | None = None
 
-        self._columns: Dict[str, Column] = {}
-        for column_name in columns or []:
+        if columns is None:
+            columns = list(self._source_column_dict.keys())
+
+        if len(self._source_column_dict) == 0:
+            raise ValueError(f"Table '{name}' does not hold any column with "
+                             f"a supported data type")
+
+        if isinstance(primary_key, MissingType):
+            primary_key = self._source_primary_key
+
+        self._columns: dict[str, Column] = {}
+        for column_name in columns:
             self.add_column(column_name)
 
         if primary_key is not None:
@@ -63,7 +83,7 @@ class Table(ABC):
         r"""The name of this table."""
         return self._name
 
-    # Data column #############################################################
+    # Column ##################################################################
 
     def has_column(self, name: str) -> bool:
         r"""Returns ``True`` if this table holds a column with name ``name``;
@@ -85,7 +105,7 @@ class Table(ABC):
         return self._columns[name]
 
     @property
-    def columns(self) -> List[Column]:
+    def columns(self) -> list[Column]:
         r"""Returns a list of :class:`Column` objects that represent the
         columns in this table.
         """
@@ -104,20 +124,24 @@ class Table(ABC):
             raise KeyError(f"Column '{name}' already exists in table "
                            f"'{self.name}'")
 
-        if not self._has_source_column(name):
+        if name not in self._source_column_dict:
             raise KeyError(f"Column '{name}' does not exist in the underlying "
                            f"source table")
 
-        try:
-            dtype = self._get_source_dtype(name)
-        except Exception as e:
-            raise RuntimeError(f"Could not obtain data type for column "
-                               f"'{name}' in table '{self.name}'. Change "
-                               f"the data type of the column in the source "
-                               f"table or remove it from the table.") from e
+        dtype = self._source_column_dict[name].dtype
 
         try:
-            stype = self._get_source_stype(name, dtype)
+            ser = self._sample_df[name]
+            if contains_id(ser, name, dtype):
+                stype = Stype.ID
+            elif contains_timestamp(ser, name, dtype):
+                stype = Stype.timestamp
+            elif contains_multicategorical(ser, name, dtype):
+                stype = Stype.multicategorical
+            elif contains_categorical(ser, name, dtype):
+                stype = Stype.categorical
+            else:
+                stype = dtype.default_stype
         except Exception as e:
             raise RuntimeError(f"Could not obtain semantic type for column "
                                f"'{name}' in table '{self.name}'. Change "
@@ -126,8 +150,8 @@ class Table(ABC):
 
         self._columns[name] = Column(
             name=name,
-            dtype=dtype,
             stype=stype,
+            dtype=dtype,
         )
 
         return self._columns[name]
@@ -163,7 +187,7 @@ class Table(ABC):
         return self._primary_key is not None
 
     @property
-    def primary_key(self) -> Optional[Column]:
+    def primary_key(self) -> Column | None:
         r"""The primary key column of this table.
 
         The getter returns the primary key column of this table, or ``None`` if
@@ -178,7 +202,7 @@ class Table(ABC):
         return self[self._primary_key]
 
     @primary_key.setter
-    def primary_key(self, name: Optional[str]) -> None:
+    def primary_key(self, name: str | None) -> None:
         if name is not None and name == self._time_column:
             raise ValueError(f"Cannot specify column '{name}' as a primary "
                              f"key since it is already defined to be a time "
@@ -208,7 +232,7 @@ class Table(ABC):
         return self._time_column is not None
 
     @property
-    def time_column(self) -> Optional[Column]:
+    def time_column(self) -> Column | None:
         r"""The time column of this table.
 
         The getter returns the time column of this table, or ``None`` if no
@@ -223,7 +247,7 @@ class Table(ABC):
         return self[self._time_column]
 
     @time_column.setter
-    def time_column(self, name: Optional[str]) -> None:
+    def time_column(self, name: str | None) -> None:
         if name is not None and name == self._primary_key:
             raise ValueError(f"Cannot specify column '{name}' as a time "
                              f"column since it is already defined to be a "
@@ -253,7 +277,7 @@ class Table(ABC):
         return self._end_time_column is not None
 
     @property
-    def end_time_column(self) -> Optional[Column]:
+    def end_time_column(self) -> Column | None:
         r"""The end time column of this table.
 
         The getter returns the end time column of this table, or ``None`` if no
@@ -269,7 +293,7 @@ class Table(ABC):
         return self[self._end_time_column]
 
     @end_time_column.setter
-    def end_time_column(self, name: Optional[str]) -> None:
+    def end_time_column(self, name: str | None) -> None:
         if name is not None and name == self._primary_key:
             raise ValueError(f"Cannot specify column '{name}' as an end time "
                              f"column since it is already defined to be a "
@@ -338,10 +362,16 @@ class Table(ABC):
 
     def print_metadata(self) -> None:
         r"""Prints the :meth:`~metadata` of this table."""
-        num_rows = self._num_rows()
-        num_rows_repr = ' ({num_rows:,} rows)' if num_rows is not None else ''
+        num_rows_repr = ''
+        if self._num_rows is not None:
+            num_rows_repr = ' ({self._num_rows:,} rows)'
 
-        if in_notebook():
+        if in_snowflake_notebook():
+            import streamlit as st
+            md_repr = f"### 🏷️ Metadata of Table `{self.name}`{num_rows_repr}"
+            st.markdown(md_repr)
+            st.dataframe(self.metadata, hide_index=True)
+        elif in_notebook():
             from IPython.display import Markdown, display
             md_repr = f"### 🏷️ Metadata of Table `{self.name}`{num_rows_repr}"
             display(Markdown(md_repr))
@@ -357,8 +387,83 @@ class Table(ABC):
             print(f"🏷️ Metadata of Table '{self.name}'{num_rows_repr}")
             print(self.metadata.to_string(index=False))
 
+    def infer_primary_key(self, verbose: bool = True) -> Self:
+        r"""Infers the primary key in this table.
+
+        Args:
+            verbose: Whether to print verbose output.
+        """
+        if self.has_primary_key():
+            return self
+
+        def _set_primary_key(primary_key: str) -> None:
+            self.primary_key = primary_key
+            if verbose:
+                print(f"Detected primary key '{primary_key}' in table "
+                      f"'{self.name}'")
+
+        if primary_key := self._source_primary_key:
+            _set_primary_key(primary_key)
+            return self
+
+        unique_keys = [
+            column.name for column in self._source_column_dict.values()
+            if column.is_unique_key
+        ]
+        if len(unique_keys) == 1:  # NOTE No composite keys yet.
+            _set_primary_key(unique_keys[0])
+            return self
+
+        candidates = [
+            column.name for column in self.columns if column.stype == Stype.ID
+        ]
+        if len(candidates) == 0:
+            for column in self.columns:
+                if self.name.lower() == column.name.lower():
+                    candidates.append(column.name)
+                elif (self.name.lower().endswith('s')
+                      and self.name.lower()[:-1] == column.name.lower()):
+                    candidates.append(column.name)
+
+        if primary_key := infer_primary_key(
+                table_name=self.name,
+                df=self._sample_df,
+                candidates=candidates,
+        ):
+            _set_primary_key(primary_key)
+            return self
+
+        return self
+
+    def infer_time_column(self, verbose: bool = True) -> Self:
+        r"""Infers the time column in this table.
+
+        Args:
+            verbose: Whether to print verbose output.
+        """
+        if self.has_time_column():
+            return self
+
+        candidates = [
+            column.name for column in self.columns
+            if column.stype == Stype.timestamp
+            and column.name != self._end_time_column
+        ]
+
+        if time_column := infer_time_column(
+                df=self._sample_df,
+                candidates=candidates,
+        ):
+            self.time_column = time_column
+
+            if verbose:
+                print(f"Detected time column '{time_column}' in table "
+                      f"'{self.name}'")
+
+        return self
+
     def infer_metadata(self, verbose: bool = True) -> Self:
-        r"""Infers metadata, *i.e.*, primary keys and time columns, in the
+        r"""Infers metadata, *i.e.*, primary keys and time columns, in this
         table.
 
         Args:
@@ -366,38 +471,15 @@ class Table(ABC):
         """
         logs = []
 
-        # Try to detect primary key if not set:
         if not self.has_primary_key():
+            self.infer_primary_key(verbose=False)
+            if self.has_primary_key():
+                logs.append(f"primary key '{self._primary_key}'")
 
-            def is_candidate(column: Column) -> bool:
-                if column.stype == Stype.ID:
-                    return True
-                if all(column.stype != Stype.ID for column in self.columns):
-                    if self.name == column.name:
-                        return True
-                    if (self.name.endswith('s')
-                            and self.name[:-1] == column.name):
-                        return True
-                return False
-
-            candidates = [
-                column.name for column in self.columns if is_candidate(column)
-            ]
-
-            if primary_key := self._infer_primary_key(candidates):
-                self.primary_key = primary_key
-                logs.append(f"primary key '{primary_key}'")
-
-        # Try to detect time column if not set:
         if not self.has_time_column():
-            candidates = [
-                column.name for column in self.columns
-                if column.stype == Stype.timestamp
-                and column.name != self._end_time_column
-            ]
-            if time_column := self._infer_time_column(candidates):
-                self.time_column = time_column
-                logs.append(f"time column '{time_column}'")
+            self.infer_time_column(verbose=False)
+            if self.has_time_column():
+                logs.append(f"time column '{self._time_column}'")
 
         if verbose and len(logs) > 0:
             print(f"Detected {' and '.join(logs)} in table '{self.name}'")
@@ -418,6 +500,17 @@ class Table(ABC):
             end_time_col=self._end_time_column,
         )
 
+    @property
+    def _source_primary_key(self) -> str | None:
+        primary_keys = [
+            column.name for column in self._source_column_dict.values()
+            if column.is_primary_key
+        ]
+        if len(primary_keys) == 1:  # NOTE No composite keys yet.
+            return primary_keys[0]
+
+        return None
+
     # Python builtins #########################################################
 
     def __hash__(self) -> int:
@@ -446,28 +539,33 @@ class Table(ABC):
                 f'  end_time_column={self._end_time_column},\n'
                 f')')
 
-    # Abstract method #########################################################
+    # Abstract Methods ########################################################
 
+    @property
     @abstractmethod
-    def _has_source_column(self, name: str) -> bool:
-        pass
+    def backend(self) -> DataBackend:
+        r"""The data backend of this table."""
 
-    @abstractmethod
-    def _get_source_dtype(self, name: str) -> Dtype:
-        pass
+    @cached_property
+    def _source_column_dict(self) -> dict[str, SourceColumn]:
+        return {col.name: col for col in self._get_source_columns()}
 
     @abstractmethod
-    def _get_source_stype(self, name: str, dtype: Dtype) -> Stype:
+    def _get_source_columns(self) -> list[SourceColumn]:
         pass
 
-    @abstractmethod
-    def _infer_primary_key(self, candidates: List[str]) -> Optional[str]:
-        pass
+    @cached_property
+    def _sample_df(self) -> pd.DataFrame:
+        return self._get_sample_df()
 
     @abstractmethod
-    def _infer_time_column(self, candidates: List[str]) -> Optional[str]:
+    def _get_sample_df(self) -> pd.DataFrame:
         pass
 
+    @cached_property
+    def _num_rows(self) -> int | None:
+        return self._get_num_rows()
+
     @abstractmethod
-    def _num_rows(self) -> Optional[int]:
+    def _get_num_rows(self) -> int | None:
         pass