kumoai 2.13.0.dev202512011731__cp312-cp312-macosx_11_0_arm64.whl → 2.13.0.dev202512031731__cp312-cp312-macosx_11_0_arm64.whl

kumoai/experimental/rfm/graph.py

@@ -2,8 +2,10 @@ import contextlib
 import io
 import warnings
 from collections import defaultdict
+from dataclasses import dataclass, field
 from importlib.util import find_spec
-from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
 
 import pandas as pd
 from kumoapi.graph import ColumnKey, ColumnKeyGroup, GraphDefinition
@@ -14,9 +16,18 @@ from typing_extensions import Self
 from kumoai import in_notebook
 from kumoai.experimental.rfm import Table
 from kumoai.graph import Edge
+from kumoai.mixin import CastMixin
 
 if TYPE_CHECKING:
     import graphviz
+    from adbc_driver_sqlite.dbapi import AdbcSqliteConnection
+    from snowflake.connector import SnowflakeConnection
+
+
+@dataclass
+class SqliteConnectionConfig(CastMixin):
+    uri: Union[str, Path]
+    kwargs: Dict[str, Any] = field(default_factory=dict)
 
 
 class Graph:
@@ -85,10 +96,24 @@ class Graph:
         for table in tables:
             self.add_table(table)
 
+        for table in tables:
+            for fkey in table._source_foreign_key_dict.values():
+                if fkey.name not in table or fkey.dst_table not in self:
+                    continue
+                if self[fkey.dst_table].primary_key is None:
+                    self[fkey.dst_table].primary_key = fkey.primary_key
+                elif self[fkey.dst_table]._primary_key != fkey.primary_key:
+                    raise ValueError(f"Found duplicate primary key definition "
+                                     f"'{self[fkey.dst_table]._primary_key}' "
+                                     f"and '{fkey.primary_key}' in table "
+                                     f"'{fkey.dst_table}'.")
+                self.link(table.name, fkey.name, fkey.dst_table)
+
         for edge in (edges or []):
             _edge = Edge._cast(edge)
             assert _edge is not None
-            self.link(*_edge)
+            if _edge not in self._edges:
+                self.link(*_edge)
 
     @classmethod
     def from_data(
@@ -101,7 +126,7 @@ class Graph:
         r"""Creates a :class:`Graph` from a dictionary of
         :class:`pandas.DataFrame` objects.
 
-        Automatically infers table metadata and links.
+        Automatically infers table metadata and links by default.
 
         .. code-block:: python
 
@@ -121,50 +146,180 @@ class Graph:
             ...     "table3": df3,
             ... })
 
-            >>> # Inspect table metadata:
-            >>> for table in graph.tables.values():
-            ...     table.print_metadata()
-
-            >>> # Visualize graph (if graphviz is installed):
-            >>> graph.visualize()
-
         Args:
             df_dict: A dictionary of data frames, where the keys are the names
                 of the tables and the values hold table data.
+            edges: An optional list of :class:`~kumoai.graph.Edge` objects to
+                add to the graph. If not provided, edges will be automatically
+                inferred from the data in case ``infer_metadata=True``.
             infer_metadata: Whether to infer metadata for all tables in the
                 graph.
+            verbose: Whether to print verbose output.
+        """
+        from kumoai.experimental.rfm.backend.local import LocalTable
+        tables = [LocalTable(df, name) for name, df in df_dict.items()]
+
+        graph = cls(tables, edges=edges or [])
+
+        if infer_metadata:
+            graph.infer_metadata(False)
+
+            if edges is None:
+                graph.infer_links(False)
+
+        if verbose:
+            graph.print_metadata()
+            graph.print_links()
+
+        return graph
+
+    @classmethod
+    def from_sqlite(
+        cls,
+        connection: Union[
+            'AdbcSqliteConnection',
+            SqliteConnectionConfig,
+            str,
+            Path,
+            Dict[str, Any],
+        ],
+        table_names: Optional[Sequence[str]] = None,
+        edges: Optional[Sequence[Edge]] = None,
+        infer_metadata: bool = True,
+        verbose: bool = True,
+    ) -> Self:
+        r"""Creates a :class:`Graph` from a :class:`sqlite` database.
+
+        Automatically infers table metadata and links by default.
+
+        .. code-block:: python
+
+            >>> # doctest: +SKIP
+            >>> import kumoai.experimental.rfm as rfm
+
+            >>> # Create a graph from a SQLite database:
+            >>> graph = rfm.Graph.from_sqlite('data.db')
+
+        Args:
+            connection: An open connection from
+                :meth:`~kumoai.experimental.rfm.backend.sqlite.connect` or the
+                path to the database file.
+            table_names: Set of table names to include. If ``None``, will add
+                all tables present in the database.
             edges: An optional list of :class:`~kumoai.graph.Edge` objects to
                 add to the graph. If not provided, edges will be automatically
-                inferred from the data.
+                inferred from the data in case ``infer_metadata=True``.
+            infer_metadata: Whether to infer metadata for all tables in the
+                graph.
             verbose: Whether to print verbose output.
+        """
+        from kumoai.experimental.rfm.backend.sqlite import (
+            Connection,
+            SQLiteTable,
+            connect,
+        )
+
+        if not isinstance(connection, Connection):
+            connection = SqliteConnectionConfig._cast(connection)
+            assert isinstance(connection, SqliteConnectionConfig)
+            connection = connect(connection.uri, **connection.kwargs)
+        assert isinstance(connection, Connection)
+
+        if table_names is None:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT name FROM sqlite_master "
+                               "WHERE type='table'")
+                table_names = [row[0] for row in cursor.fetchall()]
+
+        tables = [SQLiteTable(connection, name) for name in table_names]
 
-        Note:
-            This method will automatically infer metadata and links for the
-            graph.
+        graph = cls(tables, edges=edges or [])
+
+        if infer_metadata:
+            graph.infer_metadata(False)
+
+            if edges is None:
+                graph.infer_links(False)
+
+        if verbose:
+            graph.print_metadata()
+            graph.print_links()
+
+        return graph
+
+    @classmethod
+    def from_snowflake(
+        cls,
+        connection: Union['SnowflakeConnection', Dict[str, Any], None] = None,
+        table_names: Optional[Sequence[str]] = None,
+        edges: Optional[Sequence[Edge]] = None,
+        infer_metadata: bool = True,
+        verbose: bool = True,
+    ) -> Self:
+        r"""Creates a :class:`Graph` from a :class:`snowflake` database and
+        schema.
+
+        Automatically infers table metadata and links by default.
+
+        .. code-block:: python
 
-        Example:
             >>> # doctest: +SKIP
             >>> import kumoai.experimental.rfm as rfm
-            >>> df1 = pd.DataFrame(...)
-            >>> df2 = pd.DataFrame(...)
-            >>> df3 = pd.DataFrame(...)
-            >>> graph = rfm.Graph.from_data(data={
-            ...     "table1": df1,
-            ...     "table2": df2,
-            ...     "table3": df3,
-            ... })
-            >>> graph.validate()
+
+            >>> # Create a graph directly in a Snowflake notebook:
+            >>> graph = rfm.Graph.from_snowflake()
+
+        Args:
+            connection: An open connection from
+                :meth:`~kumoai.experimental.rfm.backend.snow.connect` or the
+                :class:`snowflake` connector keyword arguments to open a new
+                connection. If ``None``, will re-use an active session in case
+                it exists, or create a new connection from credentials stored
+                in environment variables.
+            table_names: Set of table names to include. If ``None``, will add
+                all tables present in the database.
+            edges: An optional list of :class:`~kumoai.graph.Edge` objects to
+                add to the graph. If not provided, edges will be automatically
+                inferred from the data in case ``infer_metadata=True``.
+            infer_metadata: Whether to infer metadata for all tables in the
+                graph.
+            verbose: Whether to print verbose output.
         """
-        from kumoai.experimental.rfm import LocalTable
-        tables = [LocalTable(df, name) for name, df in df_dict.items()]
+        from kumoai.experimental.rfm.backend.snow import (
+            Connection,
+            SnowTable,
+            connect,
+        )
+
+        if not isinstance(connection, Connection):
+            connection = connect(**(connection or {}))
+        assert isinstance(connection, Connection)
+
+        if table_names is None:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT CURRENT_DATABASE(), CURRENT_SCHEMA()")
+                database, schema = cursor.fetchone()
+                query = f"""
+                SELECT TABLE_NAME
+                FROM {database}.INFORMATION_SCHEMA.TABLES
+                WHERE TABLE_SCHEMA = '{schema}'
+                """
+                cursor.execute(query)
+                table_names = [row[0] for row in cursor.fetchall()]
+
+        tables = [SnowTable(connection, name) for name in table_names]
 
         graph = cls(tables, edges=edges or [])
 
         if infer_metadata:
-            graph.infer_metadata(verbose)
+            graph.infer_metadata(False)
 
             if edges is None:
-                graph.infer_links(verbose)
+                graph.infer_links(False)
+
+        if verbose:
+            graph.print_metadata()
+            graph.print_links()
 
         return graph
 
@@ -439,17 +594,13 @@ class Graph:
         return self
 
     def infer_links(self, verbose: bool = True) -> Self:
-        r"""Infers links for the tables and adds them as edges to the graph.
+        r"""Infers missing links for the tables and adds them as edges to the
+        graph.
 
         Args:
             verbose: Whether to print verbose output.
-
-        Note:
-            This function expects graph edges to be undefined upfront.
         """
-        if len(self.edges) > 0:
-            warnings.warn("Cannot infer links if graph edges already exist")
-            return self
+        known_edges = {(edge.src_table, edge.fkey) for edge in self.edges}
 
         # A list of primary key candidates (+score) for every column:
         candidate_dict: dict[
@@ -474,6 +625,9 @@ class Graph:
                 src_table_name = src_table.name.lower()
 
                 for src_key in src_table.columns:
+                    if (src_table.name, src_key.name) in known_edges:
+                        continue
+
                     if src_key == src_table.primary_key:
                         continue  # Cannot link to primary key.
 
@@ -539,10 +693,9 @@ class Graph:
                         score += 1.0
 
                     # Cardinality ratio:
-                    src_num_rows = src_table._num_rows()
-                    dst_num_rows = dst_table._num_rows()
-                    if (src_num_rows is not None and dst_num_rows is not None
-                            and src_num_rows > dst_num_rows):
+                    if (src_table._num_rows is not None
+                            and dst_table._num_rows is not None
+                            and src_table._num_rows > dst_table._num_rows):
                         score += 1.0
 
                     if score < 5.0:

kumoai/experimental/rfm/infer/__init__.py

@@ -1,9 +1,15 @@
+from .dtype import infer_dtype
+from .pkey import infer_primary_key
+from .time_col import infer_time_column
 from .id import contains_id
 from .timestamp import contains_timestamp
 from .categorical import contains_categorical
 from .multicategorical import contains_multicategorical
 
 __all__ = [
+    'infer_dtype',
+    'infer_primary_key',
+    'infer_time_column',
     'contains_id',
     'contains_timestamp',
     'contains_categorical',

kumoai/experimental/rfm/infer/dtype.py

@@ -0,0 +1,90 @@
+from typing import Any, Dict
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from kumoapi.typing import Dtype
+
+PANDAS_TO_DTYPE: Dict[Any, Dtype] = {
+    np.dtype('bool'): Dtype.bool,
+    pd.BooleanDtype(): Dtype.bool,
+    pa.bool_(): Dtype.bool,
+    np.dtype('byte'): Dtype.int,
+    pd.UInt8Dtype(): Dtype.int,
+    np.dtype('int16'): Dtype.int,
+    pd.Int16Dtype(): Dtype.int,
+    np.dtype('int32'): Dtype.int,
+    pd.Int32Dtype(): Dtype.int,
+    np.dtype('int64'): Dtype.int,
+    pd.Int64Dtype(): Dtype.int,
+    np.dtype('float32'): Dtype.float,
+    pd.Float32Dtype(): Dtype.float,
+    np.dtype('float64'): Dtype.float,
+    pd.Float64Dtype(): Dtype.float,
+    np.dtype('object'): Dtype.string,
+    pd.StringDtype(storage='python'): Dtype.string,
+    pd.StringDtype(storage='pyarrow'): Dtype.string,
+    pa.string(): Dtype.string,
+    pa.binary(): Dtype.binary,
+    np.dtype('datetime64[ns]'): Dtype.date,
+    np.dtype('timedelta64[ns]'): Dtype.timedelta,
+    pa.list_(pa.float32()): Dtype.floatlist,
+    pa.list_(pa.int64()): Dtype.intlist,
+    pa.list_(pa.string()): Dtype.stringlist,
+}
+
+
+def infer_dtype(ser: pd.Series) -> Dtype:
+    """Extracts the :class:`Dtype` from a :class:`pandas.Series`.
+
+    Args:
+        ser: A :class:`pandas.Series` to analyze.
+
+    Returns:
+        The data type.
+    """
+    if pd.api.types.is_datetime64_any_dtype(ser.dtype):
+        return Dtype.date
+
+    if isinstance(ser.dtype, pd.CategoricalDtype):
+        return Dtype.string
+
+    if pd.api.types.is_object_dtype(ser.dtype):
+        index = ser.iloc[:1000].first_valid_index()
+        if index is not None and pd.api.types.is_list_like(ser[index]):
+            pos = ser.index.get_loc(index)
+            assert isinstance(pos, int)
+            ser = ser.iloc[pos:pos + 1000].dropna()
+
+            if not ser.map(pd.api.types.is_list_like).all():
+                raise ValueError("Data contains a mix of list-like and "
+                                 "non-list-like values")
+
+            # Remove all empty Python lists without known data type:
+            ser = ser[ser.map(lambda x: not isinstance(x, list) or len(x) > 0)]
+
+            # Infer unique data types in this series:
+            dtypes = ser.apply(lambda x: PANDAS_TO_DTYPE.get(
+                np.array(x).dtype, Dtype.string)).unique().tolist()
+
+            invalid_dtypes = set(dtypes) - {
+                Dtype.string,
+                Dtype.int,
+                Dtype.float,
+            }
+            if len(invalid_dtypes) > 0:
+                raise ValueError(f"Data contains unsupported list data types: "
+                                 f"{list(invalid_dtypes)}")
+
+            if Dtype.string in dtypes:
+                return Dtype.stringlist
+
+            if dtypes == [Dtype.int]:
+                return Dtype.intlist
+
+            return Dtype.floatlist
+
+    if ser.dtype not in PANDAS_TO_DTYPE:
+        raise ValueError(f"Unsupported data type '{ser.dtype}'")
+
+    return PANDAS_TO_DTYPE[ser.dtype]

kumoai/experimental/rfm/infer/pkey.py

@@ -0,0 +1,126 @@
+import re
+import warnings
+from typing import Optional
+
+import pandas as pd
+
+
+def infer_primary_key(
+    table_name: str,
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential primary key column.
+
+    Args:
+        table_name: The table name.
+        df: The pandas DataFrame to analyze.
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected primary key, or ``None`` if not found.
+    """
+    # A list of (potentially modified) table names that are eligible to match
+    # with a primary key, i.e.:
+    # - UserInfo -> User
+    # - snakecase <-> camelcase
+    # - camelcase <-> snakecase
+    # - plural <-> singular (users -> user, eligibilities -> eligibility)
+    # - verb -> noun (qualifying -> qualify)
+    _table_names = {table_name}
+    if table_name.lower().endswith('_info'):
+        _table_names.add(table_name[:-5])
+    elif table_name.lower().endswith('info'):
+        _table_names.add(table_name[:-4])
+
+    table_names = set()
+    for _table_name in _table_names:
+        table_names.add(_table_name.lower())
+        snakecase = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', _table_name)
+        snakecase = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', snakecase)
+        table_names.add(snakecase.lower())
+        camelcase = _table_name.replace('_', '')
+        table_names.add(camelcase.lower())
+        if _table_name.lower().endswith('s'):
+            table_names.add(_table_name.lower()[:-1])
+            table_names.add(snakecase.lower()[:-1])
+            table_names.add(camelcase.lower()[:-1])
+        else:
+            table_names.add(_table_name.lower() + 's')
+            table_names.add(snakecase.lower() + 's')
+            table_names.add(camelcase.lower() + 's')
+        if _table_name.lower().endswith('ies'):
+            table_names.add(_table_name.lower()[:-3] + 'y')
+            table_names.add(snakecase.lower()[:-3] + 'y')
+            table_names.add(camelcase.lower()[:-3] + 'y')
+        elif _table_name.lower().endswith('y'):
+            table_names.add(_table_name.lower()[:-1] + 'ies')
+            table_names.add(snakecase.lower()[:-1] + 'ies')
+            table_names.add(camelcase.lower()[:-1] + 'ies')
+        if _table_name.lower().endswith('ing'):
+            table_names.add(_table_name.lower()[:-3])
+            table_names.add(snakecase.lower()[:-3])
+            table_names.add(camelcase.lower()[:-3])
+
+    scores: list[tuple[str, int]] = []
+    for col_name in candidates:
+        col_name_lower = col_name.lower()
+
+        score = 0
+
+        if col_name_lower == 'id':
+            score += 4
+
+        for table_name_lower in table_names:
+
+            if col_name_lower == table_name_lower:
+                score += 4  # USER -> USER
+                break
+
+            for suffix in ['id', 'hash', 'key', 'code', 'uuid']:
+                if not col_name_lower.endswith(suffix):
+                    continue
+
+                if col_name_lower == f'{table_name_lower}_{suffix}':
+                    score += 5  # USER -> USER_ID
+                    break
+
+                if col_name_lower == f'{table_name_lower}{suffix}':
+                    score += 5  # User -> UserId
+                    break
+
+                if col_name_lower.endswith(f'{table_name_lower}_{suffix}'):
+                    score += 2
+
+                if col_name_lower.endswith(f'{table_name_lower}{suffix}'):
+                    score += 2
+
+            # `rel-bench` hard-coding :(
+            if table_name == 'studies' and col_name == 'nct_id':
+                score += 1
+
+        ser = df[col_name].iloc[:1_000_000]
+        score += 3 * (ser.nunique() / len(ser))
+
+        scores.append((col_name, score))
+
+    scores = [x for x in scores if x[-1] >= 4]
+    scores.sort(key=lambda x: x[-1], reverse=True)
+
+    if len(scores) == 0:
+        return None
+
+    if len(scores) == 1:
+        return scores[0][0]
+
+    # In case of multiple candidates, only return one if its score is unique:
+    if scores[0][1] != scores[1][1]:
+        return scores[0][0]
+
+    max_score = max(scores, key=lambda x: x[1])
+    candidates = [col_name for col_name, score in scores if score == max_score]
+    warnings.warn(f"Found multiple potential primary keys in table "
+                  f"'{table_name}': {candidates}. Please specify the primary "
+                  f"key for this table manually.")
+
+    return None

kumoai/experimental/rfm/infer/time_col.py

@@ -0,0 +1,62 @@
+import re
+import warnings
+from typing import Optional
+
+import pandas as pd
+
+
+def infer_time_column(
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential time column.
+
+    Args:
+        df: The pandas DataFrame to analyze.
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected time column, or ``None`` if not found.
+    """
+    candidates = [  # Exclude all candidates with `*last*` in column names:
+        col_name for col_name in candidates
+        if not re.search(r'(^|_)last(_|$)', col_name, re.IGNORECASE)
+    ]
+
+    if len(candidates) == 0:
+        return None
+
+    if len(candidates) == 1:
+        return candidates[0]
+
+    # If there exists a dedicated `create*` column, use it as time column:
+    create_candidates = [
+        candidate for candidate in candidates
+        if candidate.lower().startswith('create')
+    ]
+    if len(create_candidates) == 1:
+        return create_candidates[0]
+    if len(create_candidates) > 1:
+        candidates = create_candidates
+
+    # Find the most optimal time column. Usually, it is the one pointing to
+    # the oldest timestamps:
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', message='Could not infer format')
+        min_timestamp_dict = {
+            key: pd.to_datetime(df[key].iloc[:10_000], 'coerce')
+            for key in candidates
+        }
+    min_timestamp_dict = {
+        key: value.min().tz_localize(None)
+        for key, value in min_timestamp_dict.items()
+    }
+    min_timestamp_dict = {
+        key: value
+        for key, value in min_timestamp_dict.items() if not pd.isna(value)
+    }
+
+    if len(min_timestamp_dict) == 0:
+        return None
+
+    return min(min_timestamp_dict, key=min_timestamp_dict.get)  # type: ignore

kumoai/experimental/rfm/local_graph_sampler.py

@@ -1,3 +1,4 @@
+import re
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
@@ -7,7 +8,47 @@ from kumoapi.typing import Stype
 
 import kumoai.kumolib as kumolib
 from kumoai.experimental.rfm.local_graph_store import LocalGraphStore
-from kumoai.experimental.rfm.utils import normalize_text
+
+PUNCTUATION = re.compile(r"[\'\"\.,\(\)\!\?\;\:]")
+MULTISPACE = re.compile(r"\s+")
+
+
+def normalize_text(
+    ser: pd.Series,
+    max_words: Optional[int] = 50,
+) -> pd.Series:
+    r"""Normalizes text into a list of lower-case words.
+
+    Args:
+        ser: The :class:`pandas.Series` to normalize.
+        max_words: The maximum number of words to return.
+            This will auto-shrink any large text column to avoid blowing up
+            context size.
+    """
+    if len(ser) == 0 or pd.api.types.is_list_like(ser.iloc[0]):
+        return ser
+
+    def normalize_fn(line: str) -> list[str]:
+        line = PUNCTUATION.sub(" ", line)
+        line = re.sub(r"<br\s*/?>", " ", line)  # Handle <br /> or <br>
+        line = MULTISPACE.sub(" ", line)
+        words = line.split()
+        if max_words is not None:
+            words = words[:max_words]
+        return words
+
+    ser = ser.fillna('').astype(str)
+
+    if max_words is not None:
+        # We estimate the number of words as 5 characters + 1 space in an
+        # English text on average. We need this pre-filter here, as word
+        # splitting on a giant text can be very expensive:
+        ser = ser.str[:6 * max_words]
+
+    ser = ser.str.lower()
+    ser = ser.map(normalize_fn)
+
+    return ser
 
 
 class LocalGraphSampler: