kumoai 2.13.0.dev202512091732__cp311-cp311-macosx_11_0_arm64.whl → 2.14.0.dev202512191731__cp311-cp311-macosx_11_0_arm64.whl

kumoai/experimental/rfm/graph.py

@@ -1,10 +1,13 @@
 import contextlib
+import copy
 import io
 import warnings
 from collections import defaultdict
+from collections.abc import Sequence
 from dataclasses import dataclass, field
+from itertools import chain
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Any, Union
 
 import pandas as pd
 from kumoapi.graph import ColumnKey, ColumnKeyGroup, GraphDefinition
@@ -13,7 +16,12 @@ from kumoapi.typing import Stype
 from typing_extensions import Self
 
 from kumoai import in_notebook, in_snowflake_notebook
-from kumoai.experimental.rfm import Table
+from kumoai.experimental.rfm.base import (
+    ColumnExpressionSpec,
+    DataBackend,
+    SQLTable,
+    Table,
+)
 from kumoai.graph import Edge
 from kumoai.mixin import CastMixin
 
@@ -25,8 +33,8 @@ if TYPE_CHECKING:
 
 @dataclass
 class SqliteConnectionConfig(CastMixin):
-    uri: Union[str, Path]
-    kwargs: Dict[str, Any] = field(default_factory=dict)
+    uri: str | Path
+    kwargs: dict[str, Any] = field(default_factory=dict)
 
 
 class Graph:
@@ -86,27 +94,38 @@ class Graph:
     def __init__(
         self,
         tables: Sequence[Table],
-        edges: Optional[Sequence[Edge]] = None,
+        edges: Sequence[Edge] | None = None,
     ) -> None:
 
-        self._tables: Dict[str, Table] = {}
-        self._edges: List[Edge] = []
+        self._tables: dict[str, Table] = {}
+        self._edges: list[Edge] = []
 
         for table in tables:
             self.add_table(table)
 
         for table in tables:
+            if not isinstance(table, SQLTable):
+                continue
+            if '_source_column_dict' not in table.__dict__:
+                continue
             for fkey in table._source_foreign_key_dict.values():
-                if fkey.name not in table or fkey.dst_table not in self:
+                if fkey.name not in table:
+                    continue
+                if not table[fkey.name].is_physical:
+                    continue
+                dst_table_names = [
+                    table.name for table in self.tables.values()
+                    if isinstance(table, SQLTable)
+                    and table._source_name == fkey.dst_table
+                ]
+                if len(dst_table_names) != 1:
+                    continue
+                dst_table = self[dst_table_names[0]]
+                if dst_table._primary_key != fkey.primary_key:
+                    continue
+                if not dst_table[fkey.primary_key].is_physical:
                     continue
-                if self[fkey.dst_table].primary_key is None:
-                    self[fkey.dst_table].primary_key = fkey.primary_key
-                elif self[fkey.dst_table]._primary_key != fkey.primary_key:
-                    raise ValueError(f"Found duplicate primary key definition "
-                                     f"'{self[fkey.dst_table]._primary_key}' "
-                                     f"and '{fkey.primary_key}' in table "
-                                     f"'{fkey.dst_table}'.")
-                self.link(table.name, fkey.name, fkey.dst_table)
+                self.link(table.name, fkey.name, dst_table.name)
 
         for edge in (edges or []):
             _edge = Edge._cast(edge)
@@ -117,8 +136,8 @@ class Graph:
     @classmethod
     def from_data(
         cls,
-        df_dict: Dict[str, pd.DataFrame],
-        edges: Optional[Sequence[Edge]] = None,
+        df_dict: dict[str, pd.DataFrame],
+        edges: Sequence[Edge] | None = None,
         infer_metadata: bool = True,
         verbose: bool = True,
     ) -> Self:
@@ -156,15 +175,17 @@ class Graph:
             verbose: Whether to print verbose output.
         """
         from kumoai.experimental.rfm.backend.local import LocalTable
-        tables = [LocalTable(df, name) for name, df in df_dict.items()]
 
-        graph = cls(tables, edges=edges or [])
+        graph = cls(
+            tables=[LocalTable(df, name) for name, df in df_dict.items()],
+            edges=edges or [],
+        )
 
         if infer_metadata:
-            graph.infer_metadata(False)
+            graph.infer_metadata(verbose=False)
 
             if edges is None:
-                graph.infer_links(False)
+                graph.infer_links(verbose=False)
 
         if verbose:
             graph.print_metadata()
@@ -180,10 +201,10 @@ class Graph:
             SqliteConnectionConfig,
             str,
             Path,
-            Dict[str, Any],
+            dict[str, Any],
         ],
-        table_names: Optional[Sequence[str]] = None,
-        edges: Optional[Sequence[Edge]] = None,
+        tables: Sequence[str | dict[str, Any]] | None = None,
+        edges: Sequence[Edge] | None = None,
         infer_metadata: bool = True,
         verbose: bool = True,
     ) -> Self:
@@ -199,17 +220,25 @@ class Graph:
             >>> # Create a graph from a SQLite database:
             >>> graph = rfm.Graph.from_sqlite('data.db')
 
+            >>> # Fine-grained control over table specification:
+            >>> graph = rfm.Graph.from_sqlite('data.db', tables=[
+            ...     'USERS',
+            ...     dict(name='ORDERS', source_name='ORDERS_SNAPSHOT'),
+            ...     dict(name='ITEMS', primary_key='ITEM_ID'),
+            ... ])
+
         Args:
             connection: An open connection from
                 :meth:`~kumoai.experimental.rfm.backend.sqlite.connect` or the
                 path to the database file.
-            table_names: Set of table names to include. If ``None``, will add
-                all tables present in the database.
+            tables: Set of table names or :class:`SQLiteTable` keyword
+                arguments to include. If ``None``, will add all tables present
+                in the database.
             edges: An optional list of :class:`~kumoai.graph.Edge` objects to
                 add to the graph. If not provided, edges will be automatically
                 inferred from the data in case ``infer_metadata=True``.
-            infer_metadata: Whether to infer metadata for all tables in the
-                graph.
+            infer_metadata: Whether to infer missing metadata for all tables in
+                the graph.
             verbose: Whether to print verbose output.
         """
         from kumoai.experimental.rfm.backend.sqlite import (
@@ -218,27 +247,41 @@ class Graph:
             connect,
         )
 
+        internal_connection = False
         if not isinstance(connection, Connection):
             connection = SqliteConnectionConfig._cast(connection)
             assert isinstance(connection, SqliteConnectionConfig)
             connection = connect(connection.uri, **connection.kwargs)
+            internal_connection = True
         assert isinstance(connection, Connection)
 
-        if table_names is None:
+        if tables is None:
             with connection.cursor() as cursor:
                 cursor.execute("SELECT name FROM sqlite_master "
                                "WHERE type='table'")
-                table_names = [row[0] for row in cursor.fetchall()]
+                tables = [row[0] for row in cursor.fetchall()]
 
-        tables = [SQLiteTable(connection, name) for name in table_names]
+        table_kwargs: list[dict[str, Any]] = []
+        for table in tables:
+            kwargs = dict(name=table) if isinstance(table, str) else table
+            table_kwargs.append(kwargs)
+
+        graph = cls(
+            tables=[
+                SQLiteTable(connection=connection, **kwargs)
+                for kwargs in table_kwargs
+            ],
+            edges=edges or [],
+        )
 
-        graph = cls(tables, edges=edges or [])
+        if internal_connection:
+            graph._connection = connection  # type: ignore
 
         if infer_metadata:
-            graph.infer_metadata(False)
+            graph.infer_metadata(verbose=False)
 
             if edges is None:
-                graph.infer_links(False)
+                graph.infer_links(verbose=False)
 
         if verbose:
             graph.print_metadata()
@@ -249,11 +292,11 @@ class Graph:
     @classmethod
     def from_snowflake(
         cls,
-        connection: Union['SnowflakeConnection', Dict[str, Any], None] = None,
-        database: Optional[str] = None,
-        schema: Optional[str] = None,
-        table_names: Optional[Sequence[str]] = None,
-        edges: Optional[Sequence[Edge]] = None,
+        connection: Union['SnowflakeConnection', dict[str, Any], None] = None,
+        tables: Sequence[str | dict[str, Any]] | None = None,
+        database: str | None = None,
+        schema: str | None = None,
+        edges: Sequence[Edge] | None = None,
         infer_metadata: bool = True,
         verbose: bool = True,
     ) -> Self:
@@ -270,6 +313,13 @@ class Graph:
             >>> # Create a graph directly in a Snowflake notebook:
             >>> graph = rfm.Graph.from_snowflake(schema='my_schema')
 
+            >>> # Fine-grained control over table specification:
+            >>> graph = rfm.Graph.from_snowflake(tables=[
+            ...     'USERS',
+            ...     dict(name='ORDERS', source_name='ORDERS_SNAPSHOT'),
+            ...     dict(name='ITEMS', schema='OTHER_SCHEMA'),
+            ... ], database='DEFAULT_DB', schema='DEFAULT_SCHEMA')
+
         Args:
             connection: An open connection from
                 :meth:`~kumoai.experimental.rfm.backend.snow.connect` or the
@@ -277,10 +327,11 @@ class Graph:
                 connection. If ``None``, will re-use an active session in case
                 it exists, or create a new connection from credentials stored
                 in environment variables.
+            tables: Set of table names or :class:`SnowTable` keyword arguments
+                to include. If ``None``, will add all tables present in the
+                current database and schema.
             database: The database.
             schema: The schema.
-            table_names: Set of table names to include. If ``None``, will add
-                all tables present in the database.
             edges: An optional list of :class:`~kumoai.graph.Edge` objects to
                 add to the graph. If not provided, edges will be automatically
                 inferred from the data in case ``infer_metadata=True``.
@@ -298,37 +349,50 @@ class Graph:
             connection = connect(**(connection or {}))
         assert isinstance(connection, Connection)
 
-        if table_names is None:
+        if database is None or schema is None:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT CURRENT_DATABASE(), CURRENT_SCHEMA()")
+                result = cursor.fetchone()
+                database = database or result[0]
+                assert database is not None
+                schema = schema or result[1]
+
+        if tables is None:
+            if schema is None:
+                raise ValueError("No current 'schema' set. Please specify the "
+                                 "Snowflake schema manually")
+
             with connection.cursor() as cursor:
-                if database is None and schema is None:
-                    cursor.execute("SELECT CURRENT_DATABASE(), "
-                                   "CURRENT_SCHEMA()")
-                    result = cursor.fetchone()
-                    database = database or result[0]
-                    schema = schema or result[1]
                 cursor.execute(f"""
                 SELECT TABLE_NAME
                 FROM {database}.INFORMATION_SCHEMA.TABLES
                 WHERE TABLE_SCHEMA = '{schema}'
                 """)
-                table_names = [row[0] for row in cursor.fetchall()]
+                tables = [row[0] for row in cursor.fetchall()]
 
-        tables = [
-            SnowTable(
-                connection,
-                name=table_name,
-                database=database,
-                schema=schema,
-            ) for table_name in table_names
-        ]
-
-        graph = cls(tables, edges=edges or [])
+        table_kwargs: list[dict[str, Any]] = []
+        for table in tables:
+            if isinstance(table, str):
+                kwargs = dict(name=table, database=database, schema=schema)
+            else:
+                kwargs = copy.copy(table)
+                kwargs.setdefault('database', database)
+                kwargs.setdefault('schema', schema)
+            table_kwargs.append(kwargs)
+
+        graph = cls(
+            tables=[
+                SnowTable(connection=connection, **kwargs)
+                for kwargs in table_kwargs
+            ],
+            edges=edges or [],
+        )
 
         if infer_metadata:
-            graph.infer_metadata(False)
+            graph.infer_metadata(verbose=False)
 
             if edges is None:
-                graph.infer_links(False)
+                graph.infer_links(verbose=False)
 
         if verbose:
             graph.print_metadata()
@@ -340,7 +404,7 @@ class Graph:
     def from_snowflake_semantic_view(
         cls,
         semantic_view_name: str,
-        connection: Union['SnowflakeConnection', Dict[str, Any], None] = None,
+        connection: Union['SnowflakeConnection', dict[str, Any], None] = None,
         verbose: bool = True,
     ) -> Self:
         import yaml
@@ -358,43 +422,138 @@ class Graph:
         with connection.cursor() as cursor:
             cursor.execute(f"SELECT SYSTEM$READ_YAML_FROM_SEMANTIC_VIEW("
                            f"'{semantic_view_name}')")
-            view = yaml.safe_load(cursor.fetchone()[0])
+            cfg = yaml.safe_load(cursor.fetchone()[0])
 
         graph = cls(tables=[])
 
-        for table_desc in view['tables']:
-            primary_key: Optional[str] = None
-            if ('primary_key' in table_desc  # NOTE No composite keys yet.
-                    and len(table_desc['primary_key']['columns']) == 1):
-                primary_key = table_desc['primary_key']['columns'][0]
+        msgs = []
+        table_names = {table_cfg['name'] for table_cfg in cfg['tables']}
+        for table_cfg in cfg['tables']:
+            table_name = table_cfg['name']
+            source_table_name = table_cfg['base_table']['table']
+            database = table_cfg['base_table']['database']
+            schema = table_cfg['base_table']['schema']
+
+            primary_key: str | None = None
+            if 'primary_key' in table_cfg:
+                primary_key_cfg = table_cfg['primary_key']
+                if len(primary_key_cfg['columns']) == 1:
+                    primary_key = primary_key_cfg['columns'][0]
+                elif len(primary_key_cfg['columns']) > 1:
+                    msgs.append(f"Failed to add primary key for table "
+                                f"'{table_name}' since composite primary keys "
+                                f"are not yet supported")
+
+            columns: list[str] = []
+            unsupported_columns: list[str] = []
+            column_expression_specs: list[ColumnExpressionSpec] = []
+            for column_cfg in chain(
+                    table_cfg.get('dimensions', []),
+                    table_cfg.get('time_dimensions', []),
+                    table_cfg.get('facts', []),
+            ):
+                column_name = column_cfg['name']
+                column_expr = column_cfg.get('expr', None)
+                column_data_type = column_cfg.get('data_type', None)
+
+                if column_expr is None:
+                    columns.append(column_name)
+                    continue
+
+                column_expr = column_expr.replace(f'{table_name}.', '')
+
+                if column_expr == column_name:
+                    columns.append(column_name)
+                    continue
+
+                # Drop expressions that reference other tables (for now):
+                if any(f'{name}.' in column_expr for name in table_names):
+                    unsupported_columns.append(column_name)
+                    continue
+
+                spec = ColumnExpressionSpec(
+                    name=column_name,
+                    expr=column_expr,
+                    dtype=SnowTable.to_dtype(column_data_type),
+                )
+                column_expression_specs.append(spec)
+
+            if len(unsupported_columns) == 1:
+                msgs.append(f"Failed to add column '{unsupported_columns[0]}' "
+                            f"of table '{table_name}' since its expression "
+                            f"references other tables")
+            elif len(unsupported_columns) > 1:
+                msgs.append(f"Failed to add columns '{unsupported_columns}' "
+                            f"of table '{table_name}' since their expressions "
+                            f"reference other tables")
 
             table = SnowTable(
                 connection,
-                name=table_desc['base_table']['table'],
-                database=table_desc['base_table']['database'],
-                schema=table_desc['base_table']['schema'],
+                name=table_name,
+                source_name=source_table_name,
+                database=database,
+                schema=schema,
+                columns=columns,
+                column_expressions=column_expression_specs,
                 primary_key=primary_key,
             )
+
+            # TODO Add a way to register time columns without heuristic usage.
+            table.infer_time_column(verbose=False)
+
             graph.add_table(table)
 
-        # TODO Find a solution to register time columns!
+        for relation_cfg in cfg.get('relationships', []):
+            name = relation_cfg['name']
+            if len(relation_cfg['relationship_columns']) != 1:
+                msgs.append(f"Failed to add relationship '{name}' since "
+                            f"composite key references are not yet supported")
+                continue
 
-        for relations in view['relationships']:
-            if len(relations['relationship_columns']) != 1:
-                continue  # NOTE No composite keys yet.
-            graph.link(
-                src_table=relations['left_table'],
-                fkey=relations['relationship_columns'][0]['left_column'],
-                dst_table=relations['right_table'],
-            )
+            left_table = relation_cfg['left_table']
+            left_key = relation_cfg['relationship_columns'][0]['left_column']
+            right_table = relation_cfg['right_table']
+            right_key = relation_cfg['relationship_columns'][0]['right_column']
+
+            if graph[right_table]._primary_key != right_key:
+                # Semantic view error - this should never be triggered:
+                msgs.append(f"Failed to add relationship '{name}' since the "
+                            f"referenced key '{right_key}' of table "
+                            f"'{right_table}' is not a primary key")
+                continue
+
+            if graph[left_table]._primary_key == left_key:
+                msgs.append(f"Failed to add relationship '{name}' since the "
+                            f"referencing key '{left_key}' of table "
+                            f"'{left_table}' is a primary key")
+                continue
+
+            if left_key not in graph[left_table]:
+                graph[left_table].add_column(left_key)
+
+            graph.link(left_table, left_key, right_table)
+
+        graph.validate()
 
         if verbose:
             graph.print_metadata()
             graph.print_links()
 
+        if len(msgs) > 0:
+            title = (f"Could not fully convert the semantic view definition "
+                     f"'{semantic_view_name}' into a graph:\n")
+            warnings.warn(title + '\n'.join(f'- {msg}' for msg in msgs))
+
         return graph
 
-    # Tables ##############################################################
+    # Backend #################################################################
+
+    @property
+    def backend(self) -> DataBackend | None:
+        backends = [table.backend for table in self._tables.values()]
+        return backends[0] if len(backends) > 0 else None
+
+    # Tables ##################################################################
 
     def has_table(self, name: str) -> bool:
         r"""Returns ``True`` if the graph has a table with name ``name``;
@@ -413,7 +572,7 @@ class Graph:
         return self.tables[name]
 
     @property
-    def tables(self) -> Dict[str, Table]:
+    def tables(self) -> dict[str, Table]:
         r"""Returns the dictionary of table objects."""
         return self._tables
 
@@ -433,13 +592,10 @@ class Graph:
             raise KeyError(f"Cannot add table with name '{table.name}' to "
                            f"this graph; table names must be globally unique.")
 
-        if len(self._tables) > 0:
-            cls = next(iter(self._tables.values())).__class__
-            if table.__class__ != cls:
-                raise ValueError(f"Cannot register a "
-                                 f"'{table.__class__.__name__}' to this "
-                                 f"graph since other tables are of type "
-                                 f"'{cls.__name__}'.")
+        if self.backend is not None and table.backend != self.backend:
+            raise ValueError(f"Cannot register a table with backend "
+                             f"'{table.backend}' to this graph since other "
+                             f"tables have backend '{self.backend}'.")
 
         self._tables[table.name] = table
 
@@ -541,7 +697,7 @@ class Graph:
     # Edges ###################################################################
 
     @property
-    def edges(self) -> List[Edge]:
+    def edges(self) -> list[Edge]:
         r"""Returns the edges of the graph."""
         return self._edges
 
@@ -556,7 +712,7 @@ class Graph:
             st.markdown("### 🕸️ Graph Links (FK ↔️ PK)")
             if len(edges) > 0:
                 st.markdown('\n'.join([
-                    f"- `{edge[2]}.{edge[3]}` ↔️ `{edge[0]}.{edge[1]}`"
+                    f"- {edge[2]}.{edge[3]} ↔️ {edge[0]}.{edge[1]}"
                     for edge in edges
                 ]))
             else:
@@ -584,9 +740,9 @@ class Graph:
 
     def link(
         self,
-        src_table: Union[str, Table],
+        src_table: str | Table,
         fkey: str,
-        dst_table: Union[str, Table],
+        dst_table: str | Table,
     ) -> Self:
         r"""Links two tables (``src_table`` and ``dst_table``) from the foreign
         key ``fkey`` in the source table to the primary key in the destination
@@ -647,9 +803,9 @@ class Graph:
 
     def unlink(
         self,
-        src_table: Union[str, Table],
+        src_table: str | Table,
         fkey: str,
-        dst_table: Union[str, Table],
+        dst_table: str | Table,
     ) -> Self:
         r"""Removes an :class:`~kumoai.graph.Edge` from the graph.
 
@@ -826,6 +982,10 @@ class Graph:
             raise ValueError("At least one table needs to be added to the "
                              "graph")
 
+        backends = {table.backend for table in self._tables.values()}
+        if len(backends) != 1:
+            raise ValueError("Found multiple table backends in the graph")
+
         for edge in self.edges:
             src_table, fkey, dst_table = edge
 
@@ -872,7 +1032,7 @@ class Graph:
 
     def visualize(
         self,
-        path: Optional[Union[str, io.BytesIO]] = None,
+        path: str | io.BytesIO | None = None,
         show_columns: bool = True,
     ) -> 'graphviz.Graph':
         r"""Visualizes the tables and edges in this graph using the
@@ -911,19 +1071,19 @@ class Graph:
                                "them as described at "
                                "https://graphviz.org/download/.")
 
-        format: Optional[str] = None
+        format: str | None = None
         if isinstance(path, str):
             format = path.split('.')[-1]
         elif isinstance(path, io.BytesIO):
             format = 'svg'
         graph = graphviz.Graph(format=format)
 
-        def left_align(keys: List[str]) -> str:
+        def left_align(keys: list[str]) -> str:
             if len(keys) == 0:
                 return ""
             return '\\l'.join(keys) + '\\l'
 
-        fkeys_dict: Dict[str, List[str]] = defaultdict(list)
+        fkeys_dict: dict[str, list[str]] = defaultdict(list)
         for src_table_name, fkey_name, _ in self.edges:
             fkeys_dict[src_table_name].append(fkey_name)
 
@@ -1019,8 +1179,8 @@ class Graph:
     # Helpers #################################################################
 
     def _to_api_graph_definition(self) -> GraphDefinition:
-        tables: Dict[str, TableDefinition] = {}
-        col_groups: List[ColumnKeyGroup] = []
+        tables: dict[str, TableDefinition] = {}
+        col_groups: list[ColumnKeyGroup] = []
         for table_name, table in self.tables.items():
             tables[table_name] = table._to_api_table_definition()
             if table.primary_key is None:
@@ -1063,3 +1223,7 @@ class Graph:
                 f'  tables={tables},\n'
                 f'  edges={edges},\n'
                 f')')
+
+    def __del__(self) -> None:
+        if hasattr(self, '_connection'):
+            self._connection.close()

kumoai/experimental/rfm/infer/__init__.py

@@ -1,17 +1,19 @@
 from .dtype import infer_dtype
-from .pkey import infer_primary_key
-from .time_col import infer_time_column
 from .id import contains_id
 from .timestamp import contains_timestamp
 from .categorical import contains_categorical
 from .multicategorical import contains_multicategorical
+from .stype import infer_stype
+from .pkey import infer_primary_key
+from .time_col import infer_time_column
 
 __all__ = [
     'infer_dtype',
-    'infer_primary_key',
-    'infer_time_column',
     'contains_id',
     'contains_timestamp',
     'contains_categorical',
     'contains_multicategorical',
+    'infer_stype',
+    'infer_primary_key',
+    'infer_time_column',
 ]

kumoai/experimental/rfm/infer/dtype.py

@@ -1,17 +1,17 @@
-from typing import Dict
-
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 from kumoapi.typing import Dtype
 
-PANDAS_TO_DTYPE: Dict[str, Dtype] = {
+PANDAS_TO_DTYPE: dict[str, Dtype] = {
     'bool': Dtype.bool,
     'boolean': Dtype.bool,
     'int8': Dtype.int,
     'int16': Dtype.int,
     'int32': Dtype.int,
     'int64': Dtype.int,
+    'float': Dtype.float,
+    'double': Dtype.float,
     'float16': Dtype.float,
     'float32': Dtype.float,
     'float64': Dtype.float,