kumoai 2.13.0.dev202512021731__cp310-cp310-win_amd64.whl → 2.13.0.dev202512040252__cp310-cp310-win_amd64.whl

kumoai/_version.py

@@ -1 +1 @@
-__version__ = '2.13.0.dev202512021731'
+__version__ = '2.13.0.dev202512040252'

kumoai/experimental/rfm/backend/local/table.py

@@ -1,14 +1,10 @@
-from typing import Any, Dict, List, Optional, Tuple
+import warnings
+from typing import List, Optional
 
-import numpy as np
 import pandas as pd
-import pyarrow as pa
-from kumoapi.typing import Dtype, Stype
-from typing_extensions import Self
 
-from kumoai.experimental.rfm import utils
-from kumoai.experimental.rfm.base import Column, Table
-from kumoai.experimental.rfm.infer import infer_stype
+from kumoai.experimental.rfm.base import SourceColumn, SourceForeignKey, Table
+from kumoai.experimental.rfm.infer import infer_dtype
 
 
 class LocalTable(Table):
@@ -62,7 +58,7 @@ class LocalTable(Table):
     ) -> None:
 
         if df.empty:
-            raise ValueError("Data frame must have at least one row")
+            raise ValueError("Data frame is empty")
         if isinstance(df.columns, pd.MultiIndex):
             raise ValueError("Data frame must not have a multi-index")
         if not df.columns.is_unique:
@@ -80,165 +76,34 @@ class LocalTable(Table):
             end_time_column=end_time_column,
         )
 
-    def infer_metadata(self, verbose: bool = True) -> Self:
-        r"""Infers metadata, *i.e.*, primary keys and time columns, in the
-        table.
-
-        Args:
-            verbose: Whether to print verbose output.
-        """
-        logs = []
-
-        # Try to detect primary key if not set:
-        if not self.has_primary_key():
-
-            def is_candidate(column: Column) -> bool:
-                if column.stype == Stype.ID:
-                    return True
-                if all(column.stype != Stype.ID for column in self.columns):
-                    if self.name == column.name:
-                        return True
-                    if (self.name.endswith('s')
-                            and self.name[:-1] == column.name):
-                        return True
-                return False
-
-            candidates = [
-                column.name for column in self.columns if is_candidate(column)
-            ]
-
-            if primary_key := utils.detect_primary_key(
-                    table_name=self.name,
-                    df=self._data,
-                    candidates=candidates,
-            ):
-                self.primary_key = primary_key
-                logs.append(f"primary key '{primary_key}'")
-
-        # Try to detect time column if not set:
-        if not self.has_time_column():
-            candidates = [
-                column.name for column in self.columns
-                if column.stype == Stype.timestamp
-                and column.name != self._end_time_column
-            ]
-            if time_column := utils.detect_time_column(self._data, candidates):
-                self.time_column = time_column
-                logs.append(f"time column '{time_column}'")
-
-        if verbose and len(logs) > 0:
-            print(f"Detected {' and '.join(logs)} in table '{self.name}'")
-
-        return self
-
-    def _has_source_column(self, name: str) -> bool:
-        return name in self._data.columns
-
-    def _get_source_dtype(self, name: str) -> Dtype:
-        return to_dtype(self._data[name])
-
-    def _get_source_stype(self, name: str, dtype: Dtype) -> Stype:
-        return infer_stype(self._data[name], name, dtype)
-
-    def _get_source_foreign_keys(self) -> List[Tuple[str, str, str]]:
+    def _get_source_columns(self) -> List[SourceColumn]:
+        source_columns: List[SourceColumn] = []
+        for column in self._data.columns:
+            ser = self._data[column]
+            try:
+                dtype = infer_dtype(ser)
+            except Exception:
+                warnings.warn(f"Data type inference for column '{column}' in "
+                              f"table '{self.name}' failed. Consider changing "
+                              f"the data type of the column to use it within "
+                              f"this table.")
+                continue
+
+            source_column = SourceColumn(
+                name=column,
+                dtype=dtype,
+                is_primary_key=False,
+                is_unique_key=False,
+            )
+            source_columns.append(source_column)
+
+        return source_columns
+
+    def _get_source_foreign_keys(self) -> List[SourceForeignKey]:
         return []
 
-    def _infer_primary_key(self, candidates: List[str]) -> Optional[str]:
-        return utils.detect_primary_key(
-            table_name=self.name,
-            df=self._data,
-            candidates=candidates,
-        )
-
-    def _infer_time_column(self, candidates: List[str]) -> Optional[str]:
-        return utils.detect_time_column(df=self._data, candidates=candidates)
+    def _get_sample_df(self) -> pd.DataFrame:
+        return self._data
 
-    def _num_rows(self) -> Optional[int]:
+    def _get_num_rows(self) -> Optional[int]:
         return len(self._data)
-
-
-# Data Type ###################################################################
-
-PANDAS_TO_DTYPE: Dict[Any, Dtype] = {
-    np.dtype('bool'): Dtype.bool,
-    pd.BooleanDtype(): Dtype.bool,
-    pa.bool_(): Dtype.bool,
-    np.dtype('byte'): Dtype.int,
-    pd.UInt8Dtype(): Dtype.int,
-    np.dtype('int16'): Dtype.int,
-    pd.Int16Dtype(): Dtype.int,
-    np.dtype('int32'): Dtype.int,
-    pd.Int32Dtype(): Dtype.int,
-    np.dtype('int64'): Dtype.int,
-    pd.Int64Dtype(): Dtype.int,
-    np.dtype('float32'): Dtype.float,
-    pd.Float32Dtype(): Dtype.float,
-    np.dtype('float64'): Dtype.float,
-    pd.Float64Dtype(): Dtype.float,
-    np.dtype('object'): Dtype.string,
-    pd.StringDtype(storage='python'): Dtype.string,
-    pd.StringDtype(storage='pyarrow'): Dtype.string,
-    pa.string(): Dtype.string,
-    pa.binary(): Dtype.binary,
-    np.dtype('datetime64[ns]'): Dtype.date,
-    np.dtype('timedelta64[ns]'): Dtype.timedelta,
-    pa.list_(pa.float32()): Dtype.floatlist,
-    pa.list_(pa.int64()): Dtype.intlist,
-    pa.list_(pa.string()): Dtype.stringlist,
-}
-
-
-def to_dtype(ser: pd.Series) -> Dtype:
-    """Extracts the :class:`Dtype` from a :class:`pandas.Series`.
-
-    Args:
-        ser: A :class:`pandas.Series` to analyze.
-
-    Returns:
-        The data type.
-    """
-    if pd.api.types.is_datetime64_any_dtype(ser.dtype):
-        return Dtype.date
-
-    if isinstance(ser.dtype, pd.CategoricalDtype):
-        return Dtype.string
-
-    if pd.api.types.is_object_dtype(ser.dtype):
-        index = ser.iloc[:1000].first_valid_index()
-        if index is not None and pd.api.types.is_list_like(ser[index]):
-            pos = ser.index.get_loc(index)
-            assert isinstance(pos, int)
-            ser = ser.iloc[pos:pos + 1000].dropna()
-
-            if not ser.map(pd.api.types.is_list_like).all():
-                raise ValueError("Data contains a mix of list-like and "
-                                 "non-list-like values")
-
-            # Remove all empty Python lists without known data type:
-            ser = ser[ser.map(lambda x: not isinstance(x, list) or len(x) > 0)]
-
-            # Infer unique data types in this series:
-            dtypes = ser.apply(lambda x: PANDAS_TO_DTYPE.get(
-                np.array(x).dtype, Dtype.string)).unique().tolist()
-
-            invalid_dtypes = set(dtypes) - {
-                Dtype.string,
-                Dtype.int,
-                Dtype.float,
-            }
-            if len(invalid_dtypes) > 0:
-                raise ValueError(f"Data contains unsupported list data types: "
-                                 f"{list(invalid_dtypes)}")
-
-            if Dtype.string in dtypes:
-                return Dtype.stringlist
-
-            if dtypes == [Dtype.int]:
-                return Dtype.intlist
-
-            return Dtype.floatlist
-
-    if ser.dtype not in PANDAS_TO_DTYPE:
-        raise ValueError(f"Unsupported data type '{ser.dtype}'")
-
-    return PANDAS_TO_DTYPE[ser.dtype]

kumoai/experimental/rfm/backend/snow/__init__.py

@@ -26,7 +26,10 @@ def connect(**kwargs: Any) -> Connection:
     return snowflake.connector.connect(**kwargs)
 
 
+from .table import SnowTable  # noqa: E402
+
 __all__ = [
     'connect',
     'Connection',
+    'SnowTable',
 ]

kumoai/experimental/rfm/backend/snow/table.py

@@ -0,0 +1,115 @@
+import re
+from typing import List, Optional, Sequence
+
+import pandas as pd
+from kumoapi.typing import Dtype
+
+from kumoai.experimental.rfm.backend.sqlite import Connection
+from kumoai.experimental.rfm.base import SourceColumn, SourceForeignKey, Table
+
+
+class SnowTable(Table):
+    r"""A table backed by a :class:`sqlite` database.
+
+    Args:
+        connection: The connection to a :class:`snowflake` database.
+        name: The name of this table.
+        columns: The selected columns of this table.
+        primary_key: The name of the primary key of this table, if it exists.
+        time_column: The name of the time column of this table, if it exists.
+        end_time_column: The name of the end time column of this table, if it
+            exists.
+    """
+    def __init__(
+        self,
+        connection: Connection,
+        name: str,
+        database: str | None = None,
+        schema: str | None = None,
+        columns: Optional[Sequence[str]] = None,
+        primary_key: Optional[str] = None,
+        time_column: Optional[str] = None,
+        end_time_column: Optional[str] = None,
+    ) -> None:
+
+        if database is not None and schema is None:
+            raise ValueError(f"Missing 'schema' for table '{name}' in "
+                             f"database '{database}'")
+
+        self._connection = connection
+        self._database = database
+        self._schema = schema
+
+        super().__init__(
+            name=name,
+            columns=columns,
+            primary_key=primary_key,
+            time_column=time_column,
+            end_time_column=end_time_column,
+        )
+
+    @property
+    def fqn_name(self) -> str:
+        names: List[str] = []
+        if self._database is not None:
+            assert self._schema is not None
+            names.extend([self._database, self._schema])
+        elif self._schema is not None:
+            names.append(self._schema)
+        names.append(self._name)
+        return '.'.join(names)
+
+    def _get_source_columns(self) -> List[SourceColumn]:
+        source_columns: List[SourceColumn] = []
+        with self._connection.cursor() as cursor:
+            try:
+                cursor.execute(f"DESCRIBE TABLE {self.fqn_name}")
+            except Exception as e:
+                raise ValueError(
+                    f"Table '{self.fqn_name}' does not exist") from e
+
+            for row in cursor.fetchall():
+                column, type, _, _, _, is_pkey, is_unique = row[:7]
+
+                type = type.strip().upper()
+                if type.startswith('NUMBER'):
+                    dtype = Dtype.int
+                elif type.startswith('VARCHAR'):
+                    dtype = Dtype.string
+                elif type == 'FLOAT':
+                    dtype = Dtype.float
+                elif type == 'BOOLEAN':
+                    dtype = Dtype.bool
+                elif re.search('DATE|TIMESTAMP', type):
+                    dtype = Dtype.date
+                else:
+                    continue
+
+                source_column = SourceColumn(
+                    name=column,
+                    dtype=dtype,
+                    is_primary_key=is_pkey.strip().upper() == 'Y',
+                    is_unique_key=is_unique.strip().upper() == 'Y',
+                )
+                source_columns.append(source_column)
+
+        return source_columns
+
+    def _get_source_foreign_keys(self) -> List[SourceForeignKey]:
+        source_fkeys: List[SourceForeignKey] = []
+        with self._connection.cursor() as cursor:
+            cursor.execute(f"SHOW IMPORTED KEYS IN TABLE {self.fqn_name}")
+            for row in cursor.fetchall():
+                _, _, _, dst_table, pkey, _, _, _, fkey = row[:9]
+                source_fkeys.append(SourceForeignKey(fkey, dst_table, pkey))
+        return source_fkeys
+
+    def _get_sample_df(self) -> pd.DataFrame:
+        with self._connection.cursor() as cursor:
+            columns = ', '.join(self._source_column_dict.keys())
+            cursor.execute(f"SELECT {columns} FROM {self.fqn_name} LIMIT 1000")
+            table = cursor.fetch_arrow_all()
+            return table.to_pandas(types_mapper=pd.ArrowDtype)
+
+    def _get_num_rows(self) -> Optional[int]:
+        return None

kumoai/experimental/rfm/backend/sqlite/table.py

@@ -1,13 +1,13 @@
 import re
-from typing import Dict, List, Optional, Sequence, Tuple
+import warnings
+from typing import List, Optional, Sequence
 
-import pyarrow as pa
-from kumoapi.typing import Dtype, Stype
-from typing_extensions import Self
+import pandas as pd
+from kumoapi.typing import Dtype
 
 from kumoai.experimental.rfm.backend.sqlite import Connection
-from kumoai.experimental.rfm.base import Table
-from kumoai.experimental.rfm.infer import infer_stype
+from kumoai.experimental.rfm.base import SourceColumn, SourceForeignKey, Table
+from kumoai.experimental.rfm.infer import infer_dtype
 
 
 class SQLiteTable(Table):
@@ -33,92 +33,69 @@ class SQLiteTable(Table):
     ) -> None:
 
         self._connection = connection
-        self._dtype_dict: Dict[str, Dtype] = {}
-
-        with connection.cursor() as cursor:
-            cursor.execute(f"PRAGMA table_info({name})")
-            for _, column, dtype, _, _, is_pkey in cursor.fetchall():
-                if bool(is_pkey):
-                    if primary_key is not None and primary_key != column:
-                        raise ValueError(f"Found duplicate primary key "
-                                         f"definition '{primary_key}' and "
-                                         f"'{column}' in table '{name}'")
-                    primary_key = column
-
-                # Determine colun affinity:
-                dtype = dtype.strip().upper()
-                if re.search('INT', dtype):
-                    self._dtype_dict[column] = Dtype.int
-                elif re.search('TEXT|CHAR|CLOB', dtype):
-                    self._dtype_dict[column] = Dtype.string
-                elif re.search('REAL|FLOA|DOUB', dtype):
-                    self._dtype_dict[column] = Dtype.float
-                else:  # NUMERIC affinity.
-                    self._dtype_dict[column] = Dtype.unsupported
-
-            if len(self._dtype_dict) > 0:
-                column_names = ', '.join(self._dtype_dict.keys())
-                cursor.execute(f"SELECT {column_names} FROM {name} "
-                               f"ORDER BY rowid LIMIT 1000")
-                self._sample = cursor.fetch_arrow_table()
-
-            for column_name in list(self._dtype_dict.keys()):
-                if self._dtype_dict[column_name] == Dtype.unsupported:
-                    dtype = self._sample[column_name].type
-                    if pa.types.is_integer(dtype):
-                        self._dtype_dict[column_name] = Dtype.int
-                    elif pa.types.is_floating(dtype):
-                        self._dtype_dict[column_name] = Dtype.float
-                    elif pa.types.is_decimal(dtype):
-                        self._dtype_dict[column_name] = Dtype.float
-                    elif pa.types.is_string(dtype):
-                        self._dtype_dict[column_name] = Dtype.string
-                    else:
-                        del self._dtype_dict[column_name]
-
-        if len(self._dtype_dict) == 0:
-            raise RuntimeError(f"Table '{name}' does not exist or does not "
-                               f"hold any column with a supported data type")
 
         super().__init__(
             name=name,
-            columns=columns or list(self._dtype_dict.keys()),
+            columns=columns,
             primary_key=primary_key,
             time_column=time_column,
             end_time_column=end_time_column,
         )
 
-    def infer_metadata(self, verbose: bool = True) -> Self:
-        r"""Infers metadata, *i.e.*, primary keys and time columns, in the
-        table.
-
-        Args:
-            verbose: Whether to print verbose output.
-        """
-        return self
-
-    def _has_source_column(self, name: str) -> bool:
-        return name in self._dtype_dict
-
-    def _get_source_dtype(self, name: str) -> Dtype:
-        return self._dtype_dict[name]
-
-    def _get_source_stype(self, name: str, dtype: Dtype) -> Stype:
-        return infer_stype(self._sample[name].to_pandas(), name, dtype)
-
-    def _get_source_foreign_keys(self) -> List[Tuple[str, str, str]]:
-        edges: List[Tuple[str, str, str]] = []
+    def _get_source_columns(self) -> List[SourceColumn]:
+        source_columns: List[SourceColumn] = []
+        with self._connection.cursor() as cursor:
+            cursor.execute(f"PRAGMA table_info({self.name})")
+            rows = cursor.fetchall()
+
+            if len(rows) == 0:
+                raise ValueError(f"Table '{self.name}' does not exist")
+
+            for _, column, type, _, _, is_pkey in rows:
+                # Determine column affinity:
+                type = type.strip().upper()
+                if re.search('INT', type):
+                    dtype = Dtype.int
+                elif re.search('TEXT|CHAR|CLOB', type):
+                    dtype = Dtype.string
+                elif re.search('REAL|FLOA|DOUB', type):
+                    dtype = Dtype.float
+                else:  # NUMERIC affinity.
+                    ser = self._sample_df[column]
+                    try:
+                        dtype = infer_dtype(ser)
+                    except Exception:
+                        warnings.warn(
+                            f"Data type inference for column '{column}' in "
+                            f"table '{self.name}' failed. Consider changing "
+                            f"the data type of the column to use it within "
+                            f"this table.")
+                        continue
+
+                source_column = SourceColumn(
+                    name=column,
+                    dtype=dtype,
+                    is_primary_key=bool(is_pkey),
+                    is_unique_key=False,
+                )
+                source_columns.append(source_column)
+
+        return source_columns
+
+    def _get_source_foreign_keys(self) -> List[SourceForeignKey]:
+        source_fkeys: List[SourceForeignKey] = []
         with self._connection.cursor() as cursor:
             cursor.execute(f"PRAGMA foreign_key_list({self.name})")
             for _, _, dst_table, fkey, pkey, _, _, _ in cursor.fetchall():
-                edges.append((fkey, dst_table, pkey))
-        return edges
-
-    def _infer_primary_key(self, candidates: List[str]) -> Optional[str]:
-        return None  # TODO
+                source_fkeys.append(SourceForeignKey(fkey, dst_table, pkey))
+        return source_fkeys
 
-    def _infer_time_column(self, candidates: List[str]) -> Optional[str]:
-        return None  # TODO
+    def _get_sample_df(self) -> pd.DataFrame:
+        with self._connection.cursor() as cursor:
+            cursor.execute(f"SELECT * FROM {self.name} "
+                           f"ORDER BY rowid LIMIT 1000")
+            table = cursor.fetch_arrow_table()
+            return table.to_pandas(types_mapper=pd.ArrowDtype)
 
-    def _num_rows(self) -> Optional[int]:
+    def _get_num_rows(self) -> Optional[int]:
         return None

kumoai/experimental/rfm/base/__init__.py

@@ -1,7 +1,10 @@
+from .source import SourceColumn, SourceForeignKey
 from .column import Column
 from .table import Table
 
 __all__ = [
+    'SourceColumn',
+    'SourceForeignKey',
     'Column',
     'Table',
 ]

kumoai/experimental/rfm/base/source.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+from kumoapi.typing import Dtype
+
+
+@dataclass
+class SourceColumn:
+    name: str
+    dtype: Dtype
+    is_primary_key: bool
+    is_unique_key: bool
+
+
+@dataclass
+class SourceForeignKey:
+    name: str
+    dst_table: str
+    primary_key: str