kumoai 2.10.0.dev202509231831__cp313-cp313-macosx_11_0_arm64.whl → 2.14.0.dev202512161731__cp313-cp313-macosx_11_0_arm64.whl

kumoai/experimental/rfm/infer/__init__.py

@@ -1,9 +1,15 @@
+from .dtype import infer_dtype
+from .pkey import infer_primary_key
+from .time_col import infer_time_column
 from .id import contains_id
 from .timestamp import contains_timestamp
 from .categorical import contains_categorical
 from .multicategorical import contains_multicategorical
 
 __all__ = [
+    'infer_dtype',
+    'infer_primary_key',
+    'infer_time_column',
     'contains_id',
     'contains_timestamp',
     'contains_categorical',

kumoai/experimental/rfm/infer/dtype.py

@@ -0,0 +1,79 @@
+from typing import Dict
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from kumoapi.typing import Dtype
+
+PANDAS_TO_DTYPE: Dict[str, Dtype] = {
+    'bool': Dtype.bool,
+    'boolean': Dtype.bool,
+    'int8': Dtype.int,
+    'int16': Dtype.int,
+    'int32': Dtype.int,
+    'int64': Dtype.int,
+    'float16': Dtype.float,
+    'float32': Dtype.float,
+    'float64': Dtype.float,
+    'object': Dtype.string,
+    'string': Dtype.string,
+    'string[python]': Dtype.string,
+    'string[pyarrow]': Dtype.string,
+    'binary': Dtype.binary,
+}
+
+
+def infer_dtype(ser: pd.Series) -> Dtype:
+    """Extracts the :class:`Dtype` from a :class:`pandas.Series`.
+
+    Args:
+        ser: A :class:`pandas.Series` to analyze.
+
+    Returns:
+        The data type.
+    """
+    if pd.api.types.is_datetime64_any_dtype(ser.dtype):
+        return Dtype.date
+    if pd.api.types.is_timedelta64_dtype(ser.dtype):
+        return Dtype.timedelta
+    if isinstance(ser.dtype, pd.CategoricalDtype):
+        return Dtype.string
+
+    if (pd.api.types.is_object_dtype(ser.dtype)
+            and not isinstance(ser.dtype, pd.ArrowDtype)):
+        index = ser.iloc[:1000].first_valid_index()
+        if index is not None and pd.api.types.is_list_like(ser[index]):
+            pos = ser.index.get_loc(index)
+            assert isinstance(pos, int)
+            ser = ser.iloc[pos:pos + 1000].dropna()
+            arr = pa.array(ser.tolist())
+            ser = pd.Series(arr, dtype=pd.ArrowDtype(arr.type))
+
+    if isinstance(ser.dtype, pd.ArrowDtype):
+        if pa.types.is_list(ser.dtype.pyarrow_dtype):
+            elem_dtype = ser.dtype.pyarrow_dtype.value_type
+            if pa.types.is_integer(elem_dtype):
+                return Dtype.intlist
+            if pa.types.is_floating(elem_dtype):
+                return Dtype.floatlist
+            if pa.types.is_decimal(elem_dtype):
+                return Dtype.floatlist
+            if pa.types.is_string(elem_dtype):
+                return Dtype.stringlist
+            if pa.types.is_null(elem_dtype):
+                return Dtype.floatlist
+
+    if isinstance(ser.dtype, np.dtype):
+        dtype_str = str(ser.dtype).lower()
+    elif isinstance(ser.dtype, pd.api.extensions.ExtensionDtype):
+        dtype_str = ser.dtype.name.lower()
+        dtype_str = dtype_str.split('[')[0]  # Remove backend metadata
+    elif isinstance(ser.dtype, pa.DataType):
+        dtype_str = str(ser.dtype).lower()
+    else:
+        dtype_str = 'object'
+
+    if dtype_str not in PANDAS_TO_DTYPE:
+        raise ValueError(f"Unsupported data type '{ser.dtype}'")
+
+    return PANDAS_TO_DTYPE[dtype_str]

kumoai/experimental/rfm/infer/pkey.py

@@ -0,0 +1,126 @@
+import re
+import warnings
+from typing import Optional
+
+import pandas as pd
+
+
+def infer_primary_key(
+    table_name: str,
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential primary key column.
+
+    Args:
+        table_name: The table name.
+        df: The pandas DataFrame to analyze.
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected primary key, or ``None`` if not found.
+    """
+    # A list of (potentially modified) table names that are eligible to match
+    # with a primary key, i.e.:
+    # - UserInfo -> User
+    # - snakecase <-> camelcase
+    # - camelcase <-> snakecase
+    # - plural <-> singular (users -> user, eligibilities -> eligibility)
+    # - verb -> noun (qualifying -> qualify)
+    _table_names = {table_name}
+    if table_name.lower().endswith('_info'):
+        _table_names.add(table_name[:-5])
+    elif table_name.lower().endswith('info'):
+        _table_names.add(table_name[:-4])
+
+    table_names = set()
+    for _table_name in _table_names:
+        table_names.add(_table_name.lower())
+        snakecase = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', _table_name)
+        snakecase = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', snakecase)
+        table_names.add(snakecase.lower())
+        camelcase = _table_name.replace('_', '')
+        table_names.add(camelcase.lower())
+        if _table_name.lower().endswith('s'):
+            table_names.add(_table_name.lower()[:-1])
+            table_names.add(snakecase.lower()[:-1])
+            table_names.add(camelcase.lower()[:-1])
+        else:
+            table_names.add(_table_name.lower() + 's')
+            table_names.add(snakecase.lower() + 's')
+            table_names.add(camelcase.lower() + 's')
+        if _table_name.lower().endswith('ies'):
+            table_names.add(_table_name.lower()[:-3] + 'y')
+            table_names.add(snakecase.lower()[:-3] + 'y')
+            table_names.add(camelcase.lower()[:-3] + 'y')
+        elif _table_name.lower().endswith('y'):
+            table_names.add(_table_name.lower()[:-1] + 'ies')
+            table_names.add(snakecase.lower()[:-1] + 'ies')
+            table_names.add(camelcase.lower()[:-1] + 'ies')
+        if _table_name.lower().endswith('ing'):
+            table_names.add(_table_name.lower()[:-3])
+            table_names.add(snakecase.lower()[:-3])
+            table_names.add(camelcase.lower()[:-3])
+
+    scores: list[tuple[str, int]] = []
+    for col_name in candidates:
+        col_name_lower = col_name.lower()
+
+        score = 0
+
+        if col_name_lower == 'id':
+            score += 4
+
+        for table_name_lower in table_names:
+
+            if col_name_lower == table_name_lower:
+                score += 4  # USER -> USER
+                break
+
+            for suffix in ['id', 'hash', 'key', 'code', 'uuid']:
+                if not col_name_lower.endswith(suffix):
+                    continue
+
+                if col_name_lower == f'{table_name_lower}_{suffix}':
+                    score += 5  # USER -> USER_ID
+                    break
+
+                if col_name_lower == f'{table_name_lower}{suffix}':
+                    score += 5  # User -> UserId
+                    break
+
+                if col_name_lower.endswith(f'{table_name_lower}_{suffix}'):
+                    score += 2
+
+                if col_name_lower.endswith(f'{table_name_lower}{suffix}'):
+                    score += 2
+
+            # `rel-bench` hard-coding :(
+            if table_name == 'studies' and col_name == 'nct_id':
+                score += 1
+
+        ser = df[col_name].iloc[:1_000_000]
+        score += 3 * (ser.nunique() / len(ser))
+
+        scores.append((col_name, score))
+
+    scores = [x for x in scores if x[-1] >= 4]
+    scores.sort(key=lambda x: x[-1], reverse=True)
+
+    if len(scores) == 0:
+        return None
+
+    if len(scores) == 1:
+        return scores[0][0]
+
+    # In case of multiple candidates, only return one if its score is unique:
+    if scores[0][1] != scores[1][1]:
+        return scores[0][0]
+
+    max_score = max(scores, key=lambda x: x[1])
+    candidates = [col_name for col_name, score in scores if score == max_score]
+    warnings.warn(f"Found multiple potential primary keys in table "
+                  f"'{table_name}': {candidates}. Please specify the primary "
+                  f"key for this table manually.")
+
+    return None

kumoai/experimental/rfm/infer/time_col.py

@@ -0,0 +1,62 @@
+import re
+import warnings
+from typing import Optional
+
+import pandas as pd
+
+
+def infer_time_column(
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential time column.
+
+    Args:
+        df: The pandas DataFrame to analyze.
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected time column, or ``None`` if not found.
+    """
+    candidates = [  # Exclude all candidates with `*last*` in column names:
+        col_name for col_name in candidates
+        if not re.search(r'(^|_)last(_|$)', col_name, re.IGNORECASE)
+    ]
+
+    if len(candidates) == 0:
+        return None
+
+    if len(candidates) == 1:
+        return candidates[0]
+
+    # If there exists a dedicated `create*` column, use it as time column:
+    create_candidates = [
+        candidate for candidate in candidates
+        if candidate.lower().startswith('create')
+    ]
+    if len(create_candidates) == 1:
+        return create_candidates[0]
+    if len(create_candidates) > 1:
+        candidates = create_candidates
+
+    # Find the most optimal time column. Usually, it is the one pointing to
+    # the oldest timestamps:
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', message='Could not infer format')
+        min_timestamp_dict = {
+            key: pd.to_datetime(df[key].iloc[:10_000], 'coerce')
+            for key in candidates
+        }
+    min_timestamp_dict = {
+        key: value.min().tz_localize(None)
+        for key, value in min_timestamp_dict.items()
+    }
+    min_timestamp_dict = {
+        key: value
+        for key, value in min_timestamp_dict.items() if not pd.isna(value)
+    }
+
+    if len(min_timestamp_dict) == 0:
+        return None
+
+    return min(min_timestamp_dict, key=min_timestamp_dict.get)  # type: ignore

kumoai/experimental/rfm/infer/timestamp.py

@@ -2,6 +2,7 @@ import re
 import warnings
 
 import pandas as pd
+from dateutil.parser import UnknownTimezoneWarning
 from kumoapi.typing import Dtype, Stype
 
 
@@ -20,9 +21,7 @@ def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool:
         column_name,
         re.IGNORECASE,
     )
-
-    if match is not None:
-        return True
+    score = 0.3 if match is not None else 0.0
 
     ser = ser.iloc[:100]
     ser = ser.dropna()
@@ -34,5 +33,9 @@ def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool:
     ser = ser.astype(str)  # Avoid parsing numbers as unix timestamps.
 
     with warnings.catch_warnings():
+        warnings.simplefilter('ignore', UnknownTimezoneWarning)
         warnings.filterwarnings('ignore', message='Could not infer format')
-        return pd.to_datetime(ser, errors='coerce').notna().all()
+        mask = pd.to_datetime(ser, errors='coerce').notna()
+        score += int(mask.sum()) / len(mask)
+
+    return score >= 1.0

kumoai/experimental/rfm/pquery/__init__.py

@@ -1,7 +1,7 @@
-from .backend import PQueryBackend
-from .pandas_backend import PQueryPandasBackend
+from .executor import PQueryExecutor
+from .pandas_executor import PQueryPandasExecutor
 
 __all__ = [
-    'PQueryBackend',
-    'PQueryPandasBackend',
+    'PQueryExecutor',
+    'PQueryPandasExecutor',
 ]

kumoai/experimental/rfm/pquery/{backend.py → executor.py}

@@ -1,23 +1,14 @@
 from abc import ABC, abstractmethod
-from typing import Dict, Generic, Optional, Tuple, TypeVar, Union
+from typing import Dict, Generic, Tuple, TypeVar
 
-from kumoapi.rfm import PQueryDefinition
-from kumoapi.rfm.pquery import (
+from kumoapi.pquery import ValidatedPredictiveQuery
+from kumoapi.pquery.AST import (
     Aggregation,
-    AggregationType,
-    BoolOp,
     Column,
     Condition,
     Filter,
-    Float,
-    FloatList,
-    Int,
-    IntList,
+    Join,
     LogicalOperation,
-    MemberOp,
-    RelOp,
-    Str,
-    StrList,
 )
 
 TableData = TypeVar('TableData')
@@ -25,58 +16,33 @@ ColumnData = TypeVar('ColumnData')
 IndexData = TypeVar('IndexData')
 
 
-class PQueryBackend(Generic[TableData, ColumnData, IndexData], ABC):
+class PQueryExecutor(Generic[TableData, ColumnData, IndexData], ABC):
     @abstractmethod
-    def eval_aggregation_type(
+    def execute_column(
         self,
-        op: AggregationType,
-        feat: Optional[ColumnData],
-        batch: IndexData,
-        batch_size: int,
+        column: Column,
+        feat_dict: Dict[str, TableData],
         filter_na: bool = True,
     ) -> Tuple[ColumnData, IndexData]:
         pass
 
     @abstractmethod
-    def eval_rel_op(
-        self,
-        left: ColumnData,
-        op: RelOp,
-        right: Union[Int, Float, Str, None],
-    ) -> ColumnData:
-        pass
-
-    @abstractmethod
-    def eval_member_op(
-        self,
-        left: ColumnData,
-        op: MemberOp,
-        right: Union[IntList, FloatList, StrList],
-    ) -> ColumnData:
-        pass
-
-    @abstractmethod
-    def eval_bool_op(
-        self,
-        left: ColumnData,
-        op: BoolOp,
-        right: Optional[ColumnData],
-    ) -> ColumnData:
-        pass
-
-    @abstractmethod
-    def eval_column(
+    def execute_aggregation(
         self,
-        column: Column,
+        aggr: Aggregation,
         feat_dict: Dict[str, TableData],
+        time_dict: Dict[str, ColumnData],
+        batch_dict: Dict[str, IndexData],
+        anchor_time: ColumnData,
         filter_na: bool = True,
+        num_forecasts: int = 1,
     ) -> Tuple[ColumnData, IndexData]:
         pass
 
     @abstractmethod
-    def eval_aggregation(
+    def execute_condition(
         self,
-        aggr: Aggregation,
+        condition: Condition,
         feat_dict: Dict[str, TableData],
         time_dict: Dict[str, ColumnData],
         batch_dict: Dict[str, IndexData],
@@ -87,9 +53,9 @@ class PQueryBackend(Generic[TableData, ColumnData, IndexData], ABC):
         pass
 
     @abstractmethod
-    def eval_condition(
+    def execute_logical_operation(
         self,
-        condition: Condition,
+        logical_operation: LogicalOperation,
         feat_dict: Dict[str, TableData],
         time_dict: Dict[str, ColumnData],
         batch_dict: Dict[str, IndexData],
@@ -100,9 +66,9 @@ class PQueryBackend(Generic[TableData, ColumnData, IndexData], ABC):
         pass
 
     @abstractmethod
-    def eval_logical_operation(
+    def execute_join(
         self,
-        logical_operation: LogicalOperation,
+        join: Join,
         feat_dict: Dict[str, TableData],
         time_dict: Dict[str, ColumnData],
         batch_dict: Dict[str, IndexData],
@@ -113,20 +79,20 @@ class PQueryBackend(Generic[TableData, ColumnData, IndexData], ABC):
         pass
 
     @abstractmethod
-    def eval_filter(
+    def execute_filter(
         self,
         filter: Filter,
         feat_dict: Dict[str, TableData],
         time_dict: Dict[str, ColumnData],
         batch_dict: Dict[str, IndexData],
         anchor_time: ColumnData,
-    ) -> IndexData:
+    ) -> Tuple[ColumnData, IndexData]:
         pass
 
     @abstractmethod
-    def eval_pquery(
+    def execute(
         self,
-        query: PQueryDefinition,
+        query: ValidatedPredictiveQuery,
         feat_dict: Dict[str, TableData],
         time_dict: Dict[str, ColumnData],
         batch_dict: Dict[str, IndexData],