kumoai 2.13.0.dev202511131731__cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

kumoai/experimental/rfm/utils.py

@@ -0,0 +1,344 @@
+import re
+import warnings
+from typing import Any, Dict, Optional
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from kumoapi.typing import Dtype, Stype
+
+from kumoai.experimental.rfm.infer import (
+    contains_categorical,
+    contains_id,
+    contains_multicategorical,
+    contains_timestamp,
+)
+
+# Mapping from pandas/numpy dtypes to Kumo Dtypes
+PANDAS_TO_DTYPE: Dict[Any, Dtype] = {
+    np.dtype('bool'): Dtype.bool,
+    pd.BooleanDtype(): Dtype.bool,
+    pa.bool_(): Dtype.bool,
+    np.dtype('byte'): Dtype.int,
+    pd.UInt8Dtype(): Dtype.int,
+    np.dtype('int16'): Dtype.int,
+    pd.Int16Dtype(): Dtype.int,
+    np.dtype('int32'): Dtype.int,
+    pd.Int32Dtype(): Dtype.int,
+    np.dtype('int64'): Dtype.int,
+    pd.Int64Dtype(): Dtype.int,
+    np.dtype('float32'): Dtype.float,
+    pd.Float32Dtype(): Dtype.float,
+    np.dtype('float64'): Dtype.float,
+    pd.Float64Dtype(): Dtype.float,
+    np.dtype('object'): Dtype.string,
+    pd.StringDtype(storage='python'): Dtype.string,
+    pd.StringDtype(storage='pyarrow'): Dtype.string,
+    pa.string(): Dtype.string,
+    pa.binary(): Dtype.binary,
+    np.dtype('datetime64[ns]'): Dtype.date,
+    np.dtype('timedelta64[ns]'): Dtype.timedelta,
+    pa.list_(pa.float32()): Dtype.floatlist,
+    pa.list_(pa.int64()): Dtype.intlist,
+    pa.list_(pa.string()): Dtype.stringlist,
+}
+
+
+def to_dtype(ser: pd.Series) -> Dtype:
+    """Extracts the :class:`Dtype` from a :class:`pandas.Series`.
+
+    Args:
+        ser: A :class:`pandas.Series` to analyze.
+
+    Returns:
+        The data type.
+    """
+    if pd.api.types.is_datetime64_any_dtype(ser.dtype):
+        return Dtype.date
+
+    if isinstance(ser.dtype, pd.CategoricalDtype):
+        return Dtype.string
+
+    if pd.api.types.is_object_dtype(ser.dtype):
+        index = ser.iloc[:1000].first_valid_index()
+        if index is not None and pd.api.types.is_list_like(ser[index]):
+            pos = ser.index.get_loc(index)
+            assert isinstance(pos, int)
+            ser = ser.iloc[pos:pos + 1000].dropna()
+
+            if not ser.map(pd.api.types.is_list_like).all():
+                raise ValueError("Data contains a mix of list-like and "
+                                 "non-list-like values")
+
+            ser = ser[ser.map(lambda x: not isinstance(x, list) or len(x) > 0)]
+
+            dtypes = ser.apply(lambda x: PANDAS_TO_DTYPE.get(
+                np.array(x).dtype, Dtype.string)).unique().tolist()
+
+            invalid_dtypes = set(dtypes) - {
+                Dtype.string,
+                Dtype.int,
+                Dtype.float,
+            }
+            if len(invalid_dtypes) > 0:
+                raise ValueError(f"Data contains unsupported list data types: "
+                                 f"{list(invalid_dtypes)}")
+
+            if Dtype.string in dtypes:
+                return Dtype.stringlist
+
+            if dtypes == [Dtype.int]:
+                return Dtype.intlist
+
+            return Dtype.floatlist
+
+    if ser.dtype not in PANDAS_TO_DTYPE:
+        raise ValueError(f"Unsupported data type '{ser.dtype}'")
+
+    return PANDAS_TO_DTYPE[ser.dtype]
+
+
+def infer_stype(ser: pd.Series, column_name: str, dtype: Dtype) -> Stype:
+    r"""Infers the semantic type of a column.
+
+    Args:
+        ser: A :class:`pandas.Series` to analyze.
+        column_name: The name of the column (used for pattern matching).
+        dtype: The data type.
+
+    Returns:
+        The semantic type.
+    """
+    if contains_id(ser, column_name, dtype):
+        return Stype.ID
+
+    if contains_timestamp(ser, column_name, dtype):
+        return Stype.timestamp
+
+    if contains_multicategorical(ser, column_name, dtype):
+        return Stype.multicategorical
+
+    if contains_categorical(ser, column_name, dtype):
+        return Stype.categorical
+
+    return dtype.default_stype
+
+
+def detect_primary_key(
+    table_name: str,
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential primary key column.
+
+    Args:
+        table_name: The table name.
+        df: The pandas DataFrame to analyze
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected primary key, or ``None`` if not found.
+    """
+    # A list of (potentially modified) table names that are eligible to match
+    # with a primary key, i.e.:
+    # - UserInfo -> User
+    # - snakecase <-> camelcase
+    # - camelcase <-> snakecase
+    # - plural <-> singular (users -> user, eligibilities -> eligibility)
+    # - verb -> noun (qualifying -> qualify)
+    _table_names = {table_name}
+    if table_name.lower().endswith('_info'):
+        _table_names.add(table_name[:-5])
+    elif table_name.lower().endswith('info'):
+        _table_names.add(table_name[:-4])
+
+    table_names = set()
+    for _table_name in _table_names:
+        table_names.add(_table_name.lower())
+        snakecase = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', _table_name)
+        snakecase = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', snakecase)
+        table_names.add(snakecase.lower())
+        camelcase = _table_name.replace('_', '')
+        table_names.add(camelcase.lower())
+        if _table_name.lower().endswith('s'):
+            table_names.add(_table_name.lower()[:-1])
+            table_names.add(snakecase.lower()[:-1])
+            table_names.add(camelcase.lower()[:-1])
+        else:
+            table_names.add(_table_name.lower() + 's')
+            table_names.add(snakecase.lower() + 's')
+            table_names.add(camelcase.lower() + 's')
+        if _table_name.lower().endswith('ies'):
+            table_names.add(_table_name.lower()[:-3] + 'y')
+            table_names.add(snakecase.lower()[:-3] + 'y')
+            table_names.add(camelcase.lower()[:-3] + 'y')
+        elif _table_name.lower().endswith('y'):
+            table_names.add(_table_name.lower()[:-1] + 'ies')
+            table_names.add(snakecase.lower()[:-1] + 'ies')
+            table_names.add(camelcase.lower()[:-1] + 'ies')
+        if _table_name.lower().endswith('ing'):
+            table_names.add(_table_name.lower()[:-3])
+            table_names.add(snakecase.lower()[:-3])
+            table_names.add(camelcase.lower()[:-3])
+
+    scores: list[tuple[str, int]] = []
+    for col_name in candidates:
+        col_name_lower = col_name.lower()
+
+        score = 0
+
+        if col_name_lower == 'id':
+            score += 4
+
+        for table_name_lower in table_names:
+
+            if col_name_lower == table_name_lower:
+                score += 4  # USER -> USER
+                break
+
+            for suffix in ['id', 'hash', 'key', 'code', 'uuid']:
+                if not col_name_lower.endswith(suffix):
+                    continue
+
+                if col_name_lower == f'{table_name_lower}_{suffix}':
+                    score += 5  # USER -> USER_ID
+                    break
+
+                if col_name_lower == f'{table_name_lower}{suffix}':
+                    score += 5  # User -> UserId
+                    break
+
+                if col_name_lower.endswith(f'{table_name_lower}_{suffix}'):
+                    score += 2
+
+                if col_name_lower.endswith(f'{table_name_lower}{suffix}'):
+                    score += 2
+
+            # `rel-bench` hard-coding :(
+            if table_name == 'studies' and col_name == 'nct_id':
+                score += 1
+
+        ser = df[col_name].iloc[:1_000_000]
+        score += 3 * (ser.nunique() / len(ser))
+
+        scores.append((col_name, score))
+
+    scores = [x for x in scores if x[-1] >= 4]
+    scores.sort(key=lambda x: x[-1], reverse=True)
+
+    if len(scores) == 0:
+        return None
+
+    if len(scores) == 1:
+        return scores[0][0]
+
+    # In case of multiple candidates, only return one if its score is unique:
+    if scores[0][1] != scores[1][1]:
+        return scores[0][0]
+
+    max_score = max(scores, key=lambda x: x[1])
+    candidates = [col_name for col_name, score in scores if score == max_score]
+    warnings.warn(f"Found multiple potential primary keys in table "
+                  f"'{table_name}': {candidates}. Please specify the primary "
+                  f"key for this table manually.")
+
+    return None
+
+
+def detect_time_column(
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential time column.
+
+    Args:
+        df: The pandas DataFrame to analyze
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected time column, or ``None`` if not found.
+    """
+    candidates = [  # Exclude all candidates with `*last*` in column names:
+        col_name for col_name in candidates
+        if not re.search(r'(^|_)last(_|$)', col_name, re.IGNORECASE)
+    ]
+
+    if len(candidates) == 0:
+        return None
+
+    if len(candidates) == 1:
+        return candidates[0]
+
+    # If there exists a dedicated `create*` column, use it as time column:
+    create_candidates = [
+        candidate for candidate in candidates
+        if candidate.lower().startswith('create')
+    ]
+    if len(create_candidates) == 1:
+        return create_candidates[0]
+    if len(create_candidates) > 1:
+        candidates = create_candidates
+
+    # Find the most optimal time column. Usually, it is the one pointing to
+    # the oldest timestamps:
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', message='Could not infer format')
+        min_timestamp_dict = {
+            key: pd.to_datetime(df[key].iloc[:10_000], 'coerce')
+            for key in candidates
+        }
+    min_timestamp_dict = {
+        key: value.min().tz_localize(None)
+        for key, value in min_timestamp_dict.items()
+    }
+    min_timestamp_dict = {
+        key: value
+        for key, value in min_timestamp_dict.items() if not pd.isna(value)
+    }
+
+    if len(min_timestamp_dict) == 0:
+        return None
+
+    return min(min_timestamp_dict, key=min_timestamp_dict.get)  # type: ignore
+
+
+PUNCTUATION = re.compile(r"[\'\"\.,\(\)\!\?\;\:]")
+MULTISPACE = re.compile(r"\s+")
+
+
+def normalize_text(
+    ser: pd.Series,
+    max_words: Optional[int] = 50,
+) -> pd.Series:
+    r"""Normalizes text into a list of lower-case words.
+
+    Args:
+        ser: The :class:`pandas.Series` to normalize.
+        max_words: The maximum number of words to return.
+            This will auto-shrink any large text column to avoid blowing up
+            context size.
+    """
+    if len(ser) == 0 or pd.api.types.is_list_like(ser.iloc[0]):
+        return ser
+
+    def normalize_fn(line: str) -> list[str]:
+        line = PUNCTUATION.sub(" ", line)
+        line = re.sub(r"<br\s*/?>", " ", line)  # Handle <br /> or <br>
+        line = MULTISPACE.sub(" ", line)
+        words = line.split()
+        if max_words is not None:
+            words = words[:max_words]
+        return words
+
+    ser = ser.fillna('').astype(str)
+
+    if max_words is not None:
+        # We estimate the number of words as 5 characters + 1 space in an
+        # English text on average. We need this pre-filter here, as word
+        # splitting on a giant text can be very expensive:
+        ser = ser.str[:6 * max_words]
+
+    ser = ser.str.lower()
+    ser = ser.map(normalize_fn)
+
+    return ser

kumoai/formatting.py

@@ -0,0 +1,30 @@
+from kumoapi.jobs import ErrorDetails
+
+
+def pretty_print_error_details(error_details: ErrorDetails) -> str:
+    """Pretty prints the ErrorDetails combining all the individual items.
+    If there are CTAs, they are also displayed after creating
+    corresponding hyperlinks.
+
+    Arguments:
+    error_details (ErrorDetails): Standard ErrorDetails response from
+        get_errors APIs.
+    """
+    out = ""
+    ctr = None
+    if len(error_details.items) != 1:
+        out += "Encountered multiple errors:\n"
+        ctr = 1
+    for error_detail in error_details.items:
+        if ctr is not None:
+            out += f'{ctr}.'
+            ctr += 1
+        if error_detail.title is not None:
+            out += f'{error_detail.title}: '
+        out += error_detail.description
+        if error_detail.cta is not None:
+            out += 'Follow the link for potential resolution:'
+            f' {error_detail.cta.url}'
+        out += '\n'
+
+    return out

kumoai/futures.py

@@ -0,0 +1,99 @@
+import asyncio
+import concurrent
+import logging
+import threading
+from abc import ABC, abstractmethod
+from asyncio.events import AbstractEventLoop
+from typing import Any, Awaitable, Coroutine, Generic, TypeVar
+
+logger = logging.getLogger(__name__)
+
+CoroFuncType = Awaitable[Any]
+
+# Kumo global event loop (our implementation of green threads for pollers and
+# other interactions with the Kumo backend that require long-running tasks).
+# Since the caller may have their own event loop that we do not want to
+# mess with, _do not_ ever call `set_event_loop` here!! Instead, be extra
+# cautious to pass this loop everywhere.
+_KUMO_EVENT_LOOP: AbstractEventLoop = asyncio.new_event_loop()
+
+
+def initialize_event_loop() -> None:
+    def _run_background_loop(loop: AbstractEventLoop) -> None:
+        asyncio.set_event_loop(loop)
+        loop.run_forever()
+
+    t = threading.Thread(target=_run_background_loop,
+                         args=(_KUMO_EVENT_LOOP, ), daemon=True)
+    t.start()
+
+
+def create_future(coro: Coroutine[Any, Any, Any]) -> concurrent.futures.Future:
+    r"""Creates a future to execute in the Kumo event loop."""
+    # NOTE this function creates a future, chains it to the output of the
+    # execution of `coro` in the Kumo event loop, and handles exceptions
+    # before scheduling to run in the loop:
+    return asyncio.run_coroutine_threadsafe(coro, _KUMO_EVENT_LOOP)
+
+
+T = TypeVar("T")
+
+
+class KumoFuture(ABC, Generic[T]):
+    r"""Abstract base class for a Kumo future object."""
+
+    # We cannot use Python future implementations (`asyncio.Future` or
+    # `concurrent.futures.Future`) as they are native to the Python
+    # implementation of asyncio and threading, and thus not easily extensible.
+    # Python additionally recommends not exposing low-level Future objects in
+    # user facing APIs.
+    @abstractmethod
+    def result(self) -> T:
+        r"""Returns the resolved state of the future.
+
+        Raises:
+            Exception:
+                If the future is complete but in a failed state due to an
+                exception being raised, this method will raise the same
+                exception.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def future(self) -> 'concurrent.futures.Future[T]':
+        r"""Returns the :obj:`concurrent.futures.Future` object wrapped by
+        this future. It is not recommended to access this object directly.
+        """
+        raise NotImplementedError
+
+    def done(self) -> bool:
+        r"""Returns :obj:`True` if this future has been resolved with
+        ``result()``, or :obj:`False` if this future is still
+        in-progress. Note that this method will return :obj:`True` if the
+        future is complete, but in a failed state, and that this method will
+        return :obj:`False` if the job is complete, but the future has not
+        been awaited.
+        """
+        return self.future().done()
+
+
+class KumoProgressFuture(KumoFuture[T]):
+    @abstractmethod
+    def _attach_internal(self, interval_s: float = 4.0) -> T:
+        raise NotImplementedError
+
+    def attach(self, interval_s: float = 4.0) -> T:
+        r"""Allows a user to attach to a running job and view its progress.
+
+        Args:
+            interval_s (float): Time interval (seconds) between polls, minimum
+                value allowed is 4 seconds.
+        """
+        try:
+            return self._attach_internal(interval_s=interval_s)
+        except Exception:
+            logger.warning(
+                "Detailed job tracking has become temporarily unavailable. "
+                "The job is continuing to proceed on the Kumo server, "
+                "and this call will complete when the job has finished.")
+            return self.result()

kumoai/graph/__init__.py

@@ -0,0 +1,12 @@
+from .column import Column, TimestampUnit
+from .table import Table
+from .graph import Graph, Edge, GraphHealthStats
+
+__all__ = [
+    'TimestampUnit',
+    'Column',
+    'Table',
+    'Graph',
+    'Edge',
+    'GraphHealthStats',
+]

kumoai/graph/column.py

@@ -0,0 +1,106 @@
+from dataclasses import dataclass
+from typing import Any, Optional, Union
+
+from kumoapi.table import TimestampUnit
+from kumoapi.typing import Dtype, Stype
+
+from kumoai.mixin import CastMixin
+
+
+@dataclass(init=False)
+class Column(CastMixin):
+    r"""A column represents metadata information for a column in a Kumo
+    :class:`~kumoai.graph.Table`. Columns can be created independent of
+    a table, or can be fetched from a table with the
+    :meth:`~kumoai.graph.Table.column` method.
+
+    .. code-block:: python
+
+        import kumoai
+
+        # Fetch a column from a `kumoai.Table`:
+        table = kumoai.Table(...)
+
+        column = table.column("col_name")
+        column = table["col_name"]  # equivalent to the above.
+
+        # Edit a column's data type:
+        print("Existing dtype: ", column.dtype)
+        column.dtype = "int"
+
+        # Edit a column's semantic type:
+        print("Existing stype: ", column.stype)
+        column.stype = "ID"
+
+    Args:
+        name: The name of this column.
+        stype: The semantic type of this column. Semantic types can be
+            specified as strings: the list of possible semantic types
+            is located at :class:`~kumoai.Stype`.
+        dtype: The data type of this column. Data types can be specified
+            as strings: the list of possible data types is located at
+            :class:`~kumoai.Dtype`.
+        timestamp_format: If this column represents a timestamp, the format
+            that the timestamp should be parsed in. The format can either be
+            a :class:`~kumoapi.table.TimestampUnit` for integer columns or a
+            string with a format identifier described
+            `here <https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html>`__
+            for a SaaS Kumo deployment and
+            `here <https://docs.snowflake.com/en/sql-reference/date-time-input-output#about-the-elements-used-in-input-and-output-formats>`__
+            for a Snowpark Container Services Kumo deployment. If left empty,
+            will be intelligently inferred by Kumo.
+    """  # noqa: E501
+    name: str
+    stype: Optional[Stype] = None
+    dtype: Optional[Dtype] = None
+    timestamp_format: Optional[Union[str, TimestampUnit]] = None
+
+    def __init__(
+        self,
+        name: str,
+        stype: Optional[Union[Stype, str]] = None,
+        dtype: Optional[Union[Dtype, str]] = None,
+        timestamp_format: Optional[Union[str, TimestampUnit]] = None,
+    ) -> None:
+        self.name = name
+        self.stype = Stype(stype) if stype is not None else None
+        self.dtype = Dtype(dtype) if dtype is not None else None
+        try:
+            self.timestamp_format = TimestampUnit(timestamp_format)
+        except ValueError:
+            self.timestamp_format = timestamp_format
+
+    def __hash__(self) -> int:
+        return hash((self.name, self.stype, self.dtype, self.timestamp_format))
+
+    def __setattr__(self, key: Any, value: Any) -> None:
+        if key == 'name' and value != getattr(self, key, value):
+            raise AttributeError("Attribute 'name' is read-only")
+        elif key == 'stype' and isinstance(value, str):
+            value = Stype(value)
+        elif key == 'dtype' and isinstance(value, str):
+            value = Dtype(value)
+        elif key == 'timestamp_format' and isinstance(value, str):
+            try:
+                value = TimestampUnit(value)
+            except ValueError:
+                pass
+        super().__setattr__(key, value)
+
+    def update(self, obj: 'Column', override: bool = True) -> 'Column':
+        for key in self.__dict__:
+            if key[0] == '_':  # Skip private attributes:
+                continue
+            value = getattr(obj, key, None)
+            if value is not None:
+                if override or getattr(self, key, None) is None:
+                    setattr(self, key, value)
+        return self
+
+    def __repr__(self) -> str:
+        out = (f"Column(name=\"{self.name}\", stype=\"{self.stype}\", "
+               f"dtype=\"{self.dtype}\"")
+        if self.timestamp_format is not None:
+            out += f", timestamp_format=\"{self.timestamp_format}\""
+        out += ")"
+        return out