kumoai 2.13.0.dev202511131731__cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl

kumoai/experimental/rfm/infer/multicategorical.py

@@ -0,0 +1,48 @@
+from collections import defaultdict
+
+import pandas as pd
+from kumoapi.typing import Dtype, Stype
+
+MAX_CAT = 100
+
+
+def contains_multicategorical(
+    ser: pd.Series,
+    column_name: str,
+    dtype: Dtype,
+) -> bool:
+
+    if not Stype.multicategorical.supports_dtype(dtype):
+        return False
+
+    if dtype == Dtype.stringlist:
+        return True
+
+    ser = ser.iloc[:500]
+    ser = ser.dropna()
+
+    num_unique: int = 0
+    if dtype == Dtype.string:
+        ser = ser.astype(str)
+        text = '\n'.join(ser)
+
+        white_list = {';', ':', '|', '\t'}
+        candidates: dict[str, int] = defaultdict(int)
+        for char in text:
+            if char in white_list:
+                candidates[char] += 1
+
+        if len(candidates) == 0:
+            return False
+
+        num_unique = ser.nunique()
+
+        sep = max(candidates, key=candidates.get)  # type: ignore
+        ser = ser.str.split(sep)
+
+    num_unique_multi = ser.explode().nunique()
+
+    if dtype.is_list():
+        return num_unique_multi <= MAX_CAT
+
+    return num_unique > 1.5 * num_unique_multi and num_unique_multi <= MAX_CAT

kumoai/experimental/rfm/infer/timestamp.py

@@ -0,0 +1,41 @@
+import re
+import warnings
+
+import pandas as pd
+from dateutil.parser import UnknownTimezoneWarning
+from kumoapi.typing import Dtype, Stype
+
+
+def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool:
+    if not Stype.timestamp.supports_dtype(dtype):
+        return False
+
+    if dtype.is_timestamp():
+        return True
+
+    column_name = column_name.lower()
+
+    match = re.search(
+        ('(^|_)(date|datetime|dt|time|timedate|timestamp|ts|'
+         'created|updated)(_|$)'),
+        column_name,
+        re.IGNORECASE,
+    )
+    score = 0.3 if match is not None else 0.0
+
+    ser = ser.iloc[:100]
+    ser = ser.dropna()
+    ser = ser[ser != '']
+
+    if len(ser) == 0:
+        return False
+
+    ser = ser.astype(str)  # Avoid parsing numbers as unix timestamps.
+
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', UnknownTimezoneWarning)
+        warnings.filterwarnings('ignore', message='Could not infer format')
+        mask = pd.to_datetime(ser, errors='coerce').notna()
+        score += int(mask.sum()) / len(mask)
+
+    return score >= 1.0