kumoai 2.13.0.dev202512021731__cp310-cp310-win_amd64.whl → 2.13.0.dev202512040252__cp310-cp310-win_amd64.whl

kumoai/experimental/rfm/infer/time_col.py

@@ -0,0 +1,62 @@
+import re
+import warnings
+from typing import Optional
+
+import pandas as pd
+
+
+def infer_time_column(
+    df: pd.DataFrame,
+    candidates: list[str],
+) -> Optional[str]:
+    r"""Auto-detect potential time column.
+
+    Args:
+        df: The pandas DataFrame to analyze.
+        candidates: A list of potential candidates.
+
+    Returns:
+        The name of the detected time column, or ``None`` if not found.
+    """
+    candidates = [  # Exclude all candidates with `*last*` in column names:
+        col_name for col_name in candidates
+        if not re.search(r'(^|_)last(_|$)', col_name, re.IGNORECASE)
+    ]
+
+    if len(candidates) == 0:
+        return None
+
+    if len(candidates) == 1:
+        return candidates[0]
+
+    # If there exists a dedicated `create*` column, use it as time column:
+    create_candidates = [
+        candidate for candidate in candidates
+        if candidate.lower().startswith('create')
+    ]
+    if len(create_candidates) == 1:
+        return create_candidates[0]
+    if len(create_candidates) > 1:
+        candidates = create_candidates
+
+    # Find the most optimal time column. Usually, it is the one pointing to
+    # the oldest timestamps:
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore', message='Could not infer format')
+        min_timestamp_dict = {
+            key: pd.to_datetime(df[key].iloc[:10_000], 'coerce')
+            for key in candidates
+        }
+    min_timestamp_dict = {
+        key: value.min().tz_localize(None)
+        for key, value in min_timestamp_dict.items()
+    }
+    min_timestamp_dict = {
+        key: value
+        for key, value in min_timestamp_dict.items() if not pd.isna(value)
+    }
+
+    if len(min_timestamp_dict) == 0:
+        return None
+
+    return min(min_timestamp_dict, key=min_timestamp_dict.get)  # type: ignore

kumoai/experimental/rfm/local_graph_sampler.py

@@ -1,3 +1,4 @@
+import re
 from typing import Dict, List, Optional, Tuple
 
 import numpy as np
@@ -7,7 +8,47 @@ from kumoapi.typing import Stype
 
 import kumoai.kumolib as kumolib
 from kumoai.experimental.rfm.local_graph_store import LocalGraphStore
-from kumoai.experimental.rfm.utils import normalize_text
+
+PUNCTUATION = re.compile(r"[\'\"\.,\(\)\!\?\;\:]")
+MULTISPACE = re.compile(r"\s+")
+
+
+def normalize_text(
+    ser: pd.Series,
+    max_words: Optional[int] = 50,
+) -> pd.Series:
+    r"""Normalizes text into a list of lower-case words.
+
+    Args:
+        ser: The :class:`pandas.Series` to normalize.
+        max_words: The maximum number of words to return.
+            This will auto-shrink any large text column to avoid blowing up
+            context size.
+    """
+    if len(ser) == 0 or pd.api.types.is_list_like(ser.iloc[0]):
+        return ser
+
+    def normalize_fn(line: str) -> list[str]:
+        line = PUNCTUATION.sub(" ", line)
+        line = re.sub(r"<br\s*/?>", " ", line)  # Handle <br /> or <br>
+        line = MULTISPACE.sub(" ", line)
+        words = line.split()
+        if max_words is not None:
+            words = words[:max_words]
+        return words
+
+    ser = ser.fillna('').astype(str)
+
+    if max_words is not None:
+        # We estimate the number of words as 5 characters + 1 space in an
+        # English text on average. We need this pre-filter here, as word
+        # splitting on a giant text can be very expensive:
+        ser = ser.str[:6 * max_words]
+
+    ser = ser.str.lower()
+    ser = ser.map(normalize_fn)
+
+    return ser
 
 
 class LocalGraphSampler:

kumoai/experimental/rfm/local_graph_store.py

@@ -7,7 +7,6 @@ from kumoapi.rfm.context import Subgraph
 from kumoapi.typing import Stype
 
 from kumoai.experimental.rfm import Graph, LocalTable
-from kumoai.experimental.rfm.utils import normalize_text
 from kumoai.utils import InteractiveProgressLogger, ProgressLogger
 
 try:
@@ -21,7 +20,6 @@ class LocalGraphStore:
     def __init__(
         self,
         graph: Graph,
-        preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
 
@@ -32,7 +30,7 @@ class LocalGraphStore:
             )
 
         with verbose as logger:
-            self.df_dict, self.mask_dict = self.sanitize(graph, preprocess)
+            self.df_dict, self.mask_dict = self.sanitize(graph)
             self.stype_dict = self.get_stype_dict(graph)
             logger.log("Sanitized input data")
 
@@ -106,7 +104,6 @@ class LocalGraphStore:
     def sanitize(
         self,
         graph: Graph,
-        preprocess: bool = False,
     ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, np.ndarray]]:
         r"""Sanitizes raw data according to table schema definition:
 
@@ -115,10 +112,6 @@ class LocalGraphStore:
         * drops timezone information from timestamps
         * drops duplicate primary keys
         * removes rows with missing primary keys or time values
-
-        If ``preprocess`` is set to ``True``, it will additionally pre-process
-        data for faster model processing. In particular, it:
-        * tokenizes any text column that is not a foreign key
         """
         df_dict: Dict[str, pd.DataFrame] = {}
         for table_name, table in graph.tables.items():
@@ -126,8 +119,6 @@ class LocalGraphStore:
             df = table._data
             df_dict[table_name] = df.copy(deep=False).reset_index(drop=True)
 
-        foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
-
         mask_dict: Dict[str, np.ndarray] = {}
         for table in graph.tables.values():
             for col in table.columns:
@@ -145,12 +136,6 @@ class LocalGraphStore:
                         ser = ser.dt.tz_localize(None)
                         df_dict[table.name][col.name] = ser
 
-                # Normalize text in advance (but exclude foreign keys):
-                if (preprocess and col.stype == Stype.text
-                        and (table.name, col.name) not in foreign_keys):
-                    ser = df_dict[table.name][col.name]
-                    df_dict[table.name][col.name] = normalize_text(ser)
-
             mask: Optional[np.ndarray] = None
             if table._time_column is not None:
                 ser = df_dict[table.name][table._time_column]

kumoai/experimental/rfm/rfm.py

@@ -150,26 +150,16 @@ class KumoRFM:
 
     Args:
         graph: The graph.
-        preprocess: Whether to pre-process the data in advance during graph
-            materialization.
-            This is a runtime trade-off between graph materialization and model
-            processing speed.
-            It can be benefical to preprocess your data once and then run many
-            queries on top to achieve maximum model speed.
-            However, if activiated, graph materialization can take potentially
-            much longer, especially on graphs with many large text columns.
-            Best to tune this option manually.
         verbose: Whether to print verbose output.
     """
     def __init__(
         self,
         graph: Graph,
-        preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
         graph = graph.validate()
         self._graph_def = graph._to_api_graph_definition()
-        self._graph_store = LocalGraphStore(graph, preprocess, verbose)
+        self._graph_store = LocalGraphStore(graph, verbose)
         self._graph_sampler = LocalGraphSampler(self._graph_store)
 
         self._client: Optional[RFMAPI] = None

{kumoai-2.13.0.dev202512021731.dist-info → kumoai-2.13.0.dev202512040252.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kumoai
-Version: 2.13.0.dev202512021731
+Version: 2.13.0.dev202512040252
 Summary: AI on the Modern Data Stack
 Author-email: "Kumo.AI" <hello@kumo.ai>
 License-Expression: MIT
@@ -42,6 +42,7 @@ Provides-Extra: sqlite
 Requires-Dist: adbc_driver_sqlite; extra == "sqlite"
 Provides-Extra: snowflake
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: pyyaml; extra == "snowflake"
 Provides-Extra: sagemaker
 Requires-Dist: boto3<2.0,>=1.30.0; extra == "sagemaker"
 Requires-Dist: mypy-boto3-sagemaker-runtime<2.0,>=1.34.0; extra == "sagemaker"

{kumoai-2.13.0.dev202512021731.dist-info → kumoai-2.13.0.dev202512040252.dist-info}/RECORD

@@ -1,13 +1,13 @@
 kumoai/__init__.py,sha256=qu-qohU2cQlManX1aZIlzA3ivKl52m-cSQBPSW8urUU,10837
 kumoai/_logging.py,sha256=qL4JbMQwKXri2f-SEJoFB8TY5ALG12S-nobGTNWxW-A,915
 kumoai/_singleton.py,sha256=i2BHWKpccNh5SJGDyU0IXsnYzJAYr8Xb0wz4c6LRbpo,861
-kumoai/_version.py,sha256=J8ZxoMuLZm4ZbBIvjU8zNKn-zWD-Pu5i4RPV-vurJZU,39
+kumoai/_version.py,sha256=16u1rVm-N2IEE7QbyS9U5nn_hjp7P_wxBIQzzAKSnDA,39
 kumoai/databricks.py,sha256=ahwJz6DWLXMkndT0XwEDBxF-hoqhidFR8wBUQ4TLZ68,490
 kumoai/exceptions.py,sha256=7TMs0SC8xrU009_Pgd4QXtSF9lxJq8MtRbeX9pcQUy4,859
 kumoai/formatting.py,sha256=o3uCnLwXPhe1KI5WV9sBgRrcU7ed4rgu_pf89GL9Nc0,983
 kumoai/futures.py,sha256=J8rtZMEYFzdn5xF_x-LAiKJz3KGL6PT02f6rq_2bOJk,3836
 kumoai/jobs.py,sha256=dCi7BAdfm2tCnonYlGU4WJokJWbh3RzFfaOX2EYCIHU,2576
-kumoai/kumolib.cp310-win_amd64.pyd,sha256=diW7mZyXCDEBISV6p-9XYxIXiwyZA3p5m-DxL6if4i4,194048
+kumoai/kumolib.cp310-win_amd64.pyd,sha256=3iE0thfrVDx0Yhh0I0li-BwZcIpQfRpaYxYMsSpYofc,194048
 kumoai/mixin.py,sha256=IaiB8SAI0VqOoMVzzIaUlqMt53-QPUK6OB0HikG-V9E,840
 kumoai/spcs.py,sha256=KWfENrwSLruprlD-QPh63uU0N6npiNrwkeKfBk3EUyQ,4260
 kumoai/artifact_export/__init__.py,sha256=UXAQI5q92ChBzWAk8o3J6pElzYHudAzFZssQXd4o7i8,247
@@ -55,27 +55,30 @@ kumoai/encoder/__init__.py,sha256=8FeP6mUyCeXxr1b8kUIi5dxe5vEXQRft9tPoaV1CBqg,18
 kumoai/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kumoai/experimental/rfm/__init__.py,sha256=EFZz6IvvskmeO85Vig6p1m_6jdimS_BkeREOndHuRsc,6247
 kumoai/experimental/rfm/authenticate.py,sha256=G89_4TMeUpr5fG_0VTzMF5sdNhaciitA1oc2loTlTmo,19321
-kumoai/experimental/rfm/graph.py,sha256=agUNrKA6gkH7Aqetn7f0Y4r1a8QJ9G3uA52WA1hKBpM,34476
-kumoai/experimental/rfm/local_graph_sampler.py,sha256=3JNpktW__nwxVKZxP4cQBgsIin7J_LNXYS7YlV36xbU,6854
-kumoai/experimental/rfm/local_graph_store.py,sha256=PoUwyMpoJhE5VjtgWYU3pV8__TcZ8oAv-zES5cIB3RQ,14251
+kumoai/experimental/rfm/graph.py,sha256=kSWve-Fn_9qERFjEpCDO5zDnngtd9T4MOhR_o46PI7s,39602
+kumoai/experimental/rfm/local_graph_sampler.py,sha256=dQ3JnuozTNeZyUFRu2h8OTMNmV1RAoaCA0gvkpgOstg,8110
+kumoai/experimental/rfm/local_graph_store.py,sha256=6jY1ciVIlnBBhZCxWwBTl7SKX1fxRIDLszwrftD0Cdk,13485
 kumoai/experimental/rfm/local_pquery_driver.py,sha256=Yd_yHIrvuDj16IC1pvsqiQvZS41vvOOCRMiuDGtN6Fk,26851
-kumoai/experimental/rfm/rfm.py,sha256=mS6LPNgl_kVHeLjmJoAW-hQhBZFKtEH4acGnjbFgee0,49445
+kumoai/experimental/rfm/rfm.py,sha256=vOnL8ecHTo1TX2B8_T8xaWGou8qYYz8DyVENu1H93mM,48834
 kumoai/experimental/rfm/sagemaker.py,sha256=sEJSyfEFBA3-7wKinBEzSooKHEn0BgPjrgRnPhYo79g,5120
-kumoai/experimental/rfm/utils.py,sha256=TN7HT_dy26Vhvd8YE90vB0hGZpTFsRMmpSZ3LcQoEIc,7626
 kumoai/experimental/rfm/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kumoai/experimental/rfm/backend/local/__init__.py,sha256=usMh0fuDxKK-aOVT1sU30BQWFS0eSkfUrhUVILisQQI,934
-kumoai/experimental/rfm/backend/local/table.py,sha256=oFf5cYu35Hcr950lvOGYjyXCmX6ICaoOBhzghCtVS1Q,8508
-kumoai/experimental/rfm/backend/snow/__init__.py,sha256=kn784OzM8tTnOnoieflOuWZZlab586v8yVVjyL9aXlc,898
+kumoai/experimental/rfm/backend/local/table.py,sha256=1PqNOROzlnK3SaZHNcU2hyzeifs0N4wssQAS3-Z0Myc,3674
+kumoai/experimental/rfm/backend/snow/__init__.py,sha256=viMeR9VWpB1kjRdSWCTNFMdM7a8Mj_Dtck1twJW8dV8,962
+kumoai/experimental/rfm/backend/snow/table.py,sha256=HVrPtCVvfsisFmq9jMovowsE5Wl5oti3O-kru7ruXlc,4312
 kumoai/experimental/rfm/backend/sqlite/__init__.py,sha256=xw5NNLrWSvUvRkD49X_9hZYjas5EuP1XDANPy0EEjOg,874
-kumoai/experimental/rfm/backend/sqlite/table.py,sha256=A-37etb2jNUV-I2tz_-7nvoHq-qdn1K7mYjsro85A1Y,5132
-kumoai/experimental/rfm/base/__init__.py,sha256=XQ_jx-dFWg95KWqHORThl9UIOcbfl-JzRy47SRwjqmc,101
+kumoai/experimental/rfm/backend/sqlite/table.py,sha256=mBiZC21gQwfR4demFrP37GmawMHfIm-G82mLQeBqIZo,3901
+kumoai/experimental/rfm/base/__init__.py,sha256=oXPkeBemtuDxRUK61-0sOT84GZB_oQ6HvaZNU1KFNaw,199
 kumoai/experimental/rfm/base/column.py,sha256=OE-PRQ8HO4uTq0e3_3eHJFfhp5nzw79zd-43g3iMh4g,2385
-kumoai/experimental/rfm/base/table.py,sha256=HAYFqvbmJ5rLicNE3j-ykxyXFvAnK_E6xmJ18HJir-Y,17534
-kumoai/experimental/rfm/infer/__init__.py,sha256=PF654hCdPSf3tFAxNegDA9aOJLBG4IT1eIjRzI0Rvwo,356
+kumoai/experimental/rfm/base/source.py,sha256=H5yN9xAwK3i_69EdqOV_x58muPGKQiI8ev5BhHQDZEo,290
+kumoai/experimental/rfm/base/table.py,sha256=glyAg4LCQdddM3lIRClJSA7qMyfoHUVAGBf1rEs6B8Y,20113
+kumoai/experimental/rfm/infer/__init__.py,sha256=qKg8or-SpgTApD6ePw1PJ4aUZPrOLTHLRCmBIJ92hrk,486
 kumoai/experimental/rfm/infer/categorical.py,sha256=bqmfrE5ZCBTcb35lA4SyAkCu3MgttAn29VBJYMBNhVg,893
+kumoai/experimental/rfm/infer/dtype.py,sha256=Hf_drluYNuN59lTSe-8GuXalg20Pv93kCktB6Hb9f74,2686
 kumoai/experimental/rfm/infer/id.py,sha256=xaJBETLZa8ttzZCsDwFSwfyCi3VYsLc_kDWT_t_6Ih4,954
 kumoai/experimental/rfm/infer/multicategorical.py,sha256=D-1KwYRkOSkBrOJr4Xa3eTCoAF9O9hPGa7Vg67V5_HU,1150
-kumoai/experimental/rfm/infer/stype.py,sha256=a5w_3WwQTctnAkFw0J9_EjFvOeLAEtnrtDbImqLeCXA,946
+kumoai/experimental/rfm/infer/pkey.py,sha256=Hvztcircd4iGdsnFU9Xi1kq_A5ONMnkAdnrpQT5svSs,4519
+kumoai/experimental/rfm/infer/time_col.py,sha256=G98Cgz1m9G9VA-ApnCmGYnJxEFwp1jfaPf3nCMOz_N0,1882
 kumoai/experimental/rfm/infer/timestamp.py,sha256=L2VxjtYTSyUBYAo4M-L08xSQlPpqnHMAVF5_vxjh3Y0,1135
 kumoai/experimental/rfm/pquery/__init__.py,sha256=RkTn0I74uXOUuOiBpa6S-_QEYctMutkUnBEfF9ztQzI,159
 kumoai/experimental/rfm/pquery/executor.py,sha256=S8wwXbAkH-YSnmEVYB8d6wyJF4JJ003mH_0zFTvOp_I,2843
@@ -101,8 +104,8 @@ kumoai/utils/__init__.py,sha256=wAKgmwtMIGuiauW9D_GGKH95K-24Kgwmld27mm4nsro,278
 kumoai/utils/datasets.py,sha256=UyAII-oAn7x3ombuvpbSQ41aVF9SYKBjQthTD-vcT2A,3011
 kumoai/utils/forecasting.py,sha256=ZgKeUCbWLOot0giAkoigwU5du8LkrwAicFOi5hVn6wg,7624
 kumoai/utils/progress_logger.py,sha256=MZsWgHd4UZQKCXiJZgQeW-Emi_BmzlCKPLPXOL_HqBo,5239
-kumoai-2.13.0.dev202512021731.dist-info/licenses/LICENSE,sha256=ZUilBDp--4vbhsEr6f_Upw9rnIx09zQ3K9fXQ0rfd6w,1111
-kumoai-2.13.0.dev202512021731.dist-info/METADATA,sha256=BYNYYVarUHxSCq5IctkyMu_VHIeu12aQ-AX34ZC9X3c,2535
-kumoai-2.13.0.dev202512021731.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
-kumoai-2.13.0.dev202512021731.dist-info/top_level.txt,sha256=YjU6UcmomoDx30vEXLsOU784ED7VztQOsFApk1SFwvs,7
-kumoai-2.13.0.dev202512021731.dist-info/RECORD,,
+kumoai-2.13.0.dev202512040252.dist-info/licenses/LICENSE,sha256=ZUilBDp--4vbhsEr6f_Upw9rnIx09zQ3K9fXQ0rfd6w,1111
+kumoai-2.13.0.dev202512040252.dist-info/METADATA,sha256=T-O--qEm_2QPzB-dDkwR6Ei7r79H7v6qQBbR1e1J8gg,2580
+kumoai-2.13.0.dev202512040252.dist-info/WHEEL,sha256=KUuBC6lxAbHCKilKua8R9W_TM71_-9Sg5uEP3uDWcoU,101
+kumoai-2.13.0.dev202512040252.dist-info/top_level.txt,sha256=YjU6UcmomoDx30vEXLsOU784ED7VztQOsFApk1SFwvs,7
+kumoai-2.13.0.dev202512040252.dist-info/RECORD,,

kumoai/experimental/rfm/infer/stype.py

@@ -1,35 +0,0 @@
-import pandas as pd
-from kumoapi.typing import Dtype, Stype
-
-from kumoai.experimental.rfm.infer import (
-    contains_categorical,
-    contains_id,
-    contains_multicategorical,
-    contains_timestamp,
-)
-
-
-def infer_stype(ser: pd.Series, column_name: str, dtype: Dtype) -> Stype:
-    r"""Infers the semantic type of a column.
-
-    Args:
-        ser: A :class:`pandas.Series` to analyze.
-        column_name: The name of the column (used for pattern matching).
-        dtype: The data type.
-
-    Returns:
-        The semantic type.
-    """
-    if contains_id(ser, column_name, dtype):
-        return Stype.ID
-
-    if contains_timestamp(ser, column_name, dtype):
-        return Stype.timestamp
-
-    if contains_multicategorical(ser, column_name, dtype):
-        return Stype.multicategorical
-
-    if contains_categorical(ser, column_name, dtype):
-        return Stype.categorical
-
-    return dtype.default_stype