kumoai 2.13.0.dev202512011731__cp312-cp312-macosx_11_0_arm64.whl → 2.13.0.dev202512031731__cp312-cp312-macosx_11_0_arm64.whl

kumoai/experimental/rfm/local_graph_store.py

@@ -7,7 +7,6 @@ from kumoapi.rfm.context import Subgraph
 from kumoapi.typing import Stype
 
 from kumoai.experimental.rfm import Graph, LocalTable
-from kumoai.experimental.rfm.utils import normalize_text
 from kumoai.utils import InteractiveProgressLogger, ProgressLogger
 
 try:
@@ -21,7 +20,6 @@ class LocalGraphStore:
     def __init__(
         self,
         graph: Graph,
-        preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
 
@@ -32,7 +30,7 @@ class LocalGraphStore:
             )
 
         with verbose as logger:
-            self.df_dict, self.mask_dict = self.sanitize(graph, preprocess)
+            self.df_dict, self.mask_dict = self.sanitize(graph)
             self.stype_dict = self.get_stype_dict(graph)
             logger.log("Sanitized input data")
 
@@ -106,7 +104,6 @@ class LocalGraphStore:
     def sanitize(
         self,
         graph: Graph,
-        preprocess: bool = False,
     ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, np.ndarray]]:
         r"""Sanitizes raw data according to table schema definition:
 
@@ -115,10 +112,6 @@ class LocalGraphStore:
         * drops timezone information from timestamps
         * drops duplicate primary keys
         * removes rows with missing primary keys or time values
-
-        If ``preprocess`` is set to ``True``, it will additionally pre-process
-        data for faster model processing. In particular, it:
-        * tokenizes any text column that is not a foreign key
         """
         df_dict: Dict[str, pd.DataFrame] = {}
         for table_name, table in graph.tables.items():
@@ -126,8 +119,6 @@ class LocalGraphStore:
             df = table._data
             df_dict[table_name] = df.copy(deep=False).reset_index(drop=True)
 
-        foreign_keys = {(edge.src_table, edge.fkey) for edge in graph.edges}
-
         mask_dict: Dict[str, np.ndarray] = {}
         for table in graph.tables.values():
             for col in table.columns:
@@ -145,12 +136,6 @@ class LocalGraphStore:
                         ser = ser.dt.tz_localize(None)
                         df_dict[table.name][col.name] = ser
 
-                # Normalize text in advance (but exclude foreign keys):
-                if (preprocess and col.stype == Stype.text
-                        and (table.name, col.name) not in foreign_keys):
-                    ser = df_dict[table.name][col.name]
-                    df_dict[table.name][col.name] = normalize_text(ser)
-
             mask: Optional[np.ndarray] = None
             if table._time_column is not None:
                 ser = df_dict[table.name][table._time_column]

kumoai/experimental/rfm/rfm.py

@@ -150,26 +150,16 @@ class KumoRFM:
 
     Args:
         graph: The graph.
-        preprocess: Whether to pre-process the data in advance during graph
-            materialization.
-            This is a runtime trade-off between graph materialization and model
-            processing speed.
-            It can be benefical to preprocess your data once and then run many
-            queries on top to achieve maximum model speed.
-            However, if activiated, graph materialization can take potentially
-            much longer, especially on graphs with many large text columns.
-            Best to tune this option manually.
         verbose: Whether to print verbose output.
     """
     def __init__(
         self,
         graph: Graph,
-        preprocess: bool = False,
         verbose: Union[bool, ProgressLogger] = True,
     ) -> None:
         graph = graph.validate()
         self._graph_def = graph._to_api_graph_definition()
-        self._graph_store = LocalGraphStore(graph, preprocess, verbose)
+        self._graph_store = LocalGraphStore(graph, verbose)
         self._graph_sampler = LocalGraphSampler(self._graph_store)
 
         self._client: Optional[RFMAPI] = None

{kumoai-2.13.0.dev202512011731.dist-info → kumoai-2.13.0.dev202512031731.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kumoai
-Version: 2.13.0.dev202512011731
+Version: 2.13.0.dev202512031731
 Summary: AI on the Modern Data Stack
 Author-email: "Kumo.AI" <hello@kumo.ai>
 License-Expression: MIT
@@ -40,6 +40,8 @@ Requires-Dist: pytest-mock; extra == "test"
 Requires-Dist: requests-mock; extra == "test"
 Provides-Extra: sqlite
 Requires-Dist: adbc_driver_sqlite; extra == "sqlite"
+Provides-Extra: snowflake
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: sagemaker
 Requires-Dist: boto3<2.0,>=1.30.0; extra == "sagemaker"
 Requires-Dist: mypy-boto3-sagemaker-runtime<2.0,>=1.34.0; extra == "sagemaker"

{kumoai-2.13.0.dev202512011731.dist-info → kumoai-2.13.0.dev202512031731.dist-info}/RECORD

@@ -1,6 +1,6 @@
 kumoai/_logging.py,sha256=U2_5ROdyk92P4xO4H2WJV8EC7dr6YxmmnM-b7QX9M7I,886
 kumoai/mixin.py,sha256=MP413xzuCqWhxAPUHmloLA3j4ZyF1tEtfi516b_hOXQ,812
-kumoai/_version.py,sha256=q8Zwyfwa6Ha3rL-zZFmN3WuqlLR8mLt6YklIO3ZtJg8,39
+kumoai/_version.py,sha256=5E8jDfy-Cd90GKsXB2iph05yeJqiO4NclFrisgQkb80,39
 kumoai/kumolib.cpython-312-darwin.so,sha256=xQvdWHx9xmQ11y3F3ywxJv6A0sDk6D3-2fQbxSdM1z4,232576
 kumoai/__init__.py,sha256=L3yOOtpSdwe3PYQlJBLkiQd3Ypp8iB5ChXkzprk3Si4,10546
 kumoai/formatting.py,sha256=jA_rLDCGKZI8WWCha-vtuLenVKTZvli99Tqpurz1H84,953
@@ -11,30 +11,35 @@ kumoai/databricks.py,sha256=e6E4lOFvZHXFwh4CO1kXU1zzDU3AapLQYMxjiHPC-HQ,476
 kumoai/spcs.py,sha256=N31d7rLa-bgYh8e2J4YzX1ScxGLqiVXrqJnCl1y4Mts,4139
 kumoai/_singleton.py,sha256=UTwrbDkoZSGB8ZelorvprPDDv9uZkUi1q_SrmsyngpQ,836
 kumoai/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kumoai/experimental/rfm/local_graph_sampler.py,sha256=5DbhL9h0usFKSJfnx7HjLMPcG54qwJ48M2tmONqxXyY,6672
+kumoai/experimental/rfm/local_graph_sampler.py,sha256=QmyEw1M6CsftQpwwGMaUgPog7lt4EUfp5Y_KXIOK_oo,7887
 kumoai/experimental/rfm/local_pquery_driver.py,sha256=aO7Jfwx9gxGKYvpqxZx1LLWdI1MhuZQOPtAITxoOQO0,26162
-kumoai/experimental/rfm/graph.py,sha256=Ff_9-rOJRSDd4bnx73CkfQjAxU4fqzzQ3CIkZOYjR8c,30729
+kumoai/experimental/rfm/graph.py,sha256=5TZVbd4agFePPSazgqViAqWmLMpxHuDsX_DqHnqaNnM,36581
 kumoai/experimental/rfm/__init__.py,sha256=slliYcrh80xPtQQ_nnsp3ny9IbmHCyirmdZUfKTdME4,6064
-kumoai/experimental/rfm/utils.py,sha256=3IiBvT_aLBkkcJh3H11_50yt_XlEzHR0cm9Kprrtl8k,11123
 kumoai/experimental/rfm/sagemaker.py,sha256=_hTrFg4qfXe7uzwqSEG_wze-IFkwn7qde9OpUodCpbc,4982
-kumoai/experimental/rfm/rfm.py,sha256=0i6UA6Ds72midVq4ngaP4y5cnxs_GrwBF93Y0pFCX5k,48304
-kumoai/experimental/rfm/local_graph_store.py,sha256=2ehCC7bttSp8cKLe5_6GaMUdF4fbYaTR0rYPVQb6rEQ,13891
+kumoai/experimental/rfm/rfm.py,sha256=FZRrYK9uoH4IoGI1hQunORp1zrpfeyi8dDqikt6Gfpk,47703
+kumoai/experimental/rfm/local_graph_store.py,sha256=l6HMQBNdSdDEL0xIGhTcmR3E_JOIfZJPHDbiD0E7GlA,13140
 kumoai/experimental/rfm/authenticate.py,sha256=FiuHMvP7V3zBZUlHMDMbNLhc-UgDZgz4hjVSTuQ7DRw,18888
 kumoai/experimental/rfm/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kumoai/experimental/rfm/backend/sqlite/__init__.py,sha256=wQSH6esLny3EJID-fl__fxpiiG5J5Pj2EgIfbD0gx5I,604
-kumoai/experimental/rfm/backend/sqlite/table.py,sha256=WAUOEDDNNAMPlbN6W3ZsSfd8OLhrUT5yRa0ypde0iRs,4627
+kumoai/experimental/rfm/backend/sqlite/__init__.py,sha256=jYmZDNAVsojuPO1Q5idFmG5N0aCB8BDyrpAoS31n9bc,844
+kumoai/experimental/rfm/backend/sqlite/table.py,sha256=fnw6whxUmzjycFatlHqwVP64tujNY8RE20ZnAaZ9TJc,3417
 kumoai/experimental/rfm/backend/local/__init__.py,sha256=9rupbsPadaOqrEInv2nh9KEQ9mK8dSkbteMXwZmsGbU,896
-kumoai/experimental/rfm/backend/local/table.py,sha256=wxYmWCy_F3j2TQ5trN7Cn2hAkVDgIcSWMJa-uFkl8G0,5140
+kumoai/experimental/rfm/backend/local/table.py,sha256=wQU28OX6-vtdBvrHcoRt8XTBDScSahVIql-evINkS6Y,3014
+kumoai/experimental/rfm/backend/snow/__init__.py,sha256=B-tG-p8WA-mBuwvK1f0S2gdRPEGwApdxlnyeVSnY2xg,927
+kumoai/experimental/rfm/backend/snow/table.py,sha256=vUqUJUfphmZIm7h1a8X0IvX-jf_wT1Oh3YRuhqT_7M8,3460
 kumoai/experimental/rfm/pquery/__init__.py,sha256=X0O3EIq5SMfBEE-ii5Cq6iDhR3s3XMXB52Cx5htoePw,152
 kumoai/experimental/rfm/pquery/pandas_executor.py,sha256=kiBJq7uVGbasG7TiqsubEl6ey3UYzZiM4bwxILqp_54,18487
 kumoai/experimental/rfm/pquery/executor.py,sha256=f7-pJhL0BgFU9E4o4gQpQyArOvyrZtwxFmks34-QOAE,2741
 kumoai/experimental/rfm/infer/multicategorical.py,sha256=0-cLpDnGryhr76QhZNO-klKokJ6MUSfxXcGdQ61oykY,1102
 kumoai/experimental/rfm/infer/categorical.py,sha256=VwNaKwKbRYkTxEJ1R6gziffC8dGsEThcDEfbi-KqW5c,853
+kumoai/experimental/rfm/infer/time_col.py,sha256=7R5Itl8RRBOr61qLpRTanIqrUVZFZcAXzDA9lCw4nx4,1820
+kumoai/experimental/rfm/infer/pkey.py,sha256=ubNqW1LIjLKiXbjXELAY3g6n2f3u2Eis_uC2DEiXFiU,4393
 kumoai/experimental/rfm/infer/id.py,sha256=ZIO0DWIoiEoS_8MVc5lkqBfkTWWQ0yGCgjkwLdaYa_Q,908
-kumoai/experimental/rfm/infer/__init__.py,sha256=xQ8_SuejIzXyn2J7bIKX3pXumFtRuEfBtE5oEDUDJjI,293
+kumoai/experimental/rfm/infer/dtype.py,sha256=IYhLyf4UoPZ-qqcUIt-enydRTnnNqY-sSim56V7uuUU,2979
+kumoai/experimental/rfm/infer/__init__.py,sha256=krdMFN8iKZlSFOl-M5MW1KuSviQV3H1E18jj2uB8g6Q,469
 kumoai/experimental/rfm/infer/timestamp.py,sha256=vM9--7eStzaGG13Y-oLYlpNJyhL6f9dp17HDXwtl_DM,1094
-kumoai/experimental/rfm/base/__init__.py,sha256=TlK19Ge4xTeuEMsZvSTA3uR2S0xnlYcJxXwWos-N4lw,94
-kumoai/experimental/rfm/base/table.py,sha256=YSjnsyn-5sX3IcCUgwed7JON7OaDH6n76PlX5GIgtVI,16946
+kumoai/experimental/rfm/base/__init__.py,sha256=-f3Ap-eUG1_JIX6NwRTZ2E3Rn0KTwt_PRYz8UcajWvg,189
+kumoai/experimental/rfm/base/table.py,sha256=CLC66JMBSJcvtvF8lecZywK-50_EzDHN6dc9ZekzpV0,19573
+kumoai/experimental/rfm/base/source.py,sha256=8_waFQVsctryHkm9BwmFZ9-vw5cXAXfjk7KDmcl_kic,272
 kumoai/experimental/rfm/base/column.py,sha256=izCJmufJcd1RSi-ptFMfrue-JYag38MJxizka7ya0-A,2319
 kumoai/encoder/__init__.py,sha256=VPGs4miBC_WfwWeOXeHhFomOUocERFavhKf5fqITcds,182
 kumoai/graph/graph.py,sha256=iyp4klPIMn2ttuEqMJvsrxKb_tmz_DTnvziIhCegduM,38291
@@ -99,8 +104,8 @@ kumoai/trainer/baseline_trainer.py,sha256=LlfViNOmswNv4c6zJJLsyv0pC2mM2WKMGYx06o
 kumoai/trainer/__init__.py,sha256=zUdFl-f-sBWmm2x8R-rdVzPBeU2FaMzUY5mkcgoTa1k,939
 kumoai/trainer/online_serving.py,sha256=9cddb5paeZaCgbUeceQdAOxysCtV5XP-KcsgFz_XR5w,9566
 kumoai/trainer/trainer.py,sha256=hBXO7gwpo3t59zKFTeIkK65B8QRmWCwO33sbDuEAPlY,20133
-kumoai-2.13.0.dev202512011731.dist-info/RECORD,,
-kumoai-2.13.0.dev202512011731.dist-info/WHEEL,sha256=V1loQ6TpxABu1APUg0MoTRBOzSKT5xVc3skizX-ovCU,136
-kumoai-2.13.0.dev202512011731.dist-info/top_level.txt,sha256=YjU6UcmomoDx30vEXLsOU784ED7VztQOsFApk1SFwvs,7
-kumoai-2.13.0.dev202512011731.dist-info/METADATA,sha256=q3ILcIOLhew8MHoXzkQ_z59UwQouGD6mk7U8jEpNGSQ,2376
-kumoai-2.13.0.dev202512011731.dist-info/licenses/LICENSE,sha256=TbWlyqRmhq9PEzCaTI0H0nWLQCCOywQM8wYH8MbjfLo,1102
+kumoai-2.13.0.dev202512031731.dist-info/RECORD,,
+kumoai-2.13.0.dev202512031731.dist-info/WHEEL,sha256=V1loQ6TpxABu1APUg0MoTRBOzSKT5xVc3skizX-ovCU,136
+kumoai-2.13.0.dev202512031731.dist-info/top_level.txt,sha256=YjU6UcmomoDx30vEXLsOU784ED7VztQOsFApk1SFwvs,7
+kumoai-2.13.0.dev202512031731.dist-info/METADATA,sha256=M7NK6i4Wz55zU6rHvkhts9ewT2dMeXtMIYnH5of8U-o,2466
+kumoai-2.13.0.dev202512031731.dist-info/licenses/LICENSE,sha256=TbWlyqRmhq9PEzCaTI0H0nWLQCCOywQM8wYH8MbjfLo,1102

kumoai/experimental/rfm/utils.py

@@ -1,344 +0,0 @@
-import re
-import warnings
-from typing import Any, Dict, Optional
-
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-from kumoapi.typing import Dtype, Stype
-
-from kumoai.experimental.rfm.infer import (
-    contains_categorical,
-    contains_id,
-    contains_multicategorical,
-    contains_timestamp,
-)
-
-# Mapping from pandas/numpy dtypes to Kumo Dtypes
-PANDAS_TO_DTYPE: Dict[Any, Dtype] = {
-    np.dtype('bool'): Dtype.bool,
-    pd.BooleanDtype(): Dtype.bool,
-    pa.bool_(): Dtype.bool,
-    np.dtype('byte'): Dtype.int,
-    pd.UInt8Dtype(): Dtype.int,
-    np.dtype('int16'): Dtype.int,
-    pd.Int16Dtype(): Dtype.int,
-    np.dtype('int32'): Dtype.int,
-    pd.Int32Dtype(): Dtype.int,
-    np.dtype('int64'): Dtype.int,
-    pd.Int64Dtype(): Dtype.int,
-    np.dtype('float32'): Dtype.float,
-    pd.Float32Dtype(): Dtype.float,
-    np.dtype('float64'): Dtype.float,
-    pd.Float64Dtype(): Dtype.float,
-    np.dtype('object'): Dtype.string,
-    pd.StringDtype(storage='python'): Dtype.string,
-    pd.StringDtype(storage='pyarrow'): Dtype.string,
-    pa.string(): Dtype.string,
-    pa.binary(): Dtype.binary,
-    np.dtype('datetime64[ns]'): Dtype.date,
-    np.dtype('timedelta64[ns]'): Dtype.timedelta,
-    pa.list_(pa.float32()): Dtype.floatlist,
-    pa.list_(pa.int64()): Dtype.intlist,
-    pa.list_(pa.string()): Dtype.stringlist,
-}
-
-
-def to_dtype(ser: pd.Series) -> Dtype:
-    """Extracts the :class:`Dtype` from a :class:`pandas.Series`.
-
-    Args:
-        ser: A :class:`pandas.Series` to analyze.
-
-    Returns:
-        The data type.
-    """
-    if pd.api.types.is_datetime64_any_dtype(ser.dtype):
-        return Dtype.date
-
-    if isinstance(ser.dtype, pd.CategoricalDtype):
-        return Dtype.string
-
-    if pd.api.types.is_object_dtype(ser.dtype):
-        index = ser.iloc[:1000].first_valid_index()
-        if index is not None and pd.api.types.is_list_like(ser[index]):
-            pos = ser.index.get_loc(index)
-            assert isinstance(pos, int)
-            ser = ser.iloc[pos:pos + 1000].dropna()
-
-            if not ser.map(pd.api.types.is_list_like).all():
-                raise ValueError("Data contains a mix of list-like and "
-                                 "non-list-like values")
-
-            ser = ser[ser.map(lambda x: not isinstance(x, list) or len(x) > 0)]
-
-            dtypes = ser.apply(lambda x: PANDAS_TO_DTYPE.get(
-                np.array(x).dtype, Dtype.string)).unique().tolist()
-
-            invalid_dtypes = set(dtypes) - {
-                Dtype.string,
-                Dtype.int,
-                Dtype.float,
-            }
-            if len(invalid_dtypes) > 0:
-                raise ValueError(f"Data contains unsupported list data types: "
-                                 f"{list(invalid_dtypes)}")
-
-            if Dtype.string in dtypes:
-                return Dtype.stringlist
-
-            if dtypes == [Dtype.int]:
-                return Dtype.intlist
-
-            return Dtype.floatlist
-
-    if ser.dtype not in PANDAS_TO_DTYPE:
-        raise ValueError(f"Unsupported data type '{ser.dtype}'")
-
-    return PANDAS_TO_DTYPE[ser.dtype]
-
-
-def infer_stype(ser: pd.Series, column_name: str, dtype: Dtype) -> Stype:
-    r"""Infers the semantic type of a column.
-
-    Args:
-        ser: A :class:`pandas.Series` to analyze.
-        column_name: The name of the column (used for pattern matching).
-        dtype: The data type.
-
-    Returns:
-        The semantic type.
-    """
-    if contains_id(ser, column_name, dtype):
-        return Stype.ID
-
-    if contains_timestamp(ser, column_name, dtype):
-        return Stype.timestamp
-
-    if contains_multicategorical(ser, column_name, dtype):
-        return Stype.multicategorical
-
-    if contains_categorical(ser, column_name, dtype):
-        return Stype.categorical
-
-    return dtype.default_stype
-
-
-def detect_primary_key(
-    table_name: str,
-    df: pd.DataFrame,
-    candidates: list[str],
-) -> Optional[str]:
-    r"""Auto-detect potential primary key column.
-
-    Args:
-        table_name: The table name.
-        df: The pandas DataFrame to analyze
-        candidates: A list of potential candidates.
-
-    Returns:
-        The name of the detected primary key, or ``None`` if not found.
-    """
-    # A list of (potentially modified) table names that are eligible to match
-    # with a primary key, i.e.:
-    # - UserInfo -> User
-    # - snakecase <-> camelcase
-    # - camelcase <-> snakecase
-    # - plural <-> singular (users -> user, eligibilities -> eligibility)
-    # - verb -> noun (qualifying -> qualify)
-    _table_names = {table_name}
-    if table_name.lower().endswith('_info'):
-        _table_names.add(table_name[:-5])
-    elif table_name.lower().endswith('info'):
-        _table_names.add(table_name[:-4])
-
-    table_names = set()
-    for _table_name in _table_names:
-        table_names.add(_table_name.lower())
-        snakecase = re.sub(r'(.)([A-Z][a-z]+)', r'\1_\2', _table_name)
-        snakecase = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', snakecase)
-        table_names.add(snakecase.lower())
-        camelcase = _table_name.replace('_', '')
-        table_names.add(camelcase.lower())
-        if _table_name.lower().endswith('s'):
-            table_names.add(_table_name.lower()[:-1])
-            table_names.add(snakecase.lower()[:-1])
-            table_names.add(camelcase.lower()[:-1])
-        else:
-            table_names.add(_table_name.lower() + 's')
-            table_names.add(snakecase.lower() + 's')
-            table_names.add(camelcase.lower() + 's')
-        if _table_name.lower().endswith('ies'):
-            table_names.add(_table_name.lower()[:-3] + 'y')
-            table_names.add(snakecase.lower()[:-3] + 'y')
-            table_names.add(camelcase.lower()[:-3] + 'y')
-        elif _table_name.lower().endswith('y'):
-            table_names.add(_table_name.lower()[:-1] + 'ies')
-            table_names.add(snakecase.lower()[:-1] + 'ies')
-            table_names.add(camelcase.lower()[:-1] + 'ies')
-        if _table_name.lower().endswith('ing'):
-            table_names.add(_table_name.lower()[:-3])
-            table_names.add(snakecase.lower()[:-3])
-            table_names.add(camelcase.lower()[:-3])
-
-    scores: list[tuple[str, int]] = []
-    for col_name in candidates:
-        col_name_lower = col_name.lower()
-
-        score = 0
-
-        if col_name_lower == 'id':
-            score += 4
-
-        for table_name_lower in table_names:
-
-            if col_name_lower == table_name_lower:
-                score += 4  # USER -> USER
-                break
-
-            for suffix in ['id', 'hash', 'key', 'code', 'uuid']:
-                if not col_name_lower.endswith(suffix):
-                    continue
-
-                if col_name_lower == f'{table_name_lower}_{suffix}':
-                    score += 5  # USER -> USER_ID
-                    break
-
-                if col_name_lower == f'{table_name_lower}{suffix}':
-                    score += 5  # User -> UserId
-                    break
-
-                if col_name_lower.endswith(f'{table_name_lower}_{suffix}'):
-                    score += 2
-
-                if col_name_lower.endswith(f'{table_name_lower}{suffix}'):
-                    score += 2
-
-            # `rel-bench` hard-coding :(
-            if table_name == 'studies' and col_name == 'nct_id':
-                score += 1
-
-        ser = df[col_name].iloc[:1_000_000]
-        score += 3 * (ser.nunique() / len(ser))
-
-        scores.append((col_name, score))
-
-    scores = [x for x in scores if x[-1] >= 4]
-    scores.sort(key=lambda x: x[-1], reverse=True)
-
-    if len(scores) == 0:
-        return None
-
-    if len(scores) == 1:
-        return scores[0][0]
-
-    # In case of multiple candidates, only return one if its score is unique:
-    if scores[0][1] != scores[1][1]:
-        return scores[0][0]
-
-    max_score = max(scores, key=lambda x: x[1])
-    candidates = [col_name for col_name, score in scores if score == max_score]
-    warnings.warn(f"Found multiple potential primary keys in table "
-                  f"'{table_name}': {candidates}. Please specify the primary "
-                  f"key for this table manually.")
-
-    return None
-
-
-def detect_time_column(
-    df: pd.DataFrame,
-    candidates: list[str],
-) -> Optional[str]:
-    r"""Auto-detect potential time column.
-
-    Args:
-        df: The pandas DataFrame to analyze
-        candidates: A list of potential candidates.
-
-    Returns:
-        The name of the detected time column, or ``None`` if not found.
-    """
-    candidates = [  # Exclude all candidates with `*last*` in column names:
-        col_name for col_name in candidates
-        if not re.search(r'(^|_)last(_|$)', col_name, re.IGNORECASE)
-    ]
-
-    if len(candidates) == 0:
-        return None
-
-    if len(candidates) == 1:
-        return candidates[0]
-
-    # If there exists a dedicated `create*` column, use it as time column:
-    create_candidates = [
-        candidate for candidate in candidates
-        if candidate.lower().startswith('create')
-    ]
-    if len(create_candidates) == 1:
-        return create_candidates[0]
-    if len(create_candidates) > 1:
-        candidates = create_candidates
-
-    # Find the most optimal time column. Usually, it is the one pointing to
-    # the oldest timestamps:
-    with warnings.catch_warnings():
-        warnings.filterwarnings('ignore', message='Could not infer format')
-        min_timestamp_dict = {
-            key: pd.to_datetime(df[key].iloc[:10_000], 'coerce')
-            for key in candidates
-        }
-    min_timestamp_dict = {
-        key: value.min().tz_localize(None)
-        for key, value in min_timestamp_dict.items()
-    }
-    min_timestamp_dict = {
-        key: value
-        for key, value in min_timestamp_dict.items() if not pd.isna(value)
-    }
-
-    if len(min_timestamp_dict) == 0:
-        return None
-
-    return min(min_timestamp_dict, key=min_timestamp_dict.get)  # type: ignore
-
-
-PUNCTUATION = re.compile(r"[\'\"\.,\(\)\!\?\;\:]")
-MULTISPACE = re.compile(r"\s+")
-
-
-def normalize_text(
-    ser: pd.Series,
-    max_words: Optional[int] = 50,
-) -> pd.Series:
-    r"""Normalizes text into a list of lower-case words.
-
-    Args:
-        ser: The :class:`pandas.Series` to normalize.
-        max_words: The maximum number of words to return.
-            This will auto-shrink any large text column to avoid blowing up
-            context size.
-    """
-    if len(ser) == 0 or pd.api.types.is_list_like(ser.iloc[0]):
-        return ser
-
-    def normalize_fn(line: str) -> list[str]:
-        line = PUNCTUATION.sub(" ", line)
-        line = re.sub(r"<br\s*/?>", " ", line)  # Handle <br /> or <br>
-        line = MULTISPACE.sub(" ", line)
-        words = line.split()
-        if max_words is not None:
-            words = words[:max_words]
-        return words
-
-    ser = ser.fillna('').astype(str)
-
-    if max_words is not None:
-        # We estimate the number of words as 5 characters + 1 space in an
-        # English text on average. We need this pre-filter here, as word
-        # splitting on a giant text can be very expensive:
-        ser = ser.str[:6 * max_words]
-
-    ser = ser.str.lower()
-    ser = ser.map(normalize_fn)
-
-    return ser