tdfs4ds 0.2.4.33__py3-none-any.whl → 0.2.4.35__py3-none-any.whl

tdfs4ds/__init__.py

@@ -1,5 +1,6 @@
-__version__ = '0.2.4.33'
+__version__ = '0.2.4.35'
 import logging
+
 # Setup the logger
 logging.basicConfig(
     level=logging.INFO,
@@ -66,7 +67,7 @@ import tdfs4ds.datasets
 import time
 
 import inspect
-import tqdm
+from tqdm.auto import tqdm  # auto picks the right frontend (notebook/terminal)
 
 from tdfs4ds.feature_store.feature_data_processing import generate_on_clause
 
@@ -671,8 +672,28 @@ def _upload_features(
     else:
         logger_safe("info", "FilterManager detected: %s filters to process", filtermanager.nb_filters)
         something_computed = False
-        for i in range(filtermanager.nb_filters):
-            filtermanager.update(i + 1)
+        for i in tqdm(
+            range(filtermanager.nb_filters),
+            total=filtermanager.nb_filters,
+            desc="Applying filters",
+            unit="filter",
+            leave=False
+        ):
+            filter_id = i + 1
+            filtermanager.update(filter_id)
+
+            # show which filter is being applied in the bar
+            try:
+                tqdm.write(f"Applying filter {filter_id}/{filtermanager.nb_filters}")
+                # If display() returns a long string, you can shorten it:
+                bar_info = str(filtermanager.display())
+                if len(bar_info) > 80:
+                    bar_info = bar_info[:77] + "..."
+                tqdm.tqdm._instances and next(iter(tqdm.tqdm._instances)).set_postfix_str(bar_info)
+            except Exception:
+                # postfix is optional; ignore errors from display() here
+                pass
+
             logger_safe("debug", "Applying filter %s/%s:\n%s",
                 i + 1, filtermanager.nb_filters, filtermanager.display())
 
@@ -685,38 +706,40 @@ def _upload_features(
 
             if do_compute or force_compute:
                 tdfs4ds.process_store.process_followup.followup_open(
-                    run_id=tdfs4ds.RUN_ID,
-                    process_type=tdfs4ds.PROCESS_TYPE,
-                    process_id=process_id,
-                    filtermanager=filtermanager
+                    run_id        = tdfs4ds.RUN_ID,
+                    process_type  = tdfs4ds.PROCESS_TYPE,
+                    process_id    = process_id,
+                    filtermanager = filtermanager
                 )
                 try:
                     prepared_features, volatile_table, features_infos = prepare_feature_ingestion(
                         df, entity_id, feature_names,
-                        feature_versions=selected_features,
-                        primary_index=primary_index,
-                        entity_null_substitute=entity_null_substitute,
-                        partitioning=partitioning
+                        feature_versions       = selected_features,
+                        primary_index          = primary_index,
+                        entity_null_substitute = entity_null_substitute,
+                        partitioning           = partitioning
                     )
+
                     store_feature(entity_id, volatile_table, entity_null_substitute,
                                   primary_index, partitioning, features_infos)
+                    
                     something_computed = True
 
                     tdfs4ds.process_store.process_followup.followup_close(
-                        run_id=tdfs4ds.RUN_ID,
-                        process_type=tdfs4ds.PROCESS_TYPE,
-                        process_id=process_id,
-                        filtermanager=filtermanager
+                        run_id        = tdfs4ds.RUN_ID,
+                        process_type  = tdfs4ds.PROCESS_TYPE,
+                        process_id    = process_id,
+                        filtermanager = filtermanager
                     )
 
                 except Exception as e:
                     logger_safe("exception", "Error with filter iteration %s: %s", i + 1, str(e))
                     tdfs4ds.process_store.process_followup.followup_close(
-                        run_id=tdfs4ds.RUN_ID,
-                        process_type=tdfs4ds.PROCESS_TYPE,
-                        process_id=process_id,
-                        status='FAILED,' + str(e).split('\n')[0],
-                        filtermanager=filtermanager
+                        run_id        = tdfs4ds.RUN_ID,
+                        process_type  = tdfs4ds.PROCESS_TYPE,
+                        process_id    = process_id,
+                        status        = 'FAILED,' + str(e).split('\n')[0],
+                        filtermanager = filtermanager
                     )
                     raise
 
@@ -1188,9 +1211,6 @@ def roll_out(process_list, time_manager, time_id_start = 1, time_id_end = None):
     >>> roll_out(process_list, time_manager, time_id_start=1, time_id_end=10)
     """
 
-    #global DISPLAY_LOGS
-    #global FEATURE_STORE_TIME
-
     # Disable display logs
     temp_DISPLAY_LOGS = tdfs4ds.DISPLAY_LOGS
     tdfs4ds.DISPLAY_LOGS = False
@@ -1198,40 +1218,43 @@ def roll_out(process_list, time_manager, time_id_start = 1, time_id_end = None):
     tdfs4ds.PROCESS_TYPE = 'ROLL_OUT'
     tdfs4ds.RUN_ID = str(uuid.uuid4())
 
-
-
     try:
+        # Define range of time steps
         if time_id_end is None:
-            pbar = tqdm.tqdm(range(time_id_start, time_manager.nb_time_steps + 1), desc="Starting")
+            time_range = range(time_id_start, time_manager.nb_time_steps + 1)
         else:
-            pbar = tqdm.tqdm(range(time_id_start, min([time_manager.nb_time_steps + 1,time_id_end+1]) ), desc="Starting")
-        # Iterate over each date in the provided list
+            time_range = range(time_id_start, min(time_manager.nb_time_steps + 1, time_id_end + 1))
+
+        # Progress bar
+        pbar = tqdm(time_range, desc="Starting rollout", unit="step")
+
         for i in pbar:
-            # Update the time manager with the new date
-            time_manager.update(time_id = i )
+            # Update time manager
+            time_manager.update(time_id=i)
             date_ = str(time_manager.display()['BUSINESS_DATE'].values[0])
-            pbar.set_description(f"Processing {date_}")
-            # Synchronize the time for the feature store with the current date
+
+            # Sync feature store time
             tdfs4ds.FEATURE_STORE_TIME = time_manager.get_date_in_the_past()
-            pbar.set_description(f"Processing {tdfs4ds.FEATURE_STORE_TIME}")
+
+            # Display current progress in tqdm
+            pbar.set_postfix(time=date_, feature_time=tdfs4ds.FEATURE_STORE_TIME)
+
             if tdfs4ds.DEBUG_MODE:
-                print('def roll_out','date_', date_)
-                print('def roll_out','time_manager.get_date_in_the_past()', time_manager.get_date_in_the_past())
-                print('def roll_out','tdfs4ds.FEATURE_STORE_TIME', tdfs4ds.FEATURE_STORE_TIME)
-            # Execute each process in the process list for the current date
+                print("roll_out | date_:", date_)
+                print("roll_out | feature_store_time:", tdfs4ds.FEATURE_STORE_TIME)
+
+            # Execute all processes for this time step
             for proc_id in process_list:
-                pbar.set_description(f"Processing {date_} process {proc_id}")
+                pbar.set_description(f"Processing {date_} | proc {proc_id}")
                 run(process_id=proc_id, force_compute=False)
 
+        # Restore settings
         tdfs4ds.DISPLAY_LOGS = temp_DISPLAY_LOGS
+
     except Exception as e:
         tdfs4ds.DISPLAY_LOGS = temp_DISPLAY_LOGS
-        # If an exception occurs, print the date and the first line of the exception message
-        #print(date_)
         print(str(e).split('\n')[0])
         tdfs4ds.PROCESS_TYPE = PROCESS_TYPE
         raise
 
-    tdfs4ds.PROCESS_TYPE = PROCESS_TYPE
-
-
+    tdfs4ds.PROCESS_TYPE = PROCESS_TYPE

tdfs4ds/feature_store/feature_data_processing.py

@@ -75,7 +75,7 @@ def generate_collect_stats(entity_id, primary_index='', partitioning=''):
 
     # Initialize the extended query with sampling and threshold settings for statistics collection
     query_extension_header = 'COLLECT STATISTICS USING SAMPLE 25 PERCENT AND THRESHOLD 15 PERCENT'
-    query_extension = []
+    query_extension        = []
 
     # Add primary index columns to the extended query
     if primary_index:
@@ -343,67 +343,167 @@ def prepare_feature_ingestion(df, entity_id, feature_names, feature_versions=Non
     # return None, None, None
 
 
-def apply_collect_stats(entity_id, primary_index, partitioning, feature_infos):
-    """
-    Applies a collect statistics operation on target tables grouped by feature table and database.
-
-    This function performs the following steps:
-    1. Sorts the `entity_id`.
-    2. Groups the feature information by feature table and database to count occurrences.
-    3. Generates collect statistics queries.
-    4. Executes the queries on the target tables while recording the execution time.
-    5. Logs the elapsed time if logging is enabled.
-
-    Args:
-        entity_id (list): A list of entity IDs to process.
-        primary_index (str): The primary index to use in the collect statistics query.
-        partitioning (str): Partitioning information for the query.
-        feature_infos (pd.DataFrame): A DataFrame containing feature information,
-            including columns 'FEATURE_TABLE', 'FEATURE_DATABASE', and 'FEATURE_ID'.
+import time
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
 
-    Returns:
-        None
-    """
-    # Sort entity IDs for consistent ordering
-    sorted_entity_id = list(entity_id.keys())
-    sorted_entity_id.sort()
+import pandas as pd
 
-    # Group target tables
-    target_tables = feature_infos[['FEATURE_TABLE', 'FEATURE_DATABASE', 'FEATURE_ID']].groupby(
-        ['FEATURE_TABLE', 'FEATURE_DATABASE']
-    ).count().reset_index()
+def apply_collect_stats(
+    entity_id: Mapping[str, Any] | Iterable[str],
+    primary_index: Optional[str],
+    partitioning: Optional[str],
+    feature_infos: pd.DataFrame,
+) -> Dict[str, Any]:
+    """
+    Run COLLECT STATS on all target feature tables, with fallbacks and timing.
+
+    Steps:
+    1) Determine a stable ordering of entity IDs (for deterministic query gen).
+    2) Group `feature_infos` by FEATURE_DATABASE + FEATURE_TABLE to get unique targets.
+    3) Generate COLLECT STATS statements via `generate_collect_stats(...)` for fallback use.
+    4) For each target table:
+        - Try a simple `COLLECT STATS ON <db>.<table>`.
+        - On failure, retry with generated statements (and optional extension).
+    5) Log a compact summary (counts + total duration) and return it as a dict.
+
+    Parameters
+    ----------
+    entity_id : Mapping[str, Any] | Iterable[str]
+        Entity identifiers used to parameterize collect-stat statements.
+        If a mapping (e.g., dict), its *keys* are used and sorted.
+        If an iterable (e.g., list/tuple), it’s sorted directly.
+    primary_index : Optional[str]
+        Primary index used by `generate_collect_stats` (may be None).
+    partitioning : Optional[str]
+        Partitioning clause used by `generate_collect_stats` (may be None).
+    feature_infos : pd.DataFrame
+        Must contain columns: 'FEATURE_TABLE', 'FEATURE_DATABASE', 'FEATURE_ID'.
+
+    Returns
+    -------
+    Dict[str, Any]
+        Summary with keys:
+        - total_tables: int
+        - ok: int
+        - retried: int
+        - failed: int
+        - duration_seconds: float
+        - duration_hms: str
+        - details: list[dict]  # per-table status entries
+    """
+    # --- Validate inputs -----------------------------------------------------
+    required_cols = {"FEATURE_TABLE", "FEATURE_DATABASE", "FEATURE_ID"}
+    missing = required_cols.difference(feature_infos.columns)
+    if missing:
+        raise ValueError(f"feature_infos is missing required columns: {sorted(missing)}")
+
+    # --- Normalize & sort entity IDs ----------------------------------------
+    if hasattr(entity_id, "keys"):
+        sorted_entity_ids = sorted(list(entity_id.keys()))
+    else:
+        sorted_entity_ids = sorted(list(entity_id))
+
+    # --- Group to unique targets --------------------------------------------
+    target_tables = (
+        feature_infos[["FEATURE_TABLE", "FEATURE_DATABASE", "FEATURE_ID"]]
+        .groupby(["FEATURE_TABLE", "FEATURE_DATABASE"])
+        .count()
+        .reset_index()
+    )
 
     if getattr(tdfs4ds, "DEBUG_MODE", False):
-        logger_safe("debug", "Target tables for COLLECT STATs: %s", target_tables[['FEATURE_DATABASE','FEATURE_TABLE']].to_dict(orient='records'))
+        logger_safe(
+            "debug",
+            "collect_stats.targets | count=%s | tables=%s",
+            len(target_tables),
+            target_tables[["FEATURE_DATABASE", "FEATURE_TABLE"]].to_dict(orient="records"),
+        )
 
-    # Generate COLLECT STATs queries
+    # --- Prepare statements --------------------------------------------------
     query_collect_stats, query_collect_stats_extension = generate_collect_stats(
-        sorted_entity_id,
+        sorted_entity_ids,
         primary_index=primary_index,
-        partitioning=partitioning
+        partitioning=partitioning,
     )
 
-    start_time = time.time()
+    # --- Execute -------------------------------------------------------------
+    started = time.perf_counter()
+    results: list[Dict[str, Any]] = []
+
+    ok = retried = failed = 0
 
-    # Execute COLLECT STATs
     for _, row in target_tables.iterrows():
-        table_fqn = f"{row['FEATURE_DATABASE']}.{row['FEATURE_TABLE']}"
+        db = row["FEATURE_DATABASE"]
+        tbl = row["FEATURE_TABLE"]
+        table_fqn = f"{db}.{tbl}"
+
         if getattr(tdfs4ds, "DEBUG_MODE", False):
-            logger_safe("debug", "Running COLLECT STATs on %s", table_fqn)
+            logger_safe("debug", "collect_stats.run | table=%s", table_fqn)
 
-        execute_query(query_collect_stats + f" ON {table_fqn}")
+        t0 = time.perf_counter()
+        status = "ok"
+        error_short = None
+        retried_flag = False
 
-        if query_collect_stats_extension is not None:
-            execute_query(query_collect_stats_extension + f" ON {table_fqn}")
+        try:
+            tdml.execute_sql(f"COLLECT STATS ON {table_fqn}")
+            ok += 1
+        except Exception as e:
+            # First attempt failed; try generated statement(s)
+            error_short = str(e).split("\n")[0]
+            logger_safe("warning", "collect_stats.initial_fail | table=%s | err=%s", table_fqn, error_short)
+
+            try:
+                execute_query(query_collect_stats + f" ON {table_fqn}")
+                retried_flag = True
+                retried += 1
+
+                if query_collect_stats_extension is not None:
+                    execute_query(query_collect_stats_extension + f" ON {table_fqn}")
+            except Exception as e2:
+                status = "failed"
+                error_short = str(e2).split("\n")[0]
+                failed += 1
+                logger_safe("error", "collect_stats.retry_fail | table=%s | err=%s", table_fqn, error_short)
+
+        dt = time.perf_counter() - t0
+        results.append(
+            {
+                "table": table_fqn,
+                "status": status,
+                "retried": retried_flag,
+                "elapsed_s": dt,
+                "error": error_short,
+            }
+        )
 
-    elapsed_time = time.time() - start_time
-    formatted_elapsed_time = seconds_to_dhms(elapsed_time)
+    # --- Final summary -------------------------------------------------------
+    elapsed = time.perf_counter() - started
+    formatted = seconds_to_dhms(elapsed)
+
+    # Structured, parseable one-liner
     logger_safe(
         "info",
-        "Storage of the prepared features - collect stats only: %s (%.3fs)",
-        formatted_elapsed_time, elapsed_time
+        "collect_stats.summary | tables=%d | ok=%d | retried=%d | failed=%d | duration=%s (%.3fs)",
+        len(target_tables),
+        ok,
+        retried,
+        failed,
+        formatted,
+        elapsed,
     )
 
+    return {
+        "total_tables": int(len(target_tables)),
+        "ok": int(ok),
+        "retried": int(retried),
+        "failed": int(failed),
+        "duration_seconds": float(elapsed),
+        "duration_hms": formatted,
+        "details": results,
+    }
+
+
 
 
 def _store_feature_update_insert(entity_id, volatile_table_name, entity_null_substitute={},primary_index=None,

tdfs4ds/feature_store/feature_store_management.py

@@ -9,6 +9,7 @@ import pandas as pd
 import tqdm
 import inspect
 import re
+from tdfs4ds import logger_safe, logger
 
 @execute_query_wrapper
 def feature_store_catalog_view_creation():
@@ -350,46 +351,47 @@ def register_features(entity_id, feature_names_types, primary_index = None, part
 
 def _register_features_merge(entity_id, feature_names_types, primary_index=None, partitioning=''):
     """
-    Registers or updates feature definitions in a Teradata database's feature catalog, associating entity identifiers
-    with feature names, types, and other metadata. This function prepares and executes SQL operations to insert new
-    feature definitions or update existing ones, considering partitioning strategies and primary index configurations.
-
-    Parameters:
-    - entity_id (dict): Specifies the entity's identifiers with keys representing attribute names. This dictionary
-                        is crucial for defining the scope and granularity of feature data.
-    - feature_names_types (dict): Maps feature names to their properties, including data types and unique identifiers.
-                                  Each value is a dictionary with keys 'type' and 'id' indicating the feature's data
-                                  type and a unique identifier, respectively.
-    - primary_index (list, optional): Identifies the primary index column(s) for the feature data. This influences
-                                      the organization and performance of database operations. If not specified,
-                                      defaults are used based on the entity_id structure.
-    - partitioning (str, optional): Describes the partitioning strategy through a string listing column names used
-                                    for partitioning. This can impact data storage and retrieval performance.
-
-    Returns:
-    pd.DataFrame: Contains details of the registered features, including names, types, IDs, and references to the
-                  respective feature store table and view names, alongside metadata about the entity and database schema.
-
-    Note:
-    - The function dynamically constructs SQL queries for inserting new features or updating existing ones in the
-      feature catalog, adapting to the provided partitioning and primary index settings.
-    - Assumes the existence of a Teradata feature catalog table in the specified schema and that the database connection
-      is correctly configured.
-    - Utilizes the tdfs4ds module for database schema configurations and valid-time temporal table considerations.
-
-    Example Usage:
-    >>> entity_id = {'customer_id': 'INTEGER'}
-    >>> feature_names_types = {'age': {'type': 'BIGINT', 'id': 1}, 'gender': {'type': 'VARCHAR_LATIN', 'id': 2}}
-    >>> registered_features = register_features(entity_id, feature_names_types)
-    >>> print(registered_features)
-
-    This example demonstrates registering features for an entity with attributes customer_id, age, and gender,
-    where age and gender features have specified types and unique IDs.
+    Register or update feature definitions in the feature catalog, with temporal support.
+
+    This function builds (or refreshes) entries in the Teradata feature catalog from a
+    mapping of feature names to their metadata, computes the target feature store table
+    and view names, stages the metadata to a temporary table, and executes a MERGE into
+    the catalog (with optional VALIDTIME support based on `tdfs4ds.FEATURE_STORE_TIME`).
+
+    Parameters
+    ----------
+    entity_id : dict[str, Any]
+        Mapping of entity-key column names to types. Only the keys (column names) are
+        required here; values are not used by this function.
+    feature_names_types : dict[str, dict]
+        Dict of feature name -> {"type": <SQL_TYPE>, "id": <int>} describing each
+        feature’s storage type and identifier in the catalog.
+    primary_index : list[str] | None, optional
+        Primary index column(s) to use when deriving the feature store table/view names.
+        If None, defaults are inferred by `get_feature_store_table_name`.
+    partitioning : str, optional
+        Partitioning expression or comma-separated column list used by
+        `get_feature_store_table_name`.
+
+    Returns
+    -------
+    pd.DataFrame
+        A dataframe of the features that were (up)registered, including:
+        FEATURE_NAME, FEATURE_TYPE, FEATURE_ID, FEATURE_TABLE, FEATURE_VIEW,
+        ENTITY_NAME, FEATURE_DATABASE, DATA_DOMAIN.
+
+    Notes
+    -----
+    - When `tdfs4ds.FEATURE_STORE_TIME is None`, uses CURRENT VALIDTIME (non-explicit start/end).
+      Otherwise uses `VALIDTIME PERIOD ('<FEATURE_STORE_TIME>', '<END_PERIOD>')` and adds
+      the valid-time start/end when inserting.
+    - Respects `tdfs4ds.DISPLAY_LOGS` via `logger_safe`.
     """
 
-    if tdfs4ds.FEATURE_STORE_TIME == None:
+    # --- VALIDTIME setup -----------------------------------------------------
+    if tdfs4ds.FEATURE_STORE_TIME is None:
         validtime_statement = 'CURRENT VALIDTIME'
-        validtime_start = 'CAST(CURRENT_TIME AS TIMESTAMP(0) WITH TIME ZONE)'
+        validtime_start = "CAST(CURRENT_TIME AS TIMESTAMP(0) WITH TIME ZONE)"
     else:
         validtime_statement = f"VALIDTIME PERIOD '({tdfs4ds.FEATURE_STORE_TIME},{tdfs4ds.END_PERIOD})'"
         validtime_start = f"CAST('{tdfs4ds.FEATURE_STORE_TIME}' AS TIMESTAMP(0) WITH TIME ZONE)"
@@ -399,154 +401,174 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
     else:
         end_period_ = tdfs4ds.END_PERIOD
 
-    if len(list(feature_names_types.keys())) == 0:
-        if tdfs4ds.DISPLAY_LOGS: print('no new feature to register')
+    # --- Input checks & early exit ------------------------------------------
+    if not feature_names_types:
+        logger_safe("info", "register_features: no new features to register")
         return
 
-    # Create a comma-separated string of entity IDs
-    entity_id_list = list(entity_id.keys())
-    entity_id_list.sort()
-    ENTITY_ID__ = ','.join([k for k in entity_id_list])
-
-    # Create a DataFrame from the feature_names_types dictionary
-    if len(feature_names_types.keys()) > 1:
-        df = pd.DataFrame(feature_names_types).transpose().reset_index()
-        df.columns = ['FEATURE_NAME', 'FEATURE_TYPE', 'FEATURE_ID']
-    else:
-        df = pd.DataFrame(columns=['FEATURE_NAME', 'FEATURE_TYPE', 'FEATURE_ID'])
-        k = list(feature_names_types.keys())[0]
-        df['FEATURE_NAME'] = [k]
-        df['FEATURE_TYPE'] = [feature_names_types[k]['type']]
-        df['FEATURE_ID'] = [feature_names_types[k]['id']]
-
-
+    # --- Entity columns (ordered, stable) -----------------------------------
+    entity_cols = sorted(list(entity_id.keys()))
+    ENTITY_ID__ = ",".join(entity_cols)
 
-    if tdfs4ds.DEBUG_MODE:
-        print('register_features', 'primary_index', primary_index)
-        print('register_features', 'partitioning', partitioning)
-        print('df', df)
-
-    # Generate the feature table and view names based on the entity ID and feature type
-    df['FEATURE_TABLE'] = df.apply(lambda row: get_feature_store_table_name(entity_id, row.iloc[1],
-                                                                            primary_index=primary_index,
-                                                                            partitioning=partitioning)[0],
-                                   axis=1)
-    df['FEATURE_VIEW'] = df.apply(lambda row: get_feature_store_table_name(entity_id, row.iloc[1],
-                                                                           primary_index=primary_index,
-                                                                           partitioning=partitioning)[1],
-                                  axis=1)
-
-    # Add additional columns to the DataFrame
-    df['ENTITY_NAME'] = ENTITY_ID__
-    df['FEATURE_DATABASE'] = tdfs4ds.SCHEMA
-    df['DATA_DOMAIN'] = tdfs4ds.DATA_DOMAIN
-
-    # Copy the DataFrame to a temporary table in Teradata
-    tdml.copy_to_sql(df, table_name='temp', schema_name=tdfs4ds.SCHEMA, if_exists='replace',
-                     primary_index='FEATURE_ID',
-                     types={'FEATURE_ID': tdml.BIGINT})
-
-
-
-    if tdfs4ds.DEBUG_MODE:
-        print("-----------_register_features_merge - df")
-        print(df)
-
-    if tdfs4ds.FEATURE_STORE_TIME == None:
+    # --- Build dataframe safely (no transpose tricks) ------------------------
+    rows = []
+    for fname, meta in feature_names_types.items():
+        try:
+            rows.append({
+                "FEATURE_NAME": fname,
+                "FEATURE_TYPE": meta["type"],
+                "FEATURE_ID":   meta["id"],
+            })
+        except KeyError as e:
+            logger_safe("error", "register_features: missing key %s in feature '%s' meta=%s", str(e), fname, meta)
+            raise
+
+    df = pd.DataFrame(rows, columns=["FEATURE_NAME", "FEATURE_TYPE", "FEATURE_ID"])
+
+    logger_safe(
+        "debug",
+        "register_features: features_count=%d | entity_cols=%s | primary_index=%s | partitioning=%s",
+        len(df),
+        entity_cols,
+        primary_index,
+        partitioning,
+    )
+
+    # --- Compute feature table & view names ---------------------------------
+    # Use apply to preserve original order; get_feature_store_table_name returns (table, view)
+    df["FEATURE_TABLE"] = df.apply(
+        lambda row: get_feature_store_table_name(
+            entity_id,
+            row["FEATURE_TYPE"],
+            primary_index=primary_index,
+            partitioning=partitioning
+        )[0],
+        axis=1
+    )
+    df["FEATURE_VIEW"] = df.apply(
+        lambda row: get_feature_store_table_name(
+            entity_id,
+            row["FEATURE_TYPE"],
+            primary_index=primary_index,
+            partitioning=partitioning
+        )[1],
+        axis=1
+    )
+
+    # --- Add catalog columns -------------------------------------------------
+    df["ENTITY_NAME"] = ENTITY_ID__
+    df["FEATURE_DATABASE"] = tdfs4ds.SCHEMA
+    df["DATA_DOMAIN"] = tdfs4ds.DATA_DOMAIN
+
+    # --- Stage to temp table -------------------------------------------------
+    tdml.copy_to_sql(
+        df,
+        table_name="temp",
+        schema_name=tdfs4ds.SCHEMA,
+        if_exists="replace",
+        primary_index="FEATURE_ID",
+        types={"FEATURE_ID": tdml.BIGINT},
+    )
+    logger_safe("debug", "register_features: staged %d rows to %s.temp", len(df), tdfs4ds.SCHEMA)
+
+    # --- Build MERGE statement ----------------------------------------------
+    if tdfs4ds.FEATURE_STORE_TIME is None:
+        # no explicit start/end in INSERT branch
         query_merge = f"""
          {validtime_statement}
-         MERGE INTO  {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME} EXISTING_FEATURES
+         MERGE INTO {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME} EXISTING_FEATURES
          USING (
              SELECT
-                CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
-            ,   A.FEATURE_NAME
-            ,   A.FEATURE_TYPE
-            ,   A.FEATURE_TABLE
-            ,   A.FEATURE_DATABASE
-            ,   A.FEATURE_VIEW
-            ,   A.ENTITY_NAME
-            ,   A.DATA_DOMAIN
-            FROM {tdfs4ds.SCHEMA}.temp A
-            LEFT JOIN {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME_VIEW} B
-            ON A.FEATURE_NAME = B.FEATURE_NAME
-            AND A.ENTITY_NAME = B.ENTITY_NAME -- modified
-            AND A.DATA_DOMAIN = B.DATA_DOMAIN
-             ) UPDATED_FEATURES
-         ON UPDATED_FEATURES.FEATURE_ID = EXISTING_FEATURES.FEATURE_ID
-         AND UPDATED_FEATURES.FEATURE_NAME = EXISTING_FEATURES.FEATURE_NAME
-         AND UPDATED_FEATURES.DATA_DOMAIN = EXISTING_FEATURES.DATA_DOMAIN
-         WHEN MATCHED THEN
-             UPDATE
-             SET
-                FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE,
-                FEATURE_TYPE     = UPDATED_FEATURES.FEATURE_TYPE,
-                FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE,
-                FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
-                --,ENTITY_NAME      = UPDATED_FEATURES.ENTITY_NAME -- modified
-         WHEN NOT MATCHED THEN
-             INSERT
+                   CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
+                 , A.FEATURE_NAME
+                 , A.FEATURE_TYPE
+                 , A.FEATURE_TABLE
+                 , A.FEATURE_DATABASE
+                 , A.FEATURE_VIEW
+                 , A.ENTITY_NAME
+                 , A.DATA_DOMAIN
+             FROM {tdfs4ds.SCHEMA}.temp A
+             LEFT JOIN {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME_VIEW} B
+               ON A.FEATURE_NAME = B.FEATURE_NAME
+              AND A.ENTITY_NAME = B.ENTITY_NAME
+              AND A.DATA_DOMAIN = B.DATA_DOMAIN
+         ) UPDATED_FEATURES
+           ON UPDATED_FEATURES.FEATURE_ID   = EXISTING_FEATURES.FEATURE_ID
+          AND UPDATED_FEATURES.FEATURE_NAME = EXISTING_FEATURES.FEATURE_NAME
+          AND UPDATED_FEATURES.DATA_DOMAIN  = EXISTING_FEATURES.DATA_DOMAIN
+         WHEN MATCHED THEN UPDATE SET
+               FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE
+             , FEATURE_TYPE     = UPDATED_FEATURES.FEATURE_TYPE
+             , FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE
+             , FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
+         WHEN NOT MATCHED THEN INSERT
              (  UPDATED_FEATURES.FEATURE_ID
-            ,   UPDATED_FEATURES.FEATURE_NAME
-            ,   UPDATED_FEATURES.FEATURE_TYPE
-            ,   UPDATED_FEATURES.FEATURE_TABLE
-            ,   UPDATED_FEATURES.FEATURE_DATABASE
-            ,   UPDATED_FEATURES.FEATURE_VIEW
-            ,   UPDATED_FEATURES.ENTITY_NAME
-            ,   UPDATED_FEATURES.DATA_DOMAIN
-            )
-         """
+              , UPDATED_FEATURES.FEATURE_NAME
+              , UPDATED_FEATURES.FEATURE_TYPE
+              , UPDATED_FEATURES.FEATURE_TABLE
+              , UPDATED_FEATURES.FEATURE_DATABASE
+              , UPDATED_FEATURES.FEATURE_VIEW
+              , UPDATED_FEATURES.ENTITY_NAME
+              , UPDATED_FEATURES.DATA_DOMAIN
+             );
+        """
     else:
+        # insert with explicit valid-time start/end
         query_merge = f"""
          {validtime_statement}
-         MERGE INTO  {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME} EXISTING_FEATURES
+         MERGE INTO {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME} EXISTING_FEATURES
          USING (
              SELECT
-                CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
-            ,   A.FEATURE_NAME
-            ,   A.FEATURE_TYPE
-            ,   A.FEATURE_TABLE
-            ,   A.FEATURE_DATABASE
-            ,   A.FEATURE_VIEW
-            ,   A.ENTITY_NAME
-            ,   A.DATA_DOMAIN
-            FROM {tdfs4ds.SCHEMA}.temp A
-            LEFT JOIN {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME_VIEW} B
-            ON A.FEATURE_NAME = B.FEATURE_NAME
-            AND A.ENTITY_NAME = B.ENTITY_NAME -- modified
-            AND A.DATA_DOMAIN = B.DATA_DOMAIN
-             ) UPDATED_FEATURES
-         ON  UPDATED_FEATURES.FEATURE_ID = EXISTING_FEATURES.FEATURE_ID
-         AND UPDATED_FEATURES.FEATURE_NAME = EXISTING_FEATURES.FEATURE_NAME
-         AND UPDATED_FEATURES.DATA_DOMAIN = EXISTING_FEATURES.DATA_DOMAIN
-         WHEN MATCHED THEN
-             UPDATE
-             SET
-                FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE,
-                FEATURE_TYPE    = UPDATED_FEATURES.FEATURE_TYPE,
-                FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE,
-                FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
-                --,ENTITY_NAME      = UPDATED_FEATURES.ENTITY_NAME -- modified
-         WHEN NOT MATCHED THEN
-             INSERT
+                   CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
+                 , A.FEATURE_NAME
+                 , A.FEATURE_TYPE
+                 , A.FEATURE_TABLE
+                 , A.FEATURE_DATABASE
+                 , A.FEATURE_VIEW
+                 , A.ENTITY_NAME
+                 , A.DATA_DOMAIN
+             FROM {tdfs4ds.SCHEMA}.temp A
+             LEFT JOIN {tdfs4ds.SCHEMA}.{tdfs4ds.FEATURE_CATALOG_NAME_VIEW} B
+               ON A.FEATURE_NAME = B.FEATURE_NAME
+              AND A.ENTITY_NAME = B.ENTITY_NAME
+              AND A.DATA_DOMAIN = B.DATA_DOMAIN
+         ) UPDATED_FEATURES
+           ON UPDATED_FEATURES.FEATURE_ID   = EXISTING_FEATURES.FEATURE_ID
+          AND UPDATED_FEATURES.FEATURE_NAME = EXISTING_FEATURES.FEATURE_NAME
+          AND UPDATED_FEATURES.DATA_DOMAIN  = EXISTING_FEATURES.DATA_DOMAIN
+         WHEN MATCHED THEN UPDATE SET
+               FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE
+             , FEATURE_TYPE     = UPDATED_FEATURES.FEATURE_TYPE
+             , FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE
+             , FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
+         WHEN NOT MATCHED THEN INSERT
              (  UPDATED_FEATURES.FEATURE_ID
-            ,   UPDATED_FEATURES.FEATURE_NAME
-            ,   UPDATED_FEATURES.FEATURE_TYPE
-            ,   UPDATED_FEATURES.FEATURE_TABLE
-            ,   UPDATED_FEATURES.FEATURE_DATABASE
-            ,   UPDATED_FEATURES.FEATURE_VIEW
-            ,   UPDATED_FEATURES.ENTITY_NAME
-            ,   UPDATED_FEATURES.DATA_DOMAIN,
-             {validtime_start},
-             '{end_period_}')
-         """
+              , UPDATED_FEATURES.FEATURE_NAME
+              , UPDATED_FEATURES.FEATURE_TYPE
+              , UPDATED_FEATURES.FEATURE_TABLE
+              , UPDATED_FEATURES.FEATURE_DATABASE
+              , UPDATED_FEATURES.FEATURE_VIEW
+              , UPDATED_FEATURES.ENTITY_NAME
+              , UPDATED_FEATURES.DATA_DOMAIN
+              , {validtime_start}
+              , '{end_period_}'
+             );
+        """
 
-    if tdfs4ds.DEBUG_MODE:
-        print("-----------_register_features_merge - query_merge")
-        print(query_merge)
-    # Execute the update and insert queries
+    logger_safe("debug", "register_features: merge_sql_preview=%s", " ".join(query_merge.split())[:400] + " ...")
+
+    # --- Execute MERGE -------------------------------------------------------
     execute_query(query_merge)
+    logger_safe(
+        "info",
+        "register_features: merged %d features into %s.%s",
+        len(df),
+        tdfs4ds.SCHEMA,
+        tdfs4ds.FEATURE_CATALOG_NAME,
+    )
 
     return df
+
 def _register_features_update_insert(entity_id, feature_names_types, primary_index = None, partitioning = ''):
     """
     Registers or updates feature definitions in a Teradata database's feature catalog, associating entity identifiers

tdfs4ds/utils/filter_management.py

@@ -308,7 +308,7 @@ class FilterManager:
         FROM {self.schema_name}.{self.table_name}
         WHERE {self.filter_id_name} = {filter_id}
         """
-        logger_safe("info", "Updating active filter | %s", ','.join([c + ':' + v for c,v in zip(select_cols_str, tdml.execute_sql(f"SEL * FROM {self.schema_name}.{self.view_name}").fetchall()[0])]))
+        logger_safe("info", "Updating active filter | %s", ','.join([f"{c}:{v}" for c,v in zip(select_cols_str, tdml.execute_sql(f"SEL * FROM {self.schema_name}.{self.view_name}").fetchall()[0])]))
 
         if getattr(tdfs4ds, "DEBUG_MODE", False):
             logger_safe("debug", "Replacing view with new filter:\n%s", query)

{tdfs4ds-0.2.4.33.dist-info → tdfs4ds-0.2.4.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tdfs4ds
-Version: 0.2.4.33
+Version: 0.2.4.35
 Summary: A python package to simplify the usage of feature store using Teradata Vantage ...
 Author: Denis Molin
 Requires-Python: >=3.6

{tdfs4ds-0.2.4.33.dist-info → tdfs4ds-0.2.4.35.dist-info}/RECORD

@@ -2,7 +2,7 @@ tdfs/__init__.py,sha256=7AcO7uB1opRCt7t2JOHworKimfAaDeO3boRW7u9Geo8,23
 tdfs/datasets.py,sha256=-b2MPEKGki2V1M8iUcoDR9uc2krIK7u1CK-EhChvihs,985
 tdfs/feature_store.py,sha256=Honu7eOAXxP4Ivz0mRlhuNkfTDzgZl5HB1WlQUwzcZ0,31354
 tdfs/data/curves.csv,sha256=q0Tm-0yu7VMK4lHvHpgi1LMeRq0lO5gJy2Q17brKbEM,112488
-tdfs4ds/__init__.py,sha256=A-MJUMl06mJBwd94ByY8DZoatCL4A8r7mqe5u6EzCMw,55010
+tdfs4ds/__init__.py,sha256=n3eGxALMqT_UmwvP_VZ8K0bdKSFAtPhe9bi7Kg0TQtA,55698
 tdfs4ds/datasets.py,sha256=LE4Gn0muwdyrIrCrbkE92cnafUML63z1lj5bFIIVzmc,3524
 tdfs4ds/feature_engineering.py,sha256=oVnZ2V_XNGE12LKC_fNfkrWSQZLgtYRmaf8Dispi6S4,7081
 tdfs4ds/feature_store.py,sha256=y-oItPZw6nBkBcGAceaATZbkLPTsvpk0OnpzTxYofDs,68576
@@ -17,22 +17,22 @@ tdfs4ds/dataset/dataset.py,sha256=J_fgfsVdR9zSOXrUOqyotqsUD-GlQMGyuld6ueov45w,76
 tdfs4ds/dataset/dataset_catalog.py,sha256=qxS2thDW2MvsRouSFaX1M0sX2J7IzBAYD8Yf22Tsd5k,16638
 tdfs4ds/feature_store/__init__.py,sha256=a7NPCkpTx40UR5LRErwnskpABG2Vuib7F5wUjaUGCnI,209
 tdfs4ds/feature_store/entity_management.py,sha256=9ltytv3yCTG84NZXBpb1Tlkf9pOxvrNb0MVidU4pwvE,10157
-tdfs4ds/feature_store/feature_data_processing.py,sha256=JEtz1UpZY7oMaonuhzC8eTbZAL8SzrnLOpo0WTEDFUM,41697
+tdfs4ds/feature_store/feature_data_processing.py,sha256=gjwypiTfwTyGyrP20v35Vu2uGIrCY80OBBeMVBsdjuk,45020
 tdfs4ds/feature_store/feature_query_retrieval.py,sha256=51c6ZNlLFiBIxNPinS8ot8bjWEIb1QV2eVg69yzVF80,35381
-tdfs4ds/feature_store/feature_store_management.py,sha256=pWM9sjppBgRIg3l1ksoDJsM1fnaZlWtnuE3JuOP_2mY,54736
+tdfs4ds/feature_store/feature_store_management.py,sha256=yXLbINYLA-lzd0t_6TzEe9a8Anlum4x8TRoxZU3FIr8,54276
 tdfs4ds/process_store/__init__.py,sha256=npHR_xju5ecGmWfYHDyteLwiU3x-cL4HD3sFK_th7xY,229
 tdfs4ds/process_store/process_followup.py,sha256=PvLcU7meg3ljBlPfuez3qwTVqpHHhVJxYxGqjgiHE8E,7265
 tdfs4ds/process_store/process_query_administration.py,sha256=AOufkJ6DFUpBiGm-6Q6Dq0Aovw31UGTscZ3Ya0ewS-0,7851
 tdfs4ds/process_store/process_registration_management.py,sha256=2fFjt4Pmh3An1BUFvRX3xABSlQrlWiEiPQStH3A9Xpk,36130
 tdfs4ds/process_store/process_store_catalog_management.py,sha256=eVUU9uanyXCUkzi2vcHbJPL9qFiXVasnCxPGr-r9EY8,16090
 tdfs4ds/utils/__init__.py,sha256=-yTMfDLZbQnIRQ64s_bczzT21tDW2A8FZeq9PX5SgFU,168
-tdfs4ds/utils/filter_management.py,sha256=JdCHkkw_L6vpmjPMMp3AY2ZwITGrwAvljHxZttgeWTg,24761
+tdfs4ds/utils/filter_management.py,sha256=5_8fYYtl8RQgbIi6L_1geNM0wJMm3t1n4QvNA5DnaQg,24760
 tdfs4ds/utils/info.py,sha256=sShnUxXMlvCtQ6xtShDhqdpTr6sMG0dZQhNBFgUENDY,12058
 tdfs4ds/utils/lineage.py,sha256=gy5M42qy5fvdWmlohAY3WPYoqAyp5VakeEmeT1YjrJQ,37839
 tdfs4ds/utils/query_management.py,sha256=nAcE8QY1GWAKgOtb-ubSfDVcnYbU7Ge8CruVRLoPtmY,6356
 tdfs4ds/utils/time_management.py,sha256=asIWvK5K81NNwAGqC-9Tv4Timscxyv0vyuPFs01whu0,31461
 tdfs4ds/utils/visualization.py,sha256=5S528KoKzzkrAdCxfy7ecyqKvAXBoibNvHwz_u5ISMs,23167
-tdfs4ds-0.2.4.33.dist-info/METADATA,sha256=0HSUyalUNwp7ZD6Z811pBaNbMb0GEAYsNSzFcAaEWnk,14326
-tdfs4ds-0.2.4.33.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-tdfs4ds-0.2.4.33.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
-tdfs4ds-0.2.4.33.dist-info/RECORD,,
+tdfs4ds-0.2.4.35.dist-info/METADATA,sha256=1gmDbv0lpgEcRd0ucWdSSyfGUTyb0-nCxVoMy9Y8JKk,14326
+tdfs4ds-0.2.4.35.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+tdfs4ds-0.2.4.35.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
+tdfs4ds-0.2.4.35.dist-info/RECORD,,