tdfs4ds 0.2.4.3__py3-none-any.whl → 0.2.4.5__py3-none-any.whl

tdfs4ds/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.2.4.3'
+__version__ = '0.2.4.5'
 import logging
 # Setup the logger
 logging.basicConfig(
@@ -310,7 +310,7 @@ def get_dataset_entity(dataset_id = None):
 def get_dataset_features(dataset_id = None):
     return DatasetCatalog(schema_name=tdfs4ds.SCHEMA, name=tdfs4ds.DATASET_CATALOG_NAME).get_dataset_features(dataset_id)
 
-def run(process_id, return_dataset = False, force_compute = False):
+def run(process_id, return_dataset = False, force_compute = False, force_varchar_length = None):
     """
     Executes a specific process from the feature store identified by the process ID.
     The function handles different process types and performs appropriate actions.
@@ -321,6 +321,10 @@ def run(process_id, return_dataset = False, force_compute = False):
                                        Default is False.
     - force_compute (bool, optional): A flag indicating whether to force computation even if data already exists.
                                       Default is False.
+    - force_varchar_length (int, optional): in order to avoid the multiplication of feature tables when dealing with the
+                                        VARCHAR type, it cast the VARCHAR features into VARCHAR(k x force_varchar_length)
+                                        where k is the smallest integer so that the original lengths is smaller or equal
+                                        to k x force_varchar_length. Default is None.
 
     Returns:
     DataFrame or None: If return_dataset is True, returns the dataset created during the process. Otherwise, returns None.
@@ -423,7 +427,8 @@ def run(process_id, return_dataset = False, force_compute = False):
             filtermanager = filtermanager,
             entity_null_substitute = entity_null_substitute,
             process_id = process_id,
-            force_compute= force_compute
+            force_compute= force_compute,
+            force_varchar_length = force_varchar_length
         )
 
     # Handling 'tdstone2 view' process type
@@ -437,7 +442,7 @@ def run(process_id, return_dataset = False, force_compute = False):
     else:
         return
 
-def upload_features(df, entity_id, feature_names, metadata={}, primary_index = None, partitioning = '', filtermanager = None, entity_null_substitute = {}, force_compute = True):
+def upload_features(df, entity_id, feature_names, metadata={}, primary_index = None, partitioning = '', filtermanager = None, entity_null_substitute = {}, force_compute = True, force_varchar_length = 1024):
     """
     Uploads feature data from a DataFrame to the feature store for a specified entity. This involves registering the
     process in the feature store, executing the necessary SQL to insert the data, and returning the resulting dataset
@@ -463,7 +468,10 @@ def upload_features(df, entity_id, feature_names, metadata={}, primary_index = N
                                                Default is an empty dictionary.
     - force_compute (bool, optional): A flag indicating whether to force computation even if data already exists.
                                       Default is True.
-
+    - force_varchar_length (int, optional): in order to avoid the multiplication of feature tables when dealing with the
+                                        VARCHAR type, it cast the VARCHAR features into VARCHAR(k x force_varchar_length)
+                                        where k is the smallest integer so that the original lengths is smaller or equal
+                                        to k x force_varchar_length. Default is 1024.
     Returns:
     DataFrame: A DataFrame representing the dataset resulting from the upload process, typically used for validation
                or further processing.
@@ -575,7 +583,7 @@ def upload_features(df, entity_id, feature_names, metadata={}, primary_index = N
 
         try:
 
-            dataset = run(process_id=process_id, return_dataset=True, force_compute = force_compute)
+            dataset = run(process_id=process_id, return_dataset=True, force_compute = force_compute, force_varchar_length = force_varchar_length)
 
         except Exception as e:
             tdfs4ds.process_store.process_followup.followup_close(
@@ -591,7 +599,7 @@ def upload_features(df, entity_id, feature_names, metadata={}, primary_index = N
     else:
 
         try:
-            run(process_id=process_id, return_dataset=False)
+            run(process_id=process_id, return_dataset=False, force_compute = force_compute, force_varchar_length = force_varchar_length)
         except Exception as e:
             tdfs4ds.process_store.process_followup.followup_close(
                 run_id       = tdfs4ds.RUN_ID,
@@ -605,7 +613,7 @@ def upload_features(df, entity_id, feature_names, metadata={}, primary_index = N
     tdfs4ds.PROCESS_TYPE = PROCESS_TYPE
 
 def _upload_features(df, entity_id, feature_names,
-                   feature_versions=FEATURE_VERSION_DEFAULT, primary_index = None, partitioning = '', filtermanager=None, entity_null_substitute={}, process_id = None, force_compute = False):
+                   feature_versions=FEATURE_VERSION_DEFAULT, primary_index = None, partitioning = '', filtermanager=None, entity_null_substitute={}, process_id = None, force_compute = False,force_varchar_length = None):
     """
     Uploads features from a DataFrame to the feature store, handling entity registration, feature type determination,
     feature registration, preparation for ingestion, and storage in the designated feature tables.
@@ -628,6 +636,11 @@ def _upload_features(df, entity_id, feature_names,
     - process_id (str, optional): An identifier for the process, used for tracking and follow-up. Default is None.
     - force_compute (bool, optional): A flag indicating whether to force computation even if data already exists.
                                       Default is False.
+    - force_varchar_length (int, optional): in order to avoid the multiplication of feature tables when dealing with the
+                                            VARCHAR type, it cast the VARCHAR features into VARCHAR(k x force_varchar_length)
+                                            where k is the smallest integer so that the original lengths is smaller or equal
+                                            to k x force_varchar_length. Default is None.
+
 
     Returns:
     DataFrame: A DataFrame representing the dataset view created in the feature store, detailing the features and their
@@ -655,7 +668,7 @@ def _upload_features(df, entity_id, feature_names,
     from tdfs4ds.feature_store.feature_store_management import register_features
     from tdfs4ds.feature_store.feature_data_processing  import prepare_feature_ingestion
     from tdfs4ds.feature_store.feature_data_processing  import store_feature, apply_collect_stats
-    from tdfs4ds.utils.info import get_column_types
+    from tdfs4ds.utils.info import get_column_types, update_varchar_length
 
     # Convert entity_id to a dictionary if it's not already one
     if type(entity_id) == list:
@@ -685,6 +698,10 @@ def _upload_features(df, entity_id, feature_names,
         entity_id=entity_id
     )
 
+    if force_varchar_length is not None:
+        print(feature_names_types)
+        feature_names_types = update_varchar_length(feature_names_types,new_varchar_length = force_varchar_length)
+
     def validate_feature_types(feature_names_types):
         """
         Validates feature data types and raises an error if any value contains 

tdfs4ds/dataset/dataset.py

@@ -0,0 +1,117 @@
+import teradataml as tdml
+from tdfs4ds.utils.info import get_feature_types_sql_format
+from tdfs4ds import logger
+
+class Dataset:
+
+    def __init__(self, view_name=None, schema_name=None, df=None):
+
+        if df is not None:
+            self.df          = df
+            df._DataFrame__execute_node_and_set_table_name(df._nodeid, df._metaexpr)
+            view_name = df._table_name
+            if '.' in view_name:            
+                self.view_name   = view_name.split('.')[1]
+                self.schema_name = view_name.split('.')[0]
+            else:
+                self.view_name   = view_name
+                self.schema_name = tdml.context.context._get_current_databasename()
+        elif view_name is not None and schema_name is not None:
+            self.view_name   = view_name
+            self.schema_name = schema_name
+            if view_name.lower() in  map(str.lower, tdml.db_list_tables(object_type='view', schema_name=self.schema_name).TableName.values):
+                self.df      = tdml.DataFrame(tdml.in_schema(schema_name, view_name))
+            else:
+                print(f"{self.view_name} not found in {self.schema_name} database")
+                self.df      = None
+        else:
+            raise ValueError("Either df or both view_name and schema_name must be provided.")
+        
+        self.valid_time   = self._get_validtime()
+        self.dataset_type = self._get_dataset_type()
+        self.entity, self.features     = self._retrieve_entities_and_features()
+        
+    
+    def get_dataframe(self):
+        return self.df
+    
+    def __repr__(self):
+        return f"Dataset(view_name={self.view_name}, schema_name={self.schema_name}, df={type(self.df)})"
+    
+    def __getattr__(self, item):
+        if self.df is not None:
+            return getattr(self.df, item)
+        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{item}'")
+    
+    def _retrieve_entities_and_features(self):
+
+        if self._get_dataset_type() == 'snapshot':
+
+            blocks = [x.split(')')[0] for x in self._get_ddl().split('(')]
+            feature_names    = [blocks[i].replace('\n','').split('AS ')[1].split('FROM')[0].strip() for i in range(1,len(blocks)) if i % 2 == 1]
+            feature_ids      = [int(blocks[i].replace('\n','').split('=')[1].split('AND')[0].strip()) for i in range(1,len(blocks)) if i % 2 == 0]
+            feature_versions = [blocks[i].replace('\n','').split('=')[2].replace("'",'').strip() for i in range(1,len(blocks)) if i % 2 == 0]
+            
+            feature_database = [blocks[i].replace('\n','').split('"')[1].strip() for i in range(1,len(blocks)) if i % 2 == 1]
+            feature_view     = [blocks[i].replace('\n','').split('"')[3].strip() for i in range(1,len(blocks)) if i % 2 == 1]
+
+            columns_types    = get_feature_types_sql_format(self.df)
+            feature_types    = [columns_types[f] for f in feature_names]
+
+            features = {}
+            for n,i,v,t,d,vv in zip(feature_names, feature_ids, feature_versions, feature_types, feature_database, feature_view):
+                features[n.upper()] = {'id' : i, 'version': v, 'type': t.upper(), 'database' : d.upper(), 'view' : vv.upper()}
+
+            
+            entity_names     = [x.strip().split('.')[1] for x in blocks[0].split('SELECT')[1].split('FROM')[0].replace('\n','').split(',') if x.strip().startswith('A1') if x.strip().split('.')[1] not in feature_names]
+            entity_types     = [columns_types[e] for e in entity_names]
+
+            entity = {}
+            for n,t in zip(entity_names, entity_types):
+                entity[n] = t
+
+            return entity, features
+        else:
+            logger.error(f"not implemented yet for dataset type: {self._get_dataset_type()}")
+            raise
+    
+    def _get_dataset_type(self):
+        return 'snapshot'
+    
+    def _get_validtime(self):
+        if self._get_dataset_type() == 'snapshot':
+            return self._get_ddl().split('\n')[4].strip()
+        else:
+            logger.error(f"not implemented yet for dataset type: {self._get_dataset_type()}")
+        return ''
+
+    def _get_feature_store_database(self):
+
+        databases = [self.features[k]['database'] for k in self.features.keys()]
+        databases = list(set(databases))
+        if len(databases) == 1:
+            self.feature_store_database = databases[0]
+        elif len(databases) > 1:
+            logger.warning(f"features are stored in multiple databases: {databases}")
+        else:
+            logger.error(f"unable to identify the feature store database")
+            raise
+        
+    
+    def _get_ddl(self):
+        return tdml.execute_sql(f"SHOW VIEW {self.schema_name}.{self.view_name}").fetchall()[0][0].replace('\r','\n')
+    
+    def show_query(self):
+        if self.df is not None:
+            print(self._get_ddl())
+
+    def info(self):
+        print("\nEntities:")
+        for key, value in self.entity.items():
+            print(f"  - {key}: {value}")
+
+        print("\nFeatures:")
+        for feature, details in self.features.items():
+            print(f"  - {feature}:")
+            for detail_key, detail_value in details.items():
+                print(f"      {detail_key}: {detail_value}")

tdfs4ds/dataset/dataset_catalog.py

@@ -0,0 +1,373 @@
+from tdfs4ds import logger
+import uuid
+from tdfs4ds.dataset.dataset import Dataset
+import teradataml as tdml
+import json
+
+class DatasetCatalog:
+
+    def __init__(self, schema_name = None, name = 'DATASET'):
+        if schema_name is None:
+            self.schema_name = tdml.context.context._get_current_databasename()
+        else:
+            self.schema_name = schema_name
+        self.name        = name
+
+        self.catalog_table_name = f"{self.schema_name}.FS_{self.name}_CATALOG"
+        self.catalog_view_name  = f"{self.schema_name}.FS_V_{self.name}_CATALOG"
+        self.entity_table_name = f"{self.schema_name}.FS_{self.name}_ENTITY"
+        self.entity_view_name  = f"{self.schema_name}.FS_V_{self.name}_ENTITY"
+        self.feature_table_name = f"{self.schema_name}.FS_{self.name}_FEATURES"
+        self.feature_view_name  = f"{self.schema_name}.FS_V_{self.name}_FEATURES"
+
+        self.creation_queries = self._creation_query()
+        if not self._exists():
+            self.create_catalog()
+
+        self.catalog = tdml.DataFrame(tdml.in_schema(self.catalog_view_name.split('.')[0],self.catalog_view_name.split('.')[1]))
+        self.entity = tdml.DataFrame(tdml.in_schema(self.entity_view_name.split('.')[0],self.entity_view_name.split('.')[1]))
+        self.features = tdml.DataFrame(tdml.in_schema(self.feature_view_name.split('.')[0],self.feature_view_name.split('.')[1]))
+
+    def __repr__(self):
+        return f"DatasetCatalog(catalog_view={self.catalog_view_name}, entity_view={self.entity_view_name}, feature_view={self.feature_view_name})"
+    
+    def __getattr__(self, item):
+        if self.catalog is not None:
+            return getattr(self.catalog, item)
+        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{item}'")
+
+    def _creation_query(self):
+
+        if self.schema_name is not None and self.name is not None:
+
+            query_dataset_catalog = f"""
+            CREATE MULTISET TABLE {self.catalog_table_name},
+            FALLBACK,
+            NO BEFORE JOURNAL,
+            NO AFTER JOURNAL,
+            CHECKSUM = DEFAULT,
+            DEFAULT MERGEBLOCKRATIO,
+            MAP = TD_MAP1
+            (
+                DATASET_ID VARCHAR(36) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                DATASET_NAME VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                DATASET_DATABASE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                DATASET_TYPE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                DATASET_VALIDTIME VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                METADATA JSON(32000) CHARACTER SET LATIN,
+                ValidStart TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                ValidEnd TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                PERIOD FOR ValidPeriod  (ValidStart, ValidEnd) AS VALIDTIME
+            )
+            PRIMARY INDEX (DATASET_ID);
+            """
+
+            query_dataset_catalog_view = f"""
+                        CREATE VIEW {self.catalog_view_name} AS
+                        LOCK ROW FOR ACCESS
+                        CURRENT VALIDTIME
+                        SELECT *
+                        FROM {self.catalog_table_name}
+            """
+            
+            query_dataset_entity = f"""
+            CREATE MULTISET TABLE {self.entity_table_name},
+            FALLBACK,
+            NO BEFORE JOURNAL,
+            NO AFTER JOURNAL,
+            CHECKSUM = DEFAULT,
+            DEFAULT MERGEBLOCKRATIO,
+            MAP = TD_MAP1
+            (
+                DATASET_ID VARCHAR(36) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                ENTITY VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                ENTITY_TYPE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                ValidStart TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                ValidEnd TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                PERIOD FOR ValidPeriod  (ValidStart, ValidEnd) AS VALIDTIME
+            )
+            PRIMARY INDEX (DATASET_ID);
+            """
+
+            query_dataset_entity_view = f"""
+                        CREATE VIEW {self.entity_view_name} AS
+                        LOCK ROW FOR ACCESS
+                        CURRENT VALIDTIME
+                        SELECT *
+                        FROM {self.entity_table_name}
+            """
+
+            query_dataset_features = f"""
+            CREATE MULTISET TABLE {self.feature_table_name},
+            FALLBACK,
+            NO BEFORE JOURNAL,
+            NO AFTER JOURNAL,
+            CHECKSUM = DEFAULT,
+            DEFAULT MERGEBLOCKRATIO,
+            MAP = TD_MAP1
+            (
+                DATASET_ID VARCHAR(36) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_ID VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_NAME VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_TYPE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_DATABASE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_VIEW VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                ValidStart TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                ValidEnd TIMESTAMP(0) WITH TIME ZONE NOT NULL,
+                PERIOD FOR ValidPeriod  (ValidStart, ValidEnd) AS VALIDTIME
+            )
+            PRIMARY INDEX (DATASET_ID);
+            """
+
+            query_dataset_feature_view = f"""
+                        CREATE VIEW {self.feature_view_name} AS
+                        LOCK ROW FOR ACCESS
+                        CURRENT VALIDTIME
+                        SELECT *
+                        FROM {self.feature_table_name}
+            """
+
+            queries = [
+                {'name' : f'{self.catalog_table_name}', 'type': 'table', 'query': query_dataset_catalog},
+                {'name' : f'{self.entity_table_name}', 'type': 'table', 'query': query_dataset_entity},
+                {'name' : f'{self.feature_table_name}', 'type': 'table', 'query': query_dataset_features},
+                {'name' : f'{self.catalog_view_name}', 'type': 'view', 'query': query_dataset_catalog_view},
+                {'name' : f'{self.entity_view_name}', 'type': 'view', 'query': query_dataset_entity_view},
+                {'name' : f'{self.feature_view_name}', 'type': 'view', 'query': query_dataset_feature_view}
+            ]
+
+            return queries
+        else:
+            logger.error('the schema name is not defined')
+            raise ValueError("the schema name is not defined")
+        
+    def _get_list_objects(self):
+        return [self.catalog_table_name, self.entity_table_name, self.feature_table_name, self.catalog_view_name, self.entity_view_name, self.feature_view_name]
+        
+    def create_catalog(self, schema_name = None):
+
+        if schema_name is not None:
+            self.schema_name = schema_name
+            self.catalog_table_name = f"{self.schema_name}.{self.name}_CATALOG"
+            self.catalog_view_name  = f"{self.schema_name}.V_{self.name}_CATALOG"
+            self.entity_table_name = f"{self.schema_name}.{self.name}_ENTITY"
+            self.entity_view_name  = f"{self.schema_name}.V_{self.name}_ENTITY"
+            self.feature_table_name = f"{self.schema_name}.{self.name}_FEATURES"
+            self.feature_view_name  = f"{self.schema_name}.V_{self.name}_FEATURES"
+
+        
+        self.creation_queries = self._creation_query()
+        already_exists = [v for v in self._get_list_objects() if v.lower().split('.')[1] in map(str.lower, tdml.db_list_tables(schema_name = self.schema_name).TableName.values)]
+
+        if len(already_exists) > 0:
+            msg = f"The dataset catalog cannot be created because these tables already exist : {already_exists}"
+            logger.error(msg)
+            raise ValueError(msg)
+        else:
+            for query in self.creation_queries:
+                logger.info(f"creation of {query['name']}")
+                tdml.execute_sql(query['query'])
+
+    def drop_catalog(self):
+
+        for query in self.creation_queries:
+            logger.info(f"drop {query['name']}")
+            if query['type'] == 'table':
+                tdml.execute_sql(f"DROP TABLE {query['name']}")
+            elif query['type'] == 'view':
+                tdml.execute_sql(f"DROP VIEW {query['name']}")
+
+    def _exists(self):
+        not_exists = [v for v in self._get_list_objects() if v.lower().split('.')[1] not in map(str.lower, tdml.db_list_tables(schema_name = self.schema_name).TableName.values)]
+        return not_exists == []
+    
+    def add_dataset(self, dataset, metadata = {}):
+
+        # if dataset exists:
+        res = self.catalog[(self.catalog.DATASET_NAME == dataset.view_name.upper())&(self.catalog.DATASET_DATABASE == dataset.schema_name.upper())]
+        if res.shape[0] == 1:
+            logger.info('this dataset is already present and will be updated')
+            print(res[['DATASET_ID', 'DATASET_NAME', 'DATASET_DATABASE']])
+            dataset_id = res[['DATASET_ID']].to_pandas().DATASET_ID.values[0]
+
+            entity = tdml.DataFrame(tdml.in_schema(self.entity_view_name.split('.')[0],self.entity_view_name.split('.')[1]))
+            existing_entity = entity[entity.DATASET_ID == dataset_id].to_pandas()
+
+            features = tdml.DataFrame(tdml.in_schema(self.feature_view_name.split('.')[0],self.feature_view_name.split('.')[1]))
+            existing_features = features[features.DATASET_ID == dataset_id].to_pandas()
+
+        elif res.shape[0] == 0:
+            dataset_id = str(uuid.uuid4())
+            existing_entity = None
+            existing_features = None
+            logger.info('the dataset is new and will be registered')
+        else:
+            logger.error('there are more that one dataset with the same id')
+            raise
+        logger.info(f'dataset is : {dataset_id}')
+
+        query_insert_catalog =  f"""
+        CURRENT VALIDTIME
+        MERGE INTO {self.catalog_table_name} EXISTING
+        USING (
+            SEL 
+                '{dataset_id}' AS DATASET_ID
+            ,   '{dataset.view_name}' AS DATASET_NAME
+            ,   '{dataset.schema_name}' AS DATASET_DATABASE
+            ,   '{dataset.dataset_type}' AS DATASET_TYPE
+            ,   '{dataset.valid_time}' AS DATASET_VALIDTIME
+            ,   '{json.dumps(metadata).replace("'", '"')}' AS METADATA
+        ) UPDATED
+        ON EXISTING.DATASET_ID = UPDATED.DATASET_ID
+        WHEN MATCHED THEN
+            UPDATE
+            SET
+                DATASET_NAME      = UPDATED.DATASET_NAME 
+            ,   DATASET_DATABASE  = UPDATED.DATASET_DATABASE 
+            ,   DATASET_TYPE      = UPDATED.DATASET_TYPE
+            ,   DATASET_VALIDTIME = UPDATED.DATASET_VALIDTIME
+            ,   METADATA          = UPDATED.METADATA
+        WHEN NOT MATCHED THEN
+        INSERT (
+            UPDATED.DATASET_ID,
+            UPDATED.DATASET_NAME,
+            UPDATED.DATASET_DATABASE,
+            UPDATED.DATASET_TYPE,
+            UPDATED.DATASET_VALIDTIME,
+            UPDATED.METADATA
+            )   
+        """
+
+        updated_entity = dataset.entity
+        if existing_entity is not None:
+            dropped_entity = [e for e in existing_entity.ENTITY.values if e.lower() not in map(str.lower, updated_entity.keys())]
+        else:
+            dropped_entity = []
+            
+        logger.info(f"entity to update : {list(updated_entity.keys())}")
+        logger.info(f"entity to drop : {dropped_entity}")
+
+        query_insert_entity = []
+        for k,v in updated_entity.items():
+            query_insert_entity_ =  f"""
+            CURRENT VALIDTIME
+            MERGE INTO {self.entity_table_name} EXISTING
+            USING (
+                SEL 
+                    '{dataset_id}' AS DATASET_ID
+                ,   '{k}' AS ENTITY
+                ,   '{v}' AS ENTITY_TYPE
+            ) UPDATED
+            ON EXISTING.DATASET_ID = UPDATED.DATASET_ID
+            AND EXISTING.ENTITY = UPDATED.ENTITY
+            WHEN MATCHED THEN
+                UPDATE
+                SET
+                    ENTITY_TYPE = UPDATED.ENTITY_TYPE 
+
+            WHEN NOT MATCHED THEN
+            INSERT (
+                UPDATED.DATASET_ID,
+                UPDATED.ENTITY,
+                UPDATED.ENTITY_TYPE
+                )   
+            """
+            query_insert_entity.append(query_insert_entity_)
+
+        for k in dropped_entity:
+            query_insert_entity_ = f"""
+            CURRENT VALIDTIME
+            DELETE {self.entity_table_name} WHERE DATASET_ID = '{dataset_id}' AND ENTITY = '{k}'
+            """
+            query_insert_entity.append(query_insert_entity_)
+
+        updated_features = dataset.features
+            
+        if existing_features is not None:
+            dropped_features = [f for f in existing_features.FEATURE_NAME.values if f.lower() not in map(str.lower, updated_features.keys())]
+        else:
+            dropped_features = []
+            
+        logger.info(f"features to update : {list(updated_features.keys())}")
+        logger.info(f"features to drop : {dropped_features}")
+
+        query_insert_features = []
+        for k,v in updated_features.items():
+            query_insert_feature_ =  f"""
+            CURRENT VALIDTIME
+            MERGE INTO {self.feature_table_name} EXISTING
+            USING (
+                SEL 
+                    '{dataset_id}' AS DATASET_ID
+                ,   {v['id']} AS FEATURE_ID
+                ,   '{k}' AS FEATURE_NAME
+                ,   '{v['type']}' AS FEATURE_TYPE
+                ,   '{v['database']}' AS FEATURE_DATABASE
+                ,   '{v['view']}' AS FEATURE_VIEW
+            ) UPDATED
+            ON EXISTING.DATASET_ID = UPDATED.DATASET_ID
+            AND EXISTING.FEATURE_NAME = UPDATED.FEATURE_NAME
+            WHEN MATCHED THEN
+                UPDATE
+                SET
+                    FEATURE_ID       = UPDATED.FEATURE_ID 
+                ,   FEATURE_TYPE     = UPDATED.FEATURE_TYPE 
+                ,   FEATURE_DATABASE = UPDATED.FEATURE_DATABASE 
+                ,   FEATURE_VIEW     = UPDATED.FEATURE_VIEW 
+            WHEN NOT MATCHED THEN
+            INSERT (
+                UPDATED.DATASET_ID,
+                UPDATED.FEATURE_ID,
+                UPDATED.FEATURE_NAME,
+                UPDATED.FEATURE_TYPE,
+                UPDATED.FEATURE_DATABASE,
+                UPDATED.FEATURE_VIEW
+            )   
+            """
+            query_insert_features.append(query_insert_feature_)
+
+        for k in dropped_entity:
+            query_insert_feature_ = f"""
+            CURRENT VALIDTIME
+            DELETE {self.feature_table_name} WHERE DATASET_ID = '{dataset_id}' AND FEATURE_NAME = '{k}'
+            """
+            query_insert_features.append(query_insert_feature_)
+
+        queries = [query_insert_catalog] + query_insert_entity + query_insert_features
+        for query in queries:
+            logger.info(query.split('\n')[2].strip())
+            tdml.execute_sql(query)
+
+    def drop_dataset(self, dataset_id):
+        if self.catalog[self.catalog.DATASET_ID == dataset_id].shape[0] == 1:
+            query_drop_feature = f"""
+            CURRENT VALIDTIME
+            DELETE {self.feature_table_name} WHERE DATASET_ID = '{dataset_id}'
+            """
+            query_drop_entity = f"""
+            CURRENT VALIDTIME
+            DELETE {self.entity_table_name} WHERE DATASET_ID = '{dataset_id}'
+            """
+            query_drop_catalog = f"""
+            CURRENT VALIDTIME
+            DELETE {self.catalog_table_name} WHERE DATASET_ID = '{dataset_id}'
+            """
+
+            for query in [query_drop_feature, query_drop_entity, query_drop_catalog]:
+                logger.info(query.split('\n')[2].strip())
+                tdml.execute_sql(query)
+            
+    def get_dataset_entity(self, dataset_id = None):
+
+        if dataset_id is None:
+            return self.entity
+        else:
+            return self.entity[self.entity.DATASET_ID == dataset_id]
+        
+
+    def get_dataset_features(self, dataset_id = None):
+
+        if dataset_id is None:
+            return self.features
+        else:
+            return self.features[self.features.DATASET_ID == dataset_id]

tdfs4ds/feature_store/feature_data_processing.py

@@ -124,6 +124,7 @@ def get_feature_id_and_conversion(list_entity_id, feature_names):
     conversion_name2id = {x[1]: x[0] for x in feature_id_names}
 
     return feature_id_names, conversion_name2id
+
 def prepare_feature_ingestion(df, entity_id, feature_names, feature_versions=None, primary_index=None, partitioning = '', entity_null_substitute={}, **kwargs):
     """
     Transforms and prepares a DataFrame for feature ingestion into a feature store by unpivoting it.

tdfs4ds/feature_store/feature_store_management.py

@@ -73,6 +73,7 @@ def feature_store_catalog_creation(if_exists='replace', comment='this table is a
 
                 FEATURE_ID BIGINT,
                 FEATURE_NAME VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+                FEATURE_TYPE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
                 FEATURE_TABLE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
                 FEATURE_DATABASE VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
                 FEATURE_VIEW VARCHAR(255) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
@@ -410,12 +411,12 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
     # Create a DataFrame from the feature_names_types dictionary
     if len(feature_names_types.keys()) > 1:
         df = pd.DataFrame(feature_names_types).transpose().reset_index()
-        df.columns = ['FEATURE_NAME', 'TYPE', 'FEATURE_ID']
+        df.columns = ['FEATURE_NAME', 'FEATURE_TYPE', 'FEATURE_ID']
     else:
         df = pd.DataFrame(columns=['FEATURE_NAME', 'TYPE', 'FEATURE_ID'])
         k = list(feature_names_types.keys())[0]
         df['FEATURE_NAME'] = [k]
-        df['TYPE'] = [feature_names_types[k]['type']]
+        df['FEATURE_TYPE'] = [feature_names_types[k]['type']]
         df['FEATURE_ID'] = [feature_names_types[k]['id']]
 
 
@@ -458,6 +459,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              SELECT
                 CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
             ,   A.FEATURE_NAME
+            ,   A.FEATURE_TYPE
             ,   A.FEATURE_TABLE
             ,   A.FEATURE_DATABASE
             ,   A.FEATURE_VIEW
@@ -476,6 +478,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              UPDATE
              SET
                 FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE,
+                FEATURE_TYPE     = UPDATED_FEATURES.FEATURE_TYPE,
                 FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE,
                 FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
                 --,ENTITY_NAME      = UPDATED_FEATURES.ENTITY_NAME -- modified
@@ -483,6 +486,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              INSERT
              (  UPDATED_FEATURES.FEATURE_ID
             ,   UPDATED_FEATURES.FEATURE_NAME
+            ,   UPDATED_FEATURES.FEATURE_TYPE
             ,   UPDATED_FEATURES.FEATURE_TABLE
             ,   UPDATED_FEATURES.FEATURE_DATABASE
             ,   UPDATED_FEATURES.FEATURE_VIEW
@@ -498,6 +502,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              SELECT
                 CASE WHEN B.FEATURE_ID IS NULL THEN A.FEATURE_ID ELSE B.FEATURE_ID END AS FEATURE_ID
             ,   A.FEATURE_NAME
+            ,   A.FEATURE_TYPE
             ,   A.FEATURE_TABLE
             ,   A.FEATURE_DATABASE
             ,   A.FEATURE_VIEW
@@ -516,6 +521,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              UPDATE
              SET
                 FEATURE_TABLE    = UPDATED_FEATURES.FEATURE_TABLE,
+                FEATURE_TYPE    = UPDATED_FEATURES.FEATURE_TYPE,
                 FEATURE_DATABASE = UPDATED_FEATURES.FEATURE_DATABASE,
                 FEATURE_VIEW     = UPDATED_FEATURES.FEATURE_VIEW
                 --,ENTITY_NAME      = UPDATED_FEATURES.ENTITY_NAME -- modified
@@ -523,6 +529,7 @@ def _register_features_merge(entity_id, feature_names_types, primary_index=None,
              INSERT
              (  UPDATED_FEATURES.FEATURE_ID
             ,   UPDATED_FEATURES.FEATURE_NAME
+            ,   UPDATED_FEATURES.FEATURE_TYPE
             ,   UPDATED_FEATURES.FEATURE_TABLE
             ,   UPDATED_FEATURES.FEATURE_DATABASE
             ,   UPDATED_FEATURES.FEATURE_VIEW

tdfs4ds/utils/info.py

@@ -2,6 +2,8 @@ import re
 
 import tdfs4ds
 import teradataml as tdml
+from tdfs4ds import logger
+import numpy as np
 
 def get_column_types(df, columns):
     """
@@ -264,4 +266,40 @@ def get_feature_types_sql_format(tddf, columns = None):
     res = tdml.DataFrame.from_query(query).to_pandas()
 
     # Return column names with their corresponding SQL data types in a dictionary
-    return {c: res[c].values[0].strip() for c in columns}
+    return {c: res[c].values[0].strip() for c in columns}
+
+def update_varchar_length(feature_types: dict, new_varchar_length: int) -> dict:
+    """
+    Updates the length of all VARCHAR fields in the feature_types dictionary based on an increment.
+    The new length is calculated as ceil(previous_length / new_varchar_length) * new_varchar_length,
+    ensuring that when new_varchar_length is equal to the current length, no change occurs.
+    
+    Args:
+        feature_types (dict): A dictionary where keys are feature names and values are dictionaries with 'type' and 'id'.
+        new_varchar_length (int): The increment value for adjusting VARCHAR lengths.
+    
+    Returns:
+        dict: A dictionary with updated VARCHAR lengths.
+    
+    Issues a warning if the new length is smaller than the original length.
+    """
+    updated_feature_types = {}
+    varchar_pattern = re.compile(r'VARCHAR\((\d+)\)', re.IGNORECASE)
+    
+    for key, value in feature_types.items():
+        type_value = value['type']
+        match = varchar_pattern.search(type_value)
+        if match:
+            original_length = int(match.group(1))
+            modified_length = int(np.ceil(original_length / new_varchar_length) * new_varchar_length)
+            
+            if modified_length < original_length:
+                logger.warning(f"Reducing VARCHAR length for {key} from {original_length} to {modified_length}")
+            
+            # Replace only the VARCHAR length
+            updated_value = varchar_pattern.sub(f'VARCHAR({modified_length})', type_value)
+            updated_feature_types[key] = {'type': updated_value, 'id': value['id']}
+        else:
+            updated_feature_types[key] = value
+    
+    return updated_feature_types

{tdfs4ds-0.2.4.3.dist-info → tdfs4ds-0.2.4.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tdfs4ds
-Version: 0.2.4.3
+Version: 0.2.4.5
 Summary: A python package to simplify the usage of feature store using Teradata Vantage ...
 Author: Denis Molin
 Requires-Python: >=3.6

{tdfs4ds-0.2.4.3.dist-info → tdfs4ds-0.2.4.5.dist-info}/RECORD

@@ -2,18 +2,21 @@ tdfs/__init__.py,sha256=7AcO7uB1opRCt7t2JOHworKimfAaDeO3boRW7u9Geo8,23
 tdfs/datasets.py,sha256=-b2MPEKGki2V1M8iUcoDR9uc2krIK7u1CK-EhChvihs,985
 tdfs/feature_store.py,sha256=Honu7eOAXxP4Ivz0mRlhuNkfTDzgZl5HB1WlQUwzcZ0,31354
 tdfs/data/curves.csv,sha256=q0Tm-0yu7VMK4lHvHpgi1LMeRq0lO5gJy2Q17brKbEM,112488
-tdfs4ds/__init__.py,sha256=OOakI_WdX1fjXTheqqLMUQY99apaGFXdEYg_SQpWQng,63986
+tdfs4ds/__init__.py,sha256=uyLZlPaGAVi41BEZke6OnknD0RDRWkcr-7nkCjFym34,65844
 tdfs4ds/datasets.py,sha256=LE4Gn0muwdyrIrCrbkE92cnafUML63z1lj5bFIIVzmc,3524
 tdfs4ds/feature_engineering.py,sha256=oVnZ2V_XNGE12LKC_fNfkrWSQZLgtYRmaf8Dispi6S4,7081
 tdfs4ds/feature_store.py,sha256=y-oItPZw6nBkBcGAceaATZbkLPTsvpk0OnpzTxYofDs,68576
 tdfs4ds/process_store.py,sha256=W97pwqOwabo062ow_LfAXZmlSkcq8xTuwhwAX1EStlQ,16939
 tdfs4ds/utils.py,sha256=xF1VP0NCgosXcKymOo_ofMMnvLEF228IxaxIl-f65uA,23312
 tdfs4ds/data/curves.csv,sha256=q0Tm-0yu7VMK4lHvHpgi1LMeRq0lO5gJy2Q17brKbEM,112488
+tdfs4ds/dataset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tdfs4ds/dataset/dataset.py,sha256=caiQwT-RtdPe5MDtsynWMm1n12OxftgMp7_BR9SCHKw,5360
+tdfs4ds/dataset/dataset_catalog.py,sha256=qxS2thDW2MvsRouSFaX1M0sX2J7IzBAYD8Yf22Tsd5k,16638
 tdfs4ds/feature_store/__init__.py,sha256=a7NPCkpTx40UR5LRErwnskpABG2Vuib7F5wUjaUGCnI,209
 tdfs4ds/feature_store/entity_management.py,sha256=9ltytv3yCTG84NZXBpb1Tlkf9pOxvrNb0MVidU4pwvE,10157
-tdfs4ds/feature_store/feature_data_processing.py,sha256=SuJeCTJF51l9-VS9WRS0oBUnxaVqba4hqjOpsCtdVs8,42352
+tdfs4ds/feature_store/feature_data_processing.py,sha256=vCviEJ0ARjaZ2KB8LUAdCyHdErYtWyng6iNiMJy9SQg,42354
 tdfs4ds/feature_store/feature_query_retrieval.py,sha256=zuHRZhL6-qyLpPS7mWgRy1WingSN5iibkbi53Q7jfAs,33834
-tdfs4ds/feature_store/feature_store_management.py,sha256=RIa3ZjKBULTovEmy3KEa0M2Rn5D6LMizDVnx4Q25S6o,55724
+tdfs4ds/feature_store/feature_store_management.py,sha256=WcgawACgC_lI880wj_FO2wV_FIp0W5WZ3x7r2-0WKdI,56121
 tdfs4ds/process_store/__init__.py,sha256=npHR_xju5ecGmWfYHDyteLwiU3x-cL4HD3sFK_th7xY,229
 tdfs4ds/process_store/process_followup.py,sha256=PvLcU7meg3ljBlPfuez3qwTVqpHHhVJxYxGqjgiHE8E,7265
 tdfs4ds/process_store/process_query_administration.py,sha256=DsIt97cBoJ7NcpQzbQt55eUFNgXGdOMm5Hh2aX5v0PY,7762
@@ -21,12 +24,12 @@ tdfs4ds/process_store/process_registration_management.py,sha256=F8VlBoL-de98KnkM
 tdfs4ds/process_store/process_store_catalog_management.py,sha256=H135RRTYn-pyWIqPVbHpuIyyvsaNrek6b1iPk8avJMI,16088
 tdfs4ds/utils/__init__.py,sha256=-yTMfDLZbQnIRQ64s_bczzT21tDW2A8FZeq9PX5SgFU,168
 tdfs4ds/utils/filter_management.py,sha256=7D47N_hnTSUVOkaV2XuKrlUFMxzWjDsCBvRYsH4lXdU,11073
-tdfs4ds/utils/info.py,sha256=lc9-rQDfM4NWnZGkSUkY_G0qYx7qnoErNKKcYMuLIRs,10554
+tdfs4ds/utils/info.py,sha256=SQR_ec4M9-5Z4erjb9_N0n8JPY1wpelgxkw3B12D1Q4,12322
 tdfs4ds/utils/lineage.py,sha256=LI-5pG7D8lO3-YFa9qA6CrEackiYugV23_Vz9IpF5xw,28670
 tdfs4ds/utils/query_management.py,sha256=nAcE8QY1GWAKgOtb-ubSfDVcnYbU7Ge8CruVRLoPtmY,6356
 tdfs4ds/utils/time_management.py,sha256=_jbwdyZH4Yr3VzbUrq6X93FpXDCDEdH0iv56vX7j8mA,8446
 tdfs4ds/utils/visualization.py,sha256=5S528KoKzzkrAdCxfy7ecyqKvAXBoibNvHwz_u5ISMs,23167
-tdfs4ds-0.2.4.3.dist-info/METADATA,sha256=dUqe-90oXLdYx2U6F-WmeQDHhAFN_vvZrFfVuYGmTn8,11944
-tdfs4ds-0.2.4.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-tdfs4ds-0.2.4.3.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
-tdfs4ds-0.2.4.3.dist-info/RECORD,,
+tdfs4ds-0.2.4.5.dist-info/METADATA,sha256=JwpkKDPO-5TgvnQFZJF8qllaVkUzcbv6dwHJPp2Sd1M,11944
+tdfs4ds-0.2.4.5.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+tdfs4ds-0.2.4.5.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
+tdfs4ds-0.2.4.5.dist-info/RECORD,,