tdfs4ds 0.2.4.2__py3-none-any.whl → 0.2.4.3__py3-none-any.whl

tdfs4ds/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.2.4.2'
+__version__ = '0.2.4.3'
 import logging
 # Setup the logger
 logging.basicConfig(
@@ -11,6 +11,7 @@ logger = logging.getLogger(__name__)
 
 from tdfs4ds.feature_store.feature_query_retrieval import get_available_entity_id_records, write_where_clause_filter
 from tdfs4ds.process_store.process_followup import follow_up_report
+from tdfs4ds.dataset.dataset_catalog import DatasetCatalog, Dataset
 
 DATA_DOMAIN             = None
 SCHEMA                  = None
@@ -19,6 +20,7 @@ FEATURE_CATALOG_NAME_VIEW    = 'FS_V_FEATURE_CATALOG'
 PROCESS_CATALOG_NAME         = 'FS_PROCESS_CATALOG'
 PROCESS_CATALOG_NAME_VIEW    = 'FS_V_PROCESS_CATALOG'
 PROCESS_CATALOG_NAME_VIEW_FEATURE_SPLIT    = 'FS_V_PROCESS_CATALOG_FEATURE_SPLIT'
+DATASET_CATALOG_NAME         = 'FS_DATASET'
 
 DATA_DISTRIBUTION_NAME  = 'FS_DATA_DISTRIBUTION'
 FOLLOW_UP_NAME          = 'FS_FOLLOW_UP'
@@ -125,6 +127,8 @@ def setup(database, if_exists='fail'):
             tdml.db_drop_table(table_name = tdfs4ds.DATA_DISTRIBUTION_NAME, schema_name=database)
         except Exception as e:
             print(str(e).split('\n')[0])
+
+        DatasetCatalog(schema_name=database, name=tdfs4ds.DATASET_CATALOG_NAME).drop_catalog()
     try:
         tdfs4ds.FEATURE_CATALOG_NAME = feature_store_catalog_creation()
         print('feature catalog table: ', tdfs4ds.FEATURE_CATALOG_NAME, ' in database ', database)
@@ -146,18 +150,22 @@ def setup(database, if_exists='fail'):
 
     tdfs4ds.feature_store.feature_store_management.feature_store_catalog_view_creation()
     tdfs4ds.process_store.process_store_catalog_management.process_store_catalog_view_creation()
+    dataset_catalog = DatasetCatalog(schema_name=database, name=tdfs4ds.DATASET_CATALOG_NAME)
+    if not dataset_catalog._exists():
+        dataset_catalog.create_catalog()
 
     return
 
 def connect(
-    database               = tdfs4ds.SCHEMA,
-    feature_catalog_name   = tdfs4ds.FEATURE_CATALOG_NAME,
-    process_catalog_name   = tdfs4ds.PROCESS_CATALOG_NAME,
-    data_distribution_name = tdfs4ds.DATA_DISTRIBUTION_NAME,
-    filter_manager_name    = tdfs4ds.FILTER_MANAGER_NAME,
-    followup_name          = tdfs4ds.FOLLOW_UP_NAME,
+    database                  = tdfs4ds.SCHEMA,
+    feature_catalog_name      = tdfs4ds.FEATURE_CATALOG_NAME,
+    process_catalog_name      = tdfs4ds.PROCESS_CATALOG_NAME,
+    data_distribution_name    = tdfs4ds.DATA_DISTRIBUTION_NAME,
+    filter_manager_name       = tdfs4ds.FILTER_MANAGER_NAME,
+    followup_name             = tdfs4ds.FOLLOW_UP_NAME,
     feature_catalog_name_view = tdfs4ds.FEATURE_CATALOG_NAME_VIEW,
-    process_catalog_name_view = tdfs4ds.PROCESS_CATALOG_NAME_VIEW
+    process_catalog_name_view = tdfs4ds.PROCESS_CATALOG_NAME_VIEW,
+    dataset_catalog_name      = tdfs4ds.DATASET_CATALOG_NAME
 ):
     """
     Configures the database environment by setting schema names and checking the existence of specified catalog tables.
@@ -197,7 +205,8 @@ def connect(
     distrib_exists = data_distribution_name.lower() in tables
     filter_manager_exists = filter_manager_name.lower() in tables
     followup_name_exists = followup_name.lower() in tables
-
+    
+    
     if followup_name_exists:
         tdfs4ds.FOLLOW_UP_NAME = followup_name
     else:
@@ -211,6 +220,7 @@ def connect(
         tdfs4ds.FILTER_MANAGER_NAME = filter_manager_name
         tdfs4ds.PROCESS_CATALOG_NAME_VIEW = process_catalog_name_view
         tdfs4ds.FEATURE_CATALOG_NAME_VIEW = feature_catalog_name_view
+        
 
         process_list = tdml.DataFrame(tdml.in_schema(database, process_catalog_name))
         if 'ENTITY_NULL_SUBSTITUTE' not in process_list.columns:
@@ -237,6 +247,11 @@ def connect(
     def is_data_distribution_temporal():
         return 'PERIOD' in tdfs4ds.utils.lineage.get_ddl(view_name=tdfs4ds.DATA_DISTRIBUTION_NAME,
                                                          schema_name=tdfs4ds.SCHEMA, object_type='table')
+    
+    tdfs4ds.DATASET_CATALOG_NAME = dataset_catalog_name
+    dataset_catalog = DatasetCatalog(schema_name=database, name=tdfs4ds.DATASET_CATALOG_NAME)
+    if not dataset_catalog._exists():
+        dataset_catalog.create_catalog()
 
     if is_data_distribution_temporal():
         tdfs4ds.DATA_DISTRIBUTION_TEMPORAL = True
@@ -279,7 +294,21 @@ def process_catalog():
     """
     return tdfs4ds.process_store.process_query_administration.list_processes()
 
+def dataset_catalog():
+    """
+    Retrieve a list of all datasets registered in the dataset store.
+
+    This function performs a query against the dataset store to gather a list of all
+    datasets that have been registered and are administrable. 
+
+    """
+    return DatasetCatalog(schema_name=tdfs4ds.SCHEMA, name=tdfs4ds.DATASET_CATALOG_NAME).catalog
+
+def get_dataset_entity(dataset_id = None):
+    return DatasetCatalog(schema_name=tdfs4ds.SCHEMA, name=tdfs4ds.DATASET_CATALOG_NAME).get_dataset_entity(dataset_id)
 
+def get_dataset_features(dataset_id = None):
+    return DatasetCatalog(schema_name=tdfs4ds.SCHEMA, name=tdfs4ds.DATASET_CATALOG_NAME).get_dataset_features(dataset_id)
 
 def run(process_id, return_dataset = False, force_compute = False):
     """
@@ -1064,6 +1093,12 @@ def build_dataset(entity_id, selected_features, view_name, schema_name=None, com
         tdfs4ds.logger.info(f"Adding a comment to the view {view_name} in the {schema_name} database.")
         tdml.execute_sql(f"COMMENT ON VIEW {schema_name}.{view_name} IS '{comment}'")
 
+    # build the dataset object
+    tdfs4ds.logger.info(f"Creation of the dataset object.")
+    dataset = Dataset(view_name=view_name, schema_name=schema_name)
+    tdfs4ds.logger.info(f"Registering of the dataset in the dataset catalog.")
+    DatasetCatalog(schema_name=tdfs4ds.SCHEMA, name=tdfs4ds.DATASET_CATALOG_NAME).add_dataset(dataset=dataset)
+
     # Return the query or the DataFrame based on the `return_query` flag
     if return_query:
         tdfs4ds.logger.info("Returning the generated dataset query.")

tdfs4ds/utils/info.py

@@ -47,7 +47,7 @@ def get_column_types(df, columns):
 
 
 
-def get_column_types_simple(df, columns):
+def get_column_types_simple(df, columns = None):
     """
     Retrieve simplified column types for specified columns from a DataFrame.
 
@@ -71,6 +71,9 @@ def get_column_types_simple(df, columns):
     """
 
     # Ensure that the columns parameter is in list format
+    if columns is None:
+        columns = df.columns
+
     if type(columns) != list:
         columns = [columns]
 
@@ -193,7 +196,7 @@ def generate_partitioning_clause(partitioning):
 {partitioning}
 )"""
 
-def get_feature_types_sql_format(tddf, columns):
+def get_feature_types_sql_format(tddf, columns = None):
     """
     Retrieve the SQL data types of specified columns from a Teradata dataframe.
 
@@ -228,6 +231,10 @@ def get_feature_types_sql_format(tddf, columns):
      'programming': 'VARCHAR(30)',
      'admitted': 'INTEGER'}
     """
+
+    if columns is None:
+        columns = tddf.columns
+        
     # Validate inputs
     if not isinstance(tddf, tdml.DataFrame):
         raise TypeError("tddf must be an instance of tdml.DataFrame")

{tdfs4ds-0.2.4.2.dist-info → tdfs4ds-0.2.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tdfs4ds
-Version: 0.2.4.2
+Version: 0.2.4.3
 Summary: A python package to simplify the usage of feature store using Teradata Vantage ...
 Author: Denis Molin
 Requires-Python: >=3.6

{tdfs4ds-0.2.4.2.dist-info → tdfs4ds-0.2.4.3.dist-info}/RECORD

@@ -2,7 +2,7 @@ tdfs/__init__.py,sha256=7AcO7uB1opRCt7t2JOHworKimfAaDeO3boRW7u9Geo8,23
 tdfs/datasets.py,sha256=-b2MPEKGki2V1M8iUcoDR9uc2krIK7u1CK-EhChvihs,985
 tdfs/feature_store.py,sha256=Honu7eOAXxP4Ivz0mRlhuNkfTDzgZl5HB1WlQUwzcZ0,31354
 tdfs/data/curves.csv,sha256=q0Tm-0yu7VMK4lHvHpgi1LMeRq0lO5gJy2Q17brKbEM,112488
-tdfs4ds/__init__.py,sha256=FJ9hllt1QYfh_5dOmt2BslPL-YoKSAb7MRaprF8Z_vU,62202
+tdfs4ds/__init__.py,sha256=OOakI_WdX1fjXTheqqLMUQY99apaGFXdEYg_SQpWQng,63986
 tdfs4ds/datasets.py,sha256=LE4Gn0muwdyrIrCrbkE92cnafUML63z1lj5bFIIVzmc,3524
 tdfs4ds/feature_engineering.py,sha256=oVnZ2V_XNGE12LKC_fNfkrWSQZLgtYRmaf8Dispi6S4,7081
 tdfs4ds/feature_store.py,sha256=y-oItPZw6nBkBcGAceaATZbkLPTsvpk0OnpzTxYofDs,68576
@@ -21,12 +21,12 @@ tdfs4ds/process_store/process_registration_management.py,sha256=F8VlBoL-de98KnkM
 tdfs4ds/process_store/process_store_catalog_management.py,sha256=H135RRTYn-pyWIqPVbHpuIyyvsaNrek6b1iPk8avJMI,16088
 tdfs4ds/utils/__init__.py,sha256=-yTMfDLZbQnIRQ64s_bczzT21tDW2A8FZeq9PX5SgFU,168
 tdfs4ds/utils/filter_management.py,sha256=7D47N_hnTSUVOkaV2XuKrlUFMxzWjDsCBvRYsH4lXdU,11073
-tdfs4ds/utils/info.py,sha256=N036s8h2AqJ7HPd6OBgLb1V3qUS6V1jtalPNW4Dld6c,10414
+tdfs4ds/utils/info.py,sha256=lc9-rQDfM4NWnZGkSUkY_G0qYx7qnoErNKKcYMuLIRs,10554
 tdfs4ds/utils/lineage.py,sha256=LI-5pG7D8lO3-YFa9qA6CrEackiYugV23_Vz9IpF5xw,28670
 tdfs4ds/utils/query_management.py,sha256=nAcE8QY1GWAKgOtb-ubSfDVcnYbU7Ge8CruVRLoPtmY,6356
 tdfs4ds/utils/time_management.py,sha256=_jbwdyZH4Yr3VzbUrq6X93FpXDCDEdH0iv56vX7j8mA,8446
 tdfs4ds/utils/visualization.py,sha256=5S528KoKzzkrAdCxfy7ecyqKvAXBoibNvHwz_u5ISMs,23167
-tdfs4ds-0.2.4.2.dist-info/METADATA,sha256=sFZ7UgcZ_2xj9XFQsMPOAhh1O8MrbWYzmijfs4f5Shk,11944
-tdfs4ds-0.2.4.2.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-tdfs4ds-0.2.4.2.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
-tdfs4ds-0.2.4.2.dist-info/RECORD,,
+tdfs4ds-0.2.4.3.dist-info/METADATA,sha256=dUqe-90oXLdYx2U6F-WmeQDHhAFN_vvZrFfVuYGmTn8,11944
+tdfs4ds-0.2.4.3.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+tdfs4ds-0.2.4.3.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
+tdfs4ds-0.2.4.3.dist-info/RECORD,,