teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/store/__init__.py

@@ -5,9 +5,8 @@ TERADATA CORPORATION CONFIDENTIAL AND TRADE SECRET
 Primary Owner: pradeep.garre@teradata.com
 Secondary Owner: aanchal.kavedia@teradata.com
 
-This file imports components from Feature Store and Vector Store.
+This file imports components from Feature Store.
 """
 
 from teradataml.store.feature_store.feature_store import FeatureStore
-from teradataml.store.feature_store.models import *
-from teradataml.store.vector_store import *
+from teradataml.store.feature_store.models import *

teradataml/store/feature_store/feature_store.py

@@ -66,8 +66,7 @@ class FeatureStore:
         self.__df_container = {}
 
         # Store the table names here. Then use this where ever required.
-        self.__table_names = {name: UtilFuncs._get_qualified_table_name(self.__repo, table_name)
-                              for name, table_name in EFS_TABLES.items()}
+        self.__table_names = EFS_TABLES
 
         # Declare getter's for getting the corresponding DataFrame's.
         self.__get_features_df = lambda : self.__get_obj_df("feature")
@@ -1560,6 +1559,7 @@ class FeatureStore:
                 table_name = self.__table_names["{}_staging".format(type_)]
 
             res = _delete_data(table_name=table_name,
+                               schema_name=self.__repo,
                                delete_conditions=(Col("name") == name)
                                )
 
@@ -1606,11 +1606,13 @@ class FeatureStore:
 
         # remove it from xref table first.
         _delete_data(table_name=ent_table_xref,
+                     schema_name=self.__repo,
                      delete_conditions=(Col("entity_name") == name)
                      )
 
         # remove from entity table.
         res = _delete_data(table_name=ent_table,
+                           schema_name=self.__repo,
                            delete_conditions=(Col("name") == name)
                            )
 
@@ -1758,6 +1760,96 @@ class FeatureStore:
         """
         return self.__remove_obj(name=feature, type_="feature")
 
+    def delete(self):
+        """
+        DESCRIPTION:
+            Removes the FeatureStore and its components from repository.
+            Notes:
+                 * The function removes all the associated database objects along with data.
+                   Be cautious while using this function.
+                 * The function tries to remove the underlying Database also once
+                   all the Feature Store objects are removed.
+                 * The user must have permission on the database used by this Feature Store
+                    * to drop triggers.
+                    * to drop the tables.
+                    * to drop the Database.
+                 * If the user lacks any of the mentioned permissions, Teradata recommends
+                   to not use this function.
+
+        PARAMETERS:
+            None
+
+        RETURNS:
+            bool.
+
+        RAISES:
+            None
+
+        EXAMPLES:
+            # Setup FeatureStore for repo 'vfs_v1'.
+            >>> from teradataml import FeatureStore
+            >>> fs = FeatureStore("vfs_v1")
+            >>> fs.setup()
+            True
+            >>> # Delete FeatureStore.
+            >>> fs.delete()
+            True
+            >>>
+        """
+        confirmation = input("The function removes Feature Store and drops the "
+                             "corresponding repo also. Are you sure you want to proceed? (Y/N): ")
+
+        if confirmation in ["Y", "y"]:
+            return self.__drop_feature_store_objects(self.__repo)
+
+        return False
+
+    @staticmethod
+    def __drop_feature_store_objects(repo_name):
+        """
+        DESCRIPTION:
+            Removes the FeatureStore and it's components from repository.
+
+        PARAMETERS:
+            repo_name:
+                Required Argument.
+                Specifies the name of the repository.
+                Types: str
+
+        RETURNS:
+            bool
+        """
+        # Drop all the tables and staging tables.
+        tables_ = [
+            EFS_GROUP_FEATURES_SPEC["table_name"],
+            EFS_FEATURE_GROUP_SPEC["table_name"],
+            EFS_FEATURES_SPEC['table_name'],
+            EFS_ENTITY_XREF_SPEC['table_name'],
+            EFS_ENTITY_SPEC["table_name"],
+            EFS_DATA_SOURCE_SPEC["table_name"]
+        ]
+
+        tables_stg_ = [
+            EFS_FEATURES_STAGING_SPEC['table_name'],
+            EFS_ENTITY_STAGING_SPEC["table_name"],
+            EFS_ENTITY_XREF_STAGING_SPEC["table_name"],
+            EFS_DATA_SOURCE_STAGING_SPEC["table_name"],
+            EFS_FEATURE_GROUP_STAGING_SPEC["table_name"],
+            EFS_GROUP_FEATURES_STAGING_SPEC["table_name"]
+        ]
+
+        # Drop all the triggers first. So that tables can be dropped.
+        triggers = ["{}_trg".format(table) for table in tables_]
+        for trigger in triggers:
+            execute_sql("drop trigger {}.{}".format(repo_name, trigger))
+
+        for table in (tables_ + [EFS_VERSION_SPEC["table_name"]] + tables_stg_):
+            db_drop_table(table, schema_name=repo_name)
+
+        execute_sql("DROP DATABASE {}".format(repo_name))
+
+        return True
+
     def delete_feature(self, feature):
         """
         DESCRIPTION:
@@ -2134,10 +2226,12 @@ class FeatureStore:
 
         # Remove data for FeatureGroup.
         _delete_data(table_name=self.__table_names["group_features_staging"],
+                     schema_name=self.__repo,
                      delete_conditions=(Col("group_name") == fg_name)
                      )
 
         res = _delete_data(table_name=self.__table_names["feature_group_staging"],
+                           schema_name=self.__repo,
                            delete_conditions=(Col("name") == fg_name)
                            )
 
@@ -2175,25 +2269,26 @@ class FeatureStore:
             fs.__get_features_df()
         """
         if obj_type not in self.__df_container:
+            from teradataml.dataframe.dataframe import in_schema
 
             # For feature or feature_staging, join it with xref table
             # so group name appears while listing features.
             map_ = {"feature": "group_features", "feature_staging": "group_features_staging"}
             if obj_type in map_:
-                features = DataFrame(self.__table_names[obj_type])
-                features_xref = DataFrame(self.__table_names[map_[obj_type]]).select(
+                features = DataFrame(in_schema(self.__repo, self.__table_names[obj_type]))
+                features_xref = DataFrame(in_schema(self.__repo, self.__table_names[map_[obj_type]])).select(
                     ["feature_name", "group_name"])
                 df = features.join(features_xref, on="name==feature_name", how='left')
                 self.__df_container[obj_type] = df.select(features.columns+["group_name"])
             # For entity, join with xref table.
             elif obj_type == "entity" or obj_type == "entity_staging":
-                ent_df = DataFrame(self.__table_names[obj_type])
-                xref_df = DataFrame(self.__table_names["{}_xref".format(obj_type)]).select(
+                ent_df = DataFrame(in_schema(self.__repo, self.__table_names[obj_type]))
+                xref_df = DataFrame(in_schema(self.__repo, self.__table_names["{}_xref".format(obj_type)])).select(
                     ['entity_name', 'entity_column'])
                 df = ent_df.join(xref_df, on="name==entity_name", how="inner")
                 self.__df_container[obj_type] = df.select(ent_df.columns+["entity_column"])
             else:
-                self.__df_container[obj_type] = DataFrame(self.__table_names[obj_type])
+                self.__df_container[obj_type] = DataFrame(in_schema(self.__repo, self.__table_names[obj_type]))
 
         return self.__df_container[obj_type]
 

teradataml/table_operators/Apply.py

@@ -51,7 +51,8 @@ class Apply(TableOperator):
                  data_order_column=None,
                  is_local_order=False,
                  sort_ascending=True,
-                 nulls_first=True):
+                 nulls_first=True,
+                 **kwargs):
         """
         DESCRIPTION:
             The fastpath Apply table operator executes a user-installed script or
@@ -316,14 +317,6 @@ class Apply(TableOperator):
                                     is_local_order,
                                     sort_ascending,
                                     nulls_first)
-
-        # Set the variable specific to this child class.
-        self.apply_command = apply_command
-        self.env_name = env_name if env_name is not None else get_user_env()
-        self.style = style
-        self.returns = returns
-        self._skip_argument_validation = False
-
         # Create AnalyticsWrapperUtils instance which contains validation functions.
         # This is required for is_default_or_not check.
         # Rest all validation is done using _Validators
@@ -332,20 +325,56 @@ class Apply(TableOperator):
         # Perform argument validation for arguments specific to this class.
         self.__arg_info_matrix = []
 
-        self.__arg_info_matrix.append(["style", self.style, True, (str), True, ['CSV']])
-        self.__arg_info_matrix.append(["env_name", self.env_name, False, (str, UserEnv), True])
-        self.__arg_info_matrix.append(["apply_command", self.apply_command, False, (str), True])
-        self.__arg_info_matrix.append(["returns", self.returns, True, (dict), True])
-
+        self.__arg_info_matrix.append(["style", style, True, (str), True, ['CSV']])
+        self.__arg_info_matrix.append(["env_name", env_name, False, (str, UserEnv), True])
+        self.__arg_info_matrix.append(["apply_command", apply_command, False, (str), True])
+        self.__arg_info_matrix.append(["returns", returns, True, (dict), True])
+        self._skip_argument_validation = False
         # Perform the function argument validations.
         self.__apply__validate()
 
-        self.env = self.env_name if isinstance(self.env_name, UserEnv) else get_env(self.env_name)
+        # If user do not pass environment, get the default environment.
+        if env_name is None:
+            env_name = get_user_env()
+        self._open_af_env = env_name
+
+        # Set the variable specific to this child class.
+        self.apply_command = apply_command
+        self.env_name = env_name if isinstance(env_name, str) else env_name.env_name
+        self.style = style
+        self.returns = returns
+
+        # Internal variable to check if validation is required for Python and python package versions mismatch.
+        _validation_required = kwargs.pop('_validate_version', False)
+        # Interval variable to store the function name for which validation is required.
+        _func_name = kwargs.pop('_func_name', None)
+        # Internal variable to store the list of packages required for the function.
+        _packages = kwargs.pop('_packages', None)
+
+        # Check if validation for Python and python package versions mismatch is required.
+        if _validation_required:
+            # Check if the Python interpreter major versions are consistent between Vantage and local.
+            UtilFuncs._check_python_version_diff(self.env_name)
+            # Check if the package versions are consistent between Vantage and local.
+            UtilFuncs._check_package_version_diff(_func_name, _packages, self.env_name)
+
+
+    @property
+    def env(self):
+        """
+        DESCRIPTION:
+            Getter to get environment.
+
+        RETURNS:
+            bool
+
+        RAISES:
+            None
+        """
+        if isinstance(self._open_af_env, str):
+            self._open_af_env = get_env(self._open_af_env)
 
-        # User can specify object of UserEnv class. Or if environment is already created just pass
-        # remote user environment name as string.
-        if isinstance(self.env_name, UserEnv):
-            self.env_name = self.env_name.env_name
+        return self._open_af_env
 
     @property
     def skip_argument_validation(self):

teradataml/table_operators/Script.py

@@ -67,7 +67,8 @@ class Script(TableOperator):
                  data_order_column=None,
                  is_local_order=False,
                  sort_ascending=True,
-                 nulls_first=True):
+                 nulls_first=True,
+                 **kwargs):
         """
         DESCRIPTION:
             The Script table operator function executes a user-installed script or
@@ -354,6 +355,24 @@ class Script(TableOperator):
         # Perform the function validations
         self.__validate()
 
+        # Add the prefix OPENBLAS_NUM_THREADS to the script command.
+        self.script_command = f"{TableOperatorConstants.OPENBLAS_NUM_THREADS.value} {self.script_command}"
+
+        # Internal variable to check if validation is required for Python and python package versions mismatch.
+        _validation_required = kwargs.pop('_validate_version', False)
+        # Interval variable to store the function name for which validation is required.
+        _func_name = kwargs.pop('_func_name', None)
+        # Internal variable to store the list of packages required for the function.
+        _packages = kwargs.pop('_packages', None)
+
+        # Check if validation for Python and python package versions mismatch is required.
+        if _validation_required:
+            # Check if the Python interpreter major versions are consistent between Vantage and local.
+            UtilFuncs._check_python_version_diff()
+            # Check if the package versions are consistent between Vantage and local.
+            UtilFuncs._check_package_version_diff(_func_name, _packages)
+
+
     @property
     def skip_argument_validation(self):
         """
@@ -1701,7 +1720,9 @@ class Script(TableOperator):
                                                 gc_on_quit=True, quote=False,
                                                 table_type=table_type)
         try:
-            if output_style == OutputStyle.OUTPUT_TABLE.value:
+            if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query, volatile=True)
+            elif output_style == OutputStyle.OUTPUT_TABLE.value:
                 UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
             else:
                 UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)

teradataml/table_operators/TableOperator.py

@@ -458,7 +458,9 @@ class TableOperator:
                                                                           )
 
         try:
-            if output_style == OutputStyle.OUTPUT_TABLE.value:
+            if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query, volatile=True)
+            elif output_style == OutputStyle.OUTPUT_TABLE.value:
                 UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
             else:
                 UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)

teradataml/table_operators/table_operator_util.py

@@ -249,6 +249,9 @@ class _TableOperatorUtils:
         self.data_partition_column = data_partition_column
         self.data_hash_column = data_hash_column
         self.__env = None
+        self.debug = kwargs.pop('debug', False)
+        self._validate_version = False
+        self.packages = []
         # Add all entries from kwargs as class attributes.
         self.__dict__.update(kwargs)
 
@@ -464,7 +467,10 @@ class _TableOperatorUtils:
             # Remove local copy of file to free up the disk space immediately.
             # Garbage collection will take care of it as a failsafe.
             # We may end up here after the script was created, but even before executing it.
-            GarbageCollector._delete_local_file(self.script_path)
+            if not self.debug:
+                GarbageCollector._delete_local_file(self.script_path)
+            elif not self.__dict__.get('is_printed', False):
+                self._print_script_path()
 
     def __get_script_name(self):
         """
@@ -486,7 +492,7 @@ class _TableOperatorUtils:
             script_entry, script_alias, script_name, script_base_name = self.__get_script_name()
         """
         script_entry = UtilFuncs._generate_temp_script_name(prefix="" if self.operation is None else self.operation,
-                                                            extension="py")
+                                                            extension="py", gc_on_quit=(self.debug^True))
         # script_alias is the file ID.
         script_alias = UtilFuncs._teradata_unquote_arg(UtilFuncs._extract_table_name(script_entry), quote='"')
 
@@ -530,6 +536,12 @@ class _TableOperatorUtils:
         # User can provide the installation location of previous version in case they are not using the latest.
         # This fix is done for ELE-5958 - https://teradata-pe.atlassian.net/browse/ELE-5958
 
+        # If operation is map_row or map_partition, then it should validate the python and
+        # 'dill' package version mismatch.
+        if self.operation in ["map_row", "map_partition"]:
+            self._validate_version = True
+            self.packages = ['dill']
+
         from teradataml.table_operators.Script import Script
         table_op_obj = Script(data=self.data,
                               script_name=self.script_name,
@@ -546,7 +558,10 @@ class _TableOperatorUtils:
                               nulls_first=self.nulls_first,
                               charset=self.charset,
                               data_partition_column=self.data_partition_column,
-                              data_hash_column=self.data_hash_column
+                              data_hash_column=self.data_hash_column,
+                              _validate_version = self._validate_version,
+                              _func_name = self.operation,
+                              _packages = self.packages
                               )
         table_op_obj.check_reserved_keyword = check_reserved_keyword
 
@@ -588,6 +603,12 @@ class _TableOperatorUtils:
         EXAMPLES:
            return_obj = self.__execute_apply_table_operator()
         """
+
+        # If operation is apply, then it should validate the python and 'dill' package version mismatch.
+        if self.operation == "apply":
+            self._validate_version = True
+            self.packages = ['dill']
+
         # First create Apply Table operator object so that validations are done on inputs.
         from teradataml.table_operators.Apply import Apply
         apply_op_obj = Apply(data=self.data,
@@ -604,7 +625,10 @@ class _TableOperatorUtils:
                              nulls_first=self.nulls_first,
                              data_partition_column=self.data_partition_column,
                              data_hash_column=self.data_hash_column,
-                             style=self.style
+                             style=self.style,
+                             _validate_version=self._validate_version,
+                             _func_name = self.operation,
+                             _packages = self.packages
                              )
 
         # APPLY operator requires installation and deletion of script file.
@@ -670,8 +694,33 @@ class _TableOperatorUtils:
                               TableOperatorConstants.MAP_PARTITION_OP.value,
                               TableOperatorConstants.APPLY_OP.value,
                               TableOperatorConstants.UDF_OP.value]:
-            GarbageCollector._delete_object_entry(
-                object_to_delete=self.script_entry,
-                object_type=TeradataConstants.TERADATA_SCRIPT,
-                remove_entry_from_gc_list=True
-            )
+            if self.debug:
+                self._print_script_path()
+                self.is_printed = True
+            else:
+                GarbageCollector._delete_object_entry(
+                    object_to_delete=self.script_entry,
+                    object_type=TeradataConstants.TERADATA_SCRIPT,
+                    remove_entry_from_gc_list=True
+                )
+    def _print_script_path(self):
+        """
+        DESCRIPTION:
+            Internal function to print the path of the script file.
+
+        PARAMETERS:
+            None.
+
+        RAISES:
+            None.
+
+        RETURNS:
+            None.
+        """
+        mssg1 = f"Path for the script {self.script_path}"
+        mssg2 = "The user should delete the script file since it is not being garbage collected."
+        mssg_len = max(len(mssg1), len(mssg2))
+        print("-" * mssg_len)
+        print(mssg1)
+        print(mssg2)
+        print("-" * mssg_len)

teradataml/utils/dtypes.py

@@ -8,6 +8,7 @@ from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_
                                 INTERVAL_HOUR, INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
                                 INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_SECOND)
 from teradatasqlalchemy import (GEOMETRY, MBB, MBR)
+from teradatasqlalchemy import VECTOR
 from teradataml.common.td_coltype_code_to_tdtype import HELP_COL_TYPE_TO_TDTYPE
 from teradataml.common.constants import TeradataTypes, PythonTypes
 from datetime import datetime, time, date
@@ -148,7 +149,7 @@ _GET_DATATYPES = {
                     INTERVAL_DAY_TO_SECOND, INTERVAL_HOUR, INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND,
                     INTERVAL_MINUTE, INTERVAL_MINUTE_TO_SECOND, INTERVAL_MONTH, INTERVAL_SECOND, INTERVAL_YEAR,
                     INTERVAL_YEAR_TO_MONTH, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP, TIME, TIMESTAMP, VARBYTE,
-                    VARCHAR, GEOMETRY, MBB, MBR],
+                    VARCHAR, GEOMETRY, MBB, MBR, VECTOR],
     'NON_NUM_DATE_INTERVAL': [BLOB, BYTE, CHAR, CLOB, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP, TIME, TIMESTAMP,
                               VARBYTE, VARCHAR, GEOMETRY, MBB, MBR],
     'NON_NUM_INTERVAL': [BLOB, BYTE, CHAR, CLOB, DATE, PERIOD_DATE, PERIOD_TIME, PERIOD_TIMESTAMP, TIME, TIMESTAMP,
@@ -293,10 +294,57 @@ class _DtypesMappers:
         VARCHAR: lambda x: "{0},{1}".format(x.__class__.__name__, x.length)
     }
 
+    # Holds mapping between string representation of teradatasqlalchemy type
+    # and actual teradatasqlalchemy type.
+    DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER = {
+        "CHAR": CHAR,
+        "VARCHAR": VARCHAR,
+
+        "BYTEINT": BYTEINT,
+        "SMALLINT": SMALLINT,
+        "INTEGER": INTEGER,
+        "BIGINT": BIGINT,
+
+        "REAL": FLOAT,
+        "FLOAT": FLOAT,
+        "DOUBLE": FLOAT,
+        "DECIMAL": DECIMAL,
+        "NUMBER": NUMBER,
+
+        "DATE": DATE,
+        "TIME": TIME,
+        "TIMESTAMP": TIMESTAMP,
+        "TIMESTAMP_WTZ": TIMESTAMP,
+
+        "BYTE": BYTE,
+        "VARBYTE": VARBYTE,
+        "BLOB": BLOB,
+        # TODO: Add CLOB type when support is added from OTF.
+
+        # TODO: Check these types when corresponding data type support
+        #  is available from OTF support or not.
+        "INTERVAL_YEAR": INTERVAL_YEAR,
+        "INTERVAL_YTM": INTERVAL_YEAR_TO_MONTH,
+        "INTERVAL_MONTH": INTERVAL_MONTH,
+        "INTERVAL_DAY": INTERVAL_DAY,
+
+        "INTERVAL_DTH": INTERVAL_DAY_TO_HOUR,
+        "INTERVAL_DTM": INTERVAL_DAY_TO_MINUTE,
+        "INTERVAL_DTS": INTERVAL_DAY_TO_SECOND,
+        "INTERVAL_HOUR": INTERVAL_HOUR,
+        "INTERVAL_HTM": INTERVAL_HOUR_TO_MINUTE,
+        "INTERVAL_HTS": INTERVAL_HOUR_TO_SECOND,
+        "INTERVAL_MINUTE": INTERVAL_MINUTE,
+        "INTERVAL_MTS": INTERVAL_MINUTE_TO_SECOND,
+        "INTERVAL_SECOND": INTERVAL_SECOND
+    }
+
+
 class _SuppArgTypes:
     VAL_ARG_DATATYPE = (str, BIGINT, BYTEINT, CHAR, DATE, DECIMAL, FLOAT, INTEGER, NUMBER, SMALLINT, TIME,
                         TIMESTAMP, VARCHAR)
 
+
 class _Dtypes:
 
     @staticmethod

teradataml/utils/internal_buffer.py

@@ -82,3 +82,41 @@ class _InternalBuffer:
         """
         if key in cls.__data:
             return cls.__data.get(key)
+
+    @classmethod
+    def remove_key(cls, key):
+        """
+        DESCRIPTION:
+            Remove a particular key from the internal buffer.
+
+        RETURNS:
+            None
+
+        RAISES:
+            None
+
+        EXAMPLES:
+            # Remove key "vs_session_id" from _InternalBuffer.
+            >>> _InternalBuffer.remove_key("vs_session_id")
+        """
+        del cls.__data[key]
+
+    @classmethod
+    def remove_keys(cls, keys):
+        """
+        DESCRIPTION:
+            Removes specified keys from the internal buffer.
+
+        RETURNS:
+            None
+
+        RAISES:
+            None
+
+        EXAMPLES:
+            # Remove keys "list_base_envs" and "default_base_env" from _InternalBuffer.
+            >>> _InternalBuffer.remove_keys(['list_base_envs', 'default_base_env'])
+        """
+        for key in keys:
+            if cls.__data.get(key) is not None:
+                del cls.__data[key]