teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

teradataml/table_operators/__init__.py

@@ -1,3 +1,2 @@
 from teradataml.table_operators.Script import Script
 from teradataml.table_operators.Apply import Apply
-from teradataml.table_operators.sandbox_container_util import setup_sandbox_env, cleanup_sandbox_env, copy_files_from_container

teradataml/table_operators/table_operator_util.py

@@ -12,7 +12,7 @@
 
 import os
 import teradataml.dataframe as tdmldf
-from teradataml.common.constants import TableOperatorConstants,\
+from teradataml.common.constants import TableOperatorConstants, \
     TeradataConstants, OutputStyle
 from teradataml import configure
 from teradataml.common.garbagecollector import GarbageCollector
@@ -107,10 +107,7 @@ class _TableOperatorUtils:
                              DataFrame in Vantage.
                     * LOCAL: Execute the function locally on sample data (at
                              most "num_rows" rows) from "data".
-                    * SANDBOX: Execute the function locally within a sandbox
-                               environment on sample data (at most "num_rows"
-                               rows) from "data".
-                Permitted values: 'IN-DB', 'LOCAL', 'SANDBOX', 'REMOTE'
+                Permitted values: 'IN-DB', 'LOCAL', 'REMOTE'
                 Types: str
 
             chunk_size:
@@ -183,7 +180,7 @@ class _TableOperatorUtils:
                 Required Argument.
                 Specifies the maximum number of sample rows to use from the
                 teradataml DataFrame to apply the user defined function to when
-                "exec_mode" is 'LOCAL' or 'SANDBOX'.
+                "exec_mode" is 'LOCAL'.
                 Types: int
 
             delimiter:
@@ -330,7 +327,7 @@ class _TableOperatorUtils:
         # Generate script name and alias, and add entry to a Garbage Collector.
         # script_entry is the string that is added to Garbage collector.
         # It has the format "<databasename>"."<file_id>".
-        self.script_entry, self.script_alias, self.script_name = self.__get_script_name()
+        self.script_entry, self.script_alias, self.script_name, self.script_base_name = self.__get_script_name()
 
         # Get the converters to use with pandas.read_csv, and to correctly
         # typecast the numeric data.
@@ -390,7 +387,7 @@ class _TableOperatorUtils:
             # needs to be deleted.
             self.__remove_file_and_delete_entry_from_gc(remove_from_sql_eng=False)
             raise
-            
+
     def execute(self):
         """
         DESCRIPTION:
@@ -439,18 +436,23 @@ class _TableOperatorUtils:
             None.
 
         RETURNS:
-            A 3-tuple of Strings (str).
+            A 4 -tuple of Strings (str).
 
         EXAMPLES:
-            script_entry, script_alias, script_name = self.__get_script_name()
+            script_entry, script_alias, script_name, script_base_name = self.__get_script_name()
         """
-        script_entry = UtilFuncs._generate_temp_script_name(prefix="" if self.operation is None else self.operation)
+        script_entry = UtilFuncs._generate_temp_script_name(prefix="" if self.operation is None else self.operation,
+                                                            extension="py")
         # script_alias is the file ID.
         script_alias = UtilFuncs._teradata_unquote_arg(UtilFuncs._extract_table_name(script_entry), quote='"')
+
         # script_name is the actual file name (basename).
-        script_name = "{script_name}.py".format(script_name=script_alias)
+        script_name = script_alias  # alias now contains extension also.
 
-        return script_entry, script_alias, script_name
+        # Extract the base name without extension.
+        from pathlib import Path
+        script_base_name = Path(script_alias).stem
+        return script_entry, script_alias, script_name, script_base_name
 
     def __execute_script_table_operator(self):
         """
@@ -505,14 +507,12 @@ class _TableOperatorUtils:
                               )
         table_op_obj.check_reserved_keyword = check_reserved_keyword
 
-
-
         if self.exec_mode.upper() == TableOperatorConstants.INDB_EXEC.value:
             # If not test mode, execute the script in Vantage.
             try:
                 # Install that file, suppressing the output message.
                 execute_sql('SET SESSION SEARCHUIFDBPATH = \"{}\"'.format(database))
-                install_file(file_identifier=self.script_alias,
+                install_file(file_identifier=self.script_base_name,
                              file_path=self.script_path,
                              suppress_output=True)
 
@@ -525,10 +525,6 @@ class _TableOperatorUtils:
                 self.__remove_file_and_delete_entry_from_gc()
         elif self.exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
             return table_op_obj.test_script(exec_mode='local')
-        else:
-            # TODO: Add Support for Sandbox mode.
-            # JIRA: https://jira.td.teradata.com/jira/browse/ELE-3277
-            raise NotImplementedError()
 
     def __execute_apply_table_operator(self):
         """
@@ -552,21 +548,21 @@ class _TableOperatorUtils:
         # First create Apply Table operator object so that validations are done on inputs.
         from teradataml.table_operators.Apply import Apply
         apply_op_obj = Apply(data=self.data,
-                              script_name=self.script_name,
-                              env_name=self.env_name,
-                              files_local_path=GarbageCollector._get_temp_dir_name(),
-                              apply_command="python3 {}".format(self.script_name),
-                              returns=self.returns,
-                              delimiter=self.delimiter,
-                              quotechar=self.quotechar,
-                              data_order_column=self.data_order_column,
-                              is_local_order=self.is_local_order,
-                              sort_ascending=self.sort_ascending,
-                              nulls_first=self.nulls_first,
-                              data_partition_column=self.data_partition_column,
-                              data_hash_column=self.data_hash_column,
-                              style=self.style
-                              )
+                             script_name=self.script_name,
+                             env_name=self.env_name,
+                             files_local_path=GarbageCollector._get_temp_dir_name(),
+                             apply_command="python3 {}".format(self.script_name),
+                             returns=self.returns,
+                             delimiter=self.delimiter,
+                             quotechar=self.quotechar,
+                             data_order_column=self.data_order_column,
+                             is_local_order=self.is_local_order,
+                             sort_ascending=self.sort_ascending,
+                             nulls_first=self.nulls_first,
+                             data_partition_column=self.data_partition_column,
+                             data_hash_column=self.data_hash_column,
+                             style=self.style
+                             )
 
         # APPLY operator requires installation and deletion of script file.
         # Get the UserEnv object and store it in a variable.
@@ -588,10 +584,6 @@ class _TableOperatorUtils:
                 raise
             finally:
                 self.__remove_file_and_delete_entry_from_gc(remove_from_sql_eng=False)
-        else:
-            # For test_mode, call apply_op_obj.test_script()
-            # with the required LOCAL or SANDBOX mode.
-            raise NotImplementedError()
 
     def __remove_file_and_delete_entry_from_gc(self,
                                                remove_from_sql_eng=True):
@@ -621,7 +613,7 @@ class _TableOperatorUtils:
         """
         if remove_from_sql_eng:
             # Remove file from Vantage, suppressing it's output.
-            remove_file(file_identifier=self.script_alias, force_remove=True,
+            remove_file(file_identifier=self.script_base_name, force_remove=True,
                         suppress_output=True)
 
         # For apply, remove file from remote user environment.

teradataml/utils/validators.py

@@ -9,6 +9,31 @@ from teradataml.common.messages import MessageCodes, Messages
 from teradataml.utils.dtypes import _Dtypes, _DtypesMappers, _ListOf, _TupleOf
 from teradataml.options.configure import configure
 from teradataml.dataframe.sql_interfaces import ColumnExpression
+from functools import wraps, reduce
+
+
+def skip_validation():
+    """
+    DESCRIPTION:
+        Define for skipping the validation.
+
+    PARAMETERS:
+        None
+
+    EXAMPLES:
+        @skip_validation(skip_all=True)
+        def validation_func(): ...
+
+    """
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            # If "skip_all" flag is set to False,
+            # skip all validation execution.
+            if not _Validators.skip_all:
+                return func(*args, **kwargs)
+        return wraps(func)(wrapper)
+    return decorator
+
 
 class _Validators:
     """
@@ -30,7 +55,12 @@ class _Validators:
         11. Validate whether the given file is not empty, given absolute file path.
         12. Validate whether "arg1" and "arg2" are mutually inclusive.
     """
+
+    # "skip_all" specifies to skip all the executions.
+    skip_all = False
+
     @staticmethod
+    @skip_validation()
     def __getTypeAsStr(type_list):
         """
         Function to convert type to string.
@@ -74,6 +104,7 @@ class _Validators:
         return type_as_str
 
     @staticmethod
+    @skip_validation()
     def _check_isinstance(obj, class_or_tuple):
         """
         Function checks whether an object is an instance of a class.
@@ -124,6 +155,7 @@ class _Validators:
                     return type(obj) == class_or_tuple
 
     @staticmethod
+    @skip_validation()
     def _validate_columnexpression_dataframe_has_columns(columns,
                                                          arg_name,
                                                          column_expression):
@@ -164,12 +196,12 @@ class _Validators:
         """
         if not columns:
             return True
-
+        from teradataml.common.utils import UtilFuncs
         # Converting columns to a list if string is passed.
         if not isinstance(columns, list):
             columns = [columns]
 
-        df_columns = [df_column.name for df_column in column_expression.table.columns]
+        df_columns = UtilFuncs._all_df_columns(column_expression)
 
         # Let's validate existence of each column one by one.
         for column_name in columns:
@@ -183,6 +215,66 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
+    def _validate_invalid_column_types(all_columns, column_arg, columns_to_check, invalid_types):
+        """
+        DESCRIPTION:
+            Function to validate the invalid types for "columns_to_check" from "all_columns".
+
+        PARAMETERS:
+            all_columns:
+                Required Argument.
+                Specifies the ColumnExpressions.
+                Types: List of ColumnExpression
+
+            column_arg:
+                Required Argument.
+                Specifies the name of the argument.
+                Types: str
+
+            columns_to_check:
+                Required Argument.
+                Specifies columns to check for invalid types.
+                Types: str OR list of str OR ColumnExpression OR list of ColumnExpression
+
+            invalid_types:
+                Required Argument.
+                Specifies list of invalid teradata types to check against "columns".
+                Types: list
+
+        RAISES:
+            ValueError
+
+        RETURNS:
+            bool
+
+        EXAMPLES:
+            _Validators._validate_invalid_column_types(columns, column_arg, invalid_types)
+        """
+        columns_and_types = {c.name.lower(): type(c.type) for c in all_columns}
+        get_col_type = lambda column: columns_and_types[column.lower()].__name__ if isinstance(
+            column, str) else column.type.__class__.__name__
+
+        invalid_types = ["{}({})".format(column if isinstance(column, str) else column.name, get_col_type(column))
+                         for column in columns_to_check if get_col_type(column)
+                         in [t.__name__ for t in _Dtypes._get_sort_unsupported_data_types()]
+                         ]
+
+        if invalid_types:
+            invalid_column_types = (col_type.__name__ for col_type in
+                                    _Dtypes._get_sort_unsupported_data_types())
+            error_message = Messages.get_message(MessageCodes.INVALID_COLUMN_DATATYPE,
+                                                 ", ".join(invalid_types),
+                                                 column_arg,
+                                                 "Unsupported",
+                                                 ", ".join(invalid_column_types))
+
+            raise ValueError(error_message)
+
+        return True
+
+    @staticmethod
+    @skip_validation()
     def _validate_dataframe_has_argument_columns(columns, column_arg, data, data_arg, is_partition_arg=False):
         """
         Function to check whether column names in columns are present in given dataframe or not.
@@ -282,6 +374,7 @@ class _Validators:
                                                                 data_arg=data_arg)
 
     @staticmethod
+    @skip_validation()
     def _validate_column_exists_in_dataframe(columns, metaexpr, case_insensitive=False, column_arg=None,
                                              data_arg=None, for_table=False):
         """
@@ -375,6 +468,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_engine(engine):
         """
         Function to validate whether the argument engine is supported or not.
@@ -403,6 +497,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_function_arguments(arg_list, skip_empty_check = None):
         """
         Method to verify that the input arguments are of valid data type except for
@@ -516,6 +611,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_input_columns_not_empty(arg, arg_name, skip_empty_check = None):
         """
         Function to check whether argument is empty string or not.
@@ -553,6 +649,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_missing_required_arguments(arg_list):
         """
         Method to check whether the required arguments passed to the function are missing
@@ -593,6 +690,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_permitted_values(arg, permitted_values, arg_name, case_insensitive=True, includeNone=True, supported_types=None):
         """
         Function to check the permitted values for the argument.
@@ -715,6 +813,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_positive_int(arg, arg_name, lbound=0, ubound=None, lbound_inclusive=False):
         """
         Validation to check arg values is a positive int.
@@ -781,6 +880,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_argument_range(arg,
                                  arg_name,
                                  lbound=None,
@@ -889,6 +989,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_vantage_version(vantage_version):
         """
         Function to verify whether the given vantage_version is
@@ -920,6 +1021,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_timebucket_duration(timebucket_duration, timebucket_duration_arg_name='timebucket_duration'):
         """
         Internal function to validate timeduration_bucket specified when creating a
@@ -976,6 +1078,7 @@ class _Validators:
                                                    'equivalent notation'))
 
     @staticmethod
+    @skip_validation()
     def _validate_column_type(df, col, col_arg, expected_types, raiseError = True):
         """
         Internal function to validate the type of an input DataFrame column against
@@ -1002,7 +1105,7 @@ class _Validators:
                 Required Argument.
                 Specifies a list of teradatasqlachemy datatypes that the column is
                 expected to be of type.
-                Types: list of SQLAlchemy types
+                Types: list of teradatasqlalchemy types
 
             raiseError:
                 Optional Argument.
@@ -1031,6 +1134,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_aggr_operation_unsupported_datatype(operation, columns, td_column_names_and_types):
         """
         Internal function to validate the for unsupported data types of an input DataFrame column for
@@ -1081,6 +1185,7 @@ class _Validators:
             raise TeradataMlException(msg, MessageCodes.TDMLDF_AGGREGATE_COMBINED_ERR)
 
     @staticmethod
+    @skip_validation()
     def _validate_str_arg_length(arg_name, arg_value, op, length):
         """
         Internal function to validate the length of a string passed as an argument.
@@ -1125,6 +1230,7 @@ class _Validators:
         return _Validators._validate_arg_length(arg_name=arg_name, arg_value=arg_value, op=op, length=length)
 
     @staticmethod
+    @skip_validation()
     def _validate_arg_length(arg_name, arg_value, op, length):
         """
         Internal function to validate the length of an argument.
@@ -1196,6 +1302,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_file_exists(file_path):
         """
         DESCRIPTION:
@@ -1224,6 +1331,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_mutually_exclusive_arguments(arg1, err_disp_arg1_name, arg2,
                                                err_disp_arg2_name, skip_all_none_check=False):
         """
@@ -1278,6 +1386,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_unexpected_column_type(df, col, col_arg, unexpected_types, check_exist=True, raise_error=True):
         """
         Internal function to validate the column existence and type of an input DataFrame column against
@@ -1363,6 +1472,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_dependent_argument(dependent_arg, dependent_arg_value, independent_arg, independent_arg_value,
                                      msg_arg_value=None):
         """
@@ -1428,6 +1538,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_py_type_for_td_type_conversion(py_type, py_type_arg_name):
         """
         DESCRIPTION:
@@ -1463,6 +1574,7 @@ class _Validators:
             raise TeradataMlException(error_msg, error_code)
 
     @staticmethod
+    @skip_validation()
     def _validate_function_install_location_is_set(option, function_type, option_name):
         """
          DESCRIPTION:
@@ -1506,6 +1618,7 @@ class _Validators:
             raise TeradataMlException(message, MessageCodes.MISSING_ARGS)
 
     @staticmethod
+    @skip_validation()
     def _check_table_exists(conn, table_name,
                             schema_name,
                             raise_error_if_does_not_exists=True,
@@ -1556,6 +1669,7 @@ class _Validators:
         return False
 
     @staticmethod
+    @skip_validation()
     def _check_empty_file(file_path):
         """
         Description:
@@ -1586,6 +1700,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_mutually_inclusive_arguments(arg1, err_disp_arg1_name, arg2,
                                                err_disp_arg2_name):
         """
@@ -1638,6 +1753,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_file_extension(file_path, extension):
         """
         DESCRIPTION:
@@ -1674,7 +1790,9 @@ class _Validators:
 
         return True
 
+
     @staticmethod
+    @skip_validation()
     def _validate_argument_is_not_None(arg, arg_name, additional_error="", reverse=False):
         """
         DESCRIPTION:
@@ -1730,6 +1848,7 @@ class _Validators:
 
 
     @staticmethod
+    @skip_validation()
     def _validate_dataframe(df, raise_error=True):
         """
         This is an internal function checks whether the dataframe is none
@@ -1774,6 +1893,7 @@ class _Validators:
 
 
     @staticmethod
+    @skip_validation()
     def _validate_column_value_length(argument_name, argument_value, allowed_length,
                                       operation='perform the operation'):
         """
@@ -1826,6 +1946,7 @@ class _Validators:
 
 
     @staticmethod
+    @skip_validation()
     def _validate_list_lengths_equal(list1, arg_name1, list2, arg_name2):
         """
         DESCRIPTION:
@@ -1872,6 +1993,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_dict_argument_key_value(arg_name, arg_dict, key_types=None, value_types=None,
                                           key_permitted_values=None, value_permitted_values=None,
                                           value_empty_string=True):
@@ -2036,6 +2158,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_http_response(http_response, valid_status_code, error_msg):
         """
         DESCRIPTION:
@@ -2076,6 +2199,7 @@ class _Validators:
         return True
 
     @staticmethod
+    @skip_validation()
     def _validate_module_presence(module_name, function_name):
         """
         DESCRIPTION: