teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

teradataml/table_operators/Script.py

@@ -14,12 +14,9 @@
 # ##################################################################
 
 import os
-import tarfile
 import teradataml.dataframe as tdmldf
 import subprocess
 import sys
-from pathlib import Path
-from datetime import datetime
 from io import StringIO
 from teradataml.common.constants import TableOperatorConstants
 from teradataml.common.wrapper_utils import AnalyticsWrapperUtils
@@ -28,7 +25,6 @@ from teradataml.common.constants import OutputStyle, TeradataConstants
 from teradataml.context.context import _get_current_databasename
 from teradataml.context.context import get_context, get_connection
 from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils
-from teradataml.common.garbagecollector import GarbageCollector
 from teradataml.dbutils.filemgr import install_file
 from teradataml.dbutils.filemgr import remove_file
 from teradataml.table_operators.table_operator_query_generator import \
@@ -39,7 +35,6 @@ from teradataml.common.messagecodes import MessageCodes
 from teradataml.utils.validators import _Validators
 from teradataml.options.display import display
 from teradataml.options.configure import configure
-from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
 from teradataml.utils.utils import execute_sql
 from teradatasqlalchemy.dialect import dialect as td_dialect
 from teradatasqlalchemy import (BYTEINT, SMALLINT, INTEGER, BIGINT, DECIMAL, FLOAT,
@@ -57,7 +52,6 @@ from teradatasqlalchemy import (INTERVAL_YEAR, INTERVAL_YEAR_TO_MONTH, INTERVAL_
 from teradataml.table_operators.TableOperator import TableOperator
 
 class Script(TableOperator):
-
     def __init__(self,
                  data=None,
                  script_name=None,
@@ -254,15 +248,7 @@ class Script(TableOperator):
             ...              charset='latin',
             ...              returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
 
-            # Run user script locally within docker container and using data from csv.
-            # This helps the user to fix script level issues outside Vantage.
-            # Setup the environment by providing local path to docker image file.
-            >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar'))
-            Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
-            Image loaded successfully.
-            Starting a container for stosandbox:1.0 image.
-            Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
-            started successfully.
+            # Run user script locally and using data from csv.
 
             >>> sto.test_script(input_data_file='../barrier.csv', data_file_delimiter=',')
 
@@ -316,6 +302,7 @@ class Script(TableOperator):
         self.sort_ascending = sort_ascending
         self.nulls_first = nulls_first
         self._check_reserved_keyword = True
+        self._skip_argument_validation = False
 
         # Create AnalyticsWrapperUtils instance which contains validation functions.
         # This is required for is_default_or_not check.
@@ -367,6 +354,38 @@ class Script(TableOperator):
         # Perform the function validations
         self.__validate()
 
+    @property
+    def skip_argument_validation(self):
+        """
+        DESCRIPTION:
+            Getter for self._skip_argument_validation.
+
+        RETURNS:
+            bool
+
+        RAISES:
+            None
+        """
+        return self._skip_argument_validation
+
+    @skip_argument_validation.setter
+    def skip_argument_validation(self, flag):
+        """
+        DESCRIPTION:
+            Setter for self._skip_argument_validation
+
+        PARAMETERS:
+            flag    Required Argument.
+                    Specifies whether the arguments should be skipped or not.
+                    Types: bool
+        RETURNS:
+            None
+
+        RAISES:
+            None
+        """
+        self._skip_argument_validation = flag
+
     @property
     def check_reserved_keyword(self):
         """
@@ -420,6 +439,8 @@ class Script(TableOperator):
         Function to validate Table Operator Function arguments, which verifies missing
         arguments, input argument and table types. Also processes the argument values.
         """
+        if self.skip_argument_validation:
+            return
         # Make sure that a non-NULL value has been supplied for all mandatory arguments
         _Validators._validate_missing_required_arguments(self.awu_matrix)
 
@@ -800,91 +821,11 @@ class Script(TableOperator):
                                      nulls_first)
         self.__validate()
 
-    def setup_sto_env(self, docker_image_location):
-        """
-        DESCRIPTION:
-            Function enables user to load already downloaded sandbox image.
-
-        PARAMETERS:
-            docker_image_location:
-                Required Argument.
-                Specifies the location of image on user's system.
-                Types: str
-
-                Note:
-                    For location to download docker image refer teradataml User Guide.
-
-        RETURNS:
-            None.
-
-        RAISES:
-            TeradataMlException
-
-        EXAMPLES:
-            # Note - Refer to User Guide for setting search path and required permissions.
-            # Load example data.
-            load_example_data("Script", ["barrier"])
-
-            # Example - The script mapper.py reads in a line of text input
-            # ("Old Macdonald Had A Farm") from csv and
-            # splits the line into individual words, emitting a new row for each word.
-
-            # Create teradataml DataFrame objects.
-            >>> barrierdf = DataFrame.from_table("barrier")
-
-            # Set SEARCHUIFDBPATH.
-            >>> execute_sql("SET SESSION SEARCHUIFDBPATH = alice;")
-
-            # Create a Script object that allows us to execute script on Vantage.
-            >>> import os
-            >>> td_path = os.path.dirname(teradataml.__file__)
-            >>> from teradatasqlalchemy import VARCHAR
-            >>> sto = Script(data=barrierdf,
-            ...              script_name='mapper.py',
-            ...              files_local_path= os.path.join(td_path, 'data', 'scripts'),
-            ...              script_command='tdpython3 ./alice/mapper.py',
-            ...              data_order_column="Id",
-            ...              is_local_order=False,
-            ...              nulls_first=False,
-            ...              sort_ascending=False,
-            ...              charset='latin',
-            ...              returns=OrderedDict([("word", VARCHAR(15)),("count_input", VARCHAR(2))]))
-
-            # Run user script locally within docker container and using data from csv.
-            # This helps the user to fix script level issues outside Vantage.
-            # Setup the environment by providing local path to docker image file.
-            >>> sto.setup_sto_env(docker_image_location='/tmp/sto_sandbox_docker_image.tar')
-            Loading image from /tmp/sto_sandbox_docker_image.tar. It may take few minutes.
-            Image loaded successfully.
-            Starting a container for stosandbox:1.0 image.
-            Container d7c73cb498c79a082180576bb5b10bb07b52efdd3026856146fc15e91147b19f
-            started successfully.
-
-        """
-        self.awu_matrix_setup = []
-        self.awu_matrix_setup.append((["docker_image_location", docker_image_location,
-                                       False, (str), True]))
-
-        # Validate missing arguments.
-        _Validators._validate_missing_required_arguments(self.awu_matrix_setup)
-
-        # Validate argument types.
-        _Validators._validate_function_arguments(self.awu_matrix_setup)
-
-        from teradataml.table_operators.sandbox_container_util import setup_sandbox_env
-        setup_sandbox_env(sandbox_image_location=docker_image_location,
-                          sandbox_image_name='stosandbox:1.0')
-
-        # Set _latest_sandbox_exists to True - which indicates sandbox image for STO
-        # exists on the system.
-        configure._latest_sandbox_exists = True
-
     def test_script(self, supporting_files=None, input_data_file=None, script_args="",
-                    exec_mode ='sandbox', **kwargs):
+                    exec_mode ='local', **kwargs):
         """
         DESCRIPTION:
-            Function enables user to run script in docker container environment outside
-            Vantage.
+            Function enables user to run script locally outside Vantage.
             Input data for user script is either read from a file or from database.
             Note:
                 1. Purpose of test_script() function is to enable the user to test their scripts for any errors without
@@ -917,10 +858,9 @@ class Script(TableOperator):
             exec_mode:
                 Optional Argument.
                 Specifies the mode in which user wants to test the script.
-                If set to 'sandbox', the user script will run within the sandbox
-                environment, else it will run locally on user's system.
-                Permitted Values: 'sandbox', 'local'
-                Default Value: 'sandbox'
+                When set to 'local', the user script will run locally on user's system.
+                Permitted Values: 'local'
+                Default Value: 'local'
                 Types: str
 
             kwargs:
@@ -954,13 +894,6 @@ class Script(TableOperator):
                         Default Value: True
                         Types: bool
 
-                    timeout:
-                        Optional Argument.
-                        Specifies the timeout for docker API calls when running in
-                        sandbox mode.
-                        Default Value: 5000
-                        Types: int
-
                     data_file_quote_char:
                         Optional Argument.
                         Specifies the quotechar used in the input data file.
@@ -1031,13 +964,13 @@ class Script(TableOperator):
         EXAMPLES:
             # Assumption - sto is Script() object. Please refer to help(Script)
             # for creating Script object.
-            # Run user script in sandbox mode with input from data file.
+            # Run user script in local mode with input from data file.
 
             >>> sto.test_script(input_data_file='../barrier.csv',
             ...                 data_file_delimiter=',',
             ...                 data_file_quote_char='"',
             ...                 data_file_header=True,
-            ...                 exec_mode='sandbox')
+            ...                 exec_mode='local')
 
             ############ STDOUT Output ############
                     word  count_input
@@ -1060,22 +993,6 @@ class Script(TableOperator):
             3        Had            1
             4          A            1
             5       Farm            1
-            
-            # Run user script in sandbox mode with logmech as 'TD2'.
-            >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TD2")
-
-            # Run user script in sandbox mode with logmech as 'TDNEGO'.
-            >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="TDNEGO")
-
-            # Run user script in sandbox mode with logmech as 'LDAP'.
-            >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="LDAP")
-
-            # Run user script in sandbox mode with logmech as 'KRB5'.
-            >>> sto.test_script(script_args="4 5 10 6 480", password="alice", logmech="KRB5")
-
-            # Run user script in sandbox mode with logmech as 'JWT'.
-            >>> sto.test_script(script_args="4 5 10 6 480", password="alice",
-                                logmech='JWT', logdata='token=eyJpc...h8dA')
 
         """
         logmech_valid_values = ['TD2', 'TDNEGO', 'LDAP', 'KRB5', 'JWT']
@@ -1086,8 +1003,7 @@ class Script(TableOperator):
         awu_matrix_test.append((["input_data_file", input_data_file, True, (str), True]))
         awu_matrix_test.append((["script_args", script_args, True, (str), False]))
         awu_matrix_test.append((["exec_mode", exec_mode, True, (str), True,
-                                [TableOperatorConstants.SANDBOX_EXEC.value,
-                                 TableOperatorConstants.LOCAL_EXEC.value]]))
+                                [TableOperatorConstants.LOCAL_EXEC.value]]))
 
         data_row_limit = kwargs.pop("data_row_limit", 1000)
         awu_matrix_test.append((["data_row_limit", data_row_limit, True, (int), True]))
@@ -1103,9 +1019,6 @@ class Script(TableOperator):
         data_file_header = kwargs.pop("data_file_header", True)
         awu_matrix_test.append((["data_file_header", data_file_header, True, (bool)]))
 
-        timeout = kwargs.pop("timeout", 5000)
-        awu_matrix_test.append((["timeout", timeout, True, (int), True]))
-
         logmech = kwargs.pop("logmech", "TD2")
         awu_matrix_test.append(
             ["logmech", logmech, True, (str), True, logmech_valid_values])
@@ -1116,9 +1029,6 @@ class Script(TableOperator):
         # Validate argument types.
         _Validators._validate_function_arguments(awu_matrix_test)
 
-        # Validate timeout value.
-        _Validators._validate_positive_int(timeout, "timeout")
-
         self.__validate()
         self.__validate_for_reserved_keyword()
 
@@ -1136,14 +1046,8 @@ class Script(TableOperator):
         # Either of 'input_data_file' or 'password' argument is required.
         password = kwargs.pop("password", None)
 
-        # The check of EITHER_THIS_OR_THAT_ARGUMENT is applicable only when the exec_mode is sandbox.
-        # Hence adding the check exec_mode != "local".
         # When exec_mode is local, the connection object is used to get the values in the table.
-        if exec_mode != "local" and not (input_data_file or (self.data and password)):
-            message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
-                                           "input_data_file", "Script data and password")
-            raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
-        elif exec_mode == "local" and not (input_data_file or self.data):
+        if exec_mode == "local" and not (input_data_file or self.data):
             message = Messages.get_message(MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT,
                                            "input_data_file", "Script data")
             raise TeradataMlException(message, MessageCodes.EITHER_THIS_OR_THAT_ARGUMENT)
@@ -1172,7 +1076,6 @@ class Script(TableOperator):
 
         if exec_mode.upper() == TableOperatorConstants.LOCAL_EXEC.value:
             user_script_path = os.path.join(self.files_local_path, self.script_name)
-            import sys
             cmd = [str(sys.executable), user_script_path]
             cmd.extend(script_args)
 
@@ -1206,172 +1109,6 @@ class Script(TableOperator):
                     return self.__process_test_script_output(exec_cmd_output)
                 except Exception as exp:
                     raise
-        else:
-            # Execution Mode - sandbox.
-
-            # get the frame object of the function.
-            import inspect
-            frame = inspect.currentframe()
-
-            # Validate argument types.
-            _Validators._validate_module_presence('docker', frame.f_code.co_name)
-
-            import docker
-
-            # Read container_id from configure.sandbox_container_id, if it is None then
-            # raise an exception
-
-            container_id = configure.sandbox_container_id
-            if container_id is None:
-                message = Messages.get_message(MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
-                raise TeradataMlException(message,
-                                          MessageCodes.SANDBOX_CONTAINER_NOT_FOUND)
-
-            # Set path inside docker container. This is where files will be copied to.
-            # os.path.join() will not work here because the path is not dependent on
-            # client platform. Sandbox environment is linux based here.
-            _path_in_docker_container = "/home/tdatuser"
-            user_script_path = "{}/{}".format(_path_in_docker_container, self.script_name)
-
-            if input_data_file is not None:
-                input_file_name = os.path.basename(input_data_file)
-                input_file_path = "{}/{}".format(_path_in_docker_container,
-                                                 input_file_name)
-                # Create script_executor.
-                self._create_executor_script(user_script_path=user_script_path,
-                                              user_script_args=script_args,
-                                              data_file_path=input_file_path,
-                                              data_file_delimiter=data_file_delimiter,
-                                              data_file_quote_char=data_file_quote_char,
-                                              data_file_header=data_file_header)
-            else:
-                # Read input from db.
-                if self.data.shape[0] > data_row_limit:
-                    raise ValueError(
-                        Messages.get_message(MessageCodes.DATAFRAME_LIMIT_ERROR,
-                                             'data_row_limit', 'data_row_limit',
-                                             data_row_limit))
-                db_host = get_context().url.host
-
-                user_name = get_context().url.username
-
-                if not self.data._table_name:
-                    self.data._table_name = df_utils._execute_node_return_db_object_name(
-                        self.data._nodeid, self.data._metaexpr)
-                table_name = UtilFuncs._extract_table_name(self.data._table_name)
-
-                db_name = _get_current_databasename()
-
-                # Create script_executor.
-                self._create_executor_script(user_script_path=user_script_path,
-                                              user_script_args=script_args,
-                                              db_host=db_host,
-                                              user_name=user_name,
-                                              passwd=password,
-                                              table_name=table_name,
-                                              db_name=db_name,
-                                              logmech=logmech,
-                                              logdata=logdata)
-
-            client = docker.APIClient(timeout=timeout)
-
-            # Copy files to container indicated in configure.sandbox_container_id.
-            files_to_copy = [self.script_name]
-
-            if supporting_files is not None:
-                if isinstance(supporting_files, str):
-                    supporting_files = [supporting_files]
-
-                if len(supporting_files) == 0 \
-                        or any(file in [None, "None", ""] for file in supporting_files):
-                    raise ValueError(
-                        Messages.get_message(MessageCodes.LIST_SELECT_NONE_OR_EMPTY,
-                                             'supporting_files'))
-                else:
-                    files_to_copy.extend(supporting_files)
-
-            if input_data_file is not None:
-                files_to_copy.append(input_data_file)
-
-            for filename in files_to_copy:
-                file_path = os.path.join(self.files_local_path, filename)
-                # Check if file exists.
-                _Validators._validate_file_exists(file_path)
-
-                # Copy file to docker container.
-                self._copy_to_docker_container(client, file_path,
-                                                _path_in_docker_container,
-                                                container_id)
-
-            # Copy script_executor to docker container.
-            self._copy_to_docker_container(client, self.script_path,
-                                            _path_in_docker_container,
-                                            container_id)
-
-            script_executor_file_name = os.path.basename(self.script_path)
-            exec_cmd = ("tdpython3 {0}/{1}".format(_path_in_docker_container,
-                                                 script_executor_file_name))
-
-            try:
-                # Setup an exec instance in the container.
-                exec_cmd_create = client.exec_create(container_id, exec_cmd)
-
-                # Start exec instance and run user script.
-                exec_cmd_output = client.exec_start(exec_cmd_create, demux=True)
-
-                # Inspect the output for success or failure.
-                inspect_out = client.exec_inspect(exec_cmd_create)
-
-                # Extract the exit code.
-                exit_code = inspect_out['ExitCode']
-
-                if exec_cmd_output[0] is not None:
-                    executor_output = exec_cmd_output[0].decode()
-
-                executor_error = ""
-                if exec_cmd_output[1] is not None:
-                    executor_error = exec_cmd_output[1].decode()
-
-                # Exit code 1 indicates any error thrown by subprocess.
-                # Exit code 126 indicates permission problem or command is not executable.
-                # Exit code 127 indicates possible typos in shell script with
-                # unrecognizable characters.
-                if exit_code == 1 or exit_code == 126 or exit_code == 127:
-                    message = Messages.get_message(
-                        MessageCodes.SANDBOX_SCRIPT_ERROR).format(executor_error)
-                    raise TeradataMlException(message,
-                                              MessageCodes.SANDBOX_SCRIPT_ERROR)
-                # Exit code 2 indicates either username or password is invalid.
-                elif exit_code == 2:
-                    message = Messages.get_message(
-                        MessageCodes.SANDBOX_CONNECTION_ERROR).format(executor_error)
-                    raise TeradataMlException(message,
-                                              MessageCodes.SANDBOX_CONNECTION_ERROR)
-                # Exit code 3 indicates problem with query.
-                elif exit_code == 3:
-                    message = Messages.get_message(
-                        MessageCodes.SANDBOX_QUERY_ERROR).format(executor_error)
-                    raise TeradataMlException(message,
-                                              MessageCodes.SANDBOX_QUERY_ERROR)
-                # Exit code 4 indicates all other exceptions / errors.
-                elif exit_code == 4:
-                    message = Messages.get_message(
-                        MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
-                    raise TeradataMlException(message,
-                                              MessageCodes.SANDBOX_CONTAINER_ERROR)
-                elif exit_code != 0:
-                    # Any error other than exit code 1, 2, 3, 4
-                    message = Messages.get_message(
-                        MessageCodes.SANDBOX_CONTAINER_ERROR).format(executor_error)
-                    raise TeradataMlException(message,
-                                              MessageCodes.SANDBOX_CONTAINER_ERROR)
-                else:
-                    return self.__process_test_script_output(executor_output)
-            except Exception as exp:
-                message = Messages.get_message(
-                    MessageCodes.SANDBOX_CONTAINER_ERROR).format(str(exp))
-                raise TeradataMlException(message,
-                    MessageCodes.SANDBOX_CONTAINER_ERROR)
 
     def __local_run_user_script_input_file(self, cmd, input_file_path,
                                            data_file_delimiter='\t',