teradataml 20.0.0.2__py3-none-any.whl → 20.0.0.4__py3-none-any.whl

teradataml/table_operators/Apply.py

@@ -316,14 +316,6 @@ class Apply(TableOperator):
                                     is_local_order,
                                     sort_ascending,
                                     nulls_first)
-
-        # Set the variable specific to this child class.
-        self.apply_command = apply_command
-        self.env_name = env_name if env_name is not None else get_user_env()
-        self.style = style
-        self.returns = returns
-        self._skip_argument_validation = False
-
         # Create AnalyticsWrapperUtils instance which contains validation functions.
         # This is required for is_default_or_not check.
         # Rest all validation is done using _Validators
@@ -332,20 +324,42 @@ class Apply(TableOperator):
         # Perform argument validation for arguments specific to this class.
         self.__arg_info_matrix = []
 
-        self.__arg_info_matrix.append(["style", self.style, True, (str), True, ['CSV']])
-        self.__arg_info_matrix.append(["env_name", self.env_name, False, (str, UserEnv), True])
-        self.__arg_info_matrix.append(["apply_command", self.apply_command, False, (str), True])
-        self.__arg_info_matrix.append(["returns", self.returns, True, (dict), True])
-
+        self.__arg_info_matrix.append(["style", style, True, (str), True, ['CSV']])
+        self.__arg_info_matrix.append(["env_name", env_name, False, (str, UserEnv), True])
+        self.__arg_info_matrix.append(["apply_command", apply_command, False, (str), True])
+        self.__arg_info_matrix.append(["returns", returns, True, (dict), True])
+        self._skip_argument_validation = False
         # Perform the function argument validations.
         self.__apply__validate()
 
-        self.env = self.env_name if isinstance(self.env_name, UserEnv) else get_env(self.env_name)
+        # If user do not pass environment, get the default environment.
+        if env_name is None:
+            env_name = get_user_env()
+        self._open_af_env = env_name
+
+        # Set the variable specific to this child class.
+        self.apply_command = apply_command
+        self.env_name = env_name if isinstance(env_name, str) else env_name.env_name
+        self.style = style
+        self.returns = returns
+
+
+    @property
+    def env(self):
+        """
+        DESCRIPTION:
+            Getter to get environment.
+
+        RETURNS:
+            bool
+
+        RAISES:
+            None
+        """
+        if isinstance(self._open_af_env, str):
+            self._open_af_env = get_env(self._open_af_env)
 
-        # User can specify object of UserEnv class. Or if environment is already created just pass
-        # remote user environment name as string.
-        if isinstance(self.env_name, UserEnv):
-            self.env_name = self.env_name.env_name
+        return self._open_af_env
 
     @property
     def skip_argument_validation(self):

teradataml/table_operators/Script.py

@@ -1701,7 +1701,9 @@ class Script(TableOperator):
                                                 gc_on_quit=True, quote=False,
                                                 table_type=table_type)
         try:
-            if output_style == OutputStyle.OUTPUT_TABLE.value:
+            if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query, volatile=True)
+            elif output_style == OutputStyle.OUTPUT_TABLE.value:
                 UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
             else:
                 UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)

teradataml/table_operators/TableOperator.py

@@ -458,7 +458,9 @@ class TableOperator:
                                                                           )
 
         try:
-            if output_style == OutputStyle.OUTPUT_TABLE.value:
+            if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query, volatile=True)
+            elif output_style == OutputStyle.OUTPUT_TABLE.value:
                 UtilFuncs._create_table(tblop_stdout_temp_tablename, self._tblop_query)
             else:
                 UtilFuncs._create_view(tblop_stdout_temp_tablename, self._tblop_query)

teradataml/table_operators/query_generator.py

@@ -481,6 +481,9 @@ class QueryGenerator:
             return configure.read_nos_function_mapping.upper()
         elif "WriteNOS".lower() == function_name.lower():
             return configure.write_nos_function_mapping.upper()
+        # If Table Operator function is IMAGE2MATRIX, then return alias name as TD_IMAGE2MATRIX.
+        elif "IMAGE2MATRIX".lower() == function_name.lower():
+            return "TD_IMAGE2MATRIX"
             
         engine_name = UtilFuncs._get_engine_name(self._engine)
 

teradataml/table_operators/table_operator_query_generator.py

@@ -231,7 +231,9 @@ class TableOperatorQueryGenerator(QueryGenerator):
             using_clause = ""
             # If the function is a NOS function, then USING clause is needed.
             if self._function_name.lower() in [configure.write_nos_function_mapping.lower(),
-                                               configure.read_nos_function_mapping.lower()]: 
+                                               configure.read_nos_function_mapping.lower(),
+                                               "td_image2matrix"
+                                               ]: 
                 using_clause = "USING"
             invocation_sql = "{0}\n\t{1}{2}".format(invocation_sql, using_clause, self.__OTHER_ARG_CLAUSE)
 

teradataml/table_operators/table_operator_util.py

@@ -24,6 +24,7 @@ from teradataml.utils.utils import execute_sql
 from teradataml.utils.validators import _Validators
 from functools import partial
 from inspect import isfunction, getsource
+from pathlib import Path
 
 
 class _TableOperatorUtils:
@@ -281,7 +282,8 @@ class _TableOperatorUtils:
         """
         # Validate the user defined function.
 
-        if self.operation == TableOperatorConstants.UDF_OP.value:
+        if self.operation in [TableOperatorConstants.UDF_OP.value,\
+                              TableOperatorConstants.REGISTER_OP.value]:
             for udf_function in self.user_function:
                 if not isfunction(udf_function):
                     raise TypeError(Messages.get_message(
@@ -330,20 +332,30 @@ class _TableOperatorUtils:
         EXAMPLES:
             self.__create_user_script()
         """
-        # Generate script name and alias, and add entry to a Garbage Collector.
-        # script_entry is the string that is added to Garbage collector.
-        # It has the format "<databasename>"."<file_id>".
-        self.script_entry, self.script_alias, self.script_name, self.script_base_name = self.__get_script_name()
-
-        # Get the converters to use with pandas.read_csv, and to correctly
-        # typecast the numeric data.
-        python_input_col_types = [UtilFuncs._teradata_type_to_python_type(col.type)
-                                  for col in self.data._metaexpr.c]
-        input_converters = UtilFuncs._get_pandas_converters(python_input_col_types)
-
-        python_output_col_types = [UtilFuncs._teradata_type_to_python_type(type_)
-                                   for type_ in list(self.returns.values())]
-        output_converters = UtilFuncs._get_pandas_converters(python_output_col_types)
+        # If operation is register, then generate script name based on the
+        # user function name and return type.
+        # It has the format "tdml_udf_name_<registered_name>_udf_type_<return_type>_register.py"
+        if self.operation == TableOperatorConstants.REGISTER_OP.value:
+            registered_name = list(self.returns.keys())[0]
+            return_type = self.returns[registered_name]
+            self.script_name = "tdml_udf_name_{}_udf_type_{}_register.py".format(registered_name, return_type)
+            self.script_base_name = Path(self.script_name).stem
+        else:
+            # Generate script name and alias, and add entry to a Garbage Collector.
+            # script_entry is the string that is added to Garbage collector.
+            # It has the format "<databasename>"."<file_id>".
+            self.script_entry, self.script_alias, self.script_name, self.script_base_name = self.__get_script_name()
+
+        if self.operation not in [TableOperatorConstants.UDF_OP.value, TableOperatorConstants.REGISTER_OP.value]:
+            # Get the converters to use with pandas.read_csv, and to correctly
+            # typecast the numeric data.
+            python_input_col_types = [UtilFuncs._teradata_type_to_python_type(col.type)
+                                    for col in self.data._metaexpr.c]
+            input_converters = UtilFuncs._get_pandas_converters(python_input_col_types)
+
+            python_output_col_types = [UtilFuncs._teradata_type_to_python_type(type_)
+                                    for type_ in list(self.returns.values())]
+            output_converters = UtilFuncs._get_pandas_converters(python_output_col_types)
 
         # Create script in .teradataml directory.
         script_dir = GarbageCollector._get_temp_dir_name()
@@ -357,35 +369,16 @@ class _TableOperatorUtils:
                                             "templates")
         # Get the template.
         template = {TableOperatorConstants.APPLY_OP.value: TableOperatorConstants.APPLY_TEMPLATE.value,
-                    TableOperatorConstants.UDF_OP.value: TableOperatorConstants.UDF_TEMPLATE.value}
+                    TableOperatorConstants.UDF_OP.value: TableOperatorConstants.UDF_TEMPLATE.value,
+                    TableOperatorConstants.REGISTER_OP.value: TableOperatorConstants.REGISTER_TEMPLATE.value                    }
         template_name = template.get(self.operation, TableOperatorConstants.MAP_TEMPLATE.value)
         # Write to the script based on the template.
         try:
             with open(os.path.join(template_dir, template_name), 'r') as input_file:
                 with open(self.script_path, 'w') as output_file:
                     if self.operation == TableOperatorConstants.UDF_OP.value:
-
-                        # Function can have udf as decorator. Remove that.
-                        # The below notation
-                            # @udf
-                            # def to_upper(s):
-                            #     return s.upper()
-                            # Then source code will be as it is.
-                        # But if below notation is used,
-                            # f = udf(to_upper)
-                            # Then source code will not have udf.
-                        # So, remove first line if it comes with first notation.
-                        # For both notations if in starting function defination have any extra space. Remove that.
-                        # If multiple UDF's are there append them as a single string.
                         
-                        user_function_code = ""
-                        for udf_code in self.user_function:
-                            udf_code = getsource(udf_code)
-                            udf_code = udf_code.lstrip()
-                            if udf_code.startswith("@"):
-                                udf_code = udf_code[udf_code.find("\n")+1: ].lstrip()
-                            user_function_code += udf_code + '\n'
-
+                        user_function_code = UtilFuncs._func_to_string(self.user_function)
                         output_file.write(input_file.read().format(
                             DELIMITER=self.delimiter,
                             QUOTECHAR=self.quotechar,
@@ -396,6 +389,13 @@ class _TableOperatorUtils:
                             COLUMNS_DEFINITIONS=json.dumps(self.columns_definitions),
                             OUTPUT_TYPE_CONVERTERS=json.dumps(self.output_type_converters)
                         ))
+                    elif self.operation == TableOperatorConstants.REGISTER_OP.value:
+                        # Get the source code of the user function.
+                        user_function_code = UtilFuncs._func_to_string(self.user_function)
+                        output_file.write(input_file.read().format(
+                            FUNCTION_DEFINITION=user_function_code,
+                            FUNCTION_NAME = self.user_function[0].__name__
+                        ))   
                     else:
                         # prepare script file from template file for maprow and mappartition.
                         output_file.write(
@@ -494,7 +494,6 @@ class _TableOperatorUtils:
         script_name = script_alias  # alias now contains extension also.
 
         # Extract the base name without extension.
-        from pathlib import Path
         script_base_name = Path(script_alias).stem
         return script_entry, script_alias, script_name, script_base_name
 

teradataml/table_operators/templates/dataframe_register.template

@@ -0,0 +1,69 @@
+import json
+import sys, csv
+import datetime
+import urllib.parse
+
+td_buffer = {{}}
+
+
+{FUNCTION_DEFINITION}
+
+# Decode the URL encoded string and store it back as dictionary.
+dec = urllib.parse.unquote_plus(sys.argv[1])
+script_data = json.loads(dec)
+
+# Information that is required to help with the script usage.
+#  The delimiter to use with the input and output text.
+delimiter = script_data["delimiter"]
+#  The quotechar to use.
+quotechar = script_data["qoutechar"]
+#  The names of columns in the input teradataml DataFrame.
+_input_columns = script_data["input_cols"]
+#  The names of columns in the output teradataml DataFrame.
+_output_columns = script_data["output_cols"]
+#  The types of columns in the input/output teradataml DataFrame.
+# The mapper of output column name to function arguments
+function_args = script_data["function_args"]
+#  The definition for new columns in output.
+columns_definitions = {{_output_columns[-1]: "{FUNCTION_NAME}"}}
+output_type_converters = script_data["output_type_converters"]
+for k,v in output_type_converters.items():
+    if v == 'datetime.date' or v == 'datetime.time' or v == 'datetime.datetime':
+        output_type_converters[k] = 'str'
+output_type_converters = {{k:getattr(__builtins__, v) for k,v in output_type_converters.items()}}
+
+
+
+# The entry point to the script.
+if __name__ == "__main__":
+
+    records = csv.reader(sys.stdin.readlines(), delimiter=delimiter, quotechar=quotechar)
+    for record in records:
+        record = dict(zip(_input_columns, record))
+        out_rec = []
+        for column in _output_columns:
+
+            # If it is a new column, get the value from definition.
+            if column in columns_definitions:
+                f_args = tuple()
+                # Convert the argument types first.
+                for v in function_args[column]:
+                    if v in _input_columns:
+                        c_type_ = output_type_converters.get(v)
+                        if record[v]:
+                            # If it is a float, replace the empty character.
+                            if c_type_.__name__ == 'float':
+                                arg = output_type_converters.get(v)(record[v].replace(' ', ''))
+                            else:
+                                arg = output_type_converters.get(v)(record[v])
+                        else:
+                            arg = record[v]
+                    else:
+                        arg = v
+                    f_args = f_args + (arg, )
+                func_ = globals()[columns_definitions[column]]
+                out_rec.append(output_type_converters[column](func_(*f_args)))
+            else:
+                out_rec.append(record[column])
+
+        print("{{}}".format(delimiter).join((str(i) for i in out_rec)))

teradataml/utils/dtypes.py

@@ -293,10 +293,57 @@ class _DtypesMappers:
         VARCHAR: lambda x: "{0},{1}".format(x.__class__.__name__, x.length)
     }
 
+    # Holds mapping between string representation of teradatasqlalchemy type
+    # and actual teradatasqlalchemy type.
+    DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER = {
+        "CHAR": CHAR,
+        "VARCHAR": VARCHAR,
+
+        "BYTEINT": BYTEINT,
+        "SMALLINT": SMALLINT,
+        "INTEGER": INTEGER,
+        "BIGINT": BIGINT,
+
+        "REAL": FLOAT,
+        "FLOAT": FLOAT,
+        "DOUBLE": FLOAT,
+        "DECIMAL": DECIMAL,
+        "NUMBER": NUMBER,
+
+        "DATE": DATE,
+        "TIME": TIME,
+        "TIMESTAMP": TIMESTAMP,
+        "TIMESTAMP_WTZ": TIMESTAMP,
+
+        "BYTE": BYTE,
+        "VARBYTE": VARBYTE,
+        "BLOB": BLOB,
+        # TODO: Add CLOB type when support is added from OTF.
+
+        # TODO: Check these types when corresponding data type support
+        #  is available from OTF support or not.
+        "INTERVAL_YEAR": INTERVAL_YEAR,
+        "INTERVAL_YTM": INTERVAL_YEAR_TO_MONTH,
+        "INTERVAL_MONTH": INTERVAL_MONTH,
+        "INTERVAL_DAY": INTERVAL_DAY,
+
+        "INTERVAL_DTH": INTERVAL_DAY_TO_HOUR,
+        "INTERVAL_DTM": INTERVAL_DAY_TO_MINUTE,
+        "INTERVAL_DTS": INTERVAL_DAY_TO_SECOND,
+        "INTERVAL_HOUR": INTERVAL_HOUR,
+        "INTERVAL_HTM": INTERVAL_HOUR_TO_MINUTE,
+        "INTERVAL_HTS": INTERVAL_HOUR_TO_SECOND,
+        "INTERVAL_MINUTE": INTERVAL_MINUTE,
+        "INTERVAL_MTS": INTERVAL_MINUTE_TO_SECOND,
+        "INTERVAL_SECOND": INTERVAL_SECOND
+    }
+
+
 class _SuppArgTypes:
     VAL_ARG_DATATYPE = (str, BIGINT, BYTEINT, CHAR, DATE, DECIMAL, FLOAT, INTEGER, NUMBER, SMALLINT, TIME,
                         TIMESTAMP, VARCHAR)
 
+
 class _Dtypes:
 
     @staticmethod
@@ -641,11 +688,13 @@ class _Dtypes:
 
         """
         from teradataml.dataframe.dataframe import TDSeries, TDMatrix, TDGenSeries, TDAnalyticResult
+        from teradataml.store.feature_store.feature_store import Feature
         _DtypesMappers.JSON_TD_TO_PYTHON_TYPE_MAPPER.update({"SERIES": TDSeries,
                         "MATRIX": TDMatrix,
                         "ART": TDAnalyticResult,
-                        "GENSERIES": TDGenSeries})
-
+                        "GENSERIES": TDGenSeries,
+                        "COLUMN": (str, Feature),
+                        "COLUMNS": (str, Feature)})
 
         return _DtypesMappers.JSON_TD_TO_PYTHON_TYPE_MAPPER.get(json_td_type.upper())
 

teradataml/utils/internal_buffer.py

@@ -82,3 +82,21 @@ class _InternalBuffer:
         """
         if key in cls.__data:
             return cls.__data.get(key)
+
+    @classmethod
+    def remove_key(cls, key):
+        """
+        DESCRIPTION:
+            Remove a particular key from the internal buffer.
+
+        RETURNS:
+            None
+
+        RAISES:
+            None
+
+        EXAMPLES:
+            # Remove all json objects from _InternalBuffer.
+            _InternalBuffer.remove_key("vs_session_id")
+        """
+        del cls.__data[key]

teradataml/utils/validators.py

@@ -1,3 +1,4 @@
+import enum
 import numbers
 import os
 import pandas as pd
@@ -11,6 +12,8 @@ from teradataml.options.configure import configure
 from teradataml.dataframe.sql_interfaces import ColumnExpression
 from functools import wraps, reduce
 
+from teradataml.utils.internal_buffer import _InternalBuffer
+
 
 def skip_validation():
     """
@@ -283,7 +286,8 @@ class _Validators:
 
     @staticmethod
     @skip_validation()
-    def _validate_dataframe_has_argument_columns(columns, column_arg, data, data_arg, is_partition_arg=False):
+    def _validate_dataframe_has_argument_columns(columns, column_arg, data, data_arg, is_partition_arg=False, 
+                                                 case_insensitive=False):
         """
         Function to check whether column names in columns are present in given dataframe or not.
         This function is used currently only for Analytics wrappers.
@@ -309,12 +313,19 @@ class _Validators:
                 Specifies the name of the dataframe argument.
                 Types: str
 
-            isPartitionArg:
+            is_partition_arg:
                 Optional Argument.
                 Specifies a bool argument notifying, whether argument being validate is
                 Partition argument or not.
                 Types: bool
 
+            case_insensitive:
+                Optional Argument.
+                Specifies a bool argument notifying, whether to check column names
+                in case-insensitive manner or not.
+                Default Value: False
+                Types: bool
+
         RAISES:
             TeradataMlException - TDMLDF_COLUMN_IN_ARG_NOT_FOUND column(s) does not exist in a dataframe.
 
@@ -356,7 +367,7 @@ class _Validators:
                     try:
                         # Check if its a sinlge column with one separator. For e.g. column:A.
                         # If yes, just continue.
-                        _Validators._validate_column_exists_in_dataframe(column, data._metaexpr)
+                        _Validators._validate_column_exists_in_dataframe(column, data._metaexpr, case_insensitive=case_insensitive)
                         continue
                     except:
                         # User has provided range value.
@@ -379,7 +390,8 @@ class _Validators:
                     total_columns.append(column)
 
         return _Validators._validate_column_exists_in_dataframe(total_columns, data._metaexpr, column_arg=column_arg,
-                                                                data_arg=data_arg)
+                                                                data_arg=data_arg, case_insensitive=case_insensitive)
+    
 
     @staticmethod
     @skip_validation()
@@ -545,7 +557,7 @@ class _Validators:
                 raise TypeError("Third element in argument information matrix should be bool.")
 
             if not (isinstance(args[3], tuple) or isinstance(args[3], type) or
-                    isinstance(args[3], (_ListOf, _TupleOf))):
+                    isinstance(args[3], (_ListOf, _TupleOf)) or isinstance(args[3], enum.EnumMeta)):
                 err_msg = "Fourth element in argument information matrix should be a 'tuple of types' or 'type' type."
                 raise TypeError(err_msg)
 
@@ -1395,7 +1407,8 @@ class _Validators:
 
     @staticmethod
     @skip_validation()
-    def _validate_unexpected_column_type(df, col, col_arg, unexpected_types, check_exist=True, raise_error=True):
+    def _validate_unexpected_column_type(df, col, col_arg, unexpected_types, check_exist=True, raise_error=True,
+                                         case_insensitive=False):
         """
         Internal function to validate the column existence and type of an input DataFrame column against
         a list of unexpected types.
@@ -1461,7 +1474,7 @@ class _Validators:
 
             # Check for column existence.
             if check_exist:
-                _Validators._validate_column_exists_in_dataframe(col, df._metaexpr)
+                _Validators._validate_column_exists_in_dataframe(col, df._metaexpr, case_insensitive=case_insensitive)
 
             if isinstance(df[col].type, unexpected_types):
                 if raise_error:
@@ -2274,4 +2287,82 @@ class _Validators:
                 MessageCodes.INVALID_ARG_VALUE).format(ip_address, "ip_address",
                                                        'of four numbers (each between 0 and 255) separated by periods'))
 
-        return True
+        return True
+    
+
+    @staticmethod
+    @skip_validation()
+    def _check_auth_token(func_name):
+        """
+        DESCRIPTION:
+            Check if the user has set the authentication token.
+
+        PARAMETERS:
+            func_name:
+                Required Argument.
+                Specifies the function name where the authentication token is required.
+                Types: str
+
+        RAISES:
+            TeradataMLException
+
+        RETURNS:
+            None.
+
+        EXAMPLES:
+            >>> _Validators._check_auth_token("udf")
+        """
+        if _InternalBuffer.get("auth_token") is None:
+            raise TeradataMlException(Messages.get_message(MessageCodes.SET_REQUIRED_PARAMS,\
+                                                           'Auth Token', func_name,
+                                                           'set_auth_token'),
+                                      MessageCodes.SET_REQUIRED_PARAMS)
+        
+        return True
+
+    @staticmethod
+    def _check_required_params(arg_value, arg_name, caller_func_name, target_func_name):
+        """
+        DESCRIPTION:
+            Check if the required argument is not None.
+
+        PARAMETERS:
+            arg_value:
+                Required Argument.
+                Specifies the argument value to be
+                checked for non None values.
+                Types: str, float, int, bool
+
+            arg_name:
+                Required Argument.
+                Specifies the argument name.
+                Types: str
+
+            caller_func_name:
+                Required Argument.
+                Specifies the function name which calls this function.
+                This is required for the error message.
+                Types: str
+
+            target_func_name:
+                Required Argument.
+                Specifies the function name which the user needs to call
+                so that the error is fixed.
+                This is required for the error message.
+                Types: str
+
+        RAISES:
+            TeradataMLException
+
+        RETURNS:
+            True.
+
+        EXAMPLES:
+            >>> _Validators._check_required_params("udf", "arg_name")
+        """
+        if arg_value is None:
+            raise TeradataMlException(Messages.get_message(MessageCodes.SET_REQUIRED_PARAMS, \
+                                                           arg_name, caller_func_name,
+                                                           target_func_name),
+                                      MessageCodes.SET_REQUIRED_PARAMS)
+        return True