teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.5__py3-none-any.whl

teradataml/dataframe/dataframe_utils.py

@@ -12,13 +12,14 @@ This file implements util functions of data frame.
 """
 
 import numbers
+import re
 import pandas as pd
 from collections import OrderedDict
 
 from teradataml.common.utils import UtilFuncs
 from teradataml.common.aed_utils import AedUtils
 from teradataml.common.constants import AEDConstants, PTITableConstants, \
-    SQLPattern, PythonTypes
+    SQLPattern, PythonTypes, TeradataConstants, SQLConstants
 from teradataml.common.sqlbundle import SQLBundle
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
@@ -30,6 +31,7 @@ from teradataml.dbutils.dbutils import _execute_query_and_generate_pandas_df
 
 from teradataml.options.display import display
 from teradataml.options.configure import configure
+from teradataml.utils.dtypes import _DtypesMappers
 from teradataml.utils.utils import execute_sql
 
 from teradatasqlalchemy.types import FLOAT, NUMBER, DECIMAL, PERIOD_TIMESTAMP
@@ -77,7 +79,10 @@ class DataFrameUtils():
                     is_persist = True
 
                 try:
-                    if node_query_types[index] == AEDConstants.AED_QUERY_NODE_TYPE_ML_QUERY_MULTI_OUTPUT.value or\
+                    if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                        UtilFuncs._create_table(view_names[index], queries[index], volatile=True)
+
+                    elif node_query_types[index] == AEDConstants.AED_QUERY_NODE_TYPE_ML_QUERY_MULTI_OUTPUT.value or\
                        ("OUT TABLE " in queries[index] and SQLPattern.SQLMR.value.match(queries[index])) or \
                             is_persist:
                         # TODO:: OR condition in above needs to be removed once AED support is added.
@@ -647,7 +652,7 @@ class DataFrameUtils():
         all_operations = list(set(all_operations))
         invalid_aggregates = []
         for operation in all_operations:
-            if operation not in valid_aggregate_operations \
+            if operation not in valid_aggregate_operations and not operation.startswith('percentile_') \
                     and operation not in UtilFuncs._get_valid_time_series_aggregate_operations():
                 invalid_aggregates.append(operation)
         if len(invalid_aggregates) > 0: # If any of the aggregate operations specified is not valid
@@ -730,7 +735,20 @@ class DataFrameUtils():
                 quoted_columns = UtilFuncs._process_for_teradata_keyword(kwargs[key_to_process])
                 kwargs[key_to_process] = quoted_columns
 
-            func_expression = getattr(df[column], operation)(describe_op=describe_op, **kwargs)
+            if operation.startswith('percentile_'):
+                try:
+                    _operation_value = operation.split('_')
+                    _floatvalue = float(_operation_value[1])
+                    if _floatvalue < 0.0 or _floatvalue > 1.0 or len(_operation_value)>2:
+                        raise ValueError
+                except ValueError:
+                    mssg = "Invalid aggregate operation '{}' requested on TeradataML DataFrame." \
+                           " Valid operation should be in format 'percentile_<floatvalue>' and <floatvalue> " \
+                           "should be in range [0.0, 1.0].".format(operation)
+                    raise ValueError(mssg) from None
+                func_expression = getattr(df[column], 'percentile')(percentile=_floatvalue)
+            else:
+                func_expression = getattr(df[column], operation)(describe_op=describe_op, **kwargs)
             new_column_name = column if describe_op else "{1}_{0}".format(column, operation)
             # column_supported, new_column_name, new_column_type, column_aggr_expr, invalid_column_str
             return True, new_column_name, NUMBER() if describe_op else func_expression.type, \
@@ -1291,43 +1309,32 @@ class DataFrameUtils():
 
         aggregate_expr = ", ".join(select_columns)
         return aggregate_expr, new_column_names, new_column_types
-
+    
     @staticmethod
-    def _invalid_describe_column(df, columns, metaexpr, groupby_column_list):
+    def _validate_describe_columns(columns, metaexpr, groupby_column_list):
         """
-        Internal function to validate columns provided to describe() is correct or not,
+        Internal function to validate columns provided to describe() are correct or not,
         when DataFrame is output of groupby and groupby_time.
-
         PARAMETERS:
-            df:
-                Required Argument.
-                Specifies teradataml DataFrame we are collecting statistics for.
-                Types: str
-
             columns:
                 Optional Argument.
                 Specifies the name(s) of columns we are collecting statistics for.
                 Types: str ot List of strings (str)
-
             metaexpr:
                 Required Argument.
                 Specifies the meta expression for the dataframe.
                 Types: _MetaExpression
-
             groupby_column_list:
                 Optional Argument.
                 Specifies the group by columns for the dataframe.
                 Default Values: None.
                 Types: str ot List of strings (str)
-
         Returns:
             None
-
         Raises:
             TeradataMLException
         """
-        invalid_columns = [_column for _column in groupby_column_list if columns is not None
-                           and _column in columns]
+        invalid_columns = [_column for _column in groupby_column_list if _column in columns]
         if len(invalid_columns) > 0:
             all_columns = [col.name for col in metaexpr.c]
             valid_columns = [item for item in all_columns if item not in groupby_column_list]
@@ -1849,7 +1856,10 @@ class DataFrameUtils():
             db_schema = UtilFuncs._extract_db_name(tab_name_first)
             db_table_name = UtilFuncs._extract_table_name(tab_name_first)
 
-            return DataFrame(in_schema(db_schema, db_table_name))
+            if db_schema:
+                return DataFrame(in_schema(db_schema, db_table_name))
+
+            return DataFrame(db_table_name)
 
         pids_first = None
         parent_df = None
@@ -1865,11 +1875,146 @@ class DataFrameUtils():
                 db_schema = UtilFuncs._extract_db_name(tab_name_first)
                 db_table_name = UtilFuncs._extract_table_name(tab_name_first)
 
-                parent_df = DataFrame(in_schema(db_schema, db_table_name))
+                if db_schema:
+                    parent_df = DataFrame(in_schema(db_schema, db_table_name))
+                else:
+                    parent_df = DataFrame(db_table_name)
                 pids_first = pids
             else:
                 if pids_first != pids:
                     raise TeradataMlException(Messages.get_message(MessageCodes.DFS_NO_COMMON_PARENT),
                                             MessageCodes.DFS_NO_COMMON_PARENT)
 
-        return parent_df
+        return parent_df
+
+    @staticmethod
+    def _get_sqlalchemy_type_from_str(td_type):
+        """
+        Function to get teradatasqlalchemy type from string representation of that type.
+
+        PARAMETERS:
+            td_type:
+                Required Argument.
+                Specifies string representation of teradatasqlalchemy type.
+                Types: str
+
+        RAISES:
+            ValueError
+
+        EXAMPLES:
+            >>> dt = DataFrameUtils._get_sqlalchemy_type_from_str("DECIMAL(4,4)")
+            >>> dt
+            DECIMAL(precision=4, scale=4)
+            >>> type(dt)
+            teradatasqlalchemy.types.DECIMAL
+
+            >>> dt = DataFrameUtils._get_sqlalchemy_type_from_str("VARCHAR(32000) CHARACTER SET UNICODE")
+            >>> dt
+            VARCHAR(length=32000, charset='UNICODE')
+            >>> type(dt)
+            teradatasqlalchemy.types.VARCHAR
+        """
+        # 4 groups of pattern:
+        # 1. Type name
+        # 2. Comma separated parameters enclosed in parentheses
+        # 3. Comma separated parameters without parenthesis 
+        # 4. Remaining string
+        pattern = "([A-Z0-9_]+)(\((.*)\))?(.*)"
+
+        m = re.match(pattern, td_type)
+        td_str_type = m.group(1)
+        td_str_params = m.group(3)
+        td_str_remain = m.group(4)
+
+        if m is None or td_str_type not in _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER.keys():
+            raise ValueError("Invalid Teradata type: {} from datalake".format(td_type))
+
+        if td_str_type in ["VARCHAR", "CHAR"]:
+            # If VARCHAR or CHAR, extract, length and charset from string.
+            length = int(td_str_params.split(",")[0])
+            charset = td_str_remain.strip().split(" ")[2]
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (length=length, charset=charset)
+        
+        if td_str_type in ["BLOB"]:
+            # Ignoring the charset as BLOB does not have it.
+            # If BLOB, extract length from string.
+            length = int(td_str_params.split(",")[0])
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (length=length)
+
+        if td_str_type in ["DECIMAL"]:
+            # If DECIMAL, extract precision and scale from string.
+            args = td_str_params.split(",")
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (precision=int(args[0]), scale=int(args[1]))
+
+        # TODO: Test for other data types once OTF team finalize all data types.
+        return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]()
+
+    @staticmethod
+    def _get_datalake_table_columns_info(schema, table_name, datalake):
+        """
+        Function to get column names and corresponding teradatasqlalchemy types
+        of a datalake table using results of 'help table <datalake>.<db_name>.<table_name>'
+        SQL query.
+
+        PARAMETERS:
+            schema:
+                Required Argument.
+                Specifies name of schema.
+                Types: str
+
+            table_name:
+                Required Argument.
+                Specifies name of table.
+                Types: str
+
+            datalake:
+                Required Argument.
+                Specifies name of datalake.
+                Types: str
+
+        RAISES:
+            TeradataMlException
+
+        EXAMPLES:
+            >>> DataFrameUtils._get_datalake_table_columns_info(table_name = 'sales',
+            ...                                                 schema='otftestdb',
+            ...                                                 datalake='datalake_iceberg_glue')
+            (['id', 'masters', 'gpa', 'stats', 'programming', 'admitted'],
+             [INTEGER(),
+              VARCHAR(length=2000, charset='UNICODE'),
+              FLOAT(),
+              VARCHAR(length=2000, charset='UNICODE'),
+              VARCHAR(length=2000, charset='UNICODE'),
+              INTEGER()])
+        """
+        # Get the column information from the strings type.
+        prepared = preparer(td_dialect())
+        sqlbundle = SQLBundle()
+        full_tbl_name = '{}.{}.{}'.format(prepared.quote(datalake),
+                                          prepared.quote(schema),
+                                          prepared.quote(table_name))
+        help_table_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_TABLE).format(full_tbl_name)
+
+        cur = execute_sql(help_table_sql)
+        td_types_col_index = -1
+        for i, col_metadata in enumerate(cur.description):
+            # Help Table returns column names and
+            # corresponding IcebergType, TeradataInternalType,
+            # TeradataType. We need to extract column index for
+            # 'TeradataType' column.
+            if col_metadata[0].lower() == 'teradatatype':
+                td_types_col_index = i
+
+        col_names = []
+        col_types = []
+        if td_types_col_index > -1:
+            for col_info in cur.fetchall():
+                col_names.append(col_info[0])
+                col_types.append(DataFrameUtils._get_sqlalchemy_type_from_str(col_info[td_types_col_index]))
+        else:
+            raise TeradataMlException(Messages.get_message(MessageCodes.TDMLDF_CREATE_FAIL),
+                                      MessageCodes.TDMLDF_CREATE_FAIL)
+        return col_names, col_types

teradataml/dataframe/functions.py

@@ -1,28 +1,36 @@
 import pandas as pd
 from inspect import getsource
 import re
-from types import FunctionType
+from teradataml.dataframe.copy_to import copy_to_sql
+from teradataml.dataframe.dataframe import DataFrame
 from teradataml.dbutils.filemgr import install_file, list_files, remove_file
-from teradataml.options.configure import configure
+from teradataml.utils.utils import execute_sql
 import teradatasqlalchemy as tdsqlalchemy
 from teradataml.utils.validators import _Validators
 from teradataml.dataframe.sql import _SQLColumnExpression
 from teradatasqlalchemy import VARCHAR, CLOB, CHAR
-from teradataml.common.constants import TeradataTypes
+from teradataml.common.constants import TableOperatorConstants, TeradataConstants, TeradataTypes
 from teradataml.common.utils import UtilFuncs
-from teradataml.utils.dtypes import _Dtypes
 from teradataml.dataframe.sql_interfaces import ColumnExpression
 from teradataml.table_operators.table_operator_util import _TableOperatorUtils
-from teradataml.utils.internal_buffer import _InternalBuffer
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
 from teradataml.common.messagecodes import MessageCodes
 from teradataml.scriptmgmt.lls_utils import get_env
 
-def udf(user_function=None, returns=VARCHAR(1024), env_name = None, delimiter=',', quotechar=None):
+def udf(user_function=None, returns=VARCHAR(1024), env_name = None, delimiter=',', quotechar=None, debug=False):
     """
     DESCRIPTION:
         Creates a user defined function (UDF).
+        
+        Notes: 
+            1. Date and time data types must be formatted to supported formats.
+               (See Prerequisite Input and Output Structures in Open Analytics Framework for more details.)
+            2. Packages required to run the user defined function must be installed in remote user 
+               environment using install_lib method of UserEnv class. Import statements of these
+               packages should be inside the user defined function itself.
+            3. Do not call a regular function defined outside the udf() from the user defined function.
+               The function definition and call must be inside the udf(). Look at Example 9 to understand more.
 
     PARAMETERS:
         user_function:
@@ -31,7 +39,7 @@ def udf(user_function=None, returns=VARCHAR(1024), env_name = None, delimiter=',
             teradataml DataFrame.
             Types: function
             Note:
-                1. Lambda Function are not supported.
+                Lambda functions are not supported. Re-write the lambda function as regular Python function to use with UDF.
 
         returns:
             Optional Argument.
@@ -76,21 +84,21 @@ def udf(user_function=None, returns=VARCHAR(1024), env_name = None, delimiter=',
                 * This argument cannot be same as "delimiter" argument.
                 * This argument cannot be a newline character.
 
+        debug:
+            Optional Argument.
+            Specifies whether to display the script file path generated during function execution or not. This
+            argument helps in debugging when there are any failures during function execution. When set
+            to True, function displays the path of the script and does not remove the file from local file system.
+            Otherwise, file is removed from the local file system.
+            Default Value: False
+            Types: bool
+
     RETURNS:
         ColumnExpression
 
     RAISES:
         TeradataMLException
 
-    NOTES: 
-        1. While working on date and time data types one must format these to supported formats.
-           (See Requisite Input and Output Structures in Open Analytics Framework for more details.)
-        2. Required packages to run the user defined function must be installed in remote user 
-           environment using install_lib function Of UserEnv class. Import statements of these
-           packages should be inside the user defined function itself.
-        3. One can't call a regular function defined outside the udf from the user defined function.
-           The function definition and call must be inside the udf. Look at Example 9 to understand more.
-
     EXAMPLES:
         # Load the data to run the example.
         >>> load_example_data("dataframe", "sales")
@@ -324,14 +332,14 @@ def udf(user_function=None, returns=VARCHAR(1024), env_name = None, delimiter=',
         def wrapper(f):
             def func_(*args):
                 return _SQLColumnExpression(expression=None, udf=f, udf_type=returns, udf_args=args,\
-                                            env_name=env_name, delimiter=delimiter, quotechar=quotechar)
+                                            env_name=env_name, delimiter=delimiter, quotechar=quotechar, debug=debug)
             return func_
         return wrapper
     # Notation: @udf
     else:
         def func_(*args):
             return _SQLColumnExpression(expression=None, udf=user_function, udf_type=returns, udf_args=args,\
-                                        env_name=env_name, delimiter=delimiter, quotechar=quotechar)
+                                        env_name=env_name, delimiter=delimiter, quotechar=quotechar, debug=debug)
     return func_
 
 
@@ -340,6 +348,12 @@ def register(name, user_function, returns=VARCHAR(1024)):
     DESCRIPTION:
         Registers a user defined function (UDF).
 
+        Notes: 
+            1. Date and time data types must be formatted to supported formats.
+               (See Requisite Input and Output Structures in Open Analytics Framework for more details.)
+            2. On VantageCloud Lake, user defined function is registered by default in the 'openml_env' environment.
+               User can register it in their own user environment, using the 'openml_user_env' configuration option.
+
     PARAMETERS:
         name:
             Required Argument.
@@ -351,6 +365,8 @@ def register(name, user_function, returns=VARCHAR(1024)):
             Specifies the user defined function to create a column for
             teradataml DataFrame.
             Types: function, udf
+            Note:
+                Lambda functions are not supported. Re-write the lambda function as regular Python function to use with UDF.
 
         returns:
             Optional Argument.
@@ -459,10 +475,17 @@ def call_udf(udf_name, func_args = () , **kwargs):
     DESCRIPTION:
         Call a registered user defined function (UDF).
 
+        Notes: 
+            1. Packages required to run the registered user defined function must be installed in remote user 
+               environment using install_lib method of UserEnv class. Import statements of these
+               packages should be inside the user defined function itself.
+            2. On VantageCloud Lake, user defined function runs by default in the 'openml_env' environment.
+               User can use their own user environment, using the 'openml_user_env' configuration option.
+
     PARAMETERS:
         udf_name:
             Required Argument.
-            Specifies the name of the registered user defined.
+            Specifies the name of the registered user defined function.
             Types: str
 
         func_args:
@@ -864,4 +887,96 @@ def _create_return_type(returns):
         return_str = str(returns)
     # Replace the space with underscore in the return type.
     return_str = return_str.replace(" ", "_")
-    return return_str
+    return return_str
+
+def td_range(start, end=None, step=1):
+    """
+    DESCRIPTION:
+        Creates a DataFrame with a specified range of numbers.
+
+        Notes: 
+            1. The range is inclusive of the start and exclusive of the end.
+            2. If only start is provided, then end is set to start and start is set to 0.
+        
+    PARAMETERS:
+        start:
+            Required Argument.
+            Specifies the starting number of the range.
+            Types: int
+
+        end:
+            Optional Argument.
+            Specifies the end number of the range(exclusive).
+            Default Value: None
+            Types: int
+
+        step:
+            Optional Argument.
+            Specifies the step size of the range.
+            Default Value: 1
+            Types: int
+
+    RETURNS:
+        teradataml DataFrame
+
+    RAISES:
+        TeradataMlException
+
+    EXAMPLES:
+            # Example 1: Create a DataFrame with a range of numbers from 0 to 5.
+            >>> from teradataml.dataframe.functions import td_range
+            >>> df = td_range(5)
+            >>> df.sort('id')
+               id
+            0   0
+            1   1
+            2   2
+            3   3
+            4   4
+
+            # Example 2: Create a DataFrame with a range of numbers from 5 to 1 with step size of -2.
+            >>> from teradataml.dataframe.functions import td_range
+            >>> td_range(5, 1, -2)
+               id
+            0   3
+            1   5
+
+            >>> Example 3: Create a DataFrame with a range of numbers from 1 to 5 with default step size of 1.
+            >>> from teradataml.dataframe.functions import td_range
+            >>> td_range(1, 5)
+               id
+            0   3
+            1   4
+            2   2
+            3   1
+        
+    """
+    # Validate the arguments.
+    arg_matrix = []
+    arg_matrix.append(["start", start, False, int])
+    arg_matrix.append(["end", end, True, int])
+    arg_matrix.append(["step", step, True, int])
+    _Validators._validate_function_arguments(arg_matrix)
+
+    # If only start is provided, then set end to start and start to 0.
+    if end is None:
+        end = start
+        start = 0
+
+    # If start is greater than end, then set the operation to "-" and operator to ">".
+    # If end is less than start, then set the operation to "+" and operator to "<".
+    if end < start:
+        operation, operator, step = "-", ">", -step
+    else:
+        operation, operator = "+", "<"
+
+    # Create a temporary table with the start value.
+    table_name = UtilFuncs._generate_temp_table_name(prefix="tdml_range_df",
+                                    table_type=TeradataConstants.TERADATA_TABLE)
+    execute_sql(f"CREATE MULTISET TABLE {table_name} AS (SELECT {start} AS id) WITH DATA;")
+    
+    # Create a DataFrame from the range query.
+    range_query = TableOperatorConstants.RANGE_QUERY.value \
+                        .format(table_name, step, end, operation, operator)
+    df = DataFrame.from_query(range_query)
+    return df

teradataml/dataframe/setop.py

@@ -149,7 +149,7 @@ def __check_concat_compatibility(df_list, join, sort, ignore_index):
     # Iterate on all DFs to be applied for set operation.
     for df in dfs_to_operate_on:
         # Process each column in the DF of the iteration.
-        for c in df._metaexpr.t.c:
+        for c in df._metaexpr.c:
             col_name = c.name
             # Process the column name if it is not already processed.
             # Processing of set operation is column name based so if the DF in the nth iteration had column 'xyz',
@@ -193,6 +193,8 @@ def __check_concat_compatibility(df_list, join, sort, ignore_index):
                         col_dict[col_name]['col_present'] = col_present_in_dfs
                         # The type to be used for the column is the one of the first DF it is present in.
                         col_dict[col_name]['col_type'] = col_types_in_dfs[0]
+                        # Column name stored with quotes if required.
+                        col_dict[col_name]['name'] = c.compile()
 
                         # If the type of the column in all DFs is not the same, then the operation is not lazy.
                         if not all(ctype == col_dict[col_name]['col_type']
@@ -217,6 +219,8 @@ def __check_concat_compatibility(df_list, join, sort, ignore_index):
                         col_dict[col_name]['col_present'] = col_present_in_dfs
                         # The type to be used for the column is the one of the first DF it is present in.
                         col_dict[col_name]['col_type'] = non_none_type_to_add
+                        # Column name stored with quotes if required.
+                        col_dict[col_name]['name'] = c.compile()
     
                         # If the type of the column in all DFs is not the same, then the operation is not lazy.
                         if not all(True if ctype is None else ctype == non_none_type_to_add
@@ -667,15 +671,16 @@ def concat(df_list, join='OUTER', allow_duplicates=True, sort=False, ignore_inde
 
         # Now create the list of columns for each DataFrame to concatenate
         type_compiler = td_type_compiler(td_dialect)
+
         for col_name, value in master_columns_dict.items():
             for i in range(len(col_list)):
+                # Quoting is already done for column names if column name starts with number or it is reserved keywords.
+                # Here checking again if it is teradata keyword or not for quotes.
+                column_name = UtilFuncs._process_for_teradata_keyword(value['name'])
                 if not value['col_present'][i]:
-                    col_list[i].append('CAST(NULL as {}) as {}'.format(type_compiler.process(value['col_type']),
-                                                                       UtilFuncs._teradata_quote_arg(col_name, "\"",
-                                                                                                     False)))
+                    col_list[i].append('CAST(NULL as {}) as {}'.format(type_compiler.process(value['col_type']), column_name))
                 else:
-                    col_name = UtilFuncs._process_for_teradata_keyword(col_name)
-                    col_list[i].append(col_name)
+                    col_list[i].append(column_name)
 
         input_table_columns = []
         for i in range(len(col_list)):