teradataml 20.0.0.2__py3-none-any.whl → 20.0.0.4__py3-none-any.whl

teradataml/dataframe/dataframe_utils.py

@@ -12,13 +12,14 @@ This file implements util functions of data frame.
 """
 
 import numbers
+import re
 import pandas as pd
 from collections import OrderedDict
 
 from teradataml.common.utils import UtilFuncs
 from teradataml.common.aed_utils import AedUtils
 from teradataml.common.constants import AEDConstants, PTITableConstants, \
-    SQLPattern, PythonTypes
+    SQLPattern, PythonTypes, TeradataConstants, SQLConstants
 from teradataml.common.sqlbundle import SQLBundle
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
@@ -30,6 +31,7 @@ from teradataml.dbutils.dbutils import _execute_query_and_generate_pandas_df
 
 from teradataml.options.display import display
 from teradataml.options.configure import configure
+from teradataml.utils.dtypes import _DtypesMappers
 from teradataml.utils.utils import execute_sql
 
 from teradatasqlalchemy.types import FLOAT, NUMBER, DECIMAL, PERIOD_TIMESTAMP
@@ -77,7 +79,10 @@ class DataFrameUtils():
                     is_persist = True
 
                 try:
-                    if node_query_types[index] == AEDConstants.AED_QUERY_NODE_TYPE_ML_QUERY_MULTI_OUTPUT.value or\
+                    if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                        UtilFuncs._create_table(view_names[index], queries[index], volatile=True)
+
+                    elif node_query_types[index] == AEDConstants.AED_QUERY_NODE_TYPE_ML_QUERY_MULTI_OUTPUT.value or\
                        ("OUT TABLE " in queries[index] and SQLPattern.SQLMR.value.match(queries[index])) or \
                             is_persist:
                         # TODO:: OR condition in above needs to be removed once AED support is added.
@@ -1291,43 +1296,32 @@ class DataFrameUtils():
 
         aggregate_expr = ", ".join(select_columns)
         return aggregate_expr, new_column_names, new_column_types
-
+    
     @staticmethod
-    def _invalid_describe_column(df, columns, metaexpr, groupby_column_list):
+    def _validate_describe_columns(columns, metaexpr, groupby_column_list):
         """
-        Internal function to validate columns provided to describe() is correct or not,
+        Internal function to validate columns provided to describe() are correct or not,
         when DataFrame is output of groupby and groupby_time.
-
         PARAMETERS:
-            df:
-                Required Argument.
-                Specifies teradataml DataFrame we are collecting statistics for.
-                Types: str
-
             columns:
                 Optional Argument.
                 Specifies the name(s) of columns we are collecting statistics for.
                 Types: str ot List of strings (str)
-
             metaexpr:
                 Required Argument.
                 Specifies the meta expression for the dataframe.
                 Types: _MetaExpression
-
             groupby_column_list:
                 Optional Argument.
                 Specifies the group by columns for the dataframe.
                 Default Values: None.
                 Types: str ot List of strings (str)
-
         Returns:
             None
-
         Raises:
             TeradataMLException
         """
-        invalid_columns = [_column for _column in groupby_column_list if columns is not None
-                           and _column in columns]
+        invalid_columns = [_column for _column in groupby_column_list if _column in columns]
         if len(invalid_columns) > 0:
             all_columns = [col.name for col in metaexpr.c]
             valid_columns = [item for item in all_columns if item not in groupby_column_list]
@@ -1849,7 +1843,10 @@ class DataFrameUtils():
             db_schema = UtilFuncs._extract_db_name(tab_name_first)
             db_table_name = UtilFuncs._extract_table_name(tab_name_first)
 
-            return DataFrame(in_schema(db_schema, db_table_name))
+            if db_schema:
+                return DataFrame(in_schema(db_schema, db_table_name))
+
+            return DataFrame(db_table_name)
 
         pids_first = None
         parent_df = None
@@ -1865,11 +1862,146 @@ class DataFrameUtils():
                 db_schema = UtilFuncs._extract_db_name(tab_name_first)
                 db_table_name = UtilFuncs._extract_table_name(tab_name_first)
 
-                parent_df = DataFrame(in_schema(db_schema, db_table_name))
+                if db_schema:
+                    parent_df = DataFrame(in_schema(db_schema, db_table_name))
+                else:
+                    parent_df = DataFrame(db_table_name)
                 pids_first = pids
             else:
                 if pids_first != pids:
                     raise TeradataMlException(Messages.get_message(MessageCodes.DFS_NO_COMMON_PARENT),
                                             MessageCodes.DFS_NO_COMMON_PARENT)
 
-        return parent_df
+        return parent_df
+
+    @staticmethod
+    def _get_sqlalchemy_type_from_str(td_type):
+        """
+        Function to get teradatasqlalchemy type from string representation of that type.
+
+        PARAMETERS:
+            td_type:
+                Required Argument.
+                Specifies string representation of teradatasqlalchemy type.
+                Types: str
+
+        RAISES:
+            ValueError
+
+        EXAMPLES:
+            >>> dt = DataFrameUtils._get_sqlalchemy_type_from_str("DECIMAL(4,4)")
+            >>> dt
+            DECIMAL(precision=4, scale=4)
+            >>> type(dt)
+            teradatasqlalchemy.types.DECIMAL
+
+            >>> dt = DataFrameUtils._get_sqlalchemy_type_from_str("VARCHAR(32000) CHARACTER SET UNICODE")
+            >>> dt
+            VARCHAR(length=32000, charset='UNICODE')
+            >>> type(dt)
+            teradatasqlalchemy.types.VARCHAR
+        """
+        # 4 groups of pattern:
+        # 1. Type name
+        # 2. Comma separated parameters enclosed in parentheses
+        # 3. Comma separated parameters without parenthesis 
+        # 4. Remaining string
+        pattern = "([A-Z0-9_]+)(\((.*)\))?(.*)"
+
+        m = re.match(pattern, td_type)
+        td_str_type = m.group(1)
+        td_str_params = m.group(3)
+        td_str_remain = m.group(4)
+
+        if m is None or td_str_type not in _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER.keys():
+            raise ValueError("Invalid Teradata type: {} from datalake".format(td_type))
+
+        if td_str_type in ["VARCHAR", "CHAR"]:
+            # If VARCHAR or CHAR, extract, length and charset from string.
+            length = int(td_str_params.split(",")[0])
+            charset = td_str_remain.strip().split(" ")[2]
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (length=length, charset=charset)
+        
+        if td_str_type in ["BLOB"]:
+            # Ignoring the charset as BLOB does not have it.
+            # If BLOB, extract length from string.
+            length = int(td_str_params.split(",")[0])
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (length=length)
+
+        if td_str_type in ["DECIMAL"]:
+            # If DECIMAL, extract precision and scale from string.
+            args = td_str_params.split(",")
+            return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]\
+                (precision=int(args[0]), scale=int(args[1]))
+
+        # TODO: Test for other data types once OTF team finalize all data types.
+        return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]()
+
+    @staticmethod
+    def _get_datalake_table_columns_info(schema, table_name, datalake):
+        """
+        Function to get column names and corresponding teradatasqlalchemy types
+        of a datalake table using results of 'help table <datalake>.<db_name>.<table_name>'
+        SQL query.
+
+        PARAMETERS:
+            schema:
+                Required Argument.
+                Specifies name of schema.
+                Types: str
+
+            table_name:
+                Required Argument.
+                Specifies name of table.
+                Types: str
+
+            datalake:
+                Required Argument.
+                Specifies name of datalake.
+                Types: str
+
+        RAISES:
+            TeradataMlException
+
+        EXAMPLES:
+            >>> DataFrameUtils._get_datalake_table_columns_info(table_name = 'sales',
+            ...                                                 schema='otftestdb',
+            ...                                                 datalake='datalake_iceberg_glue')
+            (['id', 'masters', 'gpa', 'stats', 'programming', 'admitted'],
+             [INTEGER(),
+              VARCHAR(length=2000, charset='UNICODE'),
+              FLOAT(),
+              VARCHAR(length=2000, charset='UNICODE'),
+              VARCHAR(length=2000, charset='UNICODE'),
+              INTEGER()])
+        """
+        # Get the column information from the strings type.
+        prepared = preparer(td_dialect())
+        sqlbundle = SQLBundle()
+        full_tbl_name = '{}.{}.{}'.format(prepared.quote(datalake),
+                                          prepared.quote(schema),
+                                          prepared.quote(table_name))
+        help_table_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_TABLE).format(full_tbl_name)
+
+        cur = execute_sql(help_table_sql)
+        td_types_col_index = -1
+        for i, col_metadata in enumerate(cur.description):
+            # Help Table returns column names and
+            # corresponding IcebergType, TeradataInternalType,
+            # TeradataType. We need to extract column index for
+            # 'TeradataType' column.
+            if col_metadata[0].lower() == 'teradatatype':
+                td_types_col_index = i
+
+        col_names = []
+        col_types = []
+        if td_types_col_index > -1:
+            for col_info in cur.fetchall():
+                col_names.append(col_info[0])
+                col_types.append(DataFrameUtils._get_sqlalchemy_type_from_str(col_info[td_types_col_index]))
+        else:
+            raise TeradataMlException(Messages.get_message(MessageCodes.TDMLDF_CREATE_FAIL),
+                                      MessageCodes.TDMLDF_CREATE_FAIL)
+        return col_names, col_types