teradataml 20.0.0.3__py3-none-any.whl → 20.0.0.4__py3-none-any.whl

teradataml/dataframe/dataframe.py

@@ -20,6 +20,9 @@ import re
 import sqlalchemy
 import sys
 import urllib.parse
+
+from sqlalchemy import Column
+
 import teradataml.context.context as tdmlctx
 
 from collections import OrderedDict, namedtuple
@@ -31,6 +34,7 @@ from teradataml.dataframe.sql_interfaces import ColumnExpression
 from teradataml.dataframe.sql_functions import case
 from teradataml.series.series import Series
 from teradatasqlalchemy.types import _TDType, BIGINT, INTEGER, PERIOD_TIMESTAMP, SMALLINT, BYTEINT, FLOAT, DECIMAL
+from teradataml.common.deprecations import argument_deprecation
 from teradataml.common.utils import UtilFuncs
 from teradataml.common.exceptions import TeradataMlException
 from teradataml.common.messages import Messages
@@ -42,6 +46,7 @@ from teradataml.dataframe.dataframe_utils import DataFrameUtils as df_utils, Dat
 from teradataml.dataframe.indexer import _LocationIndexer
 from teradataml.common.aed_utils import AedUtils
 from teradataml.options.display import display
+from teradataml.options.configure import configure
 from teradataml.dataframe.copy_to import copy_to_sql
 from teradataml.dataframe.row import _Row
 from teradataml.dataframe.setop import concat
@@ -63,7 +68,79 @@ from teradataml.common.constants import OutputStyle
 
 # TODO use logger when available on master branch
 # logger = teradatapylog.getLogger()
-in_schema = UtilFuncs._in_schema
+
+class in_schema:
+    """
+    Class takes a schema name, a table name and datalake name attributes
+    and creates an object that can be passed to DataFrame.
+    Note:
+        teradataml recommends to use this class to access table(s)/view(s),
+        from the database other than the default database.
+    """
+    def __init__(self, schema_name, table_name, datalake_name=None):
+        """
+        Constructor for in_schema class.
+
+        PARAMETERS:
+            schema_name:
+                Required Argument.
+                Specifies the schema where the table resides in.
+                Types: str
+
+            table_name:
+                Required Argument.
+                Specifies the table name or view name in Vantage.
+                Types: str
+
+            datalake_name:
+                Optional Argument.
+                Specifies the datalake name.
+                Types: str
+
+        EXAMPLES:
+            from teradataml.dataframe.dataframe import in_schema, DataFrame
+
+            # Example 1: The following example creates a DataFrame from the
+            #            existing Vantage table "dbcinfo" in the non-default
+            #            database "dbc" using the in_schema instance.
+            df = DataFrame(in_schema("dbc", "dbcinfo"))
+
+            # Example 2: The following example uses from_table() function, existing
+            #            Vantage table "dbcinfo" and non-default database "dbc" to
+            #            create a teradataml DataFrame.
+            df = DataFrame.from_table(in_schema("dbc","dbcinfo"))
+
+            # Example 3: The following example uses "in_schema" object created
+            #            with "datalake_name" argument to create DataFrame on OTF table.
+            otf_df = DataFrame(in_schema("datalake_db","datalake_table","datalake"))
+
+        """
+        self.schema_name = schema_name
+        self.table_name = table_name
+        self.datalake_name = datalake_name
+
+        awu_matrix = []
+        awu_matrix.append(["schema_name", schema_name, False, (str), True])
+        awu_matrix.append(["table_name", table_name, False, (str), True])
+        awu_matrix.append(["datalake_name", datalake_name, True, (str), True])
+
+        # Validate argument types
+        _Validators._validate_function_arguments(awu_matrix)
+
+    def __str__(self):
+        """
+        Returns the string representation of in_schema instance.
+        """
+        tbl_name = '{}.{}'.format(UtilFuncs._teradata_quote_arg(self.schema_name, "\"", False),
+                                  UtilFuncs._teradata_quote_arg(self.table_name, "\"", False))
+
+        if not self.datalake_name:
+            return tbl_name
+
+        return '{}.{}'.format(UtilFuncs._teradata_quote_arg(self.datalake_name, "\"", False), tbl_name)
+
+
+in_schema = in_schema
 
 
 class DataFrame():
@@ -166,6 +243,19 @@ class DataFrame():
         # Property to determine if table is an ART table or not.
         self._is_art = None
 
+        self._datalake = None
+        self._database = None
+        self._table = None
+        self._otf = False
+
+        if isinstance(table_name, in_schema):
+            self._table = table_name.table_name
+            self._datalake = table_name.datalake_name
+            self._database = table_name.schema_name
+            self._otf = True if self._datalake else False
+
+        table_name = str(table_name) if isinstance(table_name, in_schema) else table_name
+
         # Below matrix is list of list, where in each row contains following elements:
         # Let's take an example of following, just to get an idea:
         #   [element1, element2, element3, element4, element5, element6]
@@ -198,25 +288,45 @@ class DataFrame():
                 self._source_type = SourceType.TABLE.value
                 self._nodeid = self._aed_utils._aed_table(self._table_name)
             elif query is not None:
+                query = query.strip()
+                query = query[:-1] if query[-1] == ";" else query
+
                 self._query = query
                 self._source_type = SourceType.QUERY.value
 
-                if materialize:
-                    # If user requests to materialize the the query, then we should create a
+                temp_obj_params = {
+                    "prefix": "_frmqry_v",
+                    "use_default_database": True,
+                    "quote": False
+                }
+                __execute = UtilFuncs._create_view
+
+                if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                    # If user requests to materialize the query, then we should create a
+                    # volatile table if user intends to the same instead of view.
+                    # Volatile table does not need to be added to the GC.
+                    temp_obj_params["table_type"] = TeradataConstants.TERADATA_VOLATILE_TABLE
+                    temp_obj_params["gc_on_quit"] = False
+                    temp_obj_params["prefix"] = "_frmqry_vt"
+                    __execute = UtilFuncs._create_table
+
+                elif materialize:
+                    # If user requests to materialize the query, then we should create a
                     # table instead of view and add the same in the GarbageCollector.
-                    temp_table_name = UtilFuncs._generate_temp_table_name(prefix="_frmqry_t", use_default_database=True,
-                                                                          quote=False,
-                                                                          table_type=TeradataConstants.TERADATA_TABLE)
-                else:
-                    temp_table_name = UtilFuncs._generate_temp_table_name(prefix="_frmqry_v", use_default_database=True,
-                                                                          quote=False)
+                    temp_obj_params["table_type"] = TeradataConstants.TERADATA_TABLE
+                    temp_obj_params["gc_on_quit"] = True
+                    temp_obj_params["prefix"] = "_frmqry_t"
+                    __execute = UtilFuncs._create_table
 
+                temp_table_name = UtilFuncs._generate_temp_table_name(**temp_obj_params)
                 self._table_name = temp_table_name
+                __execute_params = (self._table_name, self._query)
+
+                if configure.temp_object_type == TeradataConstants.TERADATA_VOLATILE_TABLE:
+                    __execute_params = (self._table_name, self._query, True)
+
                 try:
-                    if materialize:
-                        UtilFuncs._create_table(self._table_name, self._query)
-                    else:
-                        UtilFuncs._create_view(self._table_name, self._query)
+                    __execute(*__execute_params)
                 except OperationalError as oe:
                     if "[Error 3707] Syntax error" in str(oe):
                         raise ValueError(Messages.get_message(
@@ -245,6 +355,9 @@ class DataFrame():
             self.__data = None
             self.__data_columns = None
             self._alias = None
+            self._plot = None
+
+            self._eda_ui = None
 
         except TeradataMlException:
             raise
@@ -334,7 +447,9 @@ class DataFrame():
         _Validators._validate_function_arguments(arg_info_matrix)
         try:
             alias_df = self._from_node(self._nodeid, self._metaexpr, self._index_label,
-                                              reuse_metaexpr=False)
+                                       reuse_metaexpr=False, _datalake=self._datalake,
+                                       _database=self._database, _table=self._table,
+                                       _otf=self._otf)
             # Assigning self attributes to newly created alias dataframe.
             alias_df._table_name = self._table_name
             alias_df._index = self._index
@@ -350,7 +465,8 @@ class DataFrame():
 
     @classmethod
     @collect_queryband(queryband="DF_fromTable")
-    def from_table(cls, table_name, index=True, index_label=None):
+    def from_table(cls, table_name, index=True, index_label=None,
+                   schema_name=None, datalake_name=None):
         """
         Class method for creating a DataFrame from a table or a view.
 
@@ -371,30 +487,48 @@ class DataFrame():
                 Column/s used for sorting.
                 Types: str
 
+            schema_name:
+                Optional Argument.
+                Specifies the schema where the table resides.
+                Types: str
+
+            datalake_name:
+                Optional Argument.
+                Specifies the datalake name.
+                Types: str
+
         EXAMPLES:
-            from teradataml.dataframe.dataframe import DataFrame
+            >>> from teradataml.dataframe.dataframe import DataFrame
 
             # Example 1: The following example creates a DataFrame from a table or
                          a view.
             # Load the example data.
-            load_example_data("dataframe","sales")
+            >>> load_example_data("dataframe","sales")
 
             # Create DataFrame from table
-            df = DataFrame.from_table('sales')
+            >>> df = DataFrame.from_table('sales')
 
             # Create DataFrame from table and without index column sorting.
-            df = DataFrame.from_table("sales", False)
+            >>> df = DataFrame.from_table("sales", False)
 
             # Create DataFrame from table and sorting using the 'accounts'
             # column.
-            df = DataFrame.from_table("sales", True, "accounts")
+            >>> df = DataFrame.from_table("sales", True, "accounts")
 
             # Example 2: The following example creates a DataFrame from existing Vantage
             #            table "dbcinfo" in the non-default database "dbc" using the
             #            in_schema() function.
 
-            from teradataml.dataframe.dataframe import in_schema
-            df = DataFrame.from_table(in_schema("dbc", "dbcinfo"))
+            >>> from teradataml.dataframe.dataframe import in_schema
+            >>> df = DataFrame.from_table(in_schema("dbc", "dbcinfo"))
+
+            # Example 3: Create a DataFrame on existing DataLake
+            #            table "lake_table" in the "datalake_database" database
+            #            in "datalake" datalake.
+
+            >>> datalake_df = DataFrame.from_table(table_name="lake_table",
+            ...                                    schema_name="datalake_database",
+            ...                                    datalake_name="datalake" )
 
         RETURNS:
             DataFrame
@@ -403,6 +537,9 @@ class DataFrame():
             TeradataMlException - TDMLDF_CREATE_FAIL
 
         """
+        if schema_name:
+            return cls(in_schema(schema_name, table_name, datalake_name))
+
         return cls(table_name, index, index_label)
 
     @classmethod
@@ -462,7 +599,7 @@ class DataFrame():
         return cls(index=index, index_label=index_label, query=query, materialize=materialize)
 
     @classmethod
-    def _from_node(cls, nodeid, metaexpr, index_label=None, undropped_index=None, reuse_metaexpr=True):
+    def _from_node(cls, nodeid, metaexpr, index_label=None, undropped_index=None, reuse_metaexpr=True, **kwargs):
         """
         Private class method for creating a DataFrame from a nodeid and parent metadata.
 
@@ -543,6 +680,11 @@ class DataFrame():
                                                  in [col.name for col in df._metaexpr.c] for elem in undropped_index):
             df._undropped_index = undropped_index
 
+        # Populate remaining attributes.
+        for arg in kwargs:
+            # Pop each argument from kwargs and assign to new DataFrame.
+            arg_value = kwargs.get(arg)
+            df.__setattr__(arg, arg_value)
         return df
 
     def create_temp_view(self, name):
@@ -670,9 +812,10 @@ class DataFrame():
         return self
 
     @collect_queryband(queryband="DF_fillna")
-    def fillna(self, value=None, columns=None, literal_value=False):
+    def fillna(self, value=None, columns=None, literal_value=False, partition_column=None):
         """
-        Method to replace the null values in a column with the value specified.
+        DESCRIPTION:
+            Method to replace the null values in a column with the value specified.
 
         PARAMETERS:
             value:
@@ -705,6 +848,12 @@ class DataFrame():
                 Default Value: False
                 Types: bool
 
+            partition_column:
+                Optional Argument.
+                Specifies the column name to partition the data.
+                Default Value: None
+                Types: str
+
         RETURNS:
             teradataml DataFrame
 
@@ -745,6 +894,26 @@ class DataFrame():
                 3    Blue Inc   90.0   50   95.0  101.0  17/01/04
                 4    Alpha Co  210.0  200  215.0  250.0  17/01/04
                 5  Orange Inc  210.0   50    NaN  250.0  17/01/04
+
+            # Example 3: Populate the null value in 'pclass' and
+            #            'fare' column with mean value with partition
+            #            column as 'sex'.
+            # Load the example data.
+            >>> load_example_data("teradataml", ["titanic"])
+            >>> df = DataFrame.from_table("titanic")
+
+            >>> df.fillna(value="mean", columns=["pclass", "fare"], partition_column="sex")
+                passenger  survived  pclass                                         name     sex   age  sibsp  parch            ticket      fare cabin embarked
+            0        284         1       3                   Dorking, Mr. Edward Arthur    male  19.0      0      0        A/5. 10482    8.0500  None        S
+            1        589         0       3                        Gilinski, Mr. Eliezer    male  22.0      0      0             14973    8.0500  None        S
+            2         17         0       3                         Rice, Master. Eugene    male   2.0      4      1            382652   29.1250  None        Q
+            3        282         0       3             Olsson, Mr. Nils Johan Goransson    male  28.0      0      0            347464    7.8542  None        S
+            4        608         1       1                  Daniel, Mr. Robert Williams    male  27.0      0      0            113804   30.5000  None        S
+            5        404         0       3               Hakkarainen, Mr. Pekka Pietari    male  28.0      1      0  STON/O2. 3101279   15.8500  None        S
+            6        427         1       2  Clarke, Mrs. Charles V (Ada Maria Winfield)  female  28.0      1      0              2003   26.0000  None        S
+            7        141         0       3                Boulos, Mrs. Joseph (Sultana)  female   NaN      0      2              2678   15.2458  None        C
+            8        610         1       1                    Shutes, Miss. Elizabeth W  female  40.0      0      0          PC 17582  153.4625  C125        S
+            9        875         1       2        Abelson, Mrs. Samuel (Hannah Wizosky)  female  28.0      1      0         P/PP 3381   24.0000  None        C
         """
         from teradataml import SimpleImputeFit, SimpleImputeTransform
 
@@ -752,6 +921,7 @@ class DataFrame():
         arg_info_matrix.append(["value", value, True, (int, float, str, dict, list)])
         arg_info_matrix.append(["columns", columns, True, (list, str, tuple)])
         arg_info_matrix.append(["literal_value", literal_value, True, (bool)])
+        arg_info_matrix.append(["partition_column", partition_column, True, (str)])
 
         # Validate argument types
         _Validators._validate_function_arguments(arg_info_matrix)
@@ -823,9 +993,15 @@ class DataFrame():
                                   literals=literals,
                                   literals_columns=literals_columns,
                                   stats=stats,
-                                  stats_columns=stats_columns)
+                                  stats_columns=stats_columns,
+                                  partition_column=partition_column)
 
-        return fit_obj.transform(data=self).result
+        impute_transform = {
+            'data': self,
+            'data_partition_column': partition_column,
+            'object_partition_column': partition_column}
+
+        return fit_obj.transform(**impute_transform).result
 
     def __execute_node_and_set_table_name(self, nodeid, metaexpr=None):
         """
@@ -924,6 +1100,7 @@ class DataFrame():
         self._column_names_and_types = []
         self._td_column_names_and_types = []
         self._td_column_names_and_sqlalchemy_types = {}
+        self._column_types = {}
 
         for col in self._metaexpr.c:
             if isinstance(col.type, sqlalchemy.sql.sqltypes.NullType):
@@ -931,9 +1108,11 @@ class DataFrame():
             else:
                 tdtype = "{}".format(col.type)
 
-            self._column_names_and_types.append((str(col.name), UtilFuncs._teradata_type_to_python_type(col.type)))
+            py_type = UtilFuncs._teradata_type_to_python_type(col.type)
+            self._column_names_and_types.append((str(col.name), py_type))
             self._td_column_names_and_types.append((str(col.name), tdtype))
             self._td_column_names_and_sqlalchemy_types[(str(col.name)).lower()] = col.type
+            self._column_types[(str(col.name)).lower()] = [py_type, col.type]
 
     def _get_metaexpr(self):
         """
@@ -952,7 +1131,24 @@ class DataFrame():
         meta = sqlalchemy.MetaData()
         db_schema = UtilFuncs._extract_db_name(self._table_name)
         db_table_name = UtilFuncs._extract_table_name(self._table_name)
-        t = sqlalchemy.Table(db_table_name, meta, schema=db_schema, autoload_with=eng)
+        if not self._datalake:
+            t = sqlalchemy.Table(db_table_name, meta, schema=db_schema, autoload_with=eng)
+            return _MetaExpression(t)
+
+        # Get metaexpression for datalake table.
+        # check existence of datalake table.
+        tdmlctx.get_connection().dialect.has_table(tdmlctx.get_connection(),
+                                                   self._table,
+                                                   schema=self._database,
+                                                   table_only=True,
+                                                   datalake=self._datalake)
+
+        # Extract column names and corresponding teradatasqlalchemy types.
+        col_names, col_types = df_utils._get_datalake_table_columns_info(self._database,
+                                                                          self._table,
+                                                                          self._datalake)
+        t = sqlalchemy.Table(self._table, meta, schema=self._database,
+                             *(Column(col_name, col_type) for col_name, col_type in zip(col_names, col_types)))
         return _MetaExpression(t)
 
     def __getattr__(self, name):
@@ -2728,9 +2924,10 @@ class DataFrame():
             msg = Messages.get_message(errcode)
             raise TeradataMlException(msg, errcode)
 
+    @argument_deprecation("20.0.0.5", "include", False, None)
     @collect_queryband(queryband="DF_describe")
     def describe(self, percentiles=[.25, .5, .75], include=None, verbose=False, distinct=False, statistics=None,
-                 columns=None):
+                 columns=None, pivot=False):
         """
         DESCRIPTION:
             Generates statistics for numeric columns. This function can be used in two modes:
@@ -2762,12 +2959,12 @@ class DataFrame():
             include:
                 Optional Argument.
                 Values can be either None or "all".
-                If the value is "all", then both numeric and non-numeric columns are included.
+                If the value is "all", both numeric and non-numeric columns are included.
                 Computes count, mean, std, min, percentiles, and max for numeric columns.
                 Computes count and unique for non-numeric columns.
                 If the value is None, only numeric columns are used for collecting statistics.
                 Note:
-                    Value 'all' is not applicable for 'Time Series Aggregate Mode'.
+                    * Value 'all' is not applicable for 'Time Series Aggregate Mode'.
                 Default Values: None
                 Types: str
 
@@ -2807,7 +3004,14 @@ class DataFrame():
                 Specifies the name(s) of the columns we are collecting statistics for.
                 Default Values: None
                 Types: str or List of str
-
+            
+            pivot:
+                Optional Argument.
+                Specifies a boolean value to pivot the output.
+                Note:
+                    * "pivot" is not supported for PTI tables.
+                Default Values: 'False'
+                Types: bool
 
         RETURNS:
             teradataml DataFrame
@@ -2829,7 +3033,7 @@ class DataFrame():
             Orange Inc  210.0  None  None   250  04/01/2017
 
             # Computes count, mean, std, min, percentiles, and max for numeric columns.
-            >>> df.describe()
+            >>> df.describe(pivot=True)
                       Apr      Feb     Mar     Jan
             func
             count       4        6       4       4
@@ -2841,8 +3045,45 @@ class DataFrame():
             75%       250    207.5  158.75   162.5
             max       250      210     215     200
 
+            # Computes count, mean, std, min, percentiles, and max for numeric columns with 
+            # default arugments.
+            >>> df.describe()
+            ATTRIBUTE   StatName            StatValue
+            Jan	        MAXIMUM	            200.0
+            Jan	        STANDARD DEVIATION	62.91528696058958
+            Jan	        PERCENTILES(25)	    125.0
+            Jan	        PERCENTILES(50)	    150.0
+            Mar	        COUNT	            4.0
+            Mar	        MINIMUM	            95.0
+            Mar	        MAXIMUM	            215.0
+            Mar	        MEAN	            147.5
+            Mar	        STANDARD DEVIATION	49.749371855331
+            Mar	        PERCENTILES(25)	    128.75
+            Mar	        PERCENTILES(50)	    140.0
+            Apr	        COUNT	            4.0
+            Apr	        MINIMUM	            101.0
+            Apr	        MAXIMUM	            250.0
+            Apr	        MEAN	            195.25
+            Apr	        STANDARD DEVIATION	70.97123830585646
+            Apr	        PERCENTILES(25)	    160.25
+            Apr	        PERCENTILES(50)	    215.0
+            Apr	        PERCENTILES(75)	    250.0
+            Feb	        COUNT	            6.0
+            Feb	        MINIMUM	            90.0
+            Feb	        MAXIMUM	            210.0
+            Feb	        MEAN	            166.66666666666666
+            Feb	        STANDARD DEVIATION	59.553897157672786
+            Feb	        PERCENTILES(25)	    117.5
+            Feb	        PERCENTILES(50)	    200.0
+            Feb	        PERCENTILES(75)	    207.5
+            Mar	        PERCENTILES(75)	    158.75
+            Jan	        PERCENTILES(75)	    162.5
+            Jan	        MEAN	            137.5
+            Jan	        MINIMUM	            50.0
+            Jan	        COUNT	            4.0
+
             # Computes count, mean, std, min, percentiles, and max for numeric columns with 30th and 60th percentiles.
-            >>> df.describe(percentiles=[.3, .6])
+            >>> df.describe(percentiles=[.3, .6], pivot=True)
                       Apr      Feb     Mar     Jan
             func
             count       4        6       4       4
@@ -2855,7 +3096,7 @@ class DataFrame():
 
             # Computes count, mean, std, min, percentiles, and max for numeric columns group by "datetime" and "Feb".
             >>> df1 = df.groupby(["datetime", "Feb"])
-            >>> df1.describe()
+            >>> df1.describe(pivot=True)
                                      Jan   Mar   Apr
             datetime   Feb   func
             04/01/2017 90.0  25%      50    95   101
@@ -2883,22 +3124,6 @@ class DataFrame():
                              min     200   215   250
                              std    None  None     0
 
-            # Computes count, mean, std, min, percentiles, and max for numeric columns and
-            # computes count and unique for non-numeric columns
-            >>> df.describe(include="all")
-                   accounts      Feb     Jan     Mar     Apr datetime
-            func
-            25%        None    117.5     125  128.75  160.25     None
-            75%        None    207.5   162.5  158.75     250     None
-            count         6        6       4       4       4        6
-            mean       None  166.667   137.5   147.5  195.25     None
-            max        None      210     200     215     250     None
-            min        None       90      50      95     101     None
-            50%        None      200     150     140     215     None
-            std        None   59.554  62.915  49.749  70.971     None
-            unique        6     None    None    None    None        1
-
-            #
             # Examples for describe() function as Time Series Aggregate.
             #
             >>> # Load the example datasets.
@@ -3081,7 +3306,7 @@ class DataFrame():
             >>>
         """
 
-        # Argument validations
+        # -------------Argument validations---------------#
         awu_matrix = []
         awu_matrix.append(["columns", columns, True, (str, list), True])
         awu_matrix.append(["percentiles", percentiles, True, (float, list)])
@@ -3090,6 +3315,7 @@ class DataFrame():
         awu_matrix.append(["distinct", distinct, True, (bool)])
         awu_matrix.append(["statistics", statistics, True, (str, list), True,
                            ["count", "mean", "min", "max", "unique", "std", "describe", "percentile"]])
+        awu_matrix.append(["pivot", pivot, True, (bool)])
 
         # Validate argument types
         _Validators._validate_function_arguments(awu_matrix)
@@ -3133,22 +3359,27 @@ class DataFrame():
         if verbose and not isinstance(self, DataFrameGroupByTime):
             raise ValueError(Messages.get_message(MessageCodes.ARG_VALUE_CLASS_DEPENDENCY).format(
                 'verbose', 'Aggregation', 'True', 'describe()', 'DataFrameGroupByTime'))
+        # -------------End of argument validations---------------#
 
         function_label = "func"
+        sort_cols = []
         try:
             self.__execute_node_and_set_table_name(self._nodeid)
 
             groupby_column_list = None
-            if isinstance(self, DataFrameGroupBy):
+            if isinstance(self, DataFrameGroupByTime) or isinstance(self, DataFrameGroupBy):
                 groupby_column_list = self.groupby_column_list
-                df_utils._invalid_describe_column(df=self, columns=columns, metaexpr=self._metaexpr,
-                                                  groupby_column_list=groupby_column_list)
+                if columns:
+                    df_utils._validate_describe_columns(columns=columns, metaexpr=self._metaexpr,
+                                                        groupby_column_list=groupby_column_list)
+                sort_cols = list(groupby_column_list)
 
-            if isinstance(self, DataFrameGroupByTime):
-                groupby_column_list = self.groupby_column_list
-                df_utils._invalid_describe_column(df=self, columns=columns, metaexpr=self._metaexpr,
-                                                  groupby_column_list=groupby_column_list)
+            # 'func' column will be always there in result.
+            sort_cols.append(function_label)
 
+            # Handle DataFrameGroupByTime using union all approach and
+            # other DataFrames using TD_UnivariateStatistics approach.
+            if isinstance(self, DataFrameGroupByTime):
                 # Construct the aggregate query.
                 agg_query = df_utils._construct_describe_query(df=self, columns=columns, metaexpr=self._metaexpr,
                                                                percentiles=percentiles, function_label=function_label,
@@ -3160,29 +3391,99 @@ class DataFrame():
                                                                timecode_column=self._timecode_column,
                                                                sequence_column=self._sequence_column,
                                                                fill=self._fill)
+
+                if groupby_column_list is not None:
+                    df = DataFrame.from_query(agg_query, index_label=sort_cols)
+                    df2 = df.sort(sort_cols)
+                    df2._metaexpr._n_rows = 100
+                    describe_df = df2
+                else:
+                    describe_df = DataFrame.from_query(agg_query, index_label=function_label)
+
+                # Check if numeric overflow can occur for result DataFrame.
+                if self._check_numeric_overflow(describe_df):
+                    result_df = self._promote_dataframe_types()
+                    describe_df = result_df.describe(pivot=True)
+                return describe_df
+
             else:
-                # Construct the aggregate query.
-                agg_query = df_utils._construct_describe_query(df=self, columns=columns, metaexpr=self._metaexpr,
-                                                               percentiles=percentiles, function_label=function_label,
-                                                               groupby_column_list=groupby_column_list, include=include,
-                                                               is_time_series_aggregate=False, verbose=verbose,
-                                                               distinct=distinct, statistics=statistics)
-
-            if groupby_column_list is not None:
-                sort_cols = [i for i in groupby_column_list]
-                sort_cols.append(function_label)
-                df = DataFrame.from_query(agg_query, index_label=sort_cols)
-                df2 = df.sort(sort_cols)
-                df2._metaexpr._n_rows = 100
-                describe_df = df2
-            else:
-                describe_df = DataFrame.from_query(agg_query, index_label=function_label)
+                # If pivot is True, then construct the aggregate query and return the result DataFrame.
+                # Otherwise, return the result DataFrame in the regular aggregate mode using UnivariateStatistics.
+
+                if pivot:
+                    # Construct the aggregate query.
+                    agg_query = df_utils._construct_describe_query(df=self, columns=columns, metaexpr=self._metaexpr,
+                                                                percentiles=percentiles, function_label=function_label,
+                                                                groupby_column_list=groupby_column_list, include=include,
+                                                                is_time_series_aggregate=False, verbose=verbose,
+                                                                distinct=distinct, statistics=statistics)
+
+                    if groupby_column_list is not None:
+                        sort_cols = [i for i in groupby_column_list]
+                        sort_cols.append(function_label)
+                        df = DataFrame.from_query(agg_query, index_label=sort_cols)
+                        df2 = df.sort(sort_cols)
+                        df2._metaexpr._n_rows = 100
+                        describe_df = df2
+                    else:
+                        describe_df = DataFrame.from_query(agg_query, index_label=function_label)
+
+                    # Check if numeric overflow can occur for result DataFrame.
+                    if self._check_numeric_overflow(describe_df):
+                        result_df = self._promote_dataframe_types()
+                        describe_df = result_df.describe(pivot=True)
+                    
+                    return describe_df
+
+                # If columns is None, then all dataframe columns are considered.
+                if columns is None:
+                    columns = self.columns
+                    # Exclude groupby columns
+                    if groupby_column_list is not None:
+                        columns = [col for col in columns if col not in groupby_column_list]
+
+                numeric_cols = []
+
+                # Extract numeric columns and their types of all columns
+                for col in self._metaexpr.c:
+                    if type(col.type) in UtilFuncs()._get_numeric_datatypes() and \
+                    col.name in columns:
+                        numeric_cols.append(col.name)
+
+                if numeric_cols:
+                    # Default statistics for 'Regular Aggregate Mode'
+                    sql_stat = ["COUNT", "MAXIMUM", "MEAN", "MINIMUM", "PERCENTILES", "STANDARD DEVIATION"]
+                    
+                    if statistics is not None:
+                        py_to_sql_func_map = {"count": "COUNT",
+                                              "max": "MAXIMUM",
+                                              "mean": "MEAN",
+                                              "unique": 'UNIQUE ENTITY COUNT',
+                                              "min": "MINIMUM",
+                                              "percentile": "PERCENTILES",
+                                              "std": "STANDARD DEVIATION"}
+                        # Convert statistics into corresponding SQL function names
+                        sql_stat = [py_to_sql_func_map[stat] for stat in UtilFuncs()._as_list(statistics)]
+
+                    # Convert percentiles to centiles for univariate statistics
+                    centiles = list(map(lambda n: int(n * 100), percentiles))
+
+                    # UnivariateStatistics parameters
+                    univar_param = {
+                        "newdata": self.select(self.columns),
+                        "target_columns": numeric_cols,
+                        "partition_columns": groupby_column_list,
+                        "centiles": centiles,
+                        "stats": sql_stat
+                    }
+
+                    from teradataml import UnivariateStatistics
+                    # Run UnivariateStatistics
+                    aggr_df = UnivariateStatistics(**univar_param).result
+
+                    # Return the result in teradataml format
+                    return aggr_df
 
-            # Check if numeric overflow can occur for result DataFrame.
-            if self._check_numeric_overflow(describe_df):
-                result_df = self._promote_dataframe_types()
-                describe_df = result_df.describe()
-            return describe_df
         except TeradataMlException:
             raise
         except Exception as err:
@@ -5765,7 +6066,35 @@ class DataFrame():
 
     def _repr_html_(self):
         """ Print method for teradataml for iPython rich display. """
+        self._generate_output_html()
+        if display.enable_ui:
+            # EDA Ui widget representation using teradatamlwidgets
+            if self._eda_ui is None:
+                from teradatamlwidgets.eda.Ui import Ui
+                self._eda_ui = Ui(df=self, html=self.html)
+            else:
+                self._eda_ui.display_ui()
+        return self.html
+
+    def get_eda_ui(self):
+        """
+        Returns the EDA representation UI.
+
+        PARAMETERS:
+            None.
 
+        EXCEPTIONS:
+            None.
+
+        RETURNS:
+            teradatamlwidgets.eda.Ui
+
+        EXAMPLE:
+            df = ui.get_eda_ui()
+        """
+        return self._eda_ui
+
+    def _generate_output_html(self, disable_types=True):
         # Check if class attributes __data and __data_columns are not None.
         # If not None, reuse the data and columns.
         # If None, generate latest results.
@@ -5778,17 +6107,25 @@ class DataFrame():
         dindent = indent + indent
 
         header_html = ['<style type="text/css">',
-                       'table {border:ridge 5px;}',
+                       'table { border:ridge 5px}',
                        'table td {border:inset 1px;}',
-                       'table tr#HeaderRow {background-color:grey; color:white;}'
+                       'table tr#HeaderRow {background-color:grey; color:white;}',
                        '</style>\n'
                        ]
         html = "\n{0}".format(indent).join(header_html)
-        html += '<html><table>\n{0}<tr id="HeaderRow">\n'.format(indent)
+        html += '<html><table style="min-width:1000px;">\n{0}<tr id="HeaderRow">\n'.format(indent)
+
+        columns_html = "</th><th>".join(self.__data_columns)
+        html += "<th>{0}</th>\n".format(columns_html)
+        html += "</tr>\n"
 
-        columns_html = "</th>\n{0}<th>".format(dindent).join(self.__data_columns)
-        html += "{0}<th>{1}</th>\n".format(dindent, columns_html)
-        html += "{0}</tr>\n".format(indent)
+        if not disable_types:
+            html += '<tr>\n'.format(indent)
+            col_types = [repr(self._td_column_names_and_sqlalchemy_types[column]) for column in
+                         self.__data_columns]
+            columns_types_html = "</td>\n{0}<td>".format(dindent).join(col_types)
+            html += "{0}<td>{1}</td>\n".format(dindent, columns_types_html)
+            html += "{0}</tr>\n".format(indent)
 
         for row in self.__data:
             row_html = ["{0}<td>{1}</td>\n".format(dindent,
@@ -5796,8 +6133,31 @@ class DataFrame():
             html += "{1}<tr>\n{0}{1}</tr>\n".format("".join(row_html), indent)
 
         html += "</table></html>"
+        self.html = html
+
+    def get_output(self, output_index=0):
+        """
+        DESCRIPTION:
+            Returns the result of analytic function when analytic function is
+            run from 'Analyze' tab in EDA UI.
+            Note:
+                * The function does not return anything if analytic function is
+                  not run from EDA UI.
+
+        PARAMETERS:
+            output_index:
+                Optional Argument.
+                Specifies the index of the output dataframe to be returned.
+                Default Value: 0
+                Types: int
+
+        RAISES:
+            IndexError
 
-        return html
+        RETURNS:
+            teradataml DataFrame object.
+        """
+        return self._eda_ui.get_output_dataframe(output_index=output_index)
 
     def __get_data_columns(self):
         """
@@ -6857,7 +7217,8 @@ class DataFrame():
                         compiled_condition = condition.compile(compile_kwargs={'include_table': True,
                                                                                'literal_binds': True,
                                                                                'table_name_kind': '_join_alias',
-                                                                               'compile_with_caller_table': True})
+                                                                               'compile_with_caller_table': True,
+                                                                               'table_only': True})
 
                         all_join_conditions.append(compiled_condition)
 
@@ -7571,14 +7932,14 @@ class DataFrame():
             _Validators._check_auth_token("udf")
             for colname, col in udf_expr.items():
                 env_name = UtilFuncs._get_env_name(col)
-                # Store the env_name and its corresponding output column   
+                # Store the env_name and its corresponding output column
                 if env_name in env_mapper:
                     env_mapper[env_name].append(colname)
                 else:
                     env_mapper[env_name] = [colname]
         else:
             env_mapper[env_name] = udf_expr.keys()
-            
+
         for env_name, cols in env_mapper.items():
             # Create a dictionary of output columns to column type.
             returns = OrderedDict([(column.name, column.type) for column in df._metaexpr.c])
@@ -7625,11 +7986,11 @@ class DataFrame():
 
             df = tbl_operators.execute()
         return df
-    
+
     def _assign_call_udf(self, call_udf_expr):
         """
         DESCRIPTION:
-            Internal function for DataFrame.assign() to execute the call_udf using 
+            Internal function for DataFrame.assign() to execute the call_udf using
             Script/Apply Table Operator and create new column for teradataml DataFrame.
 
         PARAMETER:
@@ -7656,7 +8017,7 @@ class DataFrame():
         # Create a dictionary of output columns to column type (python types).
         output_type_converters = {col_name: _Dtypes._teradata_type_to_python_type(col_type) \
                                   for col_name, col_type in returns.items()}
-       
+
         for colname, col in call_udf_expr.items():
             returns[colname] = col.type
             output_type_converters[colname] = _Dtypes._teradata_type_to_python_type(col.type)
@@ -7782,7 +8143,7 @@ class DataFrame():
                Look at Example 18 to understand more.
              8. While passing multiple udf expressions, one can not pass one column output
                as another column input in the same ``assign`` call.
-             9. If user pass multiple udf expressions, delimiter and quotechar specified in 
+             9. If user pass multiple udf expressions, delimiter and quotechar specified in
                last udf expression are considered for processing.
 
         RAISES:
@@ -8147,13 +8508,13 @@ class DataFrame():
             Red Inc     200.0  150.0  140.0    NaN  17/01/04    201.0     abc     RED INC      207
             >>>
 
-            # Example 19: Convert the values is 'accounts' column to upper case using a user 
+            # Example 19: Convert the values is 'accounts' column to upper case using a user
             #             defined function on Vantage Cloud Lake.
             # Create a Python 3.10.5 environment with given name and description in Vantage.
             >>> env = create_env('test_udf', 'python_3.10.5', 'Test environment for UDF')
             User environment 'test_udf' created.
             >>>
-            # Create a user defined functions to 'to_upper' to get the values in upper case 
+            # Create a user defined functions to 'to_upper' to get the values in upper case
             # and pass the user env to run it on.
             >>> from teradataml.dataframe.functions import udf
             >>> @udf(env_name = env)
@@ -8165,7 +8526,7 @@ class DataFrame():
             # to the DataFrame.
             >>> df.assign(upper_stats = to_upper('accounts'))
                           Feb    Jan    Mar    Apr  datetime upper_stats
-            accounts                                                    
+            accounts
             Alpha Co    210.0  200.0  215.0  250.0  17/01/04    ALPHA CO
             Blue Inc     90.0   50.0   95.0  101.0  17/01/04    BLUE INC
             Yellow Inc   90.0    NaN    NaN    NaN  17/01/04  YELLOW INC
@@ -8184,12 +8545,12 @@ class DataFrame():
             # Register the created user defined function with name "upper".
             >>> register("upper", to_upper)
             >>>
-            # Call the user defined function registered with name "upper" and assign the 
+            # Call the user defined function registered with name "upper" and assign the
             # ColumnExpression returned to the DataFrame.
             >>> res = df.assign(upper_col = call_udf("upper", ('accounts',)))
             >>> res
                           Feb    Jan    Mar    Apr  datetime   upper_col
-            accounts                                                    
+            accounts
             Alpha Co    210.0  200.0  215.0  250.0  17/01/04    ALPHA CO
             Blue Inc     90.0   50.0   95.0  101.0  17/01/04    BLUE INC
             Yellow Inc   90.0    NaN    NaN    NaN  17/01/04  YELLOW INC
@@ -8475,7 +8836,9 @@ class DataFrame():
         _Validators._validate_column_exists_in_dataframe(keys, self._metaexpr)
 
         try:
-            new_index_list = self._index_label if self._index_label is not None else []
+            
+            # Slicing creates a new list instance with the same contents.
+            new_index_list = self._index_label[:] if self._index_label is not None else []
 
             # Creating a list with requested index labels bases on append
             if append:
@@ -8490,7 +8853,7 @@ class DataFrame():
                     new_index_list = keys
 
             # Takes care of appending already existing index
-            new_index_list = list(set(new_index_list))
+            new_index_list = list(dict.fromkeys(new_index_list))
 
             # In case requested index is same as existing index, return same DF
             if new_index_list == self._index_label:
@@ -9373,15 +9736,15 @@ class DataFrame():
             TypeError, ValueError, TeradataMLException
 
         EXAMPLES:
-            >>> # Load the example datasets.
-            ... load_example_data("dataframe", ["ocean_buoys", "ocean_buoys_nonpti"])
+            # Load the example datasets.
+            >>> load_example_data("dataframe", ["ocean_buoys", "ocean_buoys_nonpti"])
             >>>
 
-            >>> # Create the required DataFrames.
-            ... # DataFrame on non-sequenced PTI table
-            ... ocean_buoys = DataFrame("ocean_buoys")
-            >>> # Check DataFrame columns and let's peek at the data
-            ... ocean_buoys.columns
+            # Create the required DataFrames.
+            # DataFrame on non-sequenced PTI table
+            >>> ocean_buoys = DataFrame("ocean_buoys")
+            # Check DataFrame columns and let's peek at the data
+            >>> ocean_buoys.columns
             ['buoyid', 'TD_TIMECODE', 'temperature', 'salinity']
             >>> ocean_buoys.head()
                                    TD_TIMECODE  temperature  salinity
@@ -9397,10 +9760,10 @@ class DataFrame():
             0       2014-01-06 08:00:00.000000         10.0        55
             0       2014-01-06 08:10:00.000000         10.0        55
 
-            >>> # DataFrame on NON-PTI table
-            ... ocean_buoys_nonpti = DataFrame("ocean_buoys_nonpti")
-            >>> # Check DataFrame columns and let's peek at the data
-            ... ocean_buoys_nonpti.columns
+            # DataFrame on NON-PTI table
+            >>> ocean_buoys_nonpti = DataFrame("ocean_buoys_nonpti")
+            # Check DataFrame columns and let's peek at the data
+            >>> ocean_buoys_nonpti.columns
             ['buoyid', 'timecode', 'temperature', 'salinity']
             >>> ocean_buoys_nonpti.head()
                                         buoyid  temperature  salinity
@@ -9974,6 +10337,15 @@ class DataFrame():
         # If user did not pass any arguments which form join conditions,
         # Merge is performed using index columns of TeradataML DataFrames
         if on is None and left_on is None and right_on is None and not use_index:
+            # DataFrames created on OTF table will not have index.
+            if self._datalake is not None or right._datalake is not None:
+                msg_code = MessageCodes.EXECUTION_FAILED
+                emsg = "Either 'on' argument or both 'left_on' and 'right_on' arguments" \
+                       " must be provided to merge DataFrames when they are created on" \
+                       " OTF table(s)."
+                error_msg = Messages.get_message(msg_code, "merge dataframes", emsg)
+                raise TeradataMlException(error_msg, msg_code)
+
             if self._index_label is None or right._index_label is None:
                 raise TeradataMlException(
                     Messages.get_message(MessageCodes.TDMLDF_INDEXES_ARE_NONE), MessageCodes.TDMLDF_INDEXES_ARE_NONE)
@@ -9981,6 +10353,12 @@ class DataFrame():
                 use_index = True
 
         if use_index:
+            if self._datalake is not None or right._datalake is not None:
+                msg_code = MessageCodes.EXECUTION_FAILED
+                emsg = "Can not use Index to merge DataFrames when they are created on OTF table(s)."
+                error_msg = Messages.get_message(msg_code, "merge dataframes", emsg)
+                raise TeradataMlException(error_msg, msg_code)
+
             if self._index_label is None or right._index_label is None:
                 raise TeradataMlException(
                     Messages.get_message(MessageCodes.TDMLDF_INDEXES_ARE_NONE), MessageCodes.TDMLDF_INDEXES_ARE_NONE)
@@ -10636,7 +11014,7 @@ class DataFrame():
                     2. seed is supported for stratify column.
                     3. Arguments "stratify_column", "seed", "id_column" are supported only 
                        for stratifying the data.
-                Types: str
+                Types: str OR Feature
             
             seed:
                 Optional Argument.
@@ -10662,7 +11040,7 @@ class DataFrame():
                        for stratifying the data.
                     2. "id_column" is supported only when "stratify_column" is used. 
                        Ignored otherwise.
-                Types: str
+                Types: str OR Feature
 
         RETURNS:
             teradataml DataFrame
@@ -12696,8 +13074,8 @@ class DataFrame():
             _Validators._validate_column_exists_in_dataframe(column_names, self._metaexpr,
                                                              False)
             column_names = list(dict.fromkeys(column_names))
-        
-        if list_td_reserved_keywords(column_names):
+
+        if list_td_reserved_keywords(column_names) or UtilFuncs._is_ascii(column_names):
             column_names = UtilFuncs._teradata_quote_arg(column_names, "\"", False)
 
         col_names_types = df_utils._get_required_columns_types_from_metaexpr(self._metaexpr, column_names)
@@ -14617,7 +14995,18 @@ class DataFrame():
             >>> plot.show()
 
         """
-        return _Plot(x=x, y=y, scale=scale, kind=kind, **kwargs)
+
+        _plot = _Plot(x=x, y=y, scale=scale, kind=kind, **kwargs)
+        # If plot is already generated, return the same plot.
+        if self._plot is None:
+            self._plot = _plot
+            return _plot
+
+        if self._plot == _plot:
+            return self._plot
+        else:
+            self._plot = _plot
+            return _plot
 
     @collect_queryband(queryband="DF_itertuples")
     def itertuples(self, name='Row', num_rows=None):
@@ -17510,11 +17899,18 @@ class _TDUAF(DataFrame):
             table_name = self._db_utils._execute_node_return_db_object_name(self._data._nodeid, self._data._metaexpr)
 
         # UAF Functions do not accept double quotes.
+        tdp = preparer(td_dialect)
         db_name = UtilFuncs._extract_db_name(table_name)
-        if db_name:
-            table_name = '"{}"."{}"'.format(db_name, UtilFuncs._extract_table_name(table_name))
+        datalake_name = UtilFuncs._extract_datalake_name(table_name)
+        if datalake_name:
+            table_name = '{}.{}.{}'.format(tdp.quote(datalake_name),
+                                           tdp.quote(db_name),
+                                           tdp.quote(UtilFuncs._extract_table_name(table_name)))
+        elif db_name:
+            table_name = '{}.{}'.format(tdp.quote(db_name),
+                                        tdp.quote(UtilFuncs._extract_table_name(table_name)))
         else:
-            table_name = UtilFuncs._extract_table_name(table_name)
+            table_name = tdp.quote(UtilFuncs._extract_table_name(table_name))
 
         sql_clauses.append("TABLE_NAME ({})")
         sql_values.append(table_name)