teradataml 17.20.0.6__py3-none-any.whl → 20.0.0.0__py3-none-any.whl

teradataml/dataframe/dataframe_utils.py

@@ -1267,7 +1267,50 @@ class DataFrameUtils():
         return aggregate_expr, new_column_names, new_column_types
 
     @staticmethod
-    def _construct_describe_query(df, metaexpr, percentiles, function_label, groupby_column_list=None,
+    def _invalid_describe_column(df, columns, metaexpr, groupby_column_list):
+        """
+        Internal function to validate columns provided to describe() is correct or not,
+        when DataFrame is output of groupby and groupby_time.
+
+        PARAMETERS:
+            df:
+                Required Argument.
+                Specifies teradataml DataFrame we are collecting statistics for.
+                Types: str
+
+            columns:
+                Optional Argument.
+                Specifies the name(s) of columns we are collecting statistics for.
+                Types: str ot List of strings (str)
+
+            metaexpr:
+                Required Argument.
+                Specifies the meta expression for the dataframe.
+                Types: _MetaExpression
+
+            groupby_column_list:
+                Optional Argument.
+                Specifies the group by columns for the dataframe.
+                Default Values: None.
+                Types: str ot List of strings (str)
+
+        Returns:
+            None
+
+        Raises:
+            TeradataMLException
+        """
+        invalid_columns = [_column for _column in groupby_column_list if columns is not None
+                           and _column in columns]
+        if len(invalid_columns) > 0:
+            all_columns = [col.name for col in metaexpr.c]
+            valid_columns = [item for item in all_columns if item not in groupby_column_list]
+            msg = Messages.get_message(MessageCodes.TDMLDF_AGGREGATE_INVALID_COLUMN). \
+                format(", ".join(invalid_columns), 'columns', ", ".join(valid_columns))
+            raise TeradataMlException(msg, MessageCodes.TDMLDF_AGGREGATE_INVALID_COLUMN)
+
+    @staticmethod
+    def _construct_describe_query(df, columns, metaexpr, percentiles, function_label, groupby_column_list=None,
                                   include=None, is_time_series_aggregate=False, verbose=False, distinct=False,
                                   statistics=None, **kwargs):
         """
@@ -1279,6 +1322,11 @@ class DataFrameUtils():
                 Specifies teradataml DataFrame we are collecting statistics for.
                 Types: str
 
+            columns:
+                Optional Argument.
+                Specifies the name(s) of columns we are collecting statistics for.
+                Types: str ot List of strings (str)
+
             metaexpr:
                 Required Argument.
                 Specifies the meta expression for the dataframe.
@@ -1388,15 +1436,16 @@ class DataFrameUtils():
         for col in metaexpr.c:
             if (include is None and type(col.type) in UtilFuncs()._get_numeric_datatypes()) or include == 'all' or statistics is not None:
                 if not(groupby is not None and col.name in groupby_column_list):
-                    col_names.append(col.name)
-                    col_types.append(col.type)
+                    if columns is None or col.name in columns:
+                        col_names.append(col.name)
+                        col_types.append(col.type)
+
 
         if len(col_names) == 0:
             raise TeradataMlException(
                 Messages.get_message(MessageCodes.TDMLDF_AGGREGATE_COMBINED_ERR,
                                      "The DataFrame does not contain numeric columns"),
                 MessageCodes.TDMLDF_AGGREGATE_COMBINED_ERR)
-
         for op in operators:
             if op == "percentile":
                 for p in percentiles:
@@ -1737,4 +1786,52 @@ class DataFrameUtils():
         stmt = text(str(stmt))
         stmt = stmt.bindparams(schema_name=schema_name)
         res = connection.execute(stmt).fetchall()
-        return [name.databasename for name in res]
+        return [name.databasename for name in res]
+
+    @staticmethod
+    def _get_common_parent_df_from_dataframes(dfs):
+        """
+        Internal function to return common parent dataframe from given list of dataframes.
+        """
+        from teradataml import DataFrame, in_schema
+        aed_utils = AedUtils()
+        if len(dfs) == 1:
+            operation = aed_utils._aed_get_node_query_type(dfs[0]._nodeid)
+            if operation == "table":
+                # Return the same dataframe if it is DataFrame object from table.
+                return dfs[0]
+
+            # If select node or any other node, then get the parent node and execute it.
+            pids = aed_utils._aed_get_parent_nodeids(dfs[0]._nodeid)
+            if not aed_utils._aed_is_node_executed(pids[0]):
+                _ = DataFrameUtils._execute_node_return_db_object_name(pids[0])
+
+            tab_name_first = aed_utils._aed_get_source_tablename(pids[0])
+
+            db_schema = UtilFuncs._extract_db_name(tab_name_first)
+            db_table_name = UtilFuncs._extract_table_name(tab_name_first)
+
+            return DataFrame(in_schema(db_schema, db_table_name))
+
+        pids_first = None
+        parent_df = None
+        for i in range(len(dfs)):
+            pids = aed_utils._aed_get_parent_nodeids(dfs[i]._nodeid)
+
+            if parent_df is None:
+                if not aed_utils._aed_is_node_executed(pids[0]):
+                    _ = DataFrameUtils._execute_node_return_db_object_name(pids[0])
+
+                tab_name_first = aed_utils._aed_get_source_tablename(pids[0])
+
+                db_schema = UtilFuncs._extract_db_name(tab_name_first)
+                db_table_name = UtilFuncs._extract_table_name(tab_name_first)
+
+                parent_df = DataFrame(in_schema(db_schema, db_table_name))
+                pids_first = pids
+            else:
+                if pids_first != pids:
+                    raise TeradataMlException(Messages.get_message(MessageCodes.DFS_NO_COMMON_PARENT),
+                                            MessageCodes.DFS_NO_COMMON_PARENT)
+
+        return parent_df

teradataml/dataframe/fastload.py

@@ -28,8 +28,10 @@ from teradataml.dataframe.copy_to import copy_to_sql, \
                                         _create_pti_table_object, _extract_column_info, \
                                         _check_columns_insertion_compatible
 from teradataml.dataframe.data_transfer import _DataTransferUtils
+from teradatasqlalchemy.telemetry.queryband import collect_queryband
 
 
+@collect_queryband(queryband="fstLd")
 def fastload(df, table_name, schema_name=None, if_exists='replace', index=False, 
              index_label=None, primary_index=None, types=None, batch_size=None, 
              save_errors=False, open_sessions=None):
@@ -292,9 +294,9 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
             _Validators._validate_function_arguments([["batch_size", batch_size,
                                                       False, (int)]])
             if batch_size < 100000:
-                warnings.warn("The batch_size provided is less than 100000. Teradata \
-                              recommends using 100000 as minimum batch size for \
-                              improved performance.")
+                warnings.warn("The batch_size provided is less than 100000. "
+                              "Teradata recommends using 100000 as minimum batch "
+                              "size for improved performance.", stacklevel=2)
 
         # If the table created must be a PTI table, then validate additional parameters
         # Note that if the required parameters for PTI are valid, then other parameters, though being validated,
@@ -360,6 +362,7 @@ def fastload(df, table_name, schema_name=None, if_exists='replace', index=False,
                                   MessageCodes.FASTLOAD_FAILS) from err
     return fl_dict
 
+
 def _insert_from_dataframe(dt_obj, table_name, batch_size):
     """
     This is an internal function used to to sequentially extract column info from DataFrame,
@@ -443,6 +446,9 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
         batch_number = 1
         num_batches = int(dt_obj.df.shape[0]/batch_size)
 
+
+        # Empty queryband buffer before SQL call.
+        UtilFuncs._set_queryband()
         for i in range(0, dt_obj.df.shape[0], batch_size):
             # Add the remaining rows to last batch after second last batch
             if (batch_number == num_batches) :
@@ -516,6 +522,7 @@ def _insert_from_dataframe(dt_obj, table_name, batch_size):
     return {"errors_dataframe": pd_err_df, "warnings_dataframe": pd_warn_df,
             "errors_table": error_tablename, "warnings_table": warn_tablename}
 
+
 def _get_batchsize(df):
     """
     This internal function calculates batch size which should be more than 100000 
@@ -537,6 +544,7 @@ def _get_batchsize(df):
     """
     return df.shape[0] if df.shape[0] <= 100000 else round(df.shape[0]/int(df.shape[0]/100000))
 
+
 def _create_table_for_fastload(df, con, table_name, schema_name=None, primary_index=None, 
                                is_pti=False, primary_time_index_name=None, timecode_column=None,
                                timezero_date=None, timebucket_duration=None, sequence_column=None,

teradataml/dataframe/setop.py

@@ -24,6 +24,8 @@ from teradatasqlalchemy.dialect import dialect as td_dialect, TeradataTypeCompil
 from teradatasqlalchemy import (GEOMETRY, MBR, MBB)
 from teradatasql import OperationalError
 
+from teradatasqlalchemy.telemetry.queryband import collect_queryband
+
 module = importlib.import_module("teradataml")
 
 def __validate_setop_args(df_list, awu_matrix, setop_type):
@@ -75,7 +77,8 @@ def __validate_setop_args(df_list, awu_matrix, setop_type):
         if not all(len(l) == the_len for l in it):
             raise TeradataMlException(Messages.get_message(MessageCodes.INVALID_DF_LENGTH),
                                       MessageCodes.INVALID_DF_LENGTH)
-    
+
+
 def __check_concat_compatibility(df_list, join, sort, ignore_index):
     """
     DESCRIPTION:
@@ -231,6 +234,7 @@ def __check_concat_compatibility(df_list, join, sort, ignore_index):
 
     return col_dict, is_lazy
 
+
 def __check_setop_if_lazy(df_list):
     """
     DESCRIPTION:
@@ -269,6 +273,7 @@ def __check_setop_if_lazy(df_list):
 
     return is_lazy
 
+
 def __process_operation(meta_data, is_lazy, setop_type, nodeid, index_label, index_to_use, class_name = None):
     """
     DESCRIPTION:
@@ -355,6 +360,7 @@ def __process_operation(meta_data, is_lazy, setop_type, nodeid, index_label, ind
             raise
 
 
+@collect_queryband(queryband="concat")
 def concat(df_list, join='OUTER', allow_duplicates=True, sort=False, ignore_index=False):
     """
     DESCRIPTION:
@@ -686,6 +692,8 @@ def concat(df_list, join='OUTER', allow_duplicates=True, sort=False, ignore_inde
         raise TeradataMlException(Messages.get_message(MessageCodes.SETOP_FAILED, setop_type),
                                   MessageCodes.SETOP_FAILED) from err
 
+
+@collect_queryband(queryband="tdIntersect")
 def td_intersect(df_list, allow_duplicates=True):
     """
     DESCRIPTION:
@@ -824,6 +832,8 @@ def td_intersect(df_list, allow_duplicates=True):
 
     return __process_setop_operation(df_list, allow_duplicates, setop_type, operation)
 
+
+@collect_queryband(queryband="tdMinus")
 def td_minus(df_list, allow_duplicates=True):
     """
     DESCRIPTION:
@@ -953,7 +963,7 @@ def td_minus(df_list, allow_duplicates=True):
     awu_matrix = []
     awu_matrix.append(["df_list", df_list, False, (list)])
     awu_matrix.append(["allow_duplicates", allow_duplicates, False, (bool)])
-    setop_type = 'td_except' if (inspect.stack()[1][3]) == 'td_except' else 'td_minus'
+    setop_type = 'td_except' if (inspect.stack()[2][3] and inspect.stack()[2][3] == 'td_except') else 'td_minus'
     operation = 'minus'
 
     # Validate Set operator arguments
@@ -961,6 +971,8 @@ def td_minus(df_list, allow_duplicates=True):
 
     return __process_setop_operation(df_list, allow_duplicates, setop_type, operation)
 
+
+@collect_queryband(queryband="tdExcept")
 def td_except(df_list, allow_duplicates=True):
     """
     DESCRIPTION:
@@ -1088,6 +1100,7 @@ def td_except(df_list, allow_duplicates=True):
     """
     return td_minus(df_list, allow_duplicates)
 
+
 def __process_setop_operation(df_list, allow_duplicates, setop_type, operation):
     """
     DESCRIPTION: