teradataml 20.0.0.4__py3-none-any.whl → 20.0.0.6__py3-none-any.whl

teradataml/dataframe/dataframe_utils.py

@@ -652,7 +652,7 @@ class DataFrameUtils():
         all_operations = list(set(all_operations))
         invalid_aggregates = []
         for operation in all_operations:
-            if operation not in valid_aggregate_operations \
+            if operation not in valid_aggregate_operations and not operation.startswith('percentile_') \
                     and operation not in UtilFuncs._get_valid_time_series_aggregate_operations():
                 invalid_aggregates.append(operation)
         if len(invalid_aggregates) > 0: # If any of the aggregate operations specified is not valid
@@ -735,7 +735,20 @@ class DataFrameUtils():
                 quoted_columns = UtilFuncs._process_for_teradata_keyword(kwargs[key_to_process])
                 kwargs[key_to_process] = quoted_columns
 
-            func_expression = getattr(df[column], operation)(describe_op=describe_op, **kwargs)
+            if operation.startswith('percentile_'):
+                try:
+                    _operation_value = operation.split('_')
+                    _floatvalue = float(_operation_value[1])
+                    if _floatvalue < 0.0 or _floatvalue > 1.0 or len(_operation_value)>2:
+                        raise ValueError
+                except ValueError:
+                    mssg = "Invalid aggregate operation '{}' requested on TeradataML DataFrame." \
+                           " Valid operation should be in format 'percentile_<floatvalue>' and <floatvalue> " \
+                           "should be in range [0.0, 1.0].".format(operation)
+                    raise ValueError(mssg) from None
+                func_expression = getattr(df[column], 'percentile')(percentile=_floatvalue)
+            else:
+                func_expression = getattr(df[column], operation)(describe_op=describe_op, **kwargs)
             new_column_name = column if describe_op else "{1}_{0}".format(column, operation)
             # column_supported, new_column_name, new_column_type, column_aggr_expr, invalid_column_str
             return True, new_column_name, NUMBER() if describe_op else func_expression.type, \
@@ -1940,7 +1953,7 @@ class DataFrameUtils():
         return _DtypesMappers.DATALAKE_STR_to_TDSQLALCHEMY_DATATYPE_MAPPER[td_str_type]()
 
     @staticmethod
-    def _get_datalake_table_columns_info(schema, table_name, datalake):
+    def _get_datalake_table_columns_info(schema, table_name, datalake, use_dialect=False):
         """
         Function to get column names and corresponding teradatasqlalchemy types
         of a datalake table using results of 'help table <datalake>.<db_name>.<table_name>'
@@ -1977,31 +1990,64 @@ class DataFrameUtils():
               VARCHAR(length=2000, charset='UNICODE'),
               INTEGER()])
         """
-        # Get the column information from the strings type.
-        prepared = preparer(td_dialect())
-        sqlbundle = SQLBundle()
-        full_tbl_name = '{}.{}.{}'.format(prepared.quote(datalake),
-                                          prepared.quote(schema),
-                                          prepared.quote(table_name))
-        help_table_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_TABLE).format(full_tbl_name)
-
-        cur = execute_sql(help_table_sql)
-        td_types_col_index = -1
-        for i, col_metadata in enumerate(cur.description):
-            # Help Table returns column names and
-            # corresponding IcebergType, TeradataInternalType,
-            # TeradataType. We need to extract column index for
-            # 'TeradataType' column.
-            if col_metadata[0].lower() == 'teradatatype':
-                td_types_col_index = i
-
         col_names = []
         col_types = []
-        if td_types_col_index > -1:
-            for col_info in cur.fetchall():
-                col_names.append(col_info[0])
-                col_types.append(DataFrameUtils._get_sqlalchemy_type_from_str(col_info[td_types_col_index]))
+        if not use_dialect:
+            # Get the column information from the strings type.
+            prepared = preparer(td_dialect())
+            sqlbundle = SQLBundle()
+            full_tbl_name = '{}.{}.{}'.format(prepared.quote(datalake),
+                                              prepared.quote(schema),
+                                              prepared.quote(table_name))
+            help_table_sql = sqlbundle._get_sql_query(SQLConstants.SQL_HELP_TABLE).format(full_tbl_name)
+
+            cur = execute_sql(help_table_sql)
+            td_types_col_index = -1
+
+            for i, col_metadata in enumerate(cur.description):
+                # Help Table returns column names and
+                # corresponding IcebergType, TeradataInternalType,
+                # TeradataType. We need to extract column index for
+                # 'TeradataType' column.
+                if col_metadata[0].lower() in ['teradatatype', 'Type']:
+                    td_types_col_index = i
+
+            if td_types_col_index > -1:
+                for col_info in cur.fetchall():
+                    col_names.append(col_info[0])
+                    col_types.append(DataFrameUtils._get_sqlalchemy_type_from_str(col_info[td_types_col_index]))
+            else:
+                raise TeradataMlException(Messages.get_message(MessageCodes.TDMLDF_CREATE_FAIL),
+                                          MessageCodes.TDMLDF_CREATE_FAIL)
         else:
-            raise TeradataMlException(Messages.get_message(MessageCodes.TDMLDF_CREATE_FAIL),
-                                      MessageCodes.TDMLDF_CREATE_FAIL)
+            new_kwarg = get_connection().dialect.__class__.__name__ + "_datalake"
+            all_col_info = get_connection().dialect.get_columns(connection=get_connection(),
+                                                                table_name=table_name,
+                                                                schema=schema,
+                                                                table_only=True,
+                                                                **{new_kwarg: datalake})
+            for col_dict in all_col_info:
+                col_names.append(col_dict.get('name', col_dict.get('Column Name')))
+                col_types.append(col_dict.get('type', col_dict.get('Type')))
+
         return col_names, col_types
+
+    @staticmethod
+    def check_otf_dataframe():
+        """Decorator for validating if DataFrame is created on OTF table or not and throw error."""
+        def decorator(method):
+            def wrapper(self, *args, **kwargs):
+                if not self._datalake:
+                    attr = getattr(type(self), method.__name__, None)
+                    caller_name = method.__name__ + '()'
+                    if isinstance(attr, property):
+                        caller_name = method.__name__
+                    raise TeradataMlException(Messages.get_message(MessageCodes.OTF_TABLE_REQUIRED,
+                                                                   caller_name),
+                                              MessageCodes.UNSUPPORTED_OPERATION)
+
+                return method(self, *args, **kwargs)
+
+            return wrapper
+
+        return decorator