fugue 0.8.2.dev1__py3-none-any.whl → 0.8.4__py3-none-any.whl

fugue/execution/api.py

@@ -15,6 +15,7 @@ from .execution_engine import (
     ExecutionEngine,
 )
 from .factory import make_execution_engine, try_get_context_execution_engine
+from .._utils.registry import fugue_plugin
 
 
 @contextmanager
@@ -120,6 +121,27 @@ def get_current_parallelism() -> int:
     return make_execution_engine().get_current_parallelism()
 
 
+@fugue_plugin
+def as_fugue_engine_df(
+    engine: ExecutionEngine, df: AnyDataFrame, schema: Any = None
+) -> DataFrame:
+    """Convert a dataframe to a Fugue engine dependent DataFrame.
+    This function is used internally by Fugue. It is not recommended
+    to use
+
+    :param engine: the ExecutionEngine to use, must not be None
+    :param df: a dataframe like object
+    :param schema: the schema of the dataframe, defaults to None
+
+    :return: the engine dependent DataFrame
+    """
+    if schema is None:
+        fdf = as_fugue_df(df)
+    else:
+        fdf = as_fugue_df(df, schema=schema)
+    return engine.to_df(fdf)
+
+
 def run_engine_function(
     func: Callable[[ExecutionEngine], Any],
     engine: AnyExecutionEngine = None,
@@ -199,10 +221,10 @@ def broadcast(
     as_fugue: bool = False,
     as_local: bool = False,
 ) -> AnyDataFrame:
-    """Broadcast the dataframe to all workers for a distributed computing framework
+    """Broadcast the dataframe to all workers of a distributed computing backend
 
     :param df: an input dataframe that can be recognized by Fugue
-    :param engine: an engine like object, defaults to None
+    :param engine: an engine-like object, defaults to None
     :param engine_conf: the configs for the engine, defaults to None
     :param as_fugue: whether to force return a Fugue DataFrame, defaults to False
     :param as_local: whether to force return a local DataFrame, defaults to False
@@ -549,11 +571,11 @@ def join(
     """
 
     def _join(e: ExecutionEngine):
-        edf1 = e.to_df(df1)
-        edf2 = e.to_df(df2)
+        edf1 = as_fugue_engine_df(e, df1)
+        edf2 = as_fugue_engine_df(e, df2)
         res = e.join(edf1, edf2, how=how, on=on)
         for odf in dfs:
-            res = e.join(res, e.to_df(odf), how=how, on=on)
+            res = e.join(res, as_fugue_engine_df(e, odf), how=how, on=on)
         return res
 
     return run_engine_function(
@@ -837,11 +859,11 @@ def union(
     """
 
     def _union(e: ExecutionEngine):
-        edf1 = e.to_df(df1)
-        edf2 = e.to_df(df2)
+        edf1 = as_fugue_engine_df(e, df1)
+        edf2 = as_fugue_engine_df(e, df2)
         res = e.union(edf1, edf2, distinct=distinct)
         for odf in dfs:
-            res = e.union(res, e.to_df(odf), distinct=distinct)
+            res = e.union(res, as_fugue_engine_df(e, odf), distinct=distinct)
         return res
 
     return run_engine_function(
@@ -885,11 +907,11 @@ def subtract(
     """
 
     def _subtract(e: ExecutionEngine):
-        edf1 = e.to_df(df1)
-        edf2 = e.to_df(df2)
+        edf1 = as_fugue_engine_df(e, df1)
+        edf2 = as_fugue_engine_df(e, df2)
         res = e.subtract(edf1, edf2, distinct=distinct)
         for odf in dfs:
-            res = e.subtract(res, e.to_df(odf), distinct=distinct)
+            res = e.subtract(res, as_fugue_engine_df(e, odf), distinct=distinct)
         return res
 
     return run_engine_function(
@@ -933,11 +955,11 @@ def intersect(
     """
 
     def _intersect(e: ExecutionEngine):
-        edf1 = e.to_df(df1)
-        edf2 = e.to_df(df2)
+        edf1 = as_fugue_engine_df(e, df1)
+        edf2 = as_fugue_engine_df(e, df2)
         res = e.intersect(edf1, edf2, distinct=distinct)
         for odf in dfs:
-            res = e.intersect(res, e.to_df(odf), distinct=distinct)
+            res = e.intersect(res, as_fugue_engine_df(e, odf), distinct=distinct)
         return res
 
     return run_engine_function(

fugue/execution/execution_engine.py

@@ -1,3 +1,4 @@
+import inspect
 import logging
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
@@ -17,8 +18,9 @@ from typing import (
 )
 from uuid import uuid4
 
-from triad import ParamDict, Schema, SerializableRLock, assert_or_throw
+from triad import ParamDict, Schema, SerializableRLock, assert_or_throw, to_uuid
 from triad.collections.fs import FileSystem
+from triad.collections.function_wrapper import AnnotatedParam
 from triad.exceptions import InvalidOperationError
 from triad.utils.convert import to_size
 from triad.utils.string import validate_triad_var_name
@@ -30,7 +32,7 @@ from fugue.collections.partition import (
     PartitionSpec,
 )
 from fugue.collections.sql import StructuredRawSQL, TempTableName
-from fugue.collections.yielded import Yielded, PhysicalYielded
+from fugue.collections.yielded import PhysicalYielded, Yielded
 from fugue.column import (
     ColumnExpr,
     SelectColumns,
@@ -40,11 +42,11 @@ from fugue.column import (
     is_agg,
 )
 from fugue.constants import _FUGUE_GLOBAL_CONF, FUGUE_SQL_DEFAULT_DIALECT
-from fugue.dataframe import AnyDataFrame, DataFrame, DataFrames
+from fugue.dataframe import AnyDataFrame, DataFrame, DataFrames, fugue_annotated_param
 from fugue.dataframe.array_dataframe import ArrayDataFrame
 from fugue.dataframe.dataframe import LocalDataFrame
 from fugue.dataframe.utils import deserialize_df, serialize_df
-from fugue.exceptions import FugueBug
+from fugue.exceptions import FugueBug, FugueWorkflowRuntimeError
 
 AnyExecutionEngine = TypeVar("AnyExecutionEngine", object, None)
 
@@ -275,6 +277,7 @@ class MapEngine(EngineFacet):
         output_schema: Any,
         partition_spec: PartitionSpec,
         on_init: Optional[Callable[[int, DataFrame], Any]] = None,
+        map_func_format_hint: Optional[str] = None,
     ) -> DataFrame:  # pragma: no cover
         """Apply a function to each partition after you partition the dataframe in a
         specified way.
@@ -287,6 +290,9 @@ class MapEngine(EngineFacet):
         :param partition_spec: partition specification
         :param on_init: callback function when the physical partition is initializaing,
           defaults to None
+        :param map_func_format_hint: the preferred data format for ``map_func``, it can
+          be ``pandas``, `pyarrow`, etc, defaults to None. Certain engines can provide
+          the most efficient map operations based on the hint.
         :return: the dataframe after the map operation
 
         .. note::
@@ -1298,6 +1304,26 @@ class ExecutionEngine(FugueEngineBase):
         return res
 
 
+@fugue_annotated_param(ExecutionEngine, "e", child_can_reuse_code=True)
+class ExecutionEngineParam(AnnotatedParam):
+    def __init__(
+        self,
+        param: Optional[inspect.Parameter],
+    ):
+        super().__init__(param)
+        self._type = self.annotation
+
+    def to_input(self, engine: Any) -> Any:
+        assert_or_throw(
+            isinstance(engine, self._type),
+            FugueWorkflowRuntimeError(f"{engine} is not of type {self._type}"),
+        )
+        return engine
+
+    def __uuid__(self) -> str:
+        return to_uuid(self.code, self.annotation, self._type)
+
+
 def _get_file_threshold(size: Any) -> int:
     if size is None:
         return -1

fugue/execution/factory.py

@@ -492,9 +492,6 @@ def make_sql_engine(
 
             # S2(engine)
             make_sql_engine("s2", engine)
-
-            # SqliteEngine(engine)
-            make_sql_engine(SqliteEngine)
     """
     if isinstance(engine, SQLEngine):
         assert_or_throw(
@@ -554,9 +551,6 @@ def parse_sql_engine(
 
             # S2(engine)
             make_sql_engine("s2", engine)
-
-            # SqliteEngine(engine)
-            make_sql_engine(SqliteEngine)
     """
     if engine is None or (isinstance(engine, str) and engine == ""):
         assert_or_throw(

fugue/execution/native_execution_engine.py

@@ -1,4 +1,3 @@
-import inspect
 import logging
 import os
 from typing import Any, Callable, Dict, List, Optional, Type, Union
@@ -6,17 +5,11 @@ from typing import Any, Callable, Dict, List, Optional, Type, Union
 import pandas as pd
 from qpd_pandas import run_sql_on_pandas
 from qpd_pandas.engine import PandasUtils
-from sqlalchemy import create_engine
 from triad import Schema
 from triad.collections.dict import IndexedOrderedDict
 from triad.collections.fs import FileSystem
 from triad.utils.assertion import assert_or_throw
 
-from fugue._utils.interfaceless import (
-    ExecutionEngineParam,
-    SimpleAnnotationConverter,
-    register_annotation_converter,
-)
 from fugue._utils.io import load_df, save_df
 from fugue.collections.partition import (
     PartitionCursor,
@@ -31,34 +24,17 @@ from fugue.dataframe import (
     LocalBoundedDataFrame,
     LocalDataFrame,
     PandasDataFrame,
-    to_local_bounded_df,
+    fugue_annotated_param,
 )
-from fugue.dataframe.utils import get_join_schemas, to_local_df
-
-from .execution_engine import ExecutionEngine, MapEngine, SQLEngine
-
-
-class SqliteEngine(SQLEngine):
-    """Sqlite execution implementation.
-
-    :param execution_engine: the execution engine this sql engine will run on
-    """
+from fugue.dataframe.dataframe import as_fugue_df
+from fugue.dataframe.utils import get_join_schemas
 
-    @property
-    def is_distributed(self) -> bool:
-        return False
-
-    @property
-    def dialect(self) -> Optional[str]:
-        return "sqlite"
-
-    def select(self, dfs: DataFrames, statement: StructuredRawSQL) -> DataFrame:
-        _dfs, _sql = self.encode(dfs, statement)
-        sql_engine = create_engine("sqlite:///:memory:")
-        for k, v in _dfs.items():
-            v.as_pandas().to_sql(k, sql_engine, if_exists="replace", index=False)
-        df = pd.read_sql_query(_sql, sql_engine)
-        return PandasDataFrame(df)
+from .execution_engine import (
+    ExecutionEngine,
+    ExecutionEngineParam,
+    MapEngine,
+    SQLEngine,
+)
 
 
 class QPDPandasEngine(SQLEngine):
@@ -105,20 +81,38 @@ class PandasMapEngine(MapEngine):
         output_schema: Any,
         partition_spec: PartitionSpec,
         on_init: Optional[Callable[[int, DataFrame], Any]] = None,
+        map_func_format_hint: Optional[str] = None,
     ) -> DataFrame:
-        if partition_spec.num_partitions != "0":
-            self.log.warning(
-                "%s doesn't respect num_partitions %s",
-                self,
-                partition_spec.num_partitions,
-            )
+        # if partition_spec.num_partitions != "0":
+        #     self.log.warning(
+        #         "%s doesn't respect num_partitions %s",
+        #         self,
+        #         partition_spec.num_partitions,
+        #     )
+        is_coarse = partition_spec.algo == "coarse"
+        presort = partition_spec.get_sorts(df.schema, with_partition_keys=is_coarse)
+        presort_keys = list(presort.keys())
+        presort_asc = list(presort.values())
+        output_schema = Schema(output_schema)
         cursor = partition_spec.get_cursor(df.schema, 0)
         if on_init is not None:
             on_init(0, df)
-        if len(partition_spec.partition_by) == 0:  # no partition
-            df = to_local_df(df)
-            cursor.set(df.peek_array(), 0, 0)
-            output_df = map_func(cursor, df)
+        if (
+            len(partition_spec.partition_by) == 0 or partition_spec.algo == "coarse"
+        ):  # no partition
+            if len(partition_spec.presort) > 0:
+                pdf = (
+                    df.as_pandas()
+                    .sort_values(presort_keys, ascending=presort_asc)
+                    .reset_index(drop=True)
+                )
+                input_df = PandasDataFrame(pdf, df.schema, pandas_df_wrapper=True)
+                cursor.set(lambda: input_df.peek_array(), cursor.partition_no + 1, 0)
+                output_df = map_func(cursor, input_df)
+            else:
+                df = df.as_local()
+                cursor.set(lambda: df.peek_array(), 0, 0)
+                output_df = map_func(cursor, df)
             if (
                 isinstance(output_df, PandasDataFrame)
                 and output_df.schema != output_schema
@@ -130,18 +124,14 @@ class PandasMapEngine(MapEngine):
                 f"mismatches given {output_schema}",
             )
             return self.to_df(output_df)  # type: ignore
-        presort = partition_spec.presort
-        presort_keys = list(presort.keys())
-        presort_asc = list(presort.values())
-        output_schema = Schema(output_schema)
 
         def _map(pdf: pd.DataFrame) -> pd.DataFrame:
-            if len(presort_keys) > 0:
+            if len(partition_spec.presort) > 0:
                 pdf = pdf.sort_values(presort_keys, ascending=presort_asc).reset_index(
                     drop=True
                 )
             input_df = PandasDataFrame(pdf, df.schema, pandas_df_wrapper=True)
-            cursor.set(input_df.peek_array(), cursor.partition_no + 1, 0)
+            cursor.set(lambda: input_df.peek_array(), cursor.partition_no + 1, 0)
             output_df = map_func(cursor, input_df)
             return output_df.as_pandas()
 
@@ -200,7 +190,7 @@ class NativeExecutionEngine(ExecutionEngine):
     def repartition(
         self, df: DataFrame, partition_spec: PartitionSpec
     ) -> DataFrame:  # pragma: no cover
-        self.log.warning("%s doesn't respect repartition", self)
+        # self.log.warning("%s doesn't respect repartition", self)
         return df
 
     def broadcast(self, df: DataFrame) -> DataFrame:
@@ -401,24 +391,11 @@ class NativeExecutionEngine(ExecutionEngine):
         save_df(df, path, format_hint=format_hint, mode=mode, fs=self.fs, **kwargs)
 
 
+@fugue_annotated_param(NativeExecutionEngine)
 class _NativeExecutionEngineParam(ExecutionEngineParam):
-    def __init__(
-        self,
-        param: Optional[inspect.Parameter],
-    ):
-        super().__init__(
-            param, annotation="NativeExecutionEngine", engine_type=NativeExecutionEngine
-        )
+    pass
 
 
 def _to_native_execution_engine_df(df: AnyDataFrame, schema: Any = None) -> DataFrame:
-    return to_local_bounded_df(df, schema)
-
-
-register_annotation_converter(
-    0.8,
-    SimpleAnnotationConverter(
-        NativeExecutionEngine,
-        lambda param: _NativeExecutionEngineParam(param),
-    ),
-)
+    fdf = as_fugue_df(df) if schema is None else as_fugue_df(df, schema=schema)
+    return fdf.as_local_bounded()

fugue/extensions/_builtins/creators.py

@@ -1,10 +1,12 @@
 from typing import Any, Callable, Optional
 
+from triad import Schema, assert_or_throw, to_uuid
+
 from fugue.collections.yielded import Yielded
 from fugue.dataframe import DataFrame
 from fugue.exceptions import FugueWorkflowCompileError
+from fugue.execution.api import as_fugue_engine_df
 from fugue.extensions.creator import Creator
-from triad import Schema, assert_or_throw, to_uuid
 
 
 class Load(Creator):
@@ -39,7 +41,7 @@ class CreateData(Creator):
     def create(self) -> DataFrame:
         if isinstance(self._df, Yielded):
             return self.execution_engine.load_yielded(self._df)
-        return self.execution_engine.to_df(self._df, schema=self._schema)
+        return as_fugue_engine_df(self.execution_engine, self._df, schema=self._schema)
 
     def _df_uid(self):
         if self._data_determiner is not None:

fugue/extensions/_builtins/outputters.py

@@ -6,7 +6,7 @@ from triad.utils.convert import to_type
 from fugue.collections.partition import PartitionCursor
 from fugue.dataframe import DataFrame, DataFrames, LocalDataFrame
 from fugue.dataframe.array_dataframe import ArrayDataFrame
-from fugue.dataframe.utils import _df_eq, to_local_bounded_df
+from fugue.dataframe.utils import _df_eq
 from fugue.exceptions import FugueWorkflowError
 from fugue.execution.execution_engine import _generate_comap_empty_dfs
 from fugue.rpc import EmptyRPCHandler, to_rpc_handler
@@ -99,6 +99,7 @@ class RunOutputTransformer(Outputter):
             output_schema=tf.output_schema,  # type: ignore
             partition_spec=tf.partition_spec,
             on_init=tr.on_init,
+            map_func_format_hint=tf.get_format_hint(),
         )
         self.execution_engine.persist(df, lazy=False)
 
@@ -135,7 +136,7 @@ class _TransformerRunner(object):
     def run(self, cursor: PartitionCursor, df: LocalDataFrame) -> LocalDataFrame:
         self.transformer._cursor = cursor  # type: ignore
         try:
-            to_local_bounded_df(self.transformer.transform(df))
+            self.transformer.transform(df).as_local_bounded()
             return ArrayDataFrame([], self.transformer.output_schema)
         except self.ignore_errors:  # type: ignore
             return ArrayDataFrame([], self.transformer.output_schema)
@@ -159,7 +160,7 @@ class _CoTransformerRunner(object):
     def run(self, cursor: PartitionCursor, dfs: DataFrames) -> LocalDataFrame:
         self.transformer._cursor = cursor  # type: ignore
         try:
-            to_local_bounded_df(self.transformer.transform(dfs))
+            self.transformer.transform(dfs).as_local_bounded()
             return ArrayDataFrame([], self.transformer.output_schema)
         except self.ignore_errors:  # type: ignore
             return ArrayDataFrame([], self.transformer.output_schema)

fugue/extensions/_builtins/processors.py

@@ -6,7 +6,6 @@ from fugue.dataframe import (
     DataFrame,
     DataFrames,
     LocalDataFrame,
-    to_local_bounded_df,
 )
 from fugue.column import ColumnExpr, SelectColumns as ColumnsSelect
 from fugue.exceptions import FugueWorkflowError
@@ -53,6 +52,7 @@ class RunTransformer(Processor):
             output_schema=tf.output_schema,  # type: ignore
             partition_spec=tf.partition_spec,
             on_init=tr.on_init,
+            map_func_format_hint=tf.get_format_hint(),
         )
 
     @no_type_check
@@ -333,7 +333,7 @@ class _TransformerRunner(object):
             return self.transformer.transform(df)
         else:
             try:
-                return to_local_bounded_df(self.transformer.transform(df))
+                return self.transformer.transform(df).as_local_bounded()
             except self.ignore_errors:  # type: ignore  # pylint: disable=E0712
                 return ArrayDataFrame([], self.transformer.output_schema)
 
@@ -363,7 +363,7 @@ class _CoTransformerRunner(object):
 
         else:
             try:
-                return to_local_bounded_df(self.transformer.transform(dfs))
+                return self.transformer.transform(dfs).as_local_bounded()
             except self.ignore_errors:  # type: ignore  # pylint: disable=E0712
                 return ArrayDataFrame([], self.transformer.output_schema)
 

fugue/extensions/creator/convert.py

@@ -7,9 +7,10 @@ from triad.utils.assertion import assert_or_throw
 from triad.utils.convert import get_caller_global_local_vars, to_function, to_instance
 from triad.utils.hash import to_uuid
 
-from fugue._utils.interfaceless import FunctionWrapper, parse_output_schema_from_comment
+from fugue._utils.interfaceless import parse_output_schema_from_comment
 from fugue._utils.registry import fugue_plugin
 from fugue.dataframe import DataFrame
+from fugue.dataframe.function_wrapper import DataFrameFunctionWrapper
 from fugue.exceptions import FugueInterfacelessError
 from fugue.extensions.creator.creator import Creator
 
@@ -200,7 +201,9 @@ class _FuncAsCreator(Creator):
         if schema is None:
             schema = parse_output_schema_from_comment(func)
         tr = _FuncAsCreator()
-        tr._wrapper = FunctionWrapper(func, "^e?x*z?$", "^[dlspq]$")  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
+            func, "^e?x*z?$", "^[dlspq]$"
+        )
         tr._engine_param = (
             tr._wrapper._params.get_value_by_index(0)
             if tr._wrapper.input_code.startswith("e")

fugue/extensions/outputter/convert.py

@@ -4,9 +4,9 @@ from typing import Any, Callable, Dict, List, Optional, no_type_check
 from triad import ParamDict, to_uuid
 from triad.utils.convert import get_caller_global_local_vars, to_function, to_instance
 
-from fugue._utils.interfaceless import FunctionWrapper
 from fugue._utils.registry import fugue_plugin
 from fugue.dataframe import DataFrames
+from fugue.dataframe.function_wrapper import DataFrameFunctionWrapper
 from fugue.exceptions import FugueInterfacelessError
 from fugue.extensions._utils import (
     load_namespace_extensions,
@@ -204,7 +204,7 @@ class _FuncAsOutputter(Outputter):
     ) -> "_FuncAsOutputter":
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputter()
-        tr._wrapper = FunctionWrapper(  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
             func, "^e?(c|[dlspq]+)x*z?$", "^n$"
         )
         tr._engine_param = (

fugue/extensions/processor/convert.py

@@ -6,9 +6,10 @@ from triad.collections import Schema
 from triad.utils.assertion import assert_or_throw
 from triad.utils.convert import get_caller_global_local_vars, to_function, to_instance
 
-from fugue._utils.interfaceless import FunctionWrapper, parse_output_schema_from_comment
+from fugue._utils.interfaceless import parse_output_schema_from_comment
 from fugue._utils.registry import fugue_plugin
 from fugue.dataframe import DataFrame, DataFrames
+from fugue.dataframe.function_wrapper import DataFrameFunctionWrapper
 from fugue.exceptions import FugueInterfacelessError
 from fugue.extensions.processor.processor import Processor
 
@@ -223,7 +224,7 @@ class _FuncAsProcessor(Processor):
             schema = parse_output_schema_from_comment(func)
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsProcessor()
-        tr._wrapper = FunctionWrapper(
+        tr._wrapper = DataFrameFunctionWrapper(
             func, "^e?(c|[dlspq]+)x*z?$", "^[dlspq]$"
         )  # type: ignore
         tr._engine_param = (

fugue/extensions/transformer/convert.py

@@ -6,13 +6,10 @@ from triad.utils.assertion import assert_arg_not_none, assert_or_throw
 from triad.utils.convert import get_caller_global_local_vars, to_function, to_instance
 from triad.utils.hash import to_uuid
 
-from fugue._utils.interfaceless import (
-    FunctionWrapper,
-    is_class_method,
-    parse_output_schema_from_comment,
-)
+from fugue._utils.interfaceless import is_class_method, parse_output_schema_from_comment
 from fugue._utils.registry import fugue_plugin
 from fugue.dataframe import ArrayDataFrame, DataFrame, DataFrames, LocalDataFrame
+from fugue.dataframe.function_wrapper import DataFrameFunctionWrapper
 from fugue.exceptions import FugueInterfacelessError
 from fugue.extensions.transformer.constants import OUTPUT_TRANSFORMER_DUMMY_SCHEMA
 from fugue.extensions.transformer.transformer import CoTransformer, Transformer
@@ -336,6 +333,9 @@ class _FuncAsTransformer(Transformer):
     def get_output_schema(self, df: DataFrame) -> Any:
         return self._parse_schema(self._output_schema_arg, df)  # type: ignore
 
+    def get_format_hint(self) -> Optional[str]:
+        return self._format_hint  # type: ignore
+
     @property
     def validation_rules(self) -> Dict[str, Any]:
         return self._validation_rules  # type: ignore
@@ -374,13 +374,14 @@ class _FuncAsTransformer(Transformer):
         validation_rules.update(parse_validation_rules_from_comment(func))
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsTransformer()
-        tr._wrapper = FunctionWrapper(  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
             func, "^[lspq][fF]?x*z?$", "^[lspq]$"
         )
         tr._output_schema_arg = schema  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
         tr._uses_callback = "f" in tr._wrapper.input_code.lower()  # type: ignore
         tr._requires_callback = "F" in tr._wrapper.input_code  # type: ignore
+        tr._format_hint = tr._wrapper.get_format_hint()  # type: ignore
         return tr
 
 
@@ -392,6 +393,9 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
     def get_output_schema(self, df: DataFrame) -> Any:
         return OUTPUT_TRANSFORMER_DUMMY_SCHEMA
 
+    def get_format_hint(self) -> Optional[str]:
+        return self._format_hint  # type: ignore
+
     @no_type_check
     def transform(self, df: LocalDataFrame) -> LocalDataFrame:
         args = [df] + _get_callback(self)
@@ -405,13 +409,14 @@ class _FuncAsOutputTransformer(_FuncAsTransformer):
         assert_or_throw(schema is None, "schema must be None for output transformers")
         validation_rules.update(parse_validation_rules_from_comment(func))
         tr = _FuncAsOutputTransformer()
-        tr._wrapper = FunctionWrapper(  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
             func, "^[lspq][fF]?x*z?$", "^[lspnq]$"
         )
         tr._output_schema_arg = None  # type: ignore
         tr._validation_rules = validation_rules  # type: ignore
         tr._uses_callback = "f" in tr._wrapper.input_code.lower()  # type: ignore
         tr._requires_callback = "F" in tr._wrapper.input_code  # type: ignore
+        tr._format_hint = tr._wrapper.get_format_hint()  # type: ignore
         return tr
 
 
@@ -423,6 +428,9 @@ class _FuncAsCoTransformer(CoTransformer):
     def get_output_schema(self, dfs: DataFrames) -> Any:
         return self._parse_schema(self._output_schema_arg, dfs)  # type: ignore
 
+    def get_format_hint(self) -> Optional[str]:
+        return self._format_hint  # type: ignore
+
     @property
     def validation_rules(self) -> ParamDict:
         return self._validation_rules  # type: ignore
@@ -494,7 +502,7 @@ class _FuncAsCoTransformer(CoTransformer):
             )
         assert_arg_not_none(schema, "schema")
         tr = _FuncAsCoTransformer()
-        tr._wrapper = FunctionWrapper(  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
             func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspq]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
@@ -502,6 +510,7 @@ class _FuncAsCoTransformer(CoTransformer):
         tr._validation_rules = {}  # type: ignore
         tr._uses_callback = "f" in tr._wrapper.input_code.lower()  # type: ignore
         tr._requires_callback = "F" in tr._wrapper.input_code  # type: ignore
+        tr._format_hint = tr._wrapper.get_format_hint()  # type: ignore
         return tr
 
 
@@ -513,6 +522,9 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
     def get_output_schema(self, dfs: DataFrames) -> Any:
         return OUTPUT_TRANSFORMER_DUMMY_SCHEMA
 
+    def get_format_hint(self) -> Optional[str]:
+        return self._format_hint  # type: ignore
+
     @no_type_check
     def transform(self, dfs: DataFrames) -> LocalDataFrame:
         cb = _get_callback(self)
@@ -549,7 +561,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
         )
 
         tr = _FuncAsOutputCoTransformer()
-        tr._wrapper = FunctionWrapper(  # type: ignore
+        tr._wrapper = DataFrameFunctionWrapper(  # type: ignore
             func, "^(c|[lspq]+)[fF]?x*z?$", "^[lspnq]$"
         )
         tr._dfs_input = tr._wrapper.input_code[0] == "c"  # type: ignore
@@ -557,6 +569,7 @@ class _FuncAsOutputCoTransformer(_FuncAsCoTransformer):
         tr._validation_rules = {}  # type: ignore
         tr._uses_callback = "f" in tr._wrapper.input_code.lower()  # type: ignore
         tr._requires_callback = "F" in tr._wrapper.input_code  # type: ignore
+        tr._format_hint = tr._wrapper.get_format_hint()  # type: ignore
         return tr