apache-hamilton 1.90.0.dev0__py3-none-any.whl

hamilton/experimental/h_dask.py

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from hamilton.plugins.h_dask import DaskDataFrameResult, DaskExecutor, DaskGraphAdapter
+
+logger = logging.getLogger(__name__)
+logger.warning(
+    "Importing from this module is deprecated. We have moved these features out of experimental! "
+    "Please use hamilton.plugins.h_dask instead."
+)
+
+__all__ = ["DaskGraphAdapter", "DaskDataFrameResult", "DaskExecutor"]

hamilton/experimental/h_databackends.py

@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""This module defines "databackends". They are essentially abstract types such as
+`AbstractPandasDataFrame` or `AbstractPandasSeries` which can be used with
+`isinstance()` or `issubclass()` without having to import pandas.
+
+It is powerful when used with `@functools.singledispatch`
+
+    ```python
+    @functools.singledispatch
+    def get_arrow_schema(df) -> pyarrow.Schema:
+        if not hasattr(df, "__dataframe__"):
+            raise NotImplementedError(f"Type {type(df)} is currently unsupported.")
+        return from_dataframe(df, allow_copy=True).schema
+
+
+    @get_arrow_schema.register
+    def _(df: h_databackends.AbstractPandasDataFrame) -> pyarrow.Schema:
+        return pyarrow.Table.from_pandas(df).schema
+
+
+    @get_arrow_schema.register
+    def _(df: h_databackends.AbstractIbisDataFrame) -> pyarrow.Schema:
+        return df.schema().to_pyarrow()
+    ```
+
+Instead of centralizing code by library / dependency, we can now centralize it
+by Hamilton feature. For example, we can have all the implementations to collect
+schemas under `schema.py` instead of spread across `pandas_extension.py`,
+polars_extension.py`, etc.
+"""
+
+import importlib
+import inspect
+
+from hamilton.experimental.databackend import AbstractBackend
+
+# TODO add a `_has__dataframe__` attribute for those that implement the
+# dataframe interchange protocol
+
+
+# PyArrow
+class AbstractPyArrowDataFrame(AbstractBackend):
+    _backends = [("pyarrow", "Table"), ("pyarrow", "RecordBatch")]
+
+
+class AbstractPyarrowColumn(AbstractBackend):
+    _backends = [("pyarrow", "Array"), ("pyarrow", "ChunkedArray")]
+
+
+# Ibis
+class AbstractIbisDataFrame(AbstractBackend):
+    _backends = [("ibis.expr.types", "Table")]
+
+
+class AbstractIbisColumn(AbstractBackend):
+    _backends = [("ibis.expr.types", "Column")]
+
+
+# Pandas
+class AbstractPandasDataFrame(AbstractBackend):
+    _backends = [("pandas", "DataFrame")]
+
+
+class AbstractPandasColumn(AbstractBackend):
+    _backends = [("pandas", "Series")]
+
+
+# Polars
+class AbstractPolarsDataFrame(AbstractBackend):
+    _backends = [("polars", "DataFrame")]
+
+
+class AbstractPolarsColumn(AbstractBackend):
+    _backends = [("polars", "Series")]
+
+
+# Polars Lazy
+class AbstractLazyPolarsDataFrame(AbstractBackend):
+    _backends = [("polars", "LazyFrame")]
+
+
+# Vaex
+class AbstractVaexDataFrame(AbstractBackend):
+    _backends = [("vaex.dataframe", "DataFrame")]
+
+
+class AbstractVaexColumn(AbstractBackend):
+    _backends = [("vaex.expression", "Expression")]
+
+
+# Dask
+class AbstractDaskDataFrame(AbstractBackend):
+    _backends = [("dask.dataframe", "DataFrame")]
+
+
+class AbstractDaskColumn(AbstractBackend):
+    _backends = [("dask.dataframe", "Series")]
+
+
+# SparkSQL
+class AbstractSparkSQLDataFrame(AbstractBackend):
+    _backends = [("pyspark.sql", "DataFrame")]
+
+
+# SparkPandas
+class AbstractSparkPandasDataFrame(AbstractBackend):
+    _backends = [("pyspark.pandas", "DataFrame")]
+
+
+class AbstractSparkPandasColumn(AbstractBackend):
+    _backends = [("pyspark.pandas", "Series")]
+
+
+# Geopandas
+class AbstractGeoPandasDataFrame(AbstractBackend):
+    _backends = [("geopandas", "GeoDataFrame")]
+
+
+class AbstractGeoPandasColumn(AbstractBackend):
+    _backends = [("geopandas", "GeoSeries")]
+
+
+# cuDF
+class AbstractCuDFDataFrame(AbstractBackend):
+    _backends = [("cudf", "DataFrame")]
+
+
+# Modin
+class AbstractModinDataFrame(AbstractBackend):
+    _backends = [("modin.pandas", "DataFrame")]
+
+
+# numpy
+class AbstractNumpyArray(AbstractBackend):
+    _backends = [("numpy", "ndarray")]
+
+
+def register_backends() -> tuple[tuple[type], tuple[type]]:
+    """Register databackends defined in this module that
+    include `DataFrame` and `Column` in their class name
+    """
+    abstract_dataframe_types = set()
+    abstract_column_types = set()
+
+    h_databackends_module = importlib.import_module(__name__)
+    for name, cls in inspect.getmembers(h_databackends_module, inspect.isclass):
+        if "DataFrame" in name:
+            abstract_dataframe_types.add(cls)
+        elif "Column" in name:
+            abstract_column_types.add(cls)
+
+    # Union[tuple()] creates a Union type object
+    DATAFRAME_TYPES = tuple(abstract_dataframe_types)
+    COLUMN_TYPES = tuple(abstract_column_types)
+    return DATAFRAME_TYPES, COLUMN_TYPES
+
+
+DATAFRAME_TYPES, COLUMN_TYPES = register_backends()

hamilton/experimental/h_ray.py

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from hamilton.plugins.h_ray import RayGraphAdapter, RayTaskExecutor
+
+logger = logging.getLogger(__name__)
+logger.warning(
+    "Importing from this module is deprecated. We have moved these features out of experimental!"
+    " Please use hamilton.plugins.h_ray instead."
+)
+
+__all__ = ["RayGraphAdapter", "RayTaskExecutor"]

hamilton/experimental/h_spark.py

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from hamilton.plugins.h_spark import (
+    KoalasDataFrameResult,
+    PySparkUDFGraphAdapter,
+    SparkKoalasGraphAdapter,
+)
+
+logger = logging.getLogger(__name__)
+logger.warning(
+    "Importing from this module is deprecated. We have moved these features out of experimental!"
+    " Please use hamilton.plugins.h_spark instead."
+)
+
+__all__ = ["KoalasDataFrameResult", "SparkKoalasGraphAdapter", "PySparkUDFGraphAdapter"]

hamilton/function_modifiers/README

@@ -0,0 +1,40 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# with_columns_base
+
+Documenting the current design flow for the `with_columns` decorator.
+
+For now, it belongs to the `NodeInjector` lifecycle since it still runs the decorated function as a node but injects the dataframe with columns appended columns as one of the parameters.
+
+The `with_columns` consists of three parts that are represented in the corresponding three abstract methods in `with_columns_base`:
+
+1. `get_initial_nodes` -- Input node(s): Either a dataframe if `pass_datafame_as` is used or extracted columns into nodes if `columns_to_pass` and is library specific.
+2. `get_subdag_nodes` -- Subdag nodes: Creating the `subdag` is outsourced to `recursive.subdag`, left flexibility to pre- and post-process since some libraries need that (see h_spark).
+3. `chain_subdag_nodes` -- Merge node: The append functionality between dataframe and selected columns is library specific.
+
+Each plugin library that can implement `with_columns` should subclass from this base class and implement the three abstract methods (four since `validate()` is also abstract). The child
+classes need to override the `init` where they call out to the parent `init` and pass in `dataframe_type` which is registered in the corresponding `extensions` and has information of what
+columns types are permitted for the given dataframe type.
+
+Keeping it for now loosely coupled to the `registry` and detached from `ResultBuilder`. The API is private, should we want to switch to `registry`, the refactoring is straightforward and shouldn't get us into trouble down the road.
+
+## NOTE
+The handling of scalars and dataframe types varies between library to library. We made the decision that such a thing should not be permissible, so all the selected columns that want to be
+appended to the original dataframe need to have the matching column type that is registered in the `registry` and set in the library extension modules.

hamilton/function_modifiers/__init__.py

@@ -0,0 +1,121 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from . import (
+    adapters,
+    base,
+    configuration,
+    delayed,
+    dependencies,
+    expanders,
+    macros,
+    metadata,
+    recursive,
+    validation,
+)
+
+logger = logging.getLogger(__name__)
+
+"""
+Annotations for modifying the way functions get added to the DAG.
+All user-facing annotation classes are lowercase as they're meant to be used
+as annotations. They are classes to hold state and subclass common functionality.
+"""
+
+# These all represent the public API for function_modifiers
+# All new user-facing decorators/helper functions should be here
+
+# Backwards-compatibility to be safe
+InvalidDecoratorException = base.InvalidDecoratorException
+
+# The config decorator
+config = configuration.config
+hamilton_exclude = configuration.hamilton_exclude()
+
+# Dependency Specification
+# Helper functions to specify dependency sources for parameterization
+value = dependencies.value
+source = dependencies.source
+group = dependencies.group
+configuration = dependencies.configuration
+
+# These aren't strictly part of the API but we should have them here for safety
+LiteralDependency = dependencies.LiteralDependency
+UpstreamDependency = dependencies.UpstreamDependency
+
+# Parameterization decorators (both the old and new ones)
+# The three "blessed" @parameterize decorators
+parameterize = expanders.parameterize
+parameterize_sources = expanders.parameterize_sources
+parameterize_values = expanders.parameterize_values
+parameterize_extract_columns = expanders.parameterize_extract_columns
+ParameterizedExtract = expanders.ParameterizedExtract
+inject = expanders.inject
+
+# The older ones that will be deprecated
+parametrized = expanders.parametrized
+parameterized_inputs = expanders.parameterized_inputs
+parametrized_input = expanders.parametrized_input
+
+# Extract decorators
+extract_columns = expanders.extract_columns
+extract_fields = expanders.extract_fields
+unpack_fields = expanders.unpack_fields
+
+# does decorator
+does = macros.does
+pipe = macros.pipe
+pipe_input = macros.pipe_input
+pipe_output = macros.pipe_output
+mutate = macros.mutate
+step = macros.step
+apply_to = macros.apply_to
+
+# resolve transform/model decorator
+dynamic_transform = macros.dynamic_transform
+model = macros.model
+
+# Metadata-specifying decorators
+tag = metadata.tag
+tag_outputs = metadata.tag_outputs
+schema = metadata.schema
+cache = metadata.cache
+
+# data quality + associated tags
+check_output = validation.check_output
+check_output_custom = validation.check_output_custom
+IS_DATA_VALIDATOR_TAG = validation.IS_DATA_VALIDATOR_TAG
+DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG = validation.DATA_VALIDATOR_ORIGINAL_OUTPUT_TAG
+
+# recursive/subdag operators
+
+subdag = recursive.subdag
+parameterized_subdag = recursive.parameterized_subdag
+
+# resolve/meta stuff -- power user features
+
+resolve = delayed.resolve
+ResolveAt = delayed.ResolveAt
+resolve_from_config = delayed.resolve_from_config
+
+# materialization stuff
+load_from = adapters.load_from
+save_to = adapters.save_to
+dataloader = adapters.dataloader
+datasaver = adapters.datasaver