apache-hamilton 1.90.0.dev0__py3-none-any.whl

hamilton/log_setup.py

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+import sys
+
+import numpy as np
+
+LOG_LEVELS = {
+    "CRITICAL": logging.CRITICAL,
+    "ERROR": logging.ERROR,
+    "WARNING": logging.WARNING,
+    "INFO": logging.INFO,
+    "DEBUG": logging.DEBUG,
+}
+
+
+# this is suboptimal but python has no public mapping of log names to levels
+
+
+def setup_logging(log_level: int = logging.INFO):
+    """Helper function to setup logging to console.
+    :param log_level: Log level to use when logging
+    """
+    root_logger = logging.getLogger("")  # root logger
+    formatter = logging.Formatter("[%(levelname)s] %(asctime)s %(name)s(%(lineno)s): %(message)s")
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setFormatter(formatter)
+    if not len(root_logger.handlers):
+        # assumes we have already been set up.
+        root_logger.addHandler(stream_handler)
+        root_logger.setLevel(log_level)
+    np.seterr(divide="ignore", invalid="ignore")

hamilton/models.py

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import abc
+from typing import Any
+
+import pandas as pd
+
+
+class DynamicTransformBase(abc.ABC):
+    """Abstract class for a dynamic transform transform as seen by hamilton.
+    These are transforms that come from configuration parameters, and define the following:
+    1. The nodes that they depend on
+    2. What the transform does
+
+    Paired with the decorator @dynamic_transform(CLS, config_item, **extra_params) one can write incredibly powerful
+    DAGs that depend on dynamic transform configs.
+    """
+
+    def __init__(self, config_parameters: Any, name: str):
+        self._config_parameters = config_parameters
+        self._name = name
+
+    @abc.abstractmethod
+    def get_dependents(self) -> list[str]:
+        """Gets the names/types of the inputs to this transform.
+        :return: A list of columns on which this model depends.
+        """
+        pass
+
+    @abc.abstractmethod
+    def compute(self, **inputs: Any) -> Any:
+        """Runs a computation based on a set of input values.
+        :param inputs: data that are inputs to the transform
+        :return: The result of the transform.
+        """
+        pass
+
+    @property
+    def config_parameters(self) -> dict[str, Any]:
+        """Accessor for configuration parameters"""
+        return self._config_parameters
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+
+# Maintained for backwards compatibility
+# Will likely keep this in some way, but might change the name or the approach
+# with the 2.0 release
+class BaseModel(DynamicTransformBase, abc.ABC):
+    """Base classes for models in hamilton
+    We define a model as anything whose inputs are configuration-driven.
+    The goal of this is to allow users to write something like:
+
+    @model(...)
+    def my_column():
+        #column that is some combinations of other columns using some previously trained model
+
+    Note that the model-training is not yet in scope."""
+
+    def compute(self, **inputs: Any) -> Any:
+        """Delegates to predict function to compute this model's return.
+
+        :param inputs: Inputs for the model
+        :return: The result of calling the model.
+        """
+        return self.predict(**inputs)
+
+    @abc.abstractmethod
+    def predict(self, **inputs: pd.Series) -> pd.Series:
+        """Runs the predict() function on the model, given the set of kwargs.
+        :param inputs: Inputs to the model.
+        :return: A series representing the output of the model.
+        """
+
+        pass

hamilton/node.py

@@ -0,0 +1,449 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import builtins
+import inspect
+import sys
+import typing
+from collections.abc import Callable
+from enum import Enum
+from typing import Any
+
+import typing_inspect
+
+from hamilton.htypes import Collect, Parallelizable
+
+"""
+Module that contains the primitive components of the graph.
+
+These get their own file because we don't like circular dependencies.
+"""
+
+
+class NodeType(Enum):
+    """
+    Specifies where this node's value originates.
+    This can be used by different adapters to flexibly execute a function graph.
+    """
+
+    STANDARD = 1  # standard dependencies
+    EXTERNAL = 2  # This node's value should be taken from cache
+    PRIOR_RUN = 3  # This node's value should be taken from a prior run.
+    EXPAND = 4
+    COLLECT = 5
+    # This is not used in a standard function graph, but it comes in handy for
+    # repeatedly running the same one.
+
+
+class DependencyType(Enum):
+    REQUIRED = 1
+    OPTIONAL = 2
+
+    @staticmethod
+    def from_parameter(param: inspect.Parameter):
+        if param.default is inspect.Parameter.empty:
+            return DependencyType.REQUIRED
+        return DependencyType.OPTIONAL
+
+
+class Node:
+    """Object representing a node of computation."""
+
+    def __init__(
+        self,
+        name: str,
+        typ: type,
+        doc_string: str = "",
+        callabl: Callable = None,
+        node_source: NodeType = NodeType.STANDARD,
+        input_types: dict[str, type | tuple[type, DependencyType]] = None,
+        tags: dict[str, Any] = None,
+        namespace: tuple[str, ...] = (),
+        originating_functions: tuple[Callable, ...] | None = None,
+        optional_values: dict[str, Any] | None = None,
+    ):
+        """Constructor for our Node object.
+
+        :param name: the name of the function.
+        :param typ: the output type of the function.
+        :param doc_string: the doc string for the function. Optional.
+        :param callabl: the actual function callable.
+        :param node_source: whether this is something someone has to pass in.
+        :param input_types: the input parameters and their types.
+        :param tags: the set of tags that this node contains.
+        """
+        if tags is None:
+            tags = dict()
+        self._tags = tags
+        self._name = name
+        self._type = typ
+        if typ is None or typ == inspect._empty:
+            raise ValueError(f"Missing type for hint for function {name}. Please add one to fix.")
+        self._callable = callabl
+        self._doc = doc_string
+        self._node_source = node_source
+        self._dependencies = []
+        self._depended_on_by = []
+        self._namespace = namespace
+        self._input_types = {}
+        self._originating_functions = originating_functions
+        self._default_parameter_values = {}
+
+        if self._node_source in (
+            NodeType.STANDARD,
+            NodeType.COLLECT,
+            NodeType.EXPAND,
+        ):
+            if input_types is not None:
+                for key, value in input_types.items():
+                    if isinstance(value, tuple):
+                        self._input_types[key] = value
+                    else:
+                        self._input_types = {
+                            key: (value, DependencyType.REQUIRED)
+                            for key, value in input_types.items()
+                        }
+                # assume optional values passed
+                self._default_parameter_values = optional_values or {}
+            else:
+                type_hint_kwargs: dict[str, Any] = {"include_extras": True}
+                if sys.version_info >= (3, 13):
+                    type_hint_kwargs["globalns"] = callabl.__globals__
+                input_types = typing.get_type_hints(callabl, **type_hint_kwargs)
+                signature = inspect.signature(callabl)
+                for key, value in signature.parameters.items():
+                    if key not in input_types:
+                        raise ValueError(
+                            f"Missing type hint for {key} in function {name}. Please add one to fix."
+                        )
+                    dep_type = DependencyType.from_parameter(value)
+                    self._input_types[key] = (
+                        input_types[key],
+                        dep_type,
+                    )
+                    if dep_type == DependencyType.OPTIONAL:
+                        # capture optional value
+                        self._default_parameter_values[key] = value.default
+        elif self.user_defined:
+            if len(self._input_types) > 0:
+                raise ValueError(
+                    f"Input types cannot be provided for user-defined node {self.name}"
+                )
+
+    @property
+    def collect_dependency(self) -> str:
+        """Returns the name of the dependency that this node collects."""
+        if self._node_source != NodeType.COLLECT:
+            raise ValueError(f"Node {self.name} is not a collect node.")
+        # gets the dependency that gets collected
+        # This should be folded into the dependency type...
+        for key, (type_, _) in self._input_types.items():
+            if typing_inspect.get_origin(type_) == Collect:
+                return key
+
+    @property
+    def namespace(self) -> tuple[str, ...]:
+        return self._namespace
+
+    @property
+    def documentation(self) -> str:
+        return self._doc
+
+    @property
+    def input_types(self) -> dict[Any, tuple[Any, DependencyType]]:
+        return self._input_types
+
+    @property
+    def default_parameter_values(self) -> dict[str, Any]:
+        """Only returns parameters for which we have optional values."""
+        return self._default_parameter_values
+
+    def requires(self, dependency: str) -> bool:
+        """Returns whether or not this node requires the given dependency.
+
+        :param dependency: Dependency we may require
+        :return: True if it is an input *and* it is required
+        """
+        return (
+            dependency in self._input_types
+            and self._input_types[dependency][1] == DependencyType.REQUIRED
+        )
+
+    @property
+    def name(self) -> str:
+        return ".".join(self.namespace + (self._name,))
+
+    @property
+    def type(self) -> Any:
+        return self._type
+
+    def set_type(self, typ: Any):
+        """Sets the type of the node"""
+        assert self.user_defined is True, "Cannot reset type of non-user-defined node"
+        self._type = typ
+
+    @property
+    def callable(self):
+        return self._callable
+
+    # TODO - deprecate in favor of the node sources above
+    @property
+    def user_defined(self):
+        return self._node_source == NodeType.EXTERNAL
+
+    @property
+    def node_role(self):
+        return self._node_source
+
+    @property
+    def dependencies(self) -> list["Node"]:
+        return self._dependencies
+
+    @property
+    def depended_on_by(self) -> list["Node"]:
+        return self._depended_on_by
+
+    @property
+    def tags(self) -> dict[str, str]:
+        return self._tags
+
+    @property
+    def originating_functions(self) -> tuple[Callable, ...] | None:
+        """Gives all functions from which this node was created. None if the data
+        is not available (it is user-defined, or we have not added it yet). Note that this can be
+        multiple in the case of subdags (the subdag function + the other function). In that case,
+        it will be in order of creation (subdag function last).
+
+        Note that this is filled in in function_modifiers.base -- see note in from_fn
+
+        :return: A Tuple consisting of functions from which this node was created.
+        """
+        return self._originating_functions
+
+    def add_originating_function(self, fn: Callable):
+        """Adds a function to the list of originating functions.
+
+        This is used in the case to attach originating functions to user-defined (i.e. external/input nodes).
+        :param fn: Function to add
+        """
+        assert self.user_defined is True, "Cannot add originating function to non-user-defined node"
+        if self._originating_functions is None:
+            self._originating_functions = (fn,)
+        else:
+            self._originating_functions += (fn,)
+
+    def add_tag(self, tag_name: str, tag_value: str):
+        self._tags[tag_name] = tag_value
+
+    def __hash__(self):
+        return hash(self._name)
+
+    def __repr__(self):
+        return f"<{self.name} {self._tags}>"
+
+    def __eq__(self, other: "Node"):
+        """Want to deeply compare nodes in a custom way.
+
+        Current user is just unit tests. But you never know :)
+
+        Note: we only compare names of dependencies because we don't want infinite recursion.
+        """
+        return (
+            isinstance(other, Node)
+            and self._name == other.name
+            and self._type == other.type
+            and self._doc == other.documentation
+            and self._tags == other.tags
+            and self.user_defined == other.user_defined
+            and [n.name for n in self.dependencies] == [o.name for o in other.dependencies]
+            and [n.name for n in self.depended_on_by] == [o.name for o in other.depended_on_by]
+            and self.node_role == other.node_role
+        )
+
+    def __ne__(self, other: "Node"):
+        return not self.__eq__(other)
+
+    def __call__(self, *args, **kwargs):
+        """Call just delegates to the callable, purely for clean syntactic sugar"""
+        return self.callable(*args, **kwargs)
+
+    @staticmethod
+    def from_fn(fn: Callable, name: str = None) -> "Node":
+        """Generates a node from a function. Optionally overrides the name.
+
+        Note that currently, the `originating_function` is externally passed in -- this
+        happens in resolve_nodes in function_modifiers.base. TBD whether we'll want it to stay there.
+
+        :param fn: Function to generate the name from
+        :param name: Name to use for the node
+        :return: The node we generated
+        """
+        if name is None:
+            name = fn.__name__
+        type_hint_kwargs = {"include_extras": True}
+        return_type = typing.get_type_hints(fn, **type_hint_kwargs).get("return")
+        if return_type is None:
+            raise ValueError(f"Missing type hint for return value in function {fn.__qualname__}.")
+        module = inspect.getmodule(fn).__name__
+        tags = {"module": module}
+
+        node_source = NodeType.STANDARD
+        # TODO - extract this into a function + clean up!
+        if typing_inspect.is_generic_type(return_type):
+            if typing_inspect.get_origin(return_type) == Parallelizable:
+                node_source = NodeType.EXPAND
+        for hint in typing.get_type_hints(fn, **type_hint_kwargs).values():
+            if typing_inspect.is_generic_type(hint):
+                if typing_inspect.get_origin(hint) == Collect:
+                    node_source = NodeType.COLLECT
+                    break
+
+        if hasattr(fn, "__config_decorated__"):
+            tags["hamilton.config"] = ",".join(fn.__config_decorated__)
+        return Node(
+            name,
+            return_type,
+            fn.__doc__ or "",
+            callabl=fn,
+            tags=tags,
+            node_source=node_source,
+        )
+
+    def copy_with(self, include_refs: bool = True, **overrides) -> "Node":
+        """Copies a node with the specified overrides for the constructor arguments.
+        Utility function for creating a node -- useful for modifying it.
+
+        :param kwargs: kwargs to use in place of the node. Passed to the constructor.
+        :param include_refs: Whether or not to include dependencies and depended_on_by
+        :return: A node copied from self with the specified keyword arguments replaced.
+        """
+        constructor_args = dict(
+            name=self.name,
+            typ=self.type,
+            doc_string=self.documentation,
+            callabl=self.callable,
+            node_source=self.node_role,
+            input_types=self.input_types.copy(),
+            tags=self.tags.copy(),
+            originating_functions=self.originating_functions,
+            optional_values=self.default_parameter_values.copy()
+            if self.default_parameter_values
+            else {},
+        )
+        constructor_args.update(**overrides)
+        out = Node(**constructor_args)
+        if include_refs:
+            out._dependencies = self._dependencies
+            out._depended_on_by = self._depended_on_by
+        return out
+
+    def copy(self, include_refs: bool = True) -> "Node":
+        """Copies a node, not modifying anything (except for the references
+        /dependencies if specified).
+
+        :param include_refs: Whether or not to include dependencies and depended_on_by
+        :return: A copy of the node.
+        """
+        """Gives a copy of the node, so we can modify it without modifying the original.
+        :return: A copy of the node.
+        """
+        return self.copy_with(include_refs)
+
+    def reassign_inputs(
+        self, input_names: dict[str, Any] = None, input_values: dict[str, Any] = None
+    ) -> "Node":
+        """Reassigns the input names of a node. Useful for applying
+        a node to a separate input if needed. Note that things can get a
+        little strange if you have multiple inputs with the same name, so
+        be careful about how you use this.
+
+        :param input_names: Input name map to reassign
+        :return: A node with the input names reassigned
+        """
+        if input_names is None:
+            input_names = {}
+        if input_values is None:
+            input_values = {}
+
+        is_async = inspect.iscoroutinefunction(self.callable)  # determine if its async
+
+        def new_callable(**kwargs) -> Any:
+            reverse_input_names = {v: k for k, v in input_names.items()}
+            kwargs = {**kwargs, **input_values}
+            return self.callable(**{reverse_input_names.get(k, k): v for k, v in kwargs.items()})
+
+        async def async_function(**kwargs):
+            return await new_callable(**kwargs)
+
+        fn_to_use = async_function if is_async else new_callable
+
+        new_input_types = {
+            input_names.get(k, k): v for k, v in self.input_types.items() if k not in input_values
+        }
+        # out = self.copy_with(callabl=new_callable, input_types=new_input_types)
+        out = self.copy_with(callabl=fn_to_use, input_types=new_input_types)
+        return out
+
+    def transform_output(
+        self, __transform: Callable[[dict[str, Any], Any], Any], __output_type: builtins.type[Any]
+    ) -> "Node":
+        """Applies a transformation on the output of the node, returning a new node.
+        Also modifies the type.
+
+        :param __transform: Transformation to apply. This is a function with two arguments:
+        (a) the kwargs passed to the node, and (b) the output of the node.
+        :param __output_type: Return type of the transformation
+        :return: A new node, with the right type/transformation
+        """
+
+        def new_callable(**kwargs) -> Any:
+            return __transform(self.callable(**kwargs), kwargs)
+
+        return self.copy_with(callabl=new_callable, typ=__output_type)
+
+
+def matches_query(
+    tags: dict[str, str | list[str]], query_dict: dict[str, str | list[str] | None]
+) -> bool:
+    """Check whether a set of node tags matches the query based on tags.
+
+    An empty dict of a query matches all tags.
+
+    :param tags: the tags of the node.
+    :param query_dict: of tag to value. If value is None, we just check that the tag exists.
+    :return: True if we have tags that match all tag queries, False otherwise.
+    """
+    # it's an AND clause between each tag and value in the query dict.
+    for tag, value in query_dict.items():
+        # if tag not in node we can return False immediately.
+        if tag not in tags:
+            return False
+        # if value is None -- we don't care about the value, just that the tag exists.
+        if value is None:
+            continue
+        node_tag_value = tags[tag]
+        if not isinstance(node_tag_value, list):
+            node_tag_value = [node_tag_value]
+        if not isinstance(value, list):
+            value = [value]
+        if set(value).intersection(set(node_tag_value)):
+            # if there is some overlap, we're good.
+            continue
+        else:
+            # else, return False.
+            return False
+    return True

hamilton/plugins/README.md

@@ -0,0 +1,48 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Plugins
+
+Apache Hamilton enables plugins -- the requirement is that the core library according to the plugin is installed, and the plugin will be registered automatically.
+It is up to the user to install the plugin via the target, E.G. `hamilton[pyspark]`, which will install the correct depdendencies.
+
+
+## Structure
+
+For each plugin, `foo`, there will be two files (likely, at some point, broken into modules), that will have the names:
+1. `foo_extensions.py` This refers to constructs that get automatically registered (dataframe/column types, data loaders, etc...) This never gets imported by the user directly.
+2. `h_foo.py` This refers to constructs that the user refers to directly, E.G. specific decorators, result builders, etc...
+
+`h_foo` is just another module. `foo_extensions` has a few special properties.
+
+1. It can opt out of specifying dataframe/column types if it doesn't make sense, by containing the variable `COLUMN_FRIENDLY_DF_TYPE = False`. This defaults to true on import.
+2. If it is true, it must contain the following methods:
+   1. `get_column_foo` -- a function to extract the column
+   2. `register_types` -- a function to register the types for that extension
+   3. `fill_with_scalar_foo` -- a function to fill a column with a scalar
+
+`register_types` must be called within the module body, as must registering any data loaders. This restriction will likely be removed and the registering pulled out to import level.
+
+Note this is not a public-facing API -- this is for managing internal plugins. If you want to add new capabilities, feel free to reach out to the team.
+
+
+## Adding a new plugin
+
+Follow the rules above. Also add an install target with the same name in `setup.py`. Once you add this, you'll also want to update [registry.py](../registry.py) to include the new plugin, and update
+the [docs](../../docs/data_adapters_extension.py) to document it.

hamilton/plugins/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.