apache-hamilton 1.90.0.dev0__py3-none-any.whl

hamilton/cli/commands.py

@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+
+from hamilton import ad_hoc_utils, driver
+from hamilton.cli import logic
+
+
+def build(modules: list[Path], context_path: Path | None = None):
+    """Build a Hamilton driver from the passed modules, and
+    load the Driver config from the context file.
+
+    Dynamic execution is enabled by default to support dataflow
+    using Parallelizable/Collect. This only matters if we are to
+    execute code.
+    """
+    context = logic.load_context(context_path) if context_path else {}
+    module_objects = [ad_hoc_utils.module_from_source(p.read_text()) for p in modules]
+    return (
+        driver.Builder()
+        .enable_dynamic_execution(allow_experimental_mode=True)
+        .with_modules(*module_objects)
+        .with_config(context.get("HAMILTON_CONFIG", {}))
+        .build()
+    )
+
+
+def diff(
+    current_dr: driver.Driver,
+    modules: list[Path],
+    git_reference: str | None = "HEAD",
+    view: bool = False,
+    output_file_path: Path = Path("./diff.png"),
+    context_path: Path | None = None,
+) -> dict:
+    """Get the diff of"""
+    context = logic.load_context(context_path) if context_path else {}
+
+    current_version = logic.hash_hamilton_nodes(current_dr)
+    current_node_to_func = logic.map_nodes_to_functions(current_dr)
+
+    reference_modules = logic.load_modules_from_git(modules, git_reference)
+    reference_dr = (
+        driver.Builder()
+        .enable_dynamic_execution(allow_experimental_mode=True)
+        .with_modules(*reference_modules)
+        .with_config(context.get("HAMILTON_CONFIG", {}))
+        .build()
+    )
+    reference_version = logic.hash_hamilton_nodes(reference_dr)
+    reference_node_to_func = logic.map_nodes_to_functions(reference_dr)
+
+    nodes_diff = logic.diff_versions(
+        reference_map=reference_version,
+        current_map=current_version,
+    )
+
+    full_diff = dict()
+    for status, node_names in nodes_diff.items():
+        full_diff[status] = dict()
+        for node_name in node_names:
+            if current_node_to_func.get(node_name):
+                func_name = current_node_to_func.get(node_name)
+            else:
+                func_name = reference_node_to_func.get(node_name)
+
+            full_diff[status][node_name] = func_name
+
+    if view:
+        dot = logic.visualize_diff(current_dr=current_dr, reference_dr=reference_dr, **nodes_diff)
+
+        # simplified logic from hamilton.graph.display()
+        output_format = "png"
+        if output_file_path:  # infer format from path
+            if output_file_path.suffix != "":
+                output_format = output_file_path.suffix.partition(".")[-1]
+
+        dot.render(output_file_path.with_suffix(""), format=output_format)
+
+    return full_diff
+
+
+def validate(dr: driver.Driver, context_path: Path) -> dict:
+    """Use driver.validate_execution() with values from the context file"""
+    context = logic.load_context(context_path)
+
+    try:
+        dr.validate_execution(
+            final_vars=context["HAMILTON_FINAL_VARS"],
+            inputs=context["HAMILTON_INPUTS"],
+            overrides=context["HAMILTON_OVERRIDES"],
+        )
+    except ValueError as e:
+        raise e
+
+    return context
+
+
+def version(dr: driver.Driver) -> dict:
+    """Get the node and dataflow versions from the instantiated Driver"""
+    nodes_hash = logic.hash_hamilton_nodes(dr)
+    dataflow_hash = logic.hash_dataflow(nodes_hash)
+    return dict(
+        nodes_hash=nodes_hash,
+        dataflow_hash=dataflow_hash,
+    )
+
+
+def view(dr: driver.Driver, output_file_path: Path = Path("dag.png")) -> None:
+    """Display all functions of the instantiated Driver"""
+    dr.display_all_functions(output_file_path)

hamilton/cli/logic.py

@@ -0,0 +1,338 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pathlib import Path
+from types import ModuleType
+
+from hamilton import driver
+
+CONFIG_HEADER = "HAMILTON_CONFIG"
+FINAL_VARS_HEADER = "HAMILTON_FINAL_VARS"
+INPUTS_HEADER = "HAMILTON_INPUTS"
+OVERRIDES_HEADER = "HAMILTON_OVERRIDES"
+
+
+def get_git_base_directory() -> str:
+    """Get the base path of the current git directory"""
+    import subprocess
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--show-toplevel"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+        if result.returncode == 0:
+            return result.stdout.strip()
+        else:
+            print("Error:", result.stderr.strip())
+            raise OSError(f"{result.stderr.strip()}")
+    except FileNotFoundError as e:
+        raise FileNotFoundError("Git command not found. Please make sure Git is installed.") from e
+
+
+def get_git_reference(git_relative_path: str | Path, git_reference: str) -> str:
+    """Get the source code from the specified file and git reference"""
+    import subprocess
+
+    try:
+        result = subprocess.run(
+            ["git", "show", f"{git_reference}:{git_relative_path}"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+
+        if result.returncode == 0:
+            return result.stdout.strip()
+        elif result.returncode == 128:
+            # TODO catch the following error:
+            # fatal: Path '{git_relative_path}' exists on disk, but not in '{git_revision}'.
+            return
+        else:
+            return
+    except FileNotFoundError as e:
+        raise FileNotFoundError("Git command not found. Please make sure Git is installed.") from e
+
+
+def version_hamilton_functions(module: ModuleType) -> dict[str, str]:
+    """Hash the source code of Hamilton functions from a module"""
+    from hamilton import graph_types, graph_utils
+
+    origins_version: dict[str, str] = dict()
+
+    for origin_name, _ in graph_utils.find_functions(module):
+        origin_callable = getattr(module, origin_name)
+        origins_version[origin_name] = graph_types.hash_source_code(origin_callable, strip=True)
+
+    return origins_version
+
+
+def hash_hamilton_nodes(dr: driver.Driver) -> dict[str, str]:
+    """Hash the source code of Hamilton functions from nodes in a Driver"""
+    from hamilton import graph_types
+
+    graph = graph_types.HamiltonGraph.from_graph(dr.graph)
+    return {n.name: n.version for n in graph.nodes}
+
+
+def map_nodes_to_functions(dr: driver.Driver) -> dict[str, str]:
+    """Get a mapping from node name to Hamilton function name"""
+    from hamilton import graph_types
+
+    graph = graph_types.HamiltonGraph.from_graph(dr.graph)
+
+    node_to_function = dict()
+    for n in graph.nodes:
+        # is None for config nodes
+        if n.originating_functions is None:
+            continue
+
+        node_callable = n.originating_functions[0]
+        node_to_function[n.name] = node_callable.__name__
+
+    return node_to_function
+
+
+def hash_dataflow(nodes_version: dict[str, str]) -> str:
+    """Create a dataflow hash from the hashes of its nodes"""
+    import hashlib
+
+    sorted_nodes = sorted(nodes_version.values())
+    return hashlib.sha256(str(sorted_nodes).encode()).hexdigest()
+
+
+def load_modules_from_git(
+    module_paths: list[Path], git_reference: str = "HEAD"
+) -> list[ModuleType]:
+    """Dynamically import modules for a git reference"""
+    from hamilton import ad_hoc_utils
+
+    git_base_dir = Path(get_git_base_directory())
+
+    modules = []
+    for module_path in module_paths:
+        relative_to_git = module_path.relative_to(git_base_dir)
+        previous_revision = get_git_reference(relative_to_git, git_reference)
+        module = ad_hoc_utils.module_from_source(previous_revision)
+        modules.append(module)
+
+    return modules
+
+
+def diff_nodes_against_functions(
+    nodes_version: dict[str, str],
+    origins_version: dict[str, str],
+    node_to_origin: dict[str, str],
+) -> dict:
+    """Compare the nodes version from a built Driver to
+    the origins version from module source code when a second
+    Driver can't be instantiated
+
+    :nodes_version: mapping from node name to its origin function hash
+    :origins_version: mapping from origin function name to origin function hash
+    :node_to_origin: mapping from node name to origin name
+    """
+    node_added = []
+    code_diff = []
+    origins_removed = []
+
+    # iterate over nodes from built Driver
+    for node_name, node_version in nodes_version.items():
+        origin_name = node_to_origin[node_name]
+        origin_version = origins_version.get(origin_name)
+
+        # if origin_version is None, the node is new
+        if origin_version is None:
+            node_added.append(node_name)
+            continue
+
+        # if node_version != origin_version, the current node's origin
+        # function existed before, but was edited since origin_version
+        if node_version != origin_version:
+            code_diff.append(node_name)
+
+    nodes_origin_version = set(nodes_version.values())
+    # iterate over origin functions from source code
+    for origin_name, origin_version in origins_version.items():
+        # if origin_version not found in nodes_version, the
+        # origin function is not used by the built Driver
+        if origin_version not in nodes_origin_version:
+            origins_removed.append(origin_name)
+
+    return dict(
+        nodes_added=node_added,
+        code_diff=code_diff,
+        origins_removed=origins_removed,
+    )
+
+
+def diff_versions(current_map: dict[str, str], reference_map: dict[str, str]) -> dict:
+    """Generic diff of two {name: hash} mappings (can be node or origin name)
+
+    :mapping_v1: mapping from node (or function) name to its function hash
+    :mapping_v2: mapping from node (or function) name to its function hash
+    """
+    current_only, reference_only, edit = [], [], []
+
+    for node_name, v1 in current_map.items():
+        v2 = reference_map.get(node_name)
+        if v2 is None:
+            current_only.append(node_name)
+            continue
+
+        if v1 != v2:
+            edit.append(node_name)
+
+    for node_name, _ in reference_map.items():
+        v1 = current_map.get(node_name)
+        if v1 is None:
+            reference_only.append(node_name)
+            continue
+
+    return dict(current_only=current_only, reference_only=reference_only, edit=edit)
+
+
+def _custom_diff_style(
+    *,
+    node,
+    node_class,
+    current_only: list[str],
+    reference_only: list[str],
+    edit: list[str],
+):
+    """Custom visualization style for the diff of 2 dataflows"""
+    if node.name in current_only:
+        style = ({"fillcolor": "greenyellow"}, node_class, "Current only")
+
+    elif node.name in reference_only:
+        style = ({"fillcolor": "tomato1"}, node_class, "Reference only")
+
+    elif node.name in edit:
+        style = ({"fillcolor": "yellow2"}, node_class, "Edited")
+
+    else:
+        style = ({}, node_class, None)
+
+    return style
+
+
+def visualize_diff(
+    current_dr: driver.Driver,
+    reference_dr: driver.Driver,
+    current_only: list[str],
+    reference_only: list[str],
+    edit: list[str],
+):
+    """Visualize the diff of 2 dataflows.
+
+    Uses the union of sets of nodes from driver 1 and driver 2.
+    """
+    import functools
+
+    from hamilton import graph
+
+    all_nodes = set(reference_dr.graph.get_nodes()).union(set(current_dr.graph.get_nodes()))
+
+    diff_style = functools.partial(
+        _custom_diff_style,
+        current_only=current_only,
+        reference_only=reference_only,
+        edit=edit,
+    )
+
+    return graph.create_graphviz_graph(
+        nodes=all_nodes,
+        comment="Diff viz",
+        graphviz_kwargs=dict(),
+        custom_style_function=diff_style,
+        node_modifiers=dict(),
+        strictly_display_only_nodes_passed_in=True,
+    )
+
+
+# TODO refactor ContextLoader to a class
+# TODO support loading from pyproject.toml
+def load_context(file_path: Path) -> dict:
+    if not file_path.exists():
+        raise FileNotFoundError(f"`{file_path}` doesn't exist.")
+
+    extension = file_path.suffix
+    if extension == ".json":
+        context = _read_json_context(file_path)
+    elif extension == ".py":
+        context = _read_py_context(file_path)
+    else:
+        raise ValueError(f"Received extension `{extension}` is unsupported.")
+
+    context = _validate_context(context)
+    return context
+
+
+def _validate_context(context: dict) -> dict:
+    if context[CONFIG_HEADER] is None:
+        context[CONFIG_HEADER] = {}
+
+    if context[FINAL_VARS_HEADER] is None:
+        context[FINAL_VARS_HEADER] = []
+
+    if context[INPUTS_HEADER] is None:
+        context[INPUTS_HEADER] = {}
+
+    if context[OVERRIDES_HEADER] is None:
+        context[OVERRIDES_HEADER] = {}
+
+    return context
+
+
+def _read_json_context(file_path: Path) -> dict:
+    """"""
+    import json
+
+    data = json.load(file_path.open())
+
+    context = {}
+    for k in [
+        CONFIG_HEADER,
+        FINAL_VARS_HEADER,
+        INPUTS_HEADER,
+        OVERRIDES_HEADER,
+    ]:
+        context[k] = data.get(k, None)
+
+    return context
+
+
+def _read_py_context(file_path: Path) -> dict:
+    import importlib
+
+    spec = importlib.util.spec_from_file_location("cli_config", file_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    context = {}
+    for k in [
+        CONFIG_HEADER,
+        FINAL_VARS_HEADER,
+        INPUTS_HEADER,
+        OVERRIDES_HEADER,
+    ]:
+        context[k] = getattr(module, k, None)
+
+    return context

hamilton/common/__init__.py

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# code in this module should no depend on much
+from collections.abc import Callable
+from typing import Any, List, Optional, Set, Tuple, Union
+
+
+def convert_output_value(
+    output_value: str | Callable | Any, module_set: set[str]
+) -> tuple[str | None, str | None]:
+    """Converts output values that one can request into strings.
+
+    It checks that if it's a function, it's in the passed in module set.
+
+    :param output_value: the value we want to convert into a string. We don't annotate driver.Variable here for
+       import reasons.
+    :param module_set: the set of modules functions could come from.
+    :return: a tuple, (string value, string error). One or the other is returned, never both.
+    """
+    if isinstance(output_value, str):
+        return output_value, None
+    elif hasattr(output_value, "name"):
+        return output_value.name, None
+    elif isinstance(output_value, Callable):
+        if output_value.__module__ in module_set:
+            return output_value.__name__, None
+        else:
+            return None, (
+                f"Function {output_value.__module__}.{output_value.__name__} is a function not "
+                f"in a "
+                f"module given to the materializer. Valid choices are {module_set}."
+            )
+    else:
+        return None, (
+            f"Materializer dependency {output_value} is not a string, a function, or a driver.Variable."
+        )
+
+
+def convert_output_values(
+    output_values: list[str | Callable | Any], module_set: set[str]
+) -> list[str]:
+    """Checks & converts outputs values to strings. This is used in building dependencies for the DAG.
+
+    :param output_values: the values to convert.
+    :param module_set: the modules any functions could come from.
+    :return: the final values
+    :raises ValueError: if there are values that can't be used/converted.
+    """
+    final_values = []
+    errors = []
+    for final_var in output_values:
+        _val, _error = convert_output_value(final_var, module_set)
+        if _val:
+            final_values.append(_val)
+        if _error:
+            errors.append(_error)
+    if errors:
+        errors.sort()
+        error_str = f"{len(errors)} errors encountered:\n  " + "\n  ".join(errors)
+        raise ValueError(error_str)
+    return final_values

hamilton/contrib/__init__.py

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""This module exists so that people can download dataflows without the sf-hamilton-contrib package.
+
+It will get clobbered when sf-hamilton-contrib is installed, which is good.
+"""
+
+import logging
+from contextlib import contextmanager
+
+__version__ = "__unknown__"  # this will be overwritten once sf-hamilton-contrib is installed.
+
+
+@contextmanager
+def catch_import_errors(module_name: str, file_location: str, logger: logging.Logger):
+    try:
+        # Yield control to the inner block which will have the import statements.
+        yield
+    except ImportError as e:
+        location = file_location[: file_location.rfind("/")]
+        logger.error("ImportError: %s", e)
+        logger.error(
+            "Please install the required packages. Options:\n"
+            f"(1): with `pip install -r {location}/requirements.txt`\n"
+        )
+        raise e

hamilton/data_quality/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.