apache-hamilton 1.90.0.dev0__py3-none-any.whl

hamilton/plugins/h_experiments/server.py

@@ -0,0 +1,375 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import functools
+import itertools
+import json
+import os
+
+import pandas as pd
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.staticfiles import StaticFiles
+from fastui import AnyComponent, FastUI, prebuilt_html
+from fastui import components as c
+from fastui.components.display import DisplayLookup, DisplayMode
+from fastui.events import GoToEvent
+from fastui.forms import SelectSearchResponse
+from pydantic import BaseModel, Field
+
+from hamilton.plugins.h_experiments.cache import JsonCache
+from hamilton.plugins.h_experiments.data_model import (
+    NodeMaterializer,
+    RunMetadata,
+    model_from_values,
+)
+
+
+def convert_graph_hash_to_version(runs):
+    """Convert graph hash to incremental id, per experiment"""
+    versioned_runs = []
+    for _, group in itertools.groupby(runs, lambda r: r.experiment):
+        group = sorted(group, key=lambda run: run.date_completed)
+
+        current_version = 1
+        current_hash = None
+        for r in group:
+            if current_hash is None:
+                current_hash = r.graph_hash
+
+            if r.graph_hash != current_hash:
+                current_hash = r.graph_hash
+                current_version += 1
+
+            r.graph_version = current_version
+            versioned_runs.append(r)
+    return versioned_runs
+
+
+def get_runs(metadata_cache_path: str) -> list[RunMetadata]:
+    """Create RunMetadata objects for all runs JSON found in cache"""
+    cache = JsonCache(cache_path=metadata_cache_path)
+    runs = [RunMetadata.model_validate_json(cache.read(run_id)) for run_id in cache.keys()]
+    runs = convert_graph_hash_to_version(runs)
+    runs = sorted(runs, key=lambda run: run.date_completed, reverse=True)
+    return runs
+
+
+# environment variables are the most convenient approach
+# to configure FastAPI application
+base_directory = os.getenv("HAMILTON_EXPERIMENTS_PATH", "")
+
+# disable Swagger UI /docs because they are currently bugged for FastUI
+app = FastAPI(docs_url=None, redoc_url=None)
+app.mount("/experiments", StaticFiles(directory=base_directory), name="experiments")
+runs = get_runs(base_directory)
+
+
+def base_page(*components: AnyComponent) -> list[AnyComponent]:
+    """Template applied to all pages. Includes: title, navigation, and footer"""
+    return [
+        c.PageTitle(text="📝 Hamilton Experiment Manager"),
+        c.Navbar(
+            title="📝 Hamilton Experiment Manager",
+            title_event=GoToEvent(url="/"),
+            class_name="+ mb-4",
+        ),
+        c.Page(components=[*components]),
+        c.Footer(
+            extra_text="Powered by Hamilton",
+            links=[
+                c.Link(
+                    components=[c.Text(text="GitHub")],
+                    on_click=GoToEvent(url="https://github.com/apache/hamilton"),
+                ),
+            ],
+        ),
+    ]
+
+
+@functools.cache
+def run_lookup():
+    """Cache a mapping of {run_id: run} to query runs"""
+    return {run.run_id: run for run in runs}
+
+
+@app.get("/api/filter/{field}", response_model=SelectSearchResponse)
+async def search_filter(field: str) -> SelectSearchResponse:
+    """Get all the unique values for a RunMetadata field to populate menus"""
+    options = {str(getattr(run, field)): getattr(run, field) for run in runs}
+    return SelectSearchResponse(options=[{"label": v, "value": v} for v in options])
+
+
+class RunFilter(BaseModel):
+    """Filter the runs_overview() page using `experiment` and `graph_version`"""
+
+    experiment: str = Field(
+        json_schema_extra={
+            "search_url": "/api/filter/experiment",
+            "placeholder": "Select Experiment ...",
+        }
+    )
+    graph_version: str = Field(
+        json_schema_extra={
+            "search_url": "/api/filter/graph_version",
+            "placeholder": "Select Graph version ...",
+        },
+    )
+
+
+@app.get("/api/runs", response_model=FastUI, response_model_exclude_none=True)
+def runs_overview(
+    experiment: str | None = None,
+    graph_version: int | None = None,
+) -> list[AnyComponent]:
+    """RunOverview page with filters for the table"""
+
+    # refresh cache to display new runs
+    runs = get_runs(os.getenv("HAMILTON_EXPERIMENTS_PATH", ""))
+
+    selection = dict()
+
+    if experiment:
+        selection["experiment"] = dict(value=experiment, label=experiment)
+
+    if graph_version:
+        selection["graph_version"] = dict(value=graph_version, label=str(graph_version))
+
+    selected_runs = runs
+    if selection:
+        for k, v in selection.items():
+            selected_runs = list(filter(lambda r: getattr(r, k) == v["value"], selected_runs))
+
+    return base_page(
+        c.ModelForm(
+            model=RunFilter,
+            submit_url="/runs",
+            initial=selection,
+            method="GOTO",
+            submit_on_change=True,
+            display_mode="inline",
+        ),
+        c.Table(
+            data=selected_runs,
+            data_model=RunMetadata,
+            columns=[
+                DisplayLookup(field="date_completed", mode=DisplayMode.datetime),
+                DisplayLookup(field="experiment"),
+                DisplayLookup(field="graph_version"),
+                DisplayLookup(field="run_id", on_click=GoToEvent(url="/run/{run_id}/")),
+            ],
+        ),
+    )
+
+
+def run_tabs(run_id) -> list[AnyComponent]:
+    """Create Metadata and Artifacts tabs for individual Run pages"""
+    return [
+        c.LinkList(
+            links=[
+                c.Link(
+                    components=[c.Text(text="Metadata")],
+                    on_click=GoToEvent(url=f"/run/{run_id}/"),
+                    active="startswith:/run/",
+                ),
+                c.Link(
+                    components=[c.Text(text="Artifacts")],
+                    on_click=GoToEvent(url=f"/artifacts/{run_id}"),
+                    active="startswith:/artifacts/",
+                ),
+            ],
+            mode="tabs",
+            class_name="+ mb-4",
+        )
+    ]
+
+
+@app.get("/api/run/{run_id}/", response_model=FastUI, response_model_exclude_none=True)
+def run_metadata(run_id: str) -> list[AnyComponent]:
+    """Individual Run > Metadata"""
+    run = run_lookup()[run_id]
+
+    return base_page(
+        c.Heading(text=run.experiment, level=2),
+        *run_tabs(run_id=run_id),
+        c.Details(
+            data=run,
+            fields=[
+                DisplayLookup(field="experiment"),
+                DisplayLookup(field="run_id"),
+                DisplayLookup(field="success"),
+                DisplayLookup(field="graph_hash"),
+                DisplayLookup(field="modules"),
+            ],
+        ),
+        c.Image(
+            src=f"/experiments/{run.experiment}/{run.run_id}/dag.png",
+            width="100%",
+            height="auto",
+            loading="lazy",
+            referrer_policy="no-referrer",
+            class_name="border rounded",
+        ),
+        c.Details(
+            data=run,
+            fields=[
+                DisplayLookup(field="config", mode=DisplayMode.json),
+                DisplayLookup(field="inputs", mode=DisplayMode.json),
+                DisplayLookup(field="overrides", mode=DisplayMode.json),
+            ],
+        ),
+    )
+
+
+@functools.cache
+def create_table_model(**kwargs):
+    """Cache the creation of a Pydantic model from a DataFrame row"""
+    return model_from_values("Table", specs=kwargs)
+
+
+def dataframe_to_table(df: pd.DataFrame, **kwargs) -> AnyComponent:
+    """Populate a FastUI table with pagination from a pandas DataFrame"""
+    df = df.reset_index()
+    Table = create_table_model(**{str(k): v for k, v in df.iloc[0].to_dict().items()})
+
+    page: int = 1 if kwargs.get("page") is None else kwargs.get("page", 1)
+    page_size = min(20, df.shape[0])
+    page_df = df.iloc[(page - 1) * page_size : page * page_size]
+
+    # create a Pydantic objects for each row
+    data = [
+        Table.model_validate({str(k): v for k, v in row.items()})
+        for row in page_df.to_dict(orient="index").values()
+    ]
+
+    return [
+        c.Table(
+            data=data,
+            data_model=Table,
+            columns=[DisplayLookup(field=str(col)) for col in df.columns],
+        ),
+        c.Pagination(page=page, page_size=20, total=df.shape[0]),
+    ]
+
+
+def artifact_components(materializer: NodeMaterializer, **kwargs) -> list[AnyComponent]:
+    """Create a FastUI component for an artifact based on the materializer type
+
+    Instead of mapping Python type -> component, the materializer/file type -> component
+    has a much lower and manageable cardinality
+    """
+    artifact_path = materializer.path.replace(base_directory, "/experiments")
+
+    if materializer.sink == "json":
+        with open(materializer.path, "r") as f:
+            data = json.load(f)
+        components = [c.Json(value=data)]
+
+    elif materializer.sink == "parquet":
+        data = pd.read_parquet(materializer.path)
+        components = dataframe_to_table(data, **kwargs)
+
+    elif materializer.sink == "csv":
+        data = pd.read_csv(materializer.path)
+        components = dataframe_to_table(data, **kwargs)
+
+    elif materializer.sink in ["plt", "plotly"]:
+        try:
+            components = [
+                c.Image(
+                    src=artifact_path,
+                    width="100%",
+                    height="auto",
+                    loading="lazy",
+                    referrer_policy="no-referrer",
+                )
+            ]
+        # TODO refactor to each default else case
+        except Exception:
+            components = [
+                c.Json(
+                    value={
+                        k: v
+                        for k, v in dict(materializer).items()
+                        if k in ["path", "sink", "data_saver"]
+                    }
+                )
+            ]
+
+    else:
+        components = [
+            c.Json(
+                value={
+                    k: v
+                    for k, v in dict(materializer).items()
+                    if k in ["path", "sink", "data_saver"]
+                }
+            )
+        ]
+
+    return components
+
+
+def artifact_tabs(run: RunMetadata) -> list[AnyComponent]:
+    """Create a tab for each artifact"""
+    links = []
+    for i, m in enumerate(run.materialized):
+        artifact_name = "-".join(m.source_nodes)
+        artifact_name, _, _ = artifact_name.partition("__")
+        links.append(
+            c.Link(
+                components=[c.Text(text=artifact_name)],
+                on_click=GoToEvent(
+                    url=f"/artifacts/{run.run_id}",
+                    query=dict(artifact_id=i),
+                ),
+                active=f"startswith:/artifacts/{run.run_id}?artifact_id={i}",
+            )
+        )
+
+    return [
+        c.LinkList(
+            links=links,
+            mode="tabs",
+            class_name="+ mb-4",
+        )
+    ]
+
+
+@app.get("/api/artifacts/{run_id}", response_model=FastUI, response_model_exclude_none=True)
+def run_artifacts(run_id: str, artifact_id: int = 0, page: int | None = None) -> list[AnyComponent]:
+    """Individual Run > Artifact"""
+    run = run_lookup()[run_id]
+
+    return base_page(
+        c.Heading(text=run.experiment, level=2),
+        *run_tabs(run_id=run_id),
+        *artifact_tabs(run=run),
+        *artifact_components(materializer=run.materialized[artifact_id], page=page),
+    )
+
+
+@app.get("/api/")
+def landing_page() -> RedirectResponse:
+    """Landing page redirects to the Run overview page"""
+    return RedirectResponse(url="/api/runs")
+
+
+@app.get("{path:path}")
+async def html_landing() -> HTMLResponse:
+    """Simple HTML page which serves the React app, comes last as it matches all paths."""
+    return HTMLResponse(prebuilt_html(title="Hamilton Experiment Manager"))

hamilton/plugins/h_kedro.py

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import inspect
+from typing import Any
+
+from kedro.pipeline.node import Node as KNode
+from kedro.pipeline.pipeline import Pipeline as KPipeline
+
+from hamilton import driver, graph
+from hamilton.function_modifiers.expanders import extract_fields
+from hamilton.lifecycle import base as lifecycle_base
+from hamilton.node import Node as HNode
+
+
+def expand_k_node(base_node: HNode, outputs: list[str]) -> list[HNode]:
+    """Manually apply `@extract_fields()` on a Hamilton node.Node for a Kedro
+    node that specifies >1 `outputs`.
+
+    The number of nodes == len(outputs) + 1 because it includes the `base_node`
+    """
+
+    def _convert_output_from_tuple_to_dict(node_result: Any, node_kwargs: dict[str, Any]):
+        return {out: v for out, v in zip(outputs, node_result, strict=False)}
+
+    # NOTE isinstance(Any, type) is False for Python < 3.11
+    extractor = extract_fields(fields={out: Any for out in outputs})
+    func = base_node.originating_functions[0]
+    if issubclass(func.__annotations__["return"], tuple):
+        base_node = base_node.transform_output(_convert_output_from_tuple_to_dict, dict)
+        func.__annotations__["return"] = dict
+
+    extractor.validate(func)
+    return list(extractor.transform_node(base_node, {}, func))
+
+
+def k_node_to_h_nodes(node: KNode) -> list[HNode]:
+    """Convert a Kedro node to a list of Hamilton nodes.
+    If the Kedro node specifies 1 output, generate 1 Hamilton node.
+    If it generate >1 output, generate len(outputs) + 1 to include the base node + extracted fields.
+    """
+    # determine if more than one output
+    node_names = []
+    if isinstance(node.outputs, list):
+        node_names.extend(node.outputs)
+    elif isinstance(node.outputs, dict):
+        node_names.extend(node.outputs.values())
+
+    # determine the base node name
+    if len(node_names) == 1:
+        base_node_name = node_names[0]
+    elif isinstance(node.outputs, str):
+        base_node_name = node.outputs
+    else:
+        base_node_name = node.func.__name__
+
+    func_sig = inspect.signature(node.func)
+    params = func_sig.parameters.values()
+    output_type = func_sig.return_annotation
+    if output_type is None:
+        # manually creating `hamilton.node.Node` doesn't accept `typ=None`
+        output_type = type[None]  # NoneType is introduced in Python 3.10
+
+    base_node = HNode(
+        name=base_node_name,
+        typ=output_type,
+        doc_string=getattr(node.func, "__doc__", ""),
+        callabl=node.func,
+        originating_functions=(node.func,),
+    )
+
+    # if Kedro node defines multiple outputs, use `@extract_fields()`
+    if len(node_names) > 1:
+        h_nodes = expand_k_node(base_node, node_names)
+    else:
+        h_nodes = [base_node]
+
+    # remap the function parameters to the node `inputs` and clean Kedro `parameters` name
+    new_params = {}
+    for param, k_input in zip(params, node.inputs, strict=False):
+        if k_input.startswith("params:"):
+            k_input = k_input.partition("params:")[-1]
+
+        new_params[param.name] = k_input
+
+    h_nodes = [n.reassign_inputs(input_names=new_params) for n in h_nodes]
+
+    return h_nodes
+
+
+def kedro_pipeline_to_driver(
+    *pipelines: KPipeline,
+    builder: driver.Builder | None = None,
+) -> driver.Driver:
+    """Convert one or mode Kedro `Pipeline` to a Hamilton `Driver`.
+    Pass a Hamilton `Builder` to include lifecycle adapters in your `Driver`.
+
+    :param pipelines: one or more Kedro `Pipeline` objects
+    :param builder: a Hamilton `Builder` to use when building the `Driver`
+    :return: the Hamilton `Driver` built from Kedro `Pipeline` objects.
+
+    .. code-block: python
+
+        from hamilton import driver
+        from hamilton.plugins import h_kedro
+
+        builder = driver.Builder().with_adapters(tracker)
+
+        dr = h_kedro.kedro_pipeline_to_driver(
+            data_science.create_pipeline(),  # Kedro Pipeline
+            data_processing.create_pipeline(),  # Kedro Pipeline
+            builder=builder
+        )
+    """
+    # generate nodes
+    h_nodes = []
+    for pipe in pipelines:
+        for node in pipe.nodes:
+            h_nodes.extend(k_node_to_h_nodes(node))
+
+    # resolve dependencies
+    h_nodes = graph.update_dependencies(
+        {n.name: n for n in h_nodes},
+        lifecycle_base.LifecycleAdapterSet(),
+    )
+
+    builder = builder or driver.Builder()
+    dr = builder.build()
+    # inject function graph in Driver
+    dr.graph = graph.FunctionGraph(
+        h_nodes, config={}, adapter=lifecycle_base.LifecycleAdapterSet(*builder.adapters)
+    )
+    # reapply lifecycle hooks
+    if dr.adapter.does_hook("post_graph_construct", is_async=False):
+        dr.adapter.call_all_lifecycle_hooks_sync(
+            "post_graph_construct", graph=dr.graph, modules=dr.graph_modules, config={}
+        )
+    return dr