genesis-flow 1.0.0__py3-none-any.whl

mlflow/models/model_config.py

@@ -0,0 +1,150 @@
+import os
+from typing import Any, Optional, Union
+
+import yaml
+
+from mlflow.exceptions import MlflowException
+from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE
+
+__mlflow_model_config__ = None
+
+
+class ModelConfig:
+    """
+    ModelConfig used in code to read a YAML configuration file or a dictionary.
+
+    Args:
+        development_config: Path to the YAML configuration file or a dictionary containing the
+                        configuration. If the configuration is not provided, an error is raised
+
+    .. code-block:: python
+        :caption: Example usage in model code
+
+        from mlflow.models import ModelConfig
+
+        # Load the configuration from a dictionary
+        config = ModelConfig(development_config={"key1": "value1"})
+        print(config.get("key1"))
+
+
+    .. code-block:: yaml
+        :caption: yaml file for model configuration
+
+        key1: value1
+        another_key:
+            - value2
+            - value3
+
+    .. code-block:: python
+        :caption: Example yaml usage in model code
+
+        from mlflow.models import ModelConfig
+
+        # Load the configuration from a file
+        config = ModelConfig(development_config="config.yaml")
+        print(config.get("key1"))
+
+
+    When invoking the ModelConfig locally in a model file, development_config can be passed in
+    which would be used as configuration for the model.
+
+
+    .. code-block:: python
+        :caption: Example to use ModelConfig when logging model as code: agent.py
+
+        import mlflow
+        from mlflow.models import ModelConfig
+
+        config = ModelConfig(development_config={"key1": "value1"})
+
+
+        class TestModel(mlflow.pyfunc.PythonModel):
+            def predict(self, context, model_input, params=None):
+                return config.get("key1")
+
+
+        mlflow.models.set_model(TestModel())
+
+
+    But this development_config configuration file will be overridden when logging a model.
+    When no model_config is passed in while logging the model, an error will be raised when
+    trying to load the model using ModelConfig.
+    Note: development_config is not used when logging the model.
+
+
+    .. code-block:: python
+        :caption: Example to use agent.py to log the model: deploy.py
+
+        model_config = {"key1": "value2"}
+        with mlflow.start_run():
+            model_info = mlflow.pyfunc.log_model(
+                name="model", python_model="agent.py", model_config=model_config
+            )
+
+        loaded_model = mlflow.pyfunc.load_model(model_info.model_uri)
+
+        # This will print "value2" as the model_config passed in while logging the model
+        print(loaded_model.predict(None))
+    """
+
+    def __init__(self, *, development_config: Optional[Union[str, dict[str, Any]]] = None):
+        config = globals().get("__mlflow_model_config__", None)
+        # Here mlflow_model_config have 3 states:
+        # 1. None, this means if the mlflow_model_config is None, use development_config if
+        # available
+        # 2. "", Empty string, this means the users explicitly didn't set the model config
+        # while logging the model so if ModelConfig is used, it should throw an error
+        # 3. A valid path, this means the users have set the model config while logging the
+        # model so use that path
+        if config is not None:
+            self.config = config
+        else:
+            self.config = development_config
+
+        if not self.config:
+            raise FileNotFoundError(
+                "Config file is not provided which is needed to load the model. "
+                "Please provide a valid path."
+            )
+
+        if not isinstance(self.config, dict) and not os.path.isfile(self.config):
+            raise FileNotFoundError(f"Config file '{self.config}' not found.")
+
+    def _read_config(self):
+        """Reads the YAML configuration file and returns its contents.
+
+        Raises:
+            FileNotFoundError: If the configuration file does not exist.
+            yaml.YAMLError: If there is an error parsing the YAML content.
+
+        Returns:
+            dict or None: The content of the YAML file as a dictionary, or None if the
+            config path is not set.
+        """
+        if isinstance(self.config, dict):
+            return self.config
+
+        with open(self.config) as file:
+            try:
+                return yaml.safe_load(file)
+            except yaml.YAMLError as e:
+                raise MlflowException(
+                    f"Error parsing YAML file: {e}", error_code=INVALID_PARAMETER_VALUE
+                )
+
+    def to_dict(self):
+        """Returns the configuration as a dictionary."""
+        return self._read_config()
+
+    def get(self, key):
+        """Gets the value of a top-level parameter in the configuration."""
+        config_data = self._read_config()
+
+        if config_data and key in config_data:
+            return config_data[key]
+        else:
+            raise KeyError(f"Key '{key}' not found in configuration: {config_data}.")
+
+
+def _set_model_config(model_config):
+    globals()["__mlflow_model_config__"] = model_config

mlflow/models/notebook_resources/agent_evaluation_template.html

@@ -0,0 +1,235 @@
+<head>
+  <link
+    rel="stylesheet"
+    href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/xcode.min.css"
+  />
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+  <script>
+    hljs.highlightAll();
+  </script>
+  <style>
+    body {
+      margin: 0;
+      font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto,
+        Helvetica Neue, Arial, Noto Sans, sans-serif, Apple Color Emoji,
+        Segoe UI Emoji, Segoe UI Symbol, Noto Color Emoji;
+      -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+      margin: 0;
+      font-weight: 400;
+      font-size: 13px;
+      line-height: 18px;
+      color: rgb(17, 23, 28);
+    }
+    code {
+      line-height: 18px;
+      font-size: 11px;
+      background: rgb(250, 250, 250) !important;
+    }
+    pre {
+      background: rgb(250, 250, 250);
+      margin: 0;
+      display: none;
+    }
+    pre.active {
+      display: unset;
+    }
+    button {
+      white-space: nowrap;
+      text-align: center;
+      position: relative;
+      cursor: pointer;
+      background: rgba(34, 114, 180, 0) !important;
+      color: rgb(34, 114, 180) !important;
+      border-color: rgba(34, 114, 180, 0) !important;
+      padding: 4px 6px !important;
+      text-decoration: none !important;
+      line-height: 20px !important;
+      box-shadow: none !important;
+      height: 32px !important;
+      display: inline-flex !important;
+      -webkit-box-align: center !important;
+      align-items: center !important;
+      -webkit-box-pack: center !important;
+      justify-content: center !important;
+      vertical-align: middle !important;
+    }
+    p {
+      margin: 0;
+      padding: 0;
+    }
+    button:hover {
+      background: rgba(34, 114, 180, 0.08) !important;
+      color: rgb(14, 83, 139) !important;
+    }
+    button:active {
+      background: rgba(34, 114, 180, 0.16) !important;
+      color: rgb(4, 53, 93) !important;
+    }
+    h1 {
+      margin-top: 4px;
+      font-size: 22px;
+    }
+    .info {
+      font-size: 12px;
+      font-weight: 500;
+      line-height: 16px;
+      color: rgb(95, 114, 129);
+    }
+    .tabs {
+      margin-top: 10px;
+      border-bottom: 1px solid rgb(209, 217, 225) !important;
+      display: flex;
+      line-height: 24px;
+    }
+    .tab {
+      font-size: 13px;
+      font-weight: 600 !important;
+      cursor: pointer;
+      margin: 0 24px 0 2px;
+      padding-left: 2px;
+    }
+    .tab:hover {
+      color: rgb(14, 83, 139) !important;
+    }
+    .tab.active {
+      border-bottom: 3px solid rgb(34, 114, 180) !important;
+    }
+    .link {
+      margin-left: 12px;
+      display: inline-block;
+      text-decoration: none;
+      color: rgb(34, 114, 180) !important;
+      font-size: 13px;
+      font-weight: 400;
+    }
+    .link:hover {
+      color: rgb(14, 83, 139) !important;
+    }
+    .link-content {
+      display: flex;
+      gap: 6px;
+      align-items: center;
+    }
+    .caret-up {
+      transform: rotate(180deg);
+    }
+  </style>
+</head>
+<body>
+  <div style="display: flex; align-items: center">
+    The logged model is compatible with the Mosaic AI Agent Framework.
+    <button onclick="toggleCode()">
+      See how to evaluate the model&nbsp;
+      <span
+        role="img"
+        id="caret"
+        aria-hidden="true"
+        class="anticon css-6xix1i"
+        style="font-size: 14px"
+        ><svg
+          xmlns="http://www.w3.org/2000/svg"
+          width="1em"
+          height="1em"
+          fill="none"
+          viewBox="0 0 16 16"
+          aria-hidden="true"
+          focusable="false"
+          class=""
+        >
+          <path
+            fill="currentColor"
+            fill-rule="evenodd"
+            d="M8 8.917 10.947 6 12 7.042 8 11 4 7.042 5.053 6z"
+            clip-rule="evenodd"
+          ></path>
+        </svg>
+      </span>
+    </button>
+  </div>
+  <div id="code" style="display: none">
+    <h1>
+      Agent evaluation
+      <a
+        class="link"
+        href="https://docs.databricks.com/en/generative-ai/agent-evaluation/synthesize-evaluation-set.html?utm_source=mlflow.log_model&utm_medium=notebook"
+        target="_blank"
+      >
+        <span class="link-content">
+          Learn more
+          <span role="img" aria-hidden="true" class="anticon css-6xix1i"
+            ><svg
+              xmlns="http://www.w3.org/2000/svg"
+              width="1em"
+              height="1em"
+              fill="none"
+              viewBox="0 0 16 16"
+              aria-hidden="true"
+              focusable="false"
+              class=""
+            >
+              <path
+                fill="currentColor"
+                d="M10 1h5v5h-1.5V3.56L8.53 8.53 7.47 7.47l4.97-4.97H10z"
+              ></path>
+              <path
+                fill="currentColor"
+                d="M1 2.75A.75.75 0 0 1 1.75 2H8v1.5H2.5v10h10V8H14v6.25a.75.75 0 0 1-.75.75H1.75a.75.75 0 0 1-.75-.75z"
+              ></path></svg></span></span
+      ></a>
+    </h1>
+    <p class="info">
+      Copy the following code snippet in a notebook cell (right click → copy)
+    </p>
+    <div class="tabs">
+      <div class="tab active" onclick="tabClicked(0)">Using synthetic data</div>
+      <div class="tab" onclick="tabClicked(1)">Using your own dataset</div>
+    </div>
+    <div style="height: 472px">
+      <pre
+        class="active"
+      ><code class="language-python">{{eval_with_synthetic_code}}</code></pre>
+
+      <pre><code class="language-python">{{eval_with_dataset_code}}</code></pre>
+    </div>
+  </div>
+  <script>
+    var codeShown = false;
+    function clip(el) {
+      var range = document.createRange();
+      range.selectNodeContents(el);
+      var sel = window.getSelection();
+      sel.removeAllRanges();
+      sel.addRange(range);
+    }
+
+    function toggleCode() {
+      if (codeShown) {
+        document.getElementById("code").style.display = "none";
+        codeShown = false;
+      } else {
+        document.getElementById("code").style.display = "block";
+        clip(document.querySelector("pre.active"));
+        codeShown = true;
+      }
+      document.getElementById("caret").classList.toggle("caret-up");
+    }
+
+    function tabClicked(tabIndex) {
+      document.querySelectorAll(".tab").forEach((tab, index) => {
+        if (index === tabIndex) {
+          tab.classList.add("active");
+        } else {
+          tab.classList.remove("active");
+        }
+      });
+      document.querySelectorAll("pre").forEach((pre, index) => {
+        if (index === tabIndex) {
+          pre.classList.add("active");
+        } else {
+          pre.classList.remove("active");
+        }
+      });
+      clip(document.querySelector("pre.active"));
+    }
+  </script>
+</body>

mlflow/models/notebook_resources/eval_with_dataset_example.py

@@ -0,0 +1,22 @@
+# ruff: noqa: F821, I001
+{{pipInstall}}
+
+import pandas as pd
+import mlflow
+
+evals = [
+    {
+        "request": {
+            "messages": [
+                {"role": "user", "content": "How do I convert a Spark DataFrame to Pandas?"}
+            ],
+        },
+        # Optional, needed for judging correctness.
+        "expected_facts": [
+            "To convert a Spark DataFrame to Pandas, you can use the toPandas() method."
+        ],
+    }
+]
+eval_result = mlflow.evaluate(
+    data=pd.DataFrame.from_records(evals), model="{{modelUri}}", model_type="databricks-agent"
+)

mlflow/models/notebook_resources/eval_with_synthetic_example.py

@@ -0,0 +1,22 @@
+# ruff: noqa: F821, I001
+{{pipInstall}}
+
+from databricks.agents.evals import generate_evals_df
+import mlflow
+
+agent_description = "A chatbot that answers questions about Databricks."
+question_guidelines = """
+# User personas
+- A developer new to the Databricks platform
+# Example questions
+- What API lets me parallelize operations over rows of a delta table?
+"""
+# TODO: Spark/Pandas DataFrame with "content" and "doc_uri" columns.
+docs = spark.table("catalog.schema.my_table_of_docs")
+evals = generate_evals_df(
+    docs=docs,
+    num_evals=25,
+    agent_description=agent_description,
+    question_guidelines=question_guidelines,
+)
+eval_result = mlflow.evaluate(data=evals, model="{{modelUri}}", model_type="databricks-agent")