ob-metaflow-extensions 1.1.151__py2.py3-none-any.whl → 1.6.2__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml

@@ -0,0 +1,337 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+title: CoreConfig
+description: |
+  Schema for defining Outerbounds Apps configuration. This schema is what we will end up using on the CLI/programmatic interface.
+  How to read this schema:
+  1. If the a property has `mutation_behavior` set to `union` then it will allow overrides of values at runtime from the CLI.
+  2. If the property has `mutation_behavior`set to `not_allowed` then either the CLI or the config file value will be used (which ever is not None). If the user supplies something in both then an error will be raised.
+  3. If a property has `experimental` set to true then a lot its validations may-be skipped and parsing handled somewhere else.
+type: object
+required:
+- name
+- port
+- commands
+properties:
+  name:
+    description: The name of the app to deploy. (validation applied)
+    type: string
+    example: myapp
+    mutation_behavior: union
+    cli_option: --name
+  port:
+    description: |-
+      Port where the app is hosted. When deployed this will be port on which we will deploy the app. (validation applied)
+    type: integer
+    example: 8000
+    mutation_behavior: union
+    cli_option: --port
+  description:
+    description: The description of the app to deploy.
+    type: string
+    example: This is a description of my app.
+    mutation_behavior: union
+    cli_option: --description
+  app_type:
+    description: The User defined type of app to deploy. Its only used for bookkeeping purposes.
+    type: string
+    example: MyCustomAgent
+    mutation_behavior: union
+    cli_option: --app-type
+  image:
+    description: The Docker image to deploy with the App.
+    type: string
+    example: python:3.10-slim
+    mutation_behavior: union
+    cli_option: --image
+  tags:
+    description: The tags of the app to deploy. (validation applied)
+    type: array
+    items:
+      type: string
+    example:
+    - foo: bar
+    - x: y
+    mutation_behavior: union
+    cli_option: --tag
+  secrets:
+    description: |-
+      Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code. (validation applied)
+    type: array
+    items:
+      type: string
+    example:
+    - hf-token
+    mutation_behavior: union
+    cli_option: --secret
+  compute_pools:
+    description: A list of compute pools to deploy the app to.
+    type: array
+    items:
+      type: string
+    example:
+    - default
+    - large
+    mutation_behavior: union
+    cli_option: --compute-pools
+  environment:
+    description: Environment variables to deploy with the App.
+    type: object
+    additionalProperties: true
+    example:
+      DEBUG: true
+      DATABASE_CONFIG:
+        host: localhost
+        port: 5432
+      ALLOWED_ORIGINS:
+      - http://localhost:3000
+      - https://myapp.com
+    mutation_behavior: union
+    cli_option: --env
+  commands:
+    description: A list of commands to run the app with.
+    type: array
+    items:
+      type: string
+    example:
+    - python app.py
+    - python app.py --foo bar
+    mutation_behavior: not_allowed
+  resources:
+    title: ResourceConfig
+    description: Resource configuration for the app.
+    type: object
+    required: []
+    properties:
+      cpu:
+        description: CPU requests (validation applied)
+        type: string
+        default: '1'
+        example: 500m
+        mutation_behavior: union
+        cli_option: --cpu
+      memory:
+        description: Memory requests (validation applied)
+        type: string
+        default: 4Gi
+        example: 512Mi
+        mutation_behavior: union
+        cli_option: --memory
+      gpu:
+        description: GPU requests (validation applied)
+        type: string
+        example: '1'
+        mutation_behavior: union
+        cli_option: --gpu
+      disk:
+        description: Storage disk size. (validation applied)
+        type: string
+        default: 20Gi
+        example: 1Gi
+        mutation_behavior: union
+        cli_option: --disk
+      shared_memory:
+        description: Shared memory (validation applied)
+        type: string
+        example: 1Gi
+        mutation_behavior: union
+        cli_option: --shared-memory
+    mutation_behavior: union
+  auth:
+    title: AuthConfig
+    description: Authentication configuration.
+    type: object
+    required: []
+    properties:
+      type:
+        description: The type of authentication to use for the app.
+        type: string
+        default: Browser
+        enum:
+        - Browser
+        - API
+        - BrowserAndApi
+        example: Browser
+        mutation_behavior: union
+        cli_option: --auth-type
+      public:
+        description: Whether the app is public or not.
+        type: boolean
+        default: true
+        example: true
+        mutation_behavior: union
+        cli_option: --public-access/--private-access
+    mutation_behavior: union
+  replicas:
+    title: ReplicaConfig
+    description: Replica configuration.
+    type: object
+    required: []
+    properties:
+      fixed:
+        description: |-
+          The fixed number of replicas to deploy the app with. If min and max are set, this will raise an error.
+        type: integer
+        example: 1
+        mutation_behavior: union
+        cli_option: --fixed-replicas
+      min:
+        description: The minimum number of replicas to deploy the app with.
+        type: integer
+        example: 1
+        mutation_behavior: union
+        cli_option: --min-replicas
+      max:
+        description: The maximum number of replicas to deploy the app with.
+        type: integer
+        example: 10
+        mutation_behavior: union
+        cli_option: --max-replicas
+      scaling_policy:
+        title: ScalingPolicyConfig
+        description: |-
+          Policies for autoscaling replicas. Available policies:
+          - Request based Autoscaling (rpm)
+        type: object
+        required: []
+        properties:
+          rpm:
+            description: |-
+              Scale up replicas when the requests per minute crosses this threshold. If nothing is provided and the replicas.max and replicas.min is set then the default rpm would be 60.
+            type: integer
+            default: 60
+            mutation_behavior: union
+            cli_option: --scaling-rpm
+        mutation_behavior: union
+    mutation_behavior: union
+  dependencies:
+    title: DependencyConfig
+    description: Dependency configuration.
+    type: object
+    required: []
+    properties:
+      from_requirements_file:
+        description: The path to the requirements.txt file to attach to the app.
+        type: string
+        example: requirements.txt
+        mutation_behavior: not_allowed
+        cli_option: --dep-from-requirements
+      from_pyproject_toml:
+        description: The path to the pyproject.toml file to attach to the app.
+        type: string
+        example: pyproject.toml
+        mutation_behavior: not_allowed
+        cli_option: --dep-from-pyproject
+      python:
+        description: The Python version to use for the app.
+        type: string
+        example: '3.10'
+        mutation_behavior: union
+        cli_option: --python
+      pypi:
+        description: |-
+          A dictionary of pypi dependencies to attach to the app. The key is the package name and the value is the version.
+        type: object
+        additionalProperties: true
+        example:
+          numpy: 1.23.0
+          pandas: ''
+        mutation_behavior: not_allowed
+        cli_option: --pypi
+      conda:
+        description: |-
+          A dictionary of conda dependencies to attach to the app. The key is the package name and the value is the version.
+        type: object
+        additionalProperties: true
+        example:
+          numpy: 1.23.0
+          pandas: ''
+        mutation_behavior: not_allowed
+        cli_option: --conda
+    mutation_behavior: union
+  package:
+    title: PackageConfig
+    description: Package configuration.
+    type: object
+    required: []
+    properties:
+      src_paths:
+        description: The path to the source code to deploy with the App.
+        type: array
+        items:
+          type: string
+        example:
+        - ./
+        mutation_behavior: union
+        cli_option: --package-src-path
+      suffixes:
+        description: A list of suffixes to add to the source code to deploy with the App.
+        type: array
+        items:
+          type: string
+        example:
+        - .py
+        - .ipynb
+        mutation_behavior: union
+        cli_option: --package-suffixes
+    mutation_behavior: union
+  no_deps:
+    description: Do not bake any dependencies. Directly used the image provided
+    type: boolean
+    default: false
+    mutation_behavior: union
+    cli_option: --no-deps
+  force_upgrade:
+    description: Force upgrade the app even if it is currently being upgraded.
+    type: boolean
+    default: false
+    mutation_behavior: union
+    cli_option: --force-upgrade
+  persistence:
+    description: The persistence mode to deploy the app with. (validation applied)
+    type: string
+    default: none
+    enum:
+    - none
+    - postgres
+    example: postgres
+    experimental: true
+    mutation_behavior: union
+    cli_option: --persistence
+  project:
+    description: The project name to deploy the app to.
+    type: string
+    example: my-project
+    experimental: true
+    mutation_behavior: union
+    cli_option: --project
+  branch:
+    description: The branch name to deploy the app to.
+    type: string
+    example: main
+    experimental: true
+    mutation_behavior: union
+    cli_option: --branch
+  models:
+    type: array
+    items:
+      type: string
+    example:
+    - asset_id: model-123
+      asset_instance_id: instance-456
+    experimental: true
+    mutation_behavior: union
+  data:
+    type: array
+    items:
+      type: string
+    example:
+    - asset_id: data-789
+      asset_instance_id: instance-101
+    experimental: true
+    mutation_behavior: union
+  generate_static_url:
+    description: Generate a static URL for the app based on its name.
+    type: boolean
+    default: false
+    mutation_behavior: union
+    cli_option: --generate-static-url

metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py

@@ -0,0 +1,115 @@
+import copy
+import json
+import os
+import shutil
+import sys
+import tempfile
+from hashlib import sha256
+from typing import List, Optional, Callable, Any
+from .app_config import AppConfig
+from .utils import TODOException
+from metaflow.metaflow_config import (
+    get_pinned_conda_libs,
+    DEFAULT_DATASTORE,
+    KUBERNETES_CONTAINER_IMAGE,
+)
+from collections import namedtuple
+
+BakingStatus = namedtuple(
+    "BakingStatus", ["image_should_be_baked", "python_path", "resolved_image"]
+)
+
+
+class ImageBakingException(Exception):
+    pass
+
+
+def _safe_open_file(path: str):
+    if not os.path.exists(path):
+        raise ImageBakingException(f"File does not exist: {path}")
+    try:
+        with open(path, "r") as f:
+            return f.read()
+    except Exception as e:
+        raise ImageBakingException(f"Failed to open file: {e}")
+
+
+def bake_deployment_image(
+    app_config: AppConfig,
+    cache_file_path: str,
+    logger: Optional[Callable[[str], Any]] = None,
+) -> BakingStatus:
+    # When do we bake an image?
+    # 1. When the user has specified something like `pypi`/`conda`
+    # 2, When the user has specified something like `from_requirements`/ `from_pyproject`
+    # TODO: add parsers for the pyproject/requirements stuff.
+    from metaflow.ob_internal import internal_bake_image  # type: ignore
+    from metaflow.plugins.pypi.parsers import (
+        requirements_txt_parser,
+        pyproject_toml_parser,
+    )
+
+    image = app_config.get("image", KUBERNETES_CONTAINER_IMAGE)
+    python_version = "%d.%d.%d" % sys.version_info[:3]
+
+    dependencies = app_config.get_state("dependencies", {})
+    pypi_packages = {}
+    conda_packages = {}
+
+    parsed_packages = {}
+
+    if dependencies.get("from_requirements_file"):
+        parsed_packages = requirements_txt_parser(
+            _safe_open_file(dependencies.get("from_requirements_file"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif dependencies.get("from_pyproject_toml"):
+        parsed_packages = pyproject_toml_parser(
+            _safe_open_file(dependencies.get("from_pyproject_toml"))
+        )
+        pypi_packages = parsed_packages.get("packages", {})
+        python_version = parsed_packages.get("python_version", python_version)
+
+    elif "pypi" in dependencies:
+        pypi_packages = dependencies.get("pypi", {}) or {}
+
+    if "conda" in dependencies:
+        conda_packages = dependencies.get("conda", {}) or {}
+    if "python" in dependencies:
+        python_version = dependencies.get("python", python_version) or python_version
+
+    python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
+    if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):
+        # Inform the user that no dependencies are being used.
+        if app_config.get_state("skip_dependencies", False) and logger:
+            logger(
+                "⏭️ Skipping baking dependencies into the image based on the --no-deps flag."
+            )
+        # TODO: Handle this a little more nicely.
+        return BakingStatus(
+            image_should_be_baked=False, resolved_image=image, python_path="python"
+        )
+
+    pinned_conda_libs = get_pinned_conda_libs(python_version, DEFAULT_DATASTORE)
+    pypi_packages.update(pinned_conda_libs)
+    _reference = app_config.get("name", "default")
+    # `image` cannot be None. If by chance it is none, FB will fart.
+    fb_response = internal_bake_image(
+        cache_file_path=cache_file_path,
+        pypi_packages=pypi_packages,
+        conda_packages=conda_packages,
+        ref=_reference,
+        python=python_version,
+        base_image=image,
+        logger=logger,
+    )
+    if fb_response.failure:
+        raise ImageBakingException(f"Failed to bake image: {fb_response.response}")
+
+    return BakingStatus(
+        image_should_be_baked=True,
+        resolved_image=fb_response.container_image,
+        python_path=fb_response.python_path,
+    )