ob-metaflow-extensions 1.1.175rc1__py2.py3-none-any.whl → 1.1.175rc3__py2.py3-none-any.whl

metaflow_extensions/outerbounds/plugins/apps/core/config_schema.yaml

@@ -1,269 +1,275 @@
-
-title: Outerbounds App Configuration Schema
+$schema: https://json-schema.org/draft/2020-12/schema
+title: CoreConfig
 description: |
   Schema for defining Outerbounds Apps configuration. This schema is what we will end up using on the CLI/programmatic interface.
   How to read this schema:
-    1. If the a property has `allow_union`:true then it will allow overrides from the cli.
-    2. If a property has `experimental` set to true then a lot its validations may-be skipped and parsing handled somewhere else.
-
-  The YAML based schema file is for Humans to change and consume. The JSON based schema file is what gets autogenerated based on pre-commit
-  hooks so that we can use within the outerbounds package. The reasons for two distinct types of files it that YAML provides the ability to
-  add comments and make readability easier. While JSON is so that we can reduce the dependency on YAML when working with apps within both
-  Metaflow and OB package.
-version: 1.0.0
+  1. If the a property has `mutation_behavior` set to `union` then it will allow overrides of values at runtime from the CLI.
+  2. If the property has `mutation_behavior`set to `not_allowed` then either the CLI or the config file value will be used (which ever is not None). If the user supplies something in both then an error will be raised.
+  3. If a property has `experimental` set to true then a lot its validations may-be skipped and parsing handled somewhere else.
 type: object
 required:
-  - name
-  - port
+- name
+- port
+- commands
 properties:
-  name: # Only used in `deploy` command
-    allow_union: true # Allow overriding name from the CLI.
+  name:
+    description: The name of the app to deploy. (validation applied)
     type: string
-    description: The name of the app to deploy.
-    maxLength: 20 # todo: check if we should allow a larger length.
-    example: "myapp"
-  port: # Only used in `deploy` command
-    allow_union: false
+    example: myapp
+    mutation_behavior: union
+  port:
+    description: |-
+      Port where the app is hosted. When deployed this will be port on which we will deploy the app. (validation applied)
     type: integer
-    description: Port where the app is hosted. When deployed this will be port on which we will deploy the app.
-    minimum: 1
-    maximum: 65535
     example: 8000
-  tags: # Only used in `deploy` command
-    allow_union: true
-    type: array
-    description: The tags of the app to deploy.
-    items:
-      type: object
-    example:
-      - foo: bar
-      - x: y
-  description: # Only used in `deploy` command
-    allow_union: true
-    type: string
+    mutation_behavior: union
+  description:
     description: The description of the app to deploy.
-    example: "This is a description of my app."
-  force_upgrade: # Only used in `deploy` command
-    allow_union: true
-    type: boolean
-    description: Whether to force upgrade the app even if it is currently being upgraded.
-    example: true
-  app_type: # Only used in `deploy` command
-    allow_union: true
     type: string
+    example: This is a description of my app.
+    mutation_behavior: union
+  app_type:
     description: The User defined type of app to deploy. Its only used for bookkeeping purposes.
-    example: "MyCustomAgent"
-  image: # Only used in `deploy` command
-    allow_union: true # We will overrwite the image if specified on the CLI.
     type: string
+    example: MyCustomAgent
+    mutation_behavior: union
+  image:
     description: The Docker image to deploy with the App.
-  secrets: # Used in `run` command
-    allow_union: true
+    type: string
+    example: python:3.10-slim
+    mutation_behavior: union
+  tags:
+    description: The tags of the app to deploy. (validation applied)
     type: array
-    description: Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code.
     items:
       type: string
-    example: ["hf-token"]
-  environment: # Used in `run` command
-    # Todo: So this part might not be best on the CLI. We should probably have a better way to handle this.
-    # In simplicity, we can just JSON dump anything that looks like a dict/list/
-    allow_union: true
-    type: object
+    example:
+    - foo: bar
+    - x: y
+    mutation_behavior: union
+  secrets:
+    description: |-
+      Outerbounds integrations to attach to the app. You can use the value you set in the `@secrets` decorator in your code. (validation applied)
+    type: array
+    items:
+      type: string
+    example:
+    - hf-token
+    mutation_behavior: union
+  compute_pools:
+    description: A list of compute pools to deploy the app to.
+    type: array
+    items:
+      type: string
+    example:
+    - default
+    - large
+    mutation_behavior: union
+  environment:
     description: Environment variables to deploy with the App.
-    additionalProperties:
-      oneOf:
-        - type: string
-        - type: number
-        - type: boolean
-        - type: object
-        - type: array # When users give arrays, or objects, we need to JSON dump them. Users need to be aware of this.
+    type: object
+    additionalProperties: true
     example:
       DEBUG: true
-      DATABASE_CONFIG: {"host": "localhost", "port": 5432}
-      ALLOWED_ORIGINS: ["http://localhost:3000", "https://myapp.com"]
-  dependencies: # Used in `run` command
-    allow_union: false
-    type: object
-    description: |
-      The dependencies to attach to the app. Only one of the properties can be specified.
-    properties:
-      from_requirements_file:
-        type: string
-        description: The path to the requirements.txt file to attach to the app.
-        example: "requirements.txt"
-      from_pyproject_toml:
-        type: string
-        description: The path to the pyproject.toml file to attach to the app.
-        example: "pyproject.toml"
-      python:
-        type: string
-        description: |
-          The Python version to use for the app.
-        example: "3.10"
-      pypi:
-        type: object
-        description: |
-          A dictionary of pypi dependencies to attach to the app.
-          The key is the package name and the value is the version.
-        example:
-          numpy: 1.23.0
-          pandas: ""
-      conda:
-        type: object
-        description: |
-          A dictionary of pypi dependencies to attach to the app.
-          The key is the package name and the value is the version.
-        example:
-          numpy: 1.23.0
-          pandas: ""
-  package:
-    allow_union: false
-    type: object
-    description: |
-      Configurations associated with packaging the app.
-    properties:
-      src_path:
-        type: string
-        description: The path to the source code to deploy with the App.
-        example: "./"
-      suffixes:
-        type: array
-        description: |
-          A list of suffixes to add to the source code to deploy with the App.
-        items:
-          type: string
-        example: [".py", ".ipynb"]
-
-  commands: # Used in `run` command
-    allow_union: false
+      DATABASE_CONFIG:
+        host: localhost
+        port: 5432
+      ALLOWED_ORIGINS:
+      - http://localhost:3000
+      - https://myapp.com
+    mutation_behavior: union
+  commands:
+    description: A list of commands to run the app with.
     type: array
-    description: A list of commands to run the app with. Cannot be configured from the CLI. Only used in `run` command.
     items:
       type: string
-    example: ["python app.py", "python app.py --foo bar"]
-  resources: # Only used in `deploy` command
-    allow_union: true # You can override CPU/Memory/GPU/Storage from the CLI.
+    example:
+    - python app.py
+    - python app.py --foo bar
+    mutation_behavior: not_allowed
+  resources:
+    title: ResourceConfig
+    description: Resource configuration for the app.
     type: object
+    required: []
     properties:
       cpu:
+        description: CPU resource request and limit. (validation applied)
         type: string
-        description: CPU resource request and limit.
-        example: "500m"
-        default: "1"
+        default: '1'
+        example: 500m
+        mutation_behavior: union
       memory:
+        description: Memory resource request and limit. (validation applied)
         type: string
-        description: Memory resource request and limit.
-        example: "512Mi"
-        default: "4Gi"
+        default: 4Gi
+        example: 512Mi
+        mutation_behavior: union
       gpu:
+        description: GPU resource request and limit. (validation applied)
         type: string
-        description: GPU resource request and limit.
-        example: "1"
+        example: '1'
+        mutation_behavior: union
       disk:
+        description: Storage resource request and limit. (validation applied)
         type: string
-        description: Storage resource request and limit.
-        example: "1Gi"
-        default: "20Gi"
+        default: 20Gi
+        example: 1Gi
+        mutation_behavior: union
+    mutation_behavior: union
+  auth:
+    title: AuthConfig
+    description: Authentication configuration.
+    type: object
+    required: []
+    properties:
+      type:
+        description: The type of authentication to use for the app.
+        type: string
+        default: Browser
+        enum:
+        - Browser
+        - API
+        example: Browser
+        mutation_behavior: union
+      public:
+        description: Whether the app is public or not.
+        type: boolean
+        default: true
+        example: true
+        mutation_behavior: union
+    mutation_behavior: union
   replicas:
-    allow_union: true
+    title: ReplicaConfig
+    description: Replica configuration.
     type: object
-    description: |
-      The number of replicas to deploy the app with.
+    required: []
     properties:
-      min:
+      fixed:
+        description: |-
+          The fixed number of replicas to deploy the app with. If min and max are set, this will raise an error.
         type: integer
+        example: 1
+        mutation_behavior: union
+      min:
         description: The minimum number of replicas to deploy the app with.
+        type: integer
         example: 1
+        mutation_behavior: union
       max:
-        type: integer
         description: The maximum number of replicas to deploy the app with.
+        type: integer
         example: 10
-  health_check: # Can be used in `run` command
+        mutation_behavior: union
+    mutation_behavior: union
+  dependencies:
+    title: DependencyConfig
+    description: Dependency configuration.
     type: object
-    # `allow_union` property means that any object in this field will be done a union with the config file if something is provided on commanline.
-    # If it is set to false, then we should throw an exception if the CLI is trying to override something specified in the config file.
-    # We will only allow unions in certains options. The rest will not allow any unions and only need to be specified in one place.
-    allow_union: false
+    required: []
     properties:
-      enabled:
-        type: boolean
-        description: Whether to enable health checks.
-        example: true
-        default: false
-      path:
+      from_requirements_file:
+        description: The path to the requirements.txt file to attach to the app.
         type: string
-        description: The path for health checks.
-        example: "/health"
-      initial_delay_seconds:
-        type: integer
-        description: Number of seconds to wait before performing the first health check.
-        example: 10
-      period_seconds:
-        type: integer
-        description: How often to perform the health check.
-        example: 30
-  compute_pools: # Only used in `deploy` command
-    allow_union: true
-    type: array
-    description: |
-      A list of compute pools to deploy the app to.
-    items:
-      type: string
-    example: ["default", "large"]
-  auth: # Only used in `deploy` command
-    allow_union: false
+        example: requirements.txt
+        mutation_behavior: not_allowed
+      from_pyproject_toml:
+        description: The path to the pyproject.toml file to attach to the app.
+        type: string
+        example: pyproject.toml
+        mutation_behavior: not_allowed
+      python:
+        description: The Python version to use for the app.
+        type: string
+        example: '3.10'
+        mutation_behavior: not_allowed
+      pypi:
+        description: |-
+          A dictionary of pypi dependencies to attach to the app. The key is the package name and the value is the version.
+        type: object
+        additionalProperties: true
+        example:
+          numpy: 1.23.0
+          pandas: ''
+        mutation_behavior: not_allowed
+      conda:
+        description: |-
+          A dictionary of conda dependencies to attach to the app. The key is the package name and the value is the version.
+        type: object
+        additionalProperties: true
+        example:
+          numpy: 1.23.0
+          pandas: ''
+        mutation_behavior: not_allowed
+    mutation_behavior: union
+  package:
+    title: PackageConfig
+    description: Package configuration.
     type: object
-    description: |
-      Auth related configurations.
+    required: []
     properties:
-      type:
+      src_path:
+        description: The path to the source code to deploy with the App.
         type: string
-        description: |
-          The type of authentication to use for the app.
-        enum: [API, Browser]
-      public:
-        type: boolean
-        description: |
-          Whether the app is public or not.
-        default: true
-      # There is an allowed perimeters property
-      # But that needs a little more thought on how
-      # to expose.
-
-  # ------------------------------------ EXPERIMENTAL ------------------------------------
-  project:
+        example: ./
+        mutation_behavior: union
+      suffixes:
+        description: A list of suffixes to add to the source code to deploy with the App.
+        type: array
+        items:
+          type: string
+        example:
+        - .py
+        - .ipynb
+        mutation_behavior: union
+    mutation_behavior: union
+  no_deps:
+    description: Do not bake any dependencies. Directly used the image provided
+    type: boolean
+    default: false
+    mutation_behavior: union
+  force_upgrade:
+    description: Force upgrade the app even if it is currently being upgraded.
+    type: boolean
+    default: false
+    mutation_behavior: union
+  persistence:
+    description: The persistence mode to deploy the app with. (validation applied)
     type: string
+    default: none
+    enum:
+    - none
+    - postgres
+    example: postgres
+    experimental: true
+    mutation_behavior: union
+  project:
     description: The project name to deploy the app to.
+    type: string
+    example: my-project
     experimental: true
-    allow_union: true
+    mutation_behavior: union
   branch:
-    type: string
     description: The branch name to deploy the app to.
+    type: string
+    example: main
     experimental: true
-    allow_union: true
-
-  models: #
+    mutation_behavior: union
+  models:
     type: array
-    description: model asset ids to include with the deployment. NO CLI Option for this Now.
-    experimental: true
-    allow_union: true
     items:
-      type: object
-      properties:
-        asset_id:
-          type: string
-        asset_instance_id:
-          type: string
-  data: #
-    type: array
-    description: data asset ids to include with the deployment.
+      type: string
+    example:
+    - asset_id: model-123
+      asset_instance_id: instance-456
     experimental: true
-    allow_union: true
+    mutation_behavior: union
+  data:
+    type: array
     items:
-      type: object
-      properties:
-        asset_id:
-          type: string
-        asset_instance_id:
-          type: string
-# ------------------------------------ EXPERIMENTAL ------------------------------------
+      type: string
+    example:
+    - asset_id: data-789
+      asset_instance_id: instance-101
+    experimental: true
+    mutation_behavior: union

metaflow_extensions/outerbounds/plugins/apps/core/dependencies.py

@@ -73,12 +73,12 @@ def bake_deployment_image(
         python_version = parsed_packages.get("python_version", python_version)
 
     elif "pypi" in dependencies:
-        pypi_packages = dependencies.get("pypi", {})
+        pypi_packages = dependencies.get("pypi", {}) or {}
 
     if "conda" in dependencies:
-        conda_packages = dependencies.get("conda", {})
+        conda_packages = dependencies.get("conda", {}) or {}
     if "python" in dependencies:
-        python_version = dependencies.get("python", python_version)
+        python_version = dependencies.get("python", python_version) or python_version
 
     python_packages_exist = len(pypi_packages) > 0 or len(conda_packages) > 0
     if (not python_packages_exist) or app_config.get_state("skip_dependencies", False):

metaflow_extensions/outerbounds/plugins/apps/core/deployer.py

@@ -0,0 +1,132 @@
+from .config import TypedCoreConfig
+
+from ._state_machine import DEPLOYMENT_READY_CONDITIONS
+from .app_config import AppConfig, AppConfigError
+from .capsule import CapsuleDeployer, list_and_filter_capsules
+from functools import partial
+import sys
+from typing import Type
+
+
+class AppDeployer(TypedCoreConfig):
+    """ """
+
+    __init__ = TypedCoreConfig.__init__
+
+    _app_config: AppConfig
+
+    _state = {}
+
+    @property
+    def app_config(self) -> AppConfig:
+        if not hasattr(self, "_app_config"):
+            self._app_config = AppConfig(self._config)
+        return self._app_config
+
+    # Things that need to be set before deploy
+    @classmethod
+    def _set_state(
+        cls,
+        perimeter: str,
+        api_url: str,
+        code_package_url: str = None,
+        code_package_key: str = None,
+        name: str = None,
+        image: str = None,
+    ):
+        cls._state["perimeter"] = perimeter
+        cls._state["api_url"] = api_url
+        cls._state["code_package_url"] = code_package_url
+        cls._state["code_package_key"] = code_package_key
+        cls._state["name"] = name
+        cls._state["image"] = image
+
+    def deploy(
+        self,
+        readiness_condition=DEPLOYMENT_READY_CONDITIONS.ATLEAST_ONE_RUNNING,
+        max_wait_time=600,
+        readiness_wait_time=10,
+        logger_fn=partial(print, file=sys.stderr),
+        status_file=None,
+        no_loader=False,
+        **kwargs,
+    ):
+        # Name setting from top level if none is set in the code
+        if self.app_config._core_config.name is None:
+            self.app_config._core_config.name = self._state["name"]
+
+        self.app_config.commit()
+
+        # Set any state that might have been passed down from the top level
+        for k, v in self._state.items():
+            if self.app_config.get_state(k) is None:
+                self.app_config.set_state(k, v)
+
+        capsule = CapsuleDeployer(
+            self.app_config,
+            self._state["api_url"],
+            create_timeout=max_wait_time,
+            debug_dir=None,
+            success_terminal_state_condition=readiness_condition,
+            readiness_wait_time=readiness_wait_time,
+            logger_fn=logger_fn,
+        )
+
+        currently_present_capsules = list_and_filter_capsules(
+            capsule.capsule_api,
+            None,
+            None,
+            capsule.name,
+            None,
+            None,
+            None,
+        )
+
+        force_upgrade = self.app_config.get_state("force_upgrade", False)
+
+        if len(currently_present_capsules) > 0:
+            # Only update the capsule if there is no upgrade in progress
+            # Only update a "already updating" capsule if the `--force-upgrade` flag is provided.
+            _curr_cap = currently_present_capsules[0]
+            this_capsule_is_being_updated = _curr_cap.get("status", {}).get(
+                "updateInProgress", False
+            )
+
+            if this_capsule_is_being_updated and not force_upgrade:
+                _upgrader = _curr_cap.get("metadata", {}).get("lastModifiedBy", None)
+                message = f"{capsule.capsule_type} is currently being upgraded"
+                if _upgrader:
+                    message = (
+                        f"{capsule.capsule_type} is currently being upgraded. Upgrade was launched by {_upgrader}. "
+                        "If you wish to force upgrade, you can do so by providing the `--force-upgrade` flag."
+                    )
+                raise AppConfigError(message)
+
+            logger_fn(
+                f"🚀 {'' if not force_upgrade else 'Force'} Upgrading {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+        else:
+            logger_fn(
+                f"🚀 Deploying {capsule.capsule_type.lower()} `{capsule.name}`....",
+            )
+
+        capsule.create()
+        final_status = capsule.wait_for_terminal_state()
+        return final_status
+
+
+class apps:
+
+    _name_prefix = None
+
+    @classmethod
+    def set_name_prefix(cls, name_prefix: str):
+        cls._name_prefix = name_prefix
+
+    @property
+    def name_prefix(self) -> str:
+        return self._name_prefix
+
+    @property
+    def Deployer(self) -> Type[AppDeployer]:
+        return AppDeployer