prefect-client 2.19.4__py3-none-any.whl → 3.0.0rc2__py3-none-any.whl

prefect/software/conda.py

@@ -1,199 +0,0 @@
-import json
-import re
-import subprocess
-import sys
-from pathlib import Path
-from typing import List, Type
-
-import yaml
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import Field, validate_arguments
-else:
-    from pydantic import Field, validate_arguments
-from typing_extensions import Self
-
-from prefect.software.base import (
-    Requirement,
-    pop_requirement_by_name,
-    remove_duplicate_requirements,
-)
-from prefect.software.pip import current_environment_requirements
-from prefect.software.python import PythonEnvironment
-from prefect.utilities.collections import listrepr
-
-# Capture each component of a conda requirement string.
-#
-#   <name><version_specifier><version><build_specifier><build>
-#
-# The specification of these requirements is more relaxed than pip requirements.
-# Notably, the version specifier is typically a single equal sign and a build can be
-# included after the version.
-#
-# For example, the requirement "requests=2.25.1=pyhd3eb1b0_0" matches as:
-#   name: "requests"
-#   version_specifier: "="
-#   version: "2.25.1"
-#   build_specifier: "="
-#   build: "pyhd3eb1b0_0"
-#
-# Regex components:
-#   name (required): any combination of numbers, letters, or dashes
-#   version_specifier (optional): one or more of `>`, `<`, and `=`
-#   version (optional): any combination of numbers, letters, or periods; not valid
-#       unless grouped with a version specifier.
-#   build_specifier (optional): just `=`.
-#   build (optional): any combination of numbers, letters, or underscores; not valid
-#       unless grouped with a build specifier.
-
-CONDA_REQUIREMENT = re.compile(
-    r"^((?P<channel>[0-9A-Za-z\_\-\/]+)::)?"
-    r"(?P<name>[0-9A-Za-z_\-\.]+)"
-    r"((?P<version_specifier>[>=<]+)(?P<version>[0-9a-zA-Z\.]+))?"
-    r"((?P<build_specifier>=)(?P<build>[0-9A-Za-z\_\[\]\=]+))?$"
-)
-
-
-class CondaRequirement(Requirement):
-    """
-    A parsed requirement for installation with conda.
-    """
-
-    def __init__(self, requirement_string: str):
-        self._requirement_string = requirement_string
-
-        parsed = CONDA_REQUIREMENT.match(requirement_string)
-        if parsed is None:
-            raise ValueError(
-                f"Invalid requirement {requirement_string!r}: could not be parsed."
-            )
-        self._parts = parsed.groupdict()
-
-        self.name = self._parts["name"]
-        self.version_specifier = self._parts["version_specifier"]
-        self.version = self._parts["version"]
-        self.build_specifier = self._parts["build_specifier"]
-        self.build = self._parts["build"]
-
-    def __str__(self) -> str:
-        return self._requirement_string
-
-
-class CondaError(RuntimeError):
-    """
-    Raised if an error occurs in conda.
-    """
-
-
-def current_environment_conda_requirements(
-    include_builds: bool = False, explicit_only: bool = True
-) -> List[CondaRequirement]:
-    """
-    Return conda requirements by exporting the current environment.
-
-    Skips any pip requirements included in the export. Only requirements that are
-    managed by conda are returned.
-    """
-    command = ["conda", "env", "export", "--json"]
-
-    if not include_builds:
-        command.append("--no-builds")
-    if explicit_only:
-        command.append("--from-history")
-
-    process = subprocess.run(command, capture_output=True)
-    parsed = json.loads(process.stdout)
-    if "error" in parsed:
-        raise CondaError(
-            "Encountered an exception while exporting the conda environment: "
-            + parsed["error"]
-        )
-
-    # If no dependencies are given, this field will not be present
-    dependencies = parsed.get("dependencies", [])
-
-    # The string check will exclude nested objects like the 'pip' subtree
-    return [CondaRequirement(dep) for dep in dependencies if isinstance(dep, str)]
-
-
-class CondaEnvironment(PythonEnvironment):
-    conda_requirements: List[CondaRequirement] = Field(default_factory=list)
-
-    @classmethod
-    def from_environment(cls: Type[Self], exclude_nested: bool = False) -> Self:
-        conda_requirements = (
-            current_environment_conda_requirements()
-            if "conda" in sys.executable
-            else []
-        )
-        pip_requirements = remove_duplicate_requirements(
-            conda_requirements,
-            current_environment_requirements(
-                exclude_nested=exclude_nested, on_uninstallable_requirement="warn"
-            ),
-        )
-        python_requirement = pop_requirement_by_name(conda_requirements, "python")
-        python_version = python_requirement.version if python_requirement else None
-
-        return cls(
-            pip_requirements=pip_requirements,
-            conda_requirements=conda_requirements,
-            python_version=python_version,
-        )
-
-    @classmethod
-    @validate_arguments
-    def from_file(cls: Type[Self], path: Path) -> Self:
-        parsed = yaml.safe_load(path.read_text())
-
-        # If no dependencies are given, this field will not be present
-        dependencies = parsed.get("dependencies", [])
-
-        # The string check will exclude nested objects like the 'pip' subtree
-        conda_requirements = [
-            CondaRequirement(dep) for dep in dependencies if isinstance(dep, str)
-        ]
-
-        python_requirement = pop_requirement_by_name(conda_requirements, "python")
-        python_version = python_requirement.version if python_requirement else None
-
-        other_requirements = {}
-
-        # Parse nested requirements. We only support 'pip' for now but we'll check for
-        # others
-        for subtree in [dep for dep in dependencies if isinstance(dep, dict)]:
-            key = list(subtree.keys())[0]
-
-            if key in other_requirements:
-                raise ValueError(
-                    "Invalid conda requirements specification. "
-                    f"Found duplicate key {key!r}."
-                )
-
-            other_requirements[key] = subtree[key]
-
-        pip_requirements = other_requirements.pop("pip", [])
-
-        if other_requirements:
-            raise ValueError(
-                "Found unsupported requirements types in file: "
-                f"{listrepr(other_requirements.keys(), ', ')}"
-            )
-
-        return cls(
-            conda_requirements=conda_requirements,
-            pip_requirements=pip_requirements,
-            python_version=python_version,
-        )
-
-    def install_commands(self) -> List[str]:
-        pip_install_commands = super().install_commands()
-
-        if not self.conda_requirements:
-            return pip_install_commands
-
-        return [
-            ["conda", "install", *(str(req) for req in self.conda_requirements)]
-        ] + pip_install_commands

prefect/software/pip.py

@@ -1,122 +0,0 @@
-import site
-import warnings
-from pathlib import Path
-from typing import Dict, List, Type, Union
-
-import packaging.requirements
-import packaging.version
-from typing_extensions import Literal, Self
-
-from prefect.software.base import Requirement
-from prefect.utilities.compat import importlib_metadata
-
-
-class PipRequirement(Requirement, packaging.requirements.Requirement):
-    """
-    A parsed Python requirement.
-
-    An extension for `packaging.version.Requirement` with Pydantic support.
-    """
-
-    @classmethod
-    def from_distribution(
-        cls: Type[Self], dist: "importlib_metadata.Distribution"
-    ) -> Self:
-        """
-        Convert a Python distribution object into a requirement
-        """
-        if _is_editable_install(dist):
-            raise ValueError(
-                f"Distribution {dist!r} is an editable installation and cannot be "
-                "used as a requirement."
-            )
-
-        return cls.validate(
-            packaging.requirements.Requirement(
-                f"{dist.metadata['name']}=={dist.version}"
-            )
-        )
-
-
-def _get_installed_distributions() -> Dict[str, "importlib_metadata.Distribution"]:
-    return {dist.name: dist for dist in importlib_metadata.distributions()}
-
-
-def _is_child_path(path: Union[Path, str], parent: Union[Path, str]) -> bool:
-    return Path(parent).resolve() in Path(path).resolve().parents
-
-
-def _is_same_path(path: Union[Path, str], other: Union[Path, str]) -> bool:
-    return Path(path).resolve() == Path(other).resolve()
-
-
-def _is_editable_install(dist: "importlib_metadata.Distribution") -> bool:
-    """
-    Determine if a distribution is an 'editable' install by scanning if it is present
-    in a site-packages directory or not; if not, we presume it is editable.
-    """
-    site_packages = site.getsitepackages() + [site.getusersitepackages()]
-
-    dist_location = dist.locate_file("")
-    for site_package_dir in site_packages:
-        if _is_same_path(dist_location, site_package_dir) or _is_child_path(
-            dist_location, site_package_dir
-        ):
-            return False
-
-    return True
-
-
-def _remove_distributions_required_by_others(
-    dists: Dict[str, "importlib_metadata.Distribution"],
-) -> Dict[str, "importlib_metadata.Distribution"]:
-    # Collect all child requirements
-    child_requirement_names = set()
-    for dist in dists.values():
-        child_requirement_names.update(dist.requires or [])
-
-    return {
-        name: dist
-        for name, dist in dists.items()
-        if name not in child_requirement_names
-    }
-
-
-def current_environment_requirements(
-    exclude_nested: bool = False,
-    on_uninstallable_requirement: Literal["ignore", "warn", "raise"] = "warn",
-) -> List[PipRequirement]:
-    dists = _get_installed_distributions()
-    if exclude_nested:
-        dists = _remove_distributions_required_by_others(dists)
-
-    requirements = []
-    uninstallable_msgs = []
-    for dist in dists.values():
-        if _is_editable_install(dist):
-            uninstallable_msgs.append(
-                f"- {dist.name}: This distribution is an editable installation."
-            )
-        else:
-            requirements.append(PipRequirement.from_distribution(dist))
-
-    if uninstallable_msgs:
-        message = (
-            "The following requirements will not be installable on another machine:\n"
-            + "\n".join(uninstallable_msgs)
-        )
-        if on_uninstallable_requirement == "ignore":
-            pass
-        elif on_uninstallable_requirement == "warn":
-            # When warning, include a note that these distributions are excluded
-            warnings.warn(message + "\nThese requirements will be ignored.")
-        elif on_uninstallable_requirement == "raise":
-            raise ValueError(message)
-        else:
-            raise ValueError(
-                "Unknown mode for `on_uninstallable_requirement`: "
-                f"{on_uninstallable_requirement!r}. "
-                "Expected one of: 'ignore', 'warn', 'raise'."
-            )
-
-    return requirements

prefect/software/python.py

@@ -1,52 +0,0 @@
-from pathlib import Path
-from typing import List, Type
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-from prefect._internal.schemas.validators import infer_python_version
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import BaseModel, Field, validate_arguments, validator
-else:
-    from pydantic import BaseModel, Field, validate_arguments, validator
-
-from typing_extensions import Self
-
-from prefect.software.pip import PipRequirement, current_environment_requirements
-
-
-class PythonEnvironment(BaseModel):
-    """
-    A specification for a Python environment.
-    """
-
-    python_version: str = None
-    pip_requirements: List[PipRequirement] = Field(default_factory=list)
-
-    @validator("python_version", pre=True, always=True)
-    def validate_python_version(cls, value):
-        return infer_python_version(value)
-
-    @classmethod
-    def from_environment(cls: Type[Self], exclude_nested: bool = False) -> Self:
-        """
-        Generate requirements from the current environment
-
-        Arguments:
-            exclude_nested: If True, only top-level requirements will be included.
-                Defaults to including all requirements.
-        """
-        pip_requirements = current_environment_requirements(
-            exclude_nested=exclude_nested, on_uninstallable_requirement="warn"
-        )
-        return cls(pip_requirements=pip_requirements)
-
-    @classmethod
-    @validate_arguments
-    def from_file(cls: Type[Self], path: Path) -> Self:
-        return PythonEnvironment(pip_requirements=path.read_text().strip().splitlines())
-
-    def install_commands(self) -> List[List[str]]:
-        if not self.pip_requirements:
-            return []
-
-        return [["pip", "install", *(str(req) for req in self.pip_requirements)]]

prefect/workers/block.py

@@ -1,218 +0,0 @@
-import shlex
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
-
-import anyio
-import anyio.abc
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-from prefect._internal.schemas.validators import validate_block_is_infrastructure
-from prefect.blocks.core import Block
-from prefect.client.schemas.objects import BlockDocument
-from prefect.utilities.collections import get_from_dict
-from prefect.workers.base import BaseWorker, BaseWorkerResult
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import BaseModel, Field, PrivateAttr, validator
-else:
-    from pydantic import BaseModel, Field, PrivateAttr, validator
-
-from prefect.client.orchestration import PrefectClient
-from prefect.client.utilities import inject_client
-from prefect.events import RelatedResource
-from prefect.events.related import object_as_related_resource, tags_as_related_resources
-from prefect.utilities.templating import apply_values
-
-if TYPE_CHECKING:
-    from prefect.client.schemas.objects import Flow, FlowRun
-    from prefect.client.schemas.responses import DeploymentResponse
-
-
-class BlockWorkerJobConfiguration(BaseModel):
-    block: Block = Field(
-        default=..., description="The infrastructure block to use for job creation."
-    )
-
-    @validator("block")
-    def _validate_infrastructure_block(cls, v):
-        return validate_block_is_infrastructure(v)
-
-    _related_objects: Dict[str, Any] = PrivateAttr(default_factory=dict)
-
-    @property
-    def is_using_a_runner(self):
-        return (
-            self.block.command is not None
-            and "prefect flow-run execute" in shlex.join(self.block.command)
-        )
-
-    @staticmethod
-    def _get_base_config_defaults(variables: dict) -> dict:
-        """Get default values from base config for all variables that have them."""
-        defaults = dict()
-        for variable_name, attrs in variables.items():
-            if "default" in attrs:
-                defaults[variable_name] = attrs["default"]
-
-        return defaults
-
-    @classmethod
-    @inject_client
-    async def from_template_and_values(
-        cls, base_job_template: dict, values: dict, client: "PrefectClient" = None
-    ):
-        """Creates a valid worker configuration object from the provided base
-        configuration and overrides.
-
-        Important: this method expects that the base_job_template was already
-        validated server-side.
-        """
-        job_config: Dict[str, Any] = base_job_template["job_configuration"]
-        variables_schema = base_job_template["variables"]
-        variables = cls._get_base_config_defaults(
-            variables_schema.get("properties", {})
-        )
-        variables.update(values)
-
-        populated_configuration = apply_values(template=job_config, values=variables)
-
-        block_document_id = get_from_dict(
-            populated_configuration, "block.$ref.block_document_id"
-        )
-        if not block_document_id:
-            raise ValueError(
-                "Base job template is invalid for this worker type because it does not"
-                " contain a block_document_id after variable resolution."
-            )
-
-        block_document = await client.read_block_document(
-            block_document_id=block_document_id
-        )
-        infrastructure_block = Block._from_block_document(block_document)
-
-        populated_configuration["block"] = infrastructure_block
-
-        return cls(**populated_configuration)
-
-    @classmethod
-    def json_template(cls) -> dict:
-        """Returns a dict with job configuration as keys and the corresponding templates as values
-
-        Defaults to using the job configuration parameter name as the template variable name.
-
-        e.g.
-        {
-            key1: '{{ key1 }}',     # default variable template
-            key2: '{{ template2 }}', # `template2` specifically provide as template
-        }
-        """
-        configuration = {}
-        properties = cls.schema()["properties"]
-        for k, v in properties.items():
-            if v.get("template"):
-                template = v["template"]
-            else:
-                template = "{{ " + k + " }}"
-            configuration[k] = template
-
-        return configuration
-
-    def _related_resources(self) -> List[RelatedResource]:
-        tags = set()
-        related = []
-
-        for kind, obj in self._related_objects.items():
-            if obj is None:
-                continue
-            if hasattr(obj, "tags"):
-                tags.update(obj.tags)
-            related.append(object_as_related_resource(kind=kind, role=kind, object=obj))
-
-        return related + tags_as_related_resources(tags)
-
-    def prepare_for_flow_run(
-        self,
-        flow_run: "FlowRun",
-        deployment: Optional["DeploymentResponse"] = None,
-        flow: Optional["Flow"] = None,
-    ):
-        self.block = self.block.prepare_for_flow_run(
-            flow_run=flow_run, deployment=deployment, flow=flow
-        )
-
-
-class BlockWorkerResult(BaseWorkerResult):
-    """Result of a block worker job"""
-
-
-class BlockWorker(BaseWorker):
-    type = "block"
-    job_configuration = BlockWorkerJobConfiguration
-
-    _description = "Execute flow runs using an infrastructure block as the job creator."
-    _display_name = "Block"
-
-    async def run(
-        self,
-        flow_run: "FlowRun",
-        configuration: BlockWorkerJobConfiguration,
-        task_status: Optional[anyio.abc.TaskStatus] = None,
-    ):
-        block = configuration.block
-
-        # logic for applying infra overrides taken from src/prefect/agent.py
-        deployment = await self._client.read_deployment(flow_run.deployment_id)
-        flow = await self._client.read_flow(deployment.flow_id)
-        infra_document = await self._client.read_block_document(
-            configuration.block._block_document_id
-        )
-
-        # this piece of logic applies any overrides that may have been set on the
-        # deployment; overrides are defined as dot.delimited paths on possibly nested
-        # attributes of the infrastructure block
-        doc_dict = infra_document.dict()
-        infra_dict = doc_dict.get("data", {})
-        for override, value in (deployment.job_variables or {}).items():
-            nested_fields = override.split(".")
-            if nested_fields == ["command"]:
-                value = shlex.split(value)
-            data = infra_dict
-            for field in nested_fields[:-1]:
-                data = data[field]
-
-            # once we reach the end, set the value
-            data[nested_fields[-1]] = value
-
-        # reconstruct the infra block
-        doc_dict["data"] = infra_dict
-        infra_document = BlockDocument(**doc_dict)
-        block = Block._from_block_document(infra_document)
-
-        block = block.prepare_for_flow_run(
-            flow_run=flow_run, deployment=deployment, flow=flow
-        )
-
-        result = await block.run(
-            task_status=task_status,
-        )
-        return BlockWorkerResult(
-            identifier=result.identifier, status_code=result.status_code
-        )
-
-    async def kill_infrastructure(
-        self,
-        infrastructure_pid: str,
-        configuration: BlockWorkerJobConfiguration,
-        grace_seconds: int = 30,
-    ):
-        block = configuration.block
-        if not hasattr(block, "kill"):
-            self._logger.error(
-                f"Flow run infrastructure block {block.type!r} "
-                "does not support killing created infrastructure. "
-                "Cancellation cannot be guaranteed."
-            )
-            return
-
-        await block.kill(
-            infrastructure_pid=infrastructure_pid, grace_seconds=grace_seconds
-        )