prefect-client 2.20.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

prefect/software/conda.py

@@ -1,199 +0,0 @@
-import json
-import re
-import subprocess
-import sys
-from pathlib import Path
-from typing import List, Type
-
-import yaml
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import Field, validate_arguments
-else:
-    from pydantic import Field, validate_arguments
-from typing_extensions import Self
-
-from prefect.software.base import (
-    Requirement,
-    pop_requirement_by_name,
-    remove_duplicate_requirements,
-)
-from prefect.software.pip import current_environment_requirements
-from prefect.software.python import PythonEnvironment
-from prefect.utilities.collections import listrepr
-
-# Capture each component of a conda requirement string.
-#
-#   <name><version_specifier><version><build_specifier><build>
-#
-# The specification of these requirements is more relaxed than pip requirements.
-# Notably, the version specifier is typically a single equal sign and a build can be
-# included after the version.
-#
-# For example, the requirement "requests=2.25.1=pyhd3eb1b0_0" matches as:
-#   name: "requests"
-#   version_specifier: "="
-#   version: "2.25.1"
-#   build_specifier: "="
-#   build: "pyhd3eb1b0_0"
-#
-# Regex components:
-#   name (required): any combination of numbers, letters, or dashes
-#   version_specifier (optional): one or more of `>`, `<`, and `=`
-#   version (optional): any combination of numbers, letters, or periods; not valid
-#       unless grouped with a version specifier.
-#   build_specifier (optional): just `=`.
-#   build (optional): any combination of numbers, letters, or underscores; not valid
-#       unless grouped with a build specifier.
-
-CONDA_REQUIREMENT = re.compile(
-    r"^((?P<channel>[0-9A-Za-z\_\-\/]+)::)?"
-    r"(?P<name>[0-9A-Za-z_\-\.]+)"
-    r"((?P<version_specifier>[>=<]+)(?P<version>[0-9a-zA-Z\.]+))?"
-    r"((?P<build_specifier>=)(?P<build>[0-9A-Za-z\_\[\]\=]+))?$"
-)
-
-
-class CondaRequirement(Requirement):
-    """
-    A parsed requirement for installation with conda.
-    """
-
-    def __init__(self, requirement_string: str):
-        self._requirement_string = requirement_string
-
-        parsed = CONDA_REQUIREMENT.match(requirement_string)
-        if parsed is None:
-            raise ValueError(
-                f"Invalid requirement {requirement_string!r}: could not be parsed."
-            )
-        self._parts = parsed.groupdict()
-
-        self.name = self._parts["name"]
-        self.version_specifier = self._parts["version_specifier"]
-        self.version = self._parts["version"]
-        self.build_specifier = self._parts["build_specifier"]
-        self.build = self._parts["build"]
-
-    def __str__(self) -> str:
-        return self._requirement_string
-
-
-class CondaError(RuntimeError):
-    """
-    Raised if an error occurs in conda.
-    """
-
-
-def current_environment_conda_requirements(
-    include_builds: bool = False, explicit_only: bool = True
-) -> List[CondaRequirement]:
-    """
-    Return conda requirements by exporting the current environment.
-
-    Skips any pip requirements included in the export. Only requirements that are
-    managed by conda are returned.
-    """
-    command = ["conda", "env", "export", "--json"]
-
-    if not include_builds:
-        command.append("--no-builds")
-    if explicit_only:
-        command.append("--from-history")
-
-    process = subprocess.run(command, capture_output=True)
-    parsed = json.loads(process.stdout)
-    if "error" in parsed:
-        raise CondaError(
-            "Encountered an exception while exporting the conda environment: "
-            + parsed["error"]
-        )
-
-    # If no dependencies are given, this field will not be present
-    dependencies = parsed.get("dependencies", [])
-
-    # The string check will exclude nested objects like the 'pip' subtree
-    return [CondaRequirement(dep) for dep in dependencies if isinstance(dep, str)]
-
-
-class CondaEnvironment(PythonEnvironment):
-    conda_requirements: List[CondaRequirement] = Field(default_factory=list)
-
-    @classmethod
-    def from_environment(cls: Type[Self], exclude_nested: bool = False) -> Self:
-        conda_requirements = (
-            current_environment_conda_requirements()
-            if "conda" in sys.executable
-            else []
-        )
-        pip_requirements = remove_duplicate_requirements(
-            conda_requirements,
-            current_environment_requirements(
-                exclude_nested=exclude_nested, on_uninstallable_requirement="warn"
-            ),
-        )
-        python_requirement = pop_requirement_by_name(conda_requirements, "python")
-        python_version = python_requirement.version if python_requirement else None
-
-        return cls(
-            pip_requirements=pip_requirements,
-            conda_requirements=conda_requirements,
-            python_version=python_version,
-        )
-
-    @classmethod
-    @validate_arguments
-    def from_file(cls: Type[Self], path: Path) -> Self:
-        parsed = yaml.safe_load(path.read_text())
-
-        # If no dependencies are given, this field will not be present
-        dependencies = parsed.get("dependencies", [])
-
-        # The string check will exclude nested objects like the 'pip' subtree
-        conda_requirements = [
-            CondaRequirement(dep) for dep in dependencies if isinstance(dep, str)
-        ]
-
-        python_requirement = pop_requirement_by_name(conda_requirements, "python")
-        python_version = python_requirement.version if python_requirement else None
-
-        other_requirements = {}
-
-        # Parse nested requirements. We only support 'pip' for now but we'll check for
-        # others
-        for subtree in [dep for dep in dependencies if isinstance(dep, dict)]:
-            key = list(subtree.keys())[0]
-
-            if key in other_requirements:
-                raise ValueError(
-                    "Invalid conda requirements specification. "
-                    f"Found duplicate key {key!r}."
-                )
-
-            other_requirements[key] = subtree[key]
-
-        pip_requirements = other_requirements.pop("pip", [])
-
-        if other_requirements:
-            raise ValueError(
-                "Found unsupported requirements types in file: "
-                f"{listrepr(other_requirements.keys(), ', ')}"
-            )
-
-        return cls(
-            conda_requirements=conda_requirements,
-            pip_requirements=pip_requirements,
-            python_version=python_version,
-        )
-
-    def install_commands(self) -> List[str]:
-        pip_install_commands = super().install_commands()
-
-        if not self.conda_requirements:
-            return pip_install_commands
-
-        return [
-            ["conda", "install", *(str(req) for req in self.conda_requirements)]
-        ] + pip_install_commands

prefect/software/pip.py

@@ -1,122 +0,0 @@
-import site
-import warnings
-from pathlib import Path
-from typing import Dict, List, Type, Union
-
-import packaging.requirements
-import packaging.version
-from typing_extensions import Literal, Self
-
-from prefect.software.base import Requirement
-from prefect.utilities.compat import importlib_metadata
-
-
-class PipRequirement(Requirement, packaging.requirements.Requirement):
-    """
-    A parsed Python requirement.
-
-    An extension for `packaging.version.Requirement` with Pydantic support.
-    """
-
-    @classmethod
-    def from_distribution(
-        cls: Type[Self], dist: "importlib_metadata.Distribution"
-    ) -> Self:
-        """
-        Convert a Python distribution object into a requirement
-        """
-        if _is_editable_install(dist):
-            raise ValueError(
-                f"Distribution {dist!r} is an editable installation and cannot be "
-                "used as a requirement."
-            )
-
-        return cls.validate(
-            packaging.requirements.Requirement(
-                f"{dist.metadata['name']}=={dist.version}"
-            )
-        )
-
-
-def _get_installed_distributions() -> Dict[str, "importlib_metadata.Distribution"]:
-    return {dist.name: dist for dist in importlib_metadata.distributions()}
-
-
-def _is_child_path(path: Union[Path, str], parent: Union[Path, str]) -> bool:
-    return Path(parent).resolve() in Path(path).resolve().parents
-
-
-def _is_same_path(path: Union[Path, str], other: Union[Path, str]) -> bool:
-    return Path(path).resolve() == Path(other).resolve()
-
-
-def _is_editable_install(dist: "importlib_metadata.Distribution") -> bool:
-    """
-    Determine if a distribution is an 'editable' install by scanning if it is present
-    in a site-packages directory or not; if not, we presume it is editable.
-    """
-    site_packages = site.getsitepackages() + [site.getusersitepackages()]
-
-    dist_location = dist.locate_file("")
-    for site_package_dir in site_packages:
-        if _is_same_path(dist_location, site_package_dir) or _is_child_path(
-            dist_location, site_package_dir
-        ):
-            return False
-
-    return True
-
-
-def _remove_distributions_required_by_others(
-    dists: Dict[str, "importlib_metadata.Distribution"],
-) -> Dict[str, "importlib_metadata.Distribution"]:
-    # Collect all child requirements
-    child_requirement_names = set()
-    for dist in dists.values():
-        child_requirement_names.update(dist.requires or [])
-
-    return {
-        name: dist
-        for name, dist in dists.items()
-        if name not in child_requirement_names
-    }
-
-
-def current_environment_requirements(
-    exclude_nested: bool = False,
-    on_uninstallable_requirement: Literal["ignore", "warn", "raise"] = "warn",
-) -> List[PipRequirement]:
-    dists = _get_installed_distributions()
-    if exclude_nested:
-        dists = _remove_distributions_required_by_others(dists)
-
-    requirements = []
-    uninstallable_msgs = []
-    for dist in dists.values():
-        if _is_editable_install(dist):
-            uninstallable_msgs.append(
-                f"- {dist.name}: This distribution is an editable installation."
-            )
-        else:
-            requirements.append(PipRequirement.from_distribution(dist))
-
-    if uninstallable_msgs:
-        message = (
-            "The following requirements will not be installable on another machine:\n"
-            + "\n".join(uninstallable_msgs)
-        )
-        if on_uninstallable_requirement == "ignore":
-            pass
-        elif on_uninstallable_requirement == "warn":
-            # When warning, include a note that these distributions are excluded
-            warnings.warn(message + "\nThese requirements will be ignored.")
-        elif on_uninstallable_requirement == "raise":
-            raise ValueError(message)
-        else:
-            raise ValueError(
-                "Unknown mode for `on_uninstallable_requirement`: "
-                f"{on_uninstallable_requirement!r}. "
-                "Expected one of: 'ignore', 'warn', 'raise'."
-            )
-
-    return requirements

prefect/software/python.py

@@ -1,52 +0,0 @@
-from pathlib import Path
-from typing import List, Type
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-from prefect._internal.schemas.validators import infer_python_version
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import BaseModel, Field, validate_arguments, validator
-else:
-    from pydantic import BaseModel, Field, validate_arguments, validator
-
-from typing_extensions import Self
-
-from prefect.software.pip import PipRequirement, current_environment_requirements
-
-
-class PythonEnvironment(BaseModel):
-    """
-    A specification for a Python environment.
-    """
-
-    python_version: str = None
-    pip_requirements: List[PipRequirement] = Field(default_factory=list)
-
-    @validator("python_version", pre=True, always=True)
-    def validate_python_version(cls, value):
-        return infer_python_version(value)
-
-    @classmethod
-    def from_environment(cls: Type[Self], exclude_nested: bool = False) -> Self:
-        """
-        Generate requirements from the current environment
-
-        Arguments:
-            exclude_nested: If True, only top-level requirements will be included.
-                Defaults to including all requirements.
-        """
-        pip_requirements = current_environment_requirements(
-            exclude_nested=exclude_nested, on_uninstallable_requirement="warn"
-        )
-        return cls(pip_requirements=pip_requirements)
-
-    @classmethod
-    @validate_arguments
-    def from_file(cls: Type[Self], path: Path) -> Self:
-        return PythonEnvironment(pip_requirements=path.read_text().strip().splitlines())
-
-    def install_commands(self) -> List[List[str]]:
-        if not self.pip_requirements:
-            return []
-
-        return [["pip", "install", *(str(req) for req in self.pip_requirements)]]

prefect/task_server.py

@@ -1,322 +0,0 @@
-import asyncio
-import inspect
-import os
-import signal
-import socket
-import sys
-from contextlib import AsyncExitStack
-from functools import partial
-from typing import List, Optional, Type
-
-import anyio
-from exceptiongroup import BaseExceptionGroup  # novermin
-from websockets.exceptions import InvalidStatusCode
-
-from prefect import Task, get_client
-from prefect._internal.concurrency.api import create_call, from_sync
-from prefect.client.schemas.objects import TaskRun
-from prefect.client.subscriptions import Subscription
-from prefect.engine import emit_task_run_state_change_event, propose_state
-from prefect.exceptions import Abort, PrefectHTTPStatusError
-from prefect.logging.loggers import get_logger
-from prefect.results import ResultFactory
-from prefect.settings import (
-    PREFECT_API_URL,
-    PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
-    PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS,
-)
-from prefect.states import Pending
-from prefect.task_engine import submit_autonomous_task_run_to_engine
-from prefect.task_runners import (
-    BaseTaskRunner,
-    ConcurrentTaskRunner,
-)
-from prefect.utilities.asyncutils import asyncnullcontext, sync_compatible
-from prefect.utilities.processutils import _register_signal
-
-logger = get_logger("task_server")
-
-
-class StopTaskServer(Exception):
-    """Raised when the task server is stopped."""
-
-    pass
-
-
-def should_try_to_read_parameters(task: Task, task_run: TaskRun) -> bool:
-    """Determines whether a task run should read parameters from the result factory."""
-    new_enough_state_details = hasattr(
-        task_run.state.state_details, "task_parameters_id"
-    )
-    task_accepts_parameters = bool(inspect.signature(task.fn).parameters)
-
-    return new_enough_state_details and task_accepts_parameters
-
-
-class TaskServer:
-    """This class is responsible for serving tasks that may be executed in the background
-    by a task runner via the traditional engine machinery.
-
-    When `start()` is called, the task server will open a websocket connection to a
-    server-side queue of scheduled task runs. When a scheduled task run is found, the
-    scheduled task run is submitted to the engine for execution with a minimal `EngineContext`
-    so that the task run can be governed by orchestration rules.
-
-    Args:
-        - tasks: A list of tasks to serve. These tasks will be submitted to the engine
-            when a scheduled task run is found.
-        - task_runner: The task runner to use for executing the tasks. Defaults to
-            `ConcurrentTaskRunner`.
-    """
-
-    def __init__(
-        self,
-        *tasks: Task,
-        task_runner: Optional[Type[BaseTaskRunner]] = None,
-    ):
-        self.tasks: List[Task] = tasks
-
-        self.task_runner: BaseTaskRunner = task_runner or ConcurrentTaskRunner()
-        self.started: bool = False
-        self.stopping: bool = False
-
-        self._client = get_client()
-        self._exit_stack = AsyncExitStack()
-
-        if not asyncio.get_event_loop().is_running():
-            raise RuntimeError(
-                "TaskServer must be initialized within an async context."
-            )
-
-        self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
-
-    @property
-    def _client_id(self) -> str:
-        return f"{socket.gethostname()}-{os.getpid()}"
-
-    def handle_sigterm(self, signum, frame):
-        """
-        Shuts down the task server when a SIGTERM is received.
-        """
-        logger.info("SIGTERM received, initiating graceful shutdown...")
-        from_sync.call_in_loop_thread(create_call(self.stop))
-
-        sys.exit(0)
-
-    @sync_compatible
-    async def start(self) -> None:
-        """
-        Starts a task server, which runs the tasks provided in the constructor.
-        """
-        _register_signal(signal.SIGTERM, self.handle_sigterm)
-
-        async with asyncnullcontext() if self.started else self:
-            logger.info("Starting task server...")
-            try:
-                await self._subscribe_to_task_scheduling()
-            except InvalidStatusCode as exc:
-                if exc.status_code == 403:
-                    logger.error(
-                        "Could not establish a connection to the `/task_runs/subscriptions/scheduled`"
-                        f" endpoint found at:\n\n {PREFECT_API_URL.value()}"
-                        "\n\nPlease double-check the values of your"
-                        " `PREFECT_API_URL` and `PREFECT_API_KEY` environment variables."
-                    )
-                else:
-                    raise
-
-    @sync_compatible
-    async def stop(self):
-        """Stops the task server's polling cycle."""
-        if not self.started:
-            raise RuntimeError(
-                "Task server has not yet started. Please start the task server by"
-                " calling .start()"
-            )
-
-        self.started = False
-        self.stopping = True
-
-        raise StopTaskServer
-
-    async def _subscribe_to_task_scheduling(self):
-        logger.info(
-            f"Subscribing to tasks: {' | '.join(t.task_key.split('.')[-1] for t in self.tasks)}"
-        )
-        async for task_run in Subscription(
-            model=TaskRun,
-            path="/task_runs/subscriptions/scheduled",
-            keys=[task.task_key for task in self.tasks],
-            client_id=self._client_id,
-        ):
-            logger.info(f"Received task run: {task_run.id} - {task_run.name}")
-            await self._submit_scheduled_task_run(task_run)
-
-    async def _submit_scheduled_task_run(self, task_run: TaskRun):
-        logger.debug(
-            f"Found task run: {task_run.name!r} in state: {task_run.state.name!r}"
-        )
-
-        task = next((t for t in self.tasks if t.task_key == task_run.task_key), None)
-
-        if not task:
-            if PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS.value():
-                logger.warning(
-                    f"Task {task_run.name!r} not found in task server registry."
-                )
-                await self._client._client.delete(f"/task_runs/{task_run.id}")
-
-            return
-
-        # The ID of the parameters for this run are stored in the Scheduled state's
-        # state_details. If there is no parameters_id, then the task was created
-        # without parameters.
-        parameters = {}
-        if should_try_to_read_parameters(task, task_run):
-            parameters_id = task_run.state.state_details.task_parameters_id
-            task.persist_result = True
-            factory = await ResultFactory.from_autonomous_task(task)
-            try:
-                parameters = await factory.read_parameters(parameters_id)
-            except Exception as exc:
-                logger.exception(
-                    f"Failed to read parameters for task run {task_run.id!r}",
-                    exc_info=exc,
-                )
-                if PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS.value():
-                    logger.info(
-                        f"Deleting task run {task_run.id!r} because it failed to submit"
-                    )
-                    await self._client._client.delete(f"/task_runs/{task_run.id}")
-                return
-
-        logger.debug(
-            f"Submitting run {task_run.name!r} of task {task.name!r} to engine"
-        )
-
-        try:
-            state = await propose_state(
-                client=get_client(),  # TODO prove that we cannot use self._client here
-                state=Pending(),
-                task_run_id=task_run.id,
-            )
-        except Abort as exc:
-            logger.exception(
-                f"Failed to submit task run {task_run.id!r} to engine", exc_info=exc
-            )
-            return
-        except PrefectHTTPStatusError as exc:
-            if exc.response.status_code == 404:
-                logger.warning(
-                    f"Task run {task_run.id!r} not found. It may have been deleted."
-                )
-                return
-            raise
-
-        if not state.is_pending():
-            logger.warning(
-                f"Cancelling submission of task run {task_run.id!r} -"
-                f" server returned a non-pending state {state.type.value!r}."
-            )
-            return
-
-        emit_task_run_state_change_event(
-            task_run=task_run,
-            initial_state=task_run.state,
-            validated_state=state,
-        )
-
-        try:
-            self._runs_task_group.start_soon(
-                partial(
-                    submit_autonomous_task_run_to_engine,
-                    task=task,
-                    task_run=task_run,
-                    parameters=parameters,
-                    task_runner=self.task_runner,
-                    client=self._client,
-                )
-            )
-        except BaseException as exc:
-            logger.exception(
-                f"Failed to submit task run {task_run.id!r} to engine", exc_info=exc
-            )
-
-    async def execute_task_run(self, task_run: TaskRun):
-        """Execute a task run in the task server."""
-        async with self if not self.started else asyncnullcontext():
-            await self._submit_scheduled_task_run(task_run)
-
-    async def __aenter__(self):
-        logger.debug("Starting task server...")
-
-        if self._client._closed:
-            self._client = get_client()
-
-        await self._exit_stack.enter_async_context(self._client)
-        await self._exit_stack.enter_async_context(self.task_runner.start())
-        await self._runs_task_group.__aenter__()
-
-        self.started = True
-        return self
-
-    async def __aexit__(self, *exc_info):
-        logger.debug("Stopping task server...")
-        self.started = False
-        await self._runs_task_group.__aexit__(*exc_info)
-        await self._exit_stack.__aexit__(*exc_info)
-
-
-@sync_compatible
-async def serve(*tasks: Task, task_runner: Optional[Type[BaseTaskRunner]] = None):
-    """Serve the provided tasks so that their runs may be submitted to and executed.
-    in the engine. Tasks do not need to be within a flow run context to be submitted.
-    You must `.submit` the same task object that you pass to `serve`.
-
-    Args:
-        - tasks: A list of tasks to serve. When a scheduled task run is found for a
-            given task, the task run will be submitted to the engine for execution.
-        - task_runner: The task runner to use for executing the tasks. Defaults to
-            `ConcurrentTaskRunner`.
-
-    Example:
-        ```python
-        from prefect import task
-        from prefect.task_server import serve
-
-        @task(log_prints=True)
-        def say(message: str):
-            print(message)
-
-        @task(log_prints=True)
-        def yell(message: str):
-            print(message.upper())
-
-        # starts a long-lived process that listens for scheduled runs of these tasks
-        if __name__ == "__main__":
-            serve(say, yell)
-        ```
-    """
-    if not PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value():
-        raise RuntimeError(
-            "To enable task scheduling, set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING"
-            " to True."
-        )
-
-    task_server = TaskServer(*tasks, task_runner=task_runner)
-    try:
-        await task_server.start()
-
-    except BaseExceptionGroup as exc:  # novermin
-        exceptions = exc.exceptions
-        n_exceptions = len(exceptions)
-        logger.error(
-            f"Task worker stopped with {n_exceptions} exception{'s' if n_exceptions != 1 else ''}:"
-            f"\n" + "\n".join(str(e) for e in exceptions)
-        )
-
-    except StopTaskServer:
-        logger.info("Task server stopped.")
-
-    except asyncio.CancelledError:
-        logger.info("Task server interrupted, stopping...")