ob-metaflow 2.9.10.1__py2.py3-none-any.whl → 2.10.2.6__py2.py3-none-any.whl

metaflow/plugins/conda/conda.py

@@ -1,247 +0,0 @@
-import errno
-import os
-import json
-import subprocess
-import time
-from distutils.version import LooseVersion
-
-from metaflow.exception import MetaflowException
-from metaflow.metaflow_config import CONDA_DEPENDENCY_RESOLVER
-from metaflow.metaflow_environment import InvalidEnvironmentException
-from metaflow.util import which
-
-
-class CondaException(MetaflowException):
-    headline = "Conda ran into an error while setting up environment."
-
-    def __init__(self, error):
-        if isinstance(error, (list,)):
-            error = "\n".join(error)
-        msg = "{error}".format(error=error)
-        super(CondaException, self).__init__(msg)
-
-
-class CondaStepException(CondaException):
-    def __init__(self, exception, step):
-        msg = "Step: {step}, Error: {error}".format(step=step, error=exception.message)
-        super(CondaStepException, self).__init__(msg)
-
-
-class Conda(object):
-    def __init__(self):
-        dependency_solver = CONDA_DEPENDENCY_RESOLVER.lower()
-        self._bin = which(dependency_solver)
-        # Check if the dependency solver exists.
-        if self._bin is None:
-            raise InvalidEnvironmentException(
-                "No %s installation found. Install %s first."
-                % (dependency_solver, dependency_solver)
-            )
-        # Check for a minimum version for conda when conda or mamba is used
-        # for dependency resolution.
-        if dependency_solver == "conda" or dependency_solver == "mamba":
-            if LooseVersion(self._info()["conda_version"]) < LooseVersion("4.6.0"):
-                msg = "Conda version 4.6.0 or newer is required."
-                if dependency_solver == "mamba":
-                    msg += " Visit https://mamba.readthedocs.io/en/latest/installation.html for installation instructions."
-                else:
-                    msg += " Visit https://docs.conda.io/en/latest/miniconda.html for installation instructions."
-                raise InvalidEnvironmentException(msg)
-        # Check if conda-forge is available as a channel to pick up Metaflow's
-        # dependencies. This check will go away once all of Metaflow's
-        # dependencies are vendored in.
-        if "conda-forge" not in "\t".join(self._info()["channels"]):
-            raise InvalidEnvironmentException(
-                "Conda channel 'conda-forge' is required. Specify it with CONDA_CHANNELS environment variable."
-            )
-
-    def create(
-        self,
-        step_name,
-        env_id,
-        deps,
-        architecture=None,
-        explicit=False,
-        disable_safety_checks=False,
-    ):
-        # Create the conda environment
-        try:
-            with CondaLock(self._env_lock_file(env_id)):
-                self._remove(env_id)
-                self._create(
-                    env_id, deps, explicit, architecture, disable_safety_checks
-                )
-                return self._deps(env_id)
-        except CondaException as e:
-            raise CondaStepException(e, step_name)
-
-    def remove(self, step_name, env_id):
-        # Remove the conda environment
-        try:
-            with CondaLock(self._env_lock_file(env_id)):
-                self._remove(env_id)
-        except CondaException as e:
-            raise CondaStepException(e, step_name)
-
-    def python(self, env_id):
-        # Get Python interpreter for the conda environment
-        return os.path.join(self._env_path(env_id), "bin/python")
-
-    def environments(self, flow):
-        # List all conda environments associated with the flow
-        envs = self._info()["envs"]
-        ret = {}
-        for env in envs:
-            name = os.path.basename(env)
-            if name.startswith("metaflow_%s" % flow):
-                ret[name] = env
-        return ret
-
-    def package_info(self, env_id):
-        # Show conda environment package configuration
-        # Not every parameter is exposed via conda cli hence this ignominy
-        metadata = os.path.join(self._env_path(env_id), "conda-meta")
-        for path, dirs, files in os.walk(metadata):
-            for file in files:
-                if file.endswith(".json"):
-                    with open(os.path.join(path, file)) as f:
-                        yield json.loads(f.read())
-
-    def _info(self):
-        return json.loads(self._call_conda(["info"]))
-
-    def _create(
-        self,
-        env_id,
-        deps,
-        explicit=False,
-        architecture=None,
-        disable_safety_checks=False,
-    ):
-        cmd = ["create", "--yes", "--no-default-packages", "--name", env_id, "--quiet"]
-        if explicit:
-            cmd.append("--no-deps")
-        cmd.extend(deps)
-        self._call_conda(
-            cmd, architecture=architecture, disable_safety_checks=disable_safety_checks
-        )
-
-    def _remove(self, env_id):
-        self._call_conda(["env", "remove", "--name", env_id, "--yes", "--quiet"])
-
-    def _install(self, env_id, deps, explicit=False):
-        cmd = ["install", "--yes", "--name", env_id, "--quiet"]
-        if explicit:
-            cmd.append("--no-deps")
-        cmd.extend(deps)
-        self._call_conda(cmd)
-
-    def _install_order(self, env_id):
-        cmd = ["list", "--name", env_id, "--explicit"]
-        response = self._call_conda(cmd).decode("utf-8")
-        emit = False
-        result = []
-        for line in response.splitlines():
-            if emit:
-                result.append(line.split("/")[-1])
-            if not emit and line == "@EXPLICIT":
-                emit = True
-        return result
-
-    def _deps(self, env_id):
-        exact_deps = []
-        urls = []
-        for package in self.package_info(env_id):
-            exact_deps.append(
-                "%s=%s=%s" % (package["name"], package["version"], package["build"])
-            )
-            urls.append(package["url"])
-        order = self._install_order(env_id)
-        return (exact_deps, urls, order)
-
-    def _env_path(self, env_id):
-        envs = self._info()["envs"]
-        for env in envs:
-            name = os.path.basename(env)
-            if name == env_id:
-                return env
-        return None
-
-    def _env_lock_file(self, env_id):
-        return os.path.join(self._info()["envs_dirs"][0], "mf_env-creation.lock")
-
-    def _call_conda(self, args, architecture=None, disable_safety_checks=False):
-        try:
-            env = {
-                "CONDA_JSON": "True",
-                "CONDA_SUBDIR": (architecture if architecture else ""),
-                "MAMBA_NO_BANNER": "1",
-                "MAMBA_JSON": "True",
-            }
-            if disable_safety_checks:
-                env["CONDA_SAFETY_CHECKS"] = "disabled"
-            return subprocess.check_output(
-                [self._bin] + args, stderr=subprocess.PIPE, env=dict(os.environ, **env)
-            ).strip()
-        except subprocess.CalledProcessError as e:
-            try:
-                output = json.loads(e.output)
-                err = [output["error"]]
-                for error in output.get("errors", []):
-                    err.append(error["error"])
-                raise CondaException(err)
-            except (TypeError, ValueError) as ve:
-                pass
-            raise CondaException(
-                "command '{cmd}' returned error ({code}): {output}, stderr={stderr}".format(
-                    cmd=e.cmd, code=e.returncode, output=e.output, stderr=e.stderr
-                )
-            )
-
-
-class CondaLock(object):
-    def __init__(self, lock, timeout=3600, delay=10):
-        self.lock = lock
-        self.locked = False
-        self.timeout = timeout
-        self.delay = delay
-
-    def _acquire(self):
-        start = time.time()
-        try:
-            os.makedirs(os.path.dirname(self.lock))
-        except OSError as x:
-            if x.errno != errno.EEXIST:
-                raise
-        while True:
-            try:
-                self.fd = os.open(self.lock, os.O_CREAT | os.O_EXCL | os.O_RDWR)
-                self.locked = True
-                break
-            except OSError as e:
-                if e.errno != errno.EEXIST:
-                    raise
-                if self.timeout is None:
-                    raise CondaException("Could not acquire lock {}".format(self.lock))
-                if (time.time() - start) >= self.timeout:
-                    raise CondaException(
-                        "Timeout occurred while acquiring lock {}".format(self.lock)
-                    )
-                time.sleep(self.delay)
-
-    def _release(self):
-        if self.locked:
-            os.close(self.fd)
-            os.unlink(self.lock)
-            self.locked = False
-
-    def __enter__(self):
-        if not self.locked:
-            self._acquire()
-        return self
-
-    def __exit__(self, type, value, traceback):
-        self.__del__()
-
-    def __del__(self):
-        self._release()

metaflow/plugins/conda/conda_environment.py

@@ -1,136 +0,0 @@
-import json
-import os
-import sys
-import tarfile
-
-from io import BytesIO
-
-from metaflow.metaflow_environment import MetaflowEnvironment
-from metaflow.exception import MetaflowException
-from metaflow.mflog import BASH_SAVE_LOGS
-
-from .conda import Conda
-from . import get_conda_manifest_path, CONDA_MAGIC_FILE
-
-
-class CondaEnvironment(MetaflowEnvironment):
-    TYPE = "conda"
-    _filecache = None
-
-    def __init__(self, flow):
-        self.flow = flow
-        self.local_root = None
-        # A conda environment sits on top of whatever default environment
-        # the user has, so we get that environment to be able to forward
-        # any calls we don't handle specifically to that one.
-        from ...plugins import ENVIRONMENTS
-        from metaflow.metaflow_config import DEFAULT_ENVIRONMENT
-
-        if DEFAULT_ENVIRONMENT == self.TYPE:
-            # If the default environment is Conda itself then fallback on
-            # the default 'default environment'
-            self.base_env = MetaflowEnvironment(self.flow)
-        else:
-            self.base_env = [
-                e
-                for e in ENVIRONMENTS + [MetaflowEnvironment]
-                if e.TYPE == DEFAULT_ENVIRONMENT
-            ][0](self.flow)
-
-    def init_environment(self, echo):
-        # Print a message for now
-        echo("Bootstrapping conda environment..." + "(this could take a few minutes)")
-        self.base_env.init_environment(echo)
-
-    def validate_environment(self, echo, datastore_type):
-        return self.base_env.validate_environment(echo, datastore_type)
-
-    def decospecs(self):
-        # Apply conda decorator and base environment's decorators to all steps
-        return ("conda",) + self.base_env.decospecs()
-
-    def _get_conda_decorator(self, step_name):
-        step = next(step for step in self.flow if step.name == step_name)
-        decorator = next(deco for deco in step.decorators if deco.name == "conda")
-        # Guaranteed to have a conda decorator because of self.decospecs()
-        return decorator
-
-    def _get_env_id(self, step_name):
-        conda_decorator = self._get_conda_decorator(step_name)
-        if conda_decorator.is_enabled():
-            return conda_decorator._env_id()
-        return None
-
-    def _get_executable(self, step_name):
-        env_id = self._get_env_id(step_name)
-        if env_id is not None:
-            return os.path.join(env_id, "bin/python -s")
-        return None
-
-    def set_local_root(self, ds_root):
-        self.local_root = ds_root
-
-    def bootstrap_commands(self, step_name, datastore_type):
-        # Bootstrap conda and execution environment for step
-        env_id = self._get_env_id(step_name)
-        if env_id is not None:
-            return [
-                "echo 'Bootstrapping environment...'",
-                'python -m metaflow.plugins.conda.batch_bootstrap "%s" %s "%s"'
-                % (self.flow.name, env_id, datastore_type),
-                "echo 'Environment bootstrapped.'",
-            ]
-        return []
-
-    def add_to_package(self):
-        files = self.base_env.add_to_package()
-        # Add conda manifest file to job package at the top level.
-        path = get_conda_manifest_path(self.local_root, self.flow.name)
-        if os.path.exists(path):
-            files.append((path, os.path.basename(path)))
-        return files
-
-    def pylint_config(self):
-        config = self.base_env.pylint_config()
-        # Disable (import-error) in pylint
-        config.append("--disable=F0401")
-        return config
-
-    def executable(self, step_name):
-        # Get relevant python interpreter for step
-        executable = self._get_executable(step_name)
-        if executable is not None:
-            return executable
-        return self.base_env.executable(step_name)
-
-    @classmethod
-    def get_client_info(cls, flow_name, metadata):
-        if cls._filecache is None:
-            from metaflow.client.filecache import FileCache
-
-            cls._filecache = FileCache()
-        info = metadata.get("code-package")
-        env_id = metadata.get("conda_env_id")
-        if info is None or env_id is None:
-            return {"type": "conda"}
-        info = json.loads(info)
-        _, blobdata = cls._filecache.get_data(
-            info["ds_type"], flow_name, info["location"], info["sha"]
-        )
-        with tarfile.open(fileobj=BytesIO(blobdata), mode="r:gz") as tar:
-            conda_file = tar.extractfile(CONDA_MAGIC_FILE)
-        if conda_file is None:
-            return {"type": "conda"}
-        info = json.loads(conda_file.read().decode("utf-8"))
-        new_info = {
-            "type": "conda",
-            "explicit": info[env_id]["explicit"],
-            "deps": info[env_id]["deps"],
-        }
-        return new_info
-
-    def get_package_commands(self, code_package_url, datastore_type):
-        return self.base_env.get_package_commands(code_package_url, datastore_type)
-
-    def get_environment_info(self, include_ext_info=False):
-        return self.base_env.get_environment_info(include_ext_info)

metaflow/plugins/conda/conda_flow_decorator.py

@@ -1,35 +0,0 @@
-from metaflow.decorators import FlowDecorator
-from metaflow.metaflow_environment import InvalidEnvironmentException
-
-from typing import Dict
-
-
-class CondaFlowDecorator(FlowDecorator):
-    """
-    Specifies the Conda environment for all steps of the flow.
-
-    Use `@conda_base` to set common libraries required by all
-    steps and use `@conda` to specify step-specific additions.
-
-    Parameters
-    ----------
-    libraries : Dict[str, str], default: {}
-        Libraries to use for this flow. The key is the name of the package
-        and the value is the version to use.
-    python : str, optional
-        Version of Python to use, e.g. '3.7.4'. A default value of None means
-        to use the current Python version.
-    disabled : bool, default: False
-        If set to True, disables Conda.
-    """
-
-    name = "conda_base"
-    defaults = {"libraries": {}, "python": None, "disabled": None}
-
-    def flow_init(
-        self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
-    ):
-        if environment.TYPE != "conda":
-            raise InvalidEnvironmentException(
-                "The *@conda* decorator requires " "--environment=conda"
-            )