expops 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- expops-0.1.3.dist-info/METADATA +826 -0
- expops-0.1.3.dist-info/RECORD +86 -0
- expops-0.1.3.dist-info/WHEEL +5 -0
- expops-0.1.3.dist-info/entry_points.txt +3 -0
- expops-0.1.3.dist-info/licenses/LICENSE +674 -0
- expops-0.1.3.dist-info/top_level.txt +1 -0
- mlops/__init__.py +0 -0
- mlops/__main__.py +11 -0
- mlops/_version.py +34 -0
- mlops/adapters/__init__.py +12 -0
- mlops/adapters/base.py +86 -0
- mlops/adapters/config_schema.py +89 -0
- mlops/adapters/custom/__init__.py +3 -0
- mlops/adapters/custom/custom_adapter.py +447 -0
- mlops/adapters/plugin_manager.py +113 -0
- mlops/adapters/sklearn/__init__.py +3 -0
- mlops/adapters/sklearn/adapter.py +94 -0
- mlops/cluster/__init__.py +3 -0
- mlops/cluster/controller.py +496 -0
- mlops/cluster/process_runner.py +91 -0
- mlops/cluster/providers.py +258 -0
- mlops/core/__init__.py +95 -0
- mlops/core/custom_model_base.py +38 -0
- mlops/core/dask_networkx_executor.py +1265 -0
- mlops/core/executor_worker.py +1239 -0
- mlops/core/experiment_tracker.py +81 -0
- mlops/core/graph_types.py +64 -0
- mlops/core/networkx_parser.py +135 -0
- mlops/core/payload_spill.py +278 -0
- mlops/core/pipeline_utils.py +162 -0
- mlops/core/process_hashing.py +216 -0
- mlops/core/step_state_manager.py +1298 -0
- mlops/core/step_system.py +956 -0
- mlops/core/workspace.py +99 -0
- mlops/environment/__init__.py +10 -0
- mlops/environment/base.py +43 -0
- mlops/environment/conda_manager.py +307 -0
- mlops/environment/factory.py +70 -0
- mlops/environment/pyenv_manager.py +146 -0
- mlops/environment/setup_env.py +31 -0
- mlops/environment/system_manager.py +66 -0
- mlops/environment/utils.py +105 -0
- mlops/environment/venv_manager.py +134 -0
- mlops/main.py +527 -0
- mlops/managers/project_manager.py +400 -0
- mlops/managers/reproducibility_manager.py +575 -0
- mlops/platform.py +996 -0
- mlops/reporting/__init__.py +16 -0
- mlops/reporting/context.py +187 -0
- mlops/reporting/entrypoint.py +292 -0
- mlops/reporting/kv_utils.py +77 -0
- mlops/reporting/registry.py +50 -0
- mlops/runtime/__init__.py +9 -0
- mlops/runtime/context.py +34 -0
- mlops/runtime/env_export.py +113 -0
- mlops/storage/__init__.py +12 -0
- mlops/storage/adapters/__init__.py +9 -0
- mlops/storage/adapters/gcp_kv_store.py +778 -0
- mlops/storage/adapters/gcs_object_store.py +96 -0
- mlops/storage/adapters/memory_store.py +240 -0
- mlops/storage/adapters/redis_store.py +438 -0
- mlops/storage/factory.py +199 -0
- mlops/storage/interfaces/__init__.py +6 -0
- mlops/storage/interfaces/kv_store.py +118 -0
- mlops/storage/path_utils.py +38 -0
- mlops/templates/premier-league/charts/plot_metrics.js +70 -0
- mlops/templates/premier-league/charts/plot_metrics.py +145 -0
- mlops/templates/premier-league/charts/requirements.txt +6 -0
- mlops/templates/premier-league/configs/cluster_config.yaml +13 -0
- mlops/templates/premier-league/configs/project_config.yaml +207 -0
- mlops/templates/premier-league/data/England CSV.csv +12154 -0
- mlops/templates/premier-league/models/premier_league_model.py +638 -0
- mlops/templates/premier-league/requirements.txt +8 -0
- mlops/templates/sklearn-basic/README.md +22 -0
- mlops/templates/sklearn-basic/charts/plot_metrics.py +85 -0
- mlops/templates/sklearn-basic/charts/requirements.txt +3 -0
- mlops/templates/sklearn-basic/configs/project_config.yaml +64 -0
- mlops/templates/sklearn-basic/data/train.csv +14 -0
- mlops/templates/sklearn-basic/models/model.py +62 -0
- mlops/templates/sklearn-basic/requirements.txt +10 -0
- mlops/web/__init__.py +3 -0
- mlops/web/server.py +585 -0
- mlops/web/ui/index.html +52 -0
- mlops/web/ui/mlops-charts.js +357 -0
- mlops/web/ui/script.js +1244 -0
- mlops/web/ui/styles.css +248 -0
mlops/core/workspace.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
ENV_WORKSPACE_DIR = "MLOPS_WORKSPACE_DIR"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_workspace_root() -> Path:
|
|
11
|
+
"""Return the workspace root directory.
|
|
12
|
+
|
|
13
|
+
The workspace is where `projects/` lives. Resolution order:
|
|
14
|
+
1) `MLOPS_WORKSPACE_DIR`
|
|
15
|
+
2) current working directory
|
|
16
|
+
"""
|
|
17
|
+
raw = os.environ.get(ENV_WORKSPACE_DIR)
|
|
18
|
+
if raw:
|
|
19
|
+
try:
|
|
20
|
+
return Path(raw).expanduser().resolve()
|
|
21
|
+
except Exception:
|
|
22
|
+
return Path(raw)
|
|
23
|
+
return Path.cwd()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_projects_root(workspace_root: Optional[Path] = None) -> Path:
|
|
27
|
+
root = workspace_root or get_workspace_root()
|
|
28
|
+
return root / "projects"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def resolve_relative_path(
|
|
32
|
+
p: str | Path,
|
|
33
|
+
*,
|
|
34
|
+
project_root: Optional[Path] = None,
|
|
35
|
+
workspace_root: Optional[Path] = None,
|
|
36
|
+
) -> Path:
|
|
37
|
+
"""Resolve a user-provided path against likely bases.
|
|
38
|
+
|
|
39
|
+
- Absolute paths are returned as-is.
|
|
40
|
+
- Relative paths are tried against:
|
|
41
|
+
1) current working directory
|
|
42
|
+
2) `project_root` (if provided)
|
|
43
|
+
3) `workspace_root` / `MLOPS_WORKSPACE_DIR` (if provided/available)
|
|
44
|
+
|
|
45
|
+
Returns a Path even if it does not exist (best-effort).
|
|
46
|
+
"""
|
|
47
|
+
path = Path(p)
|
|
48
|
+
if path.is_absolute():
|
|
49
|
+
return path
|
|
50
|
+
|
|
51
|
+
# 1) As given relative to CWD
|
|
52
|
+
try:
|
|
53
|
+
if path.exists():
|
|
54
|
+
return path
|
|
55
|
+
except Exception:
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
# 2) Relative to project root
|
|
59
|
+
if project_root is not None:
|
|
60
|
+
try:
|
|
61
|
+
cand = (project_root / path)
|
|
62
|
+
if cand.exists():
|
|
63
|
+
return cand
|
|
64
|
+
except Exception:
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
# 3) Relative to workspace root
|
|
68
|
+
wr = workspace_root or get_workspace_root()
|
|
69
|
+
try:
|
|
70
|
+
cand = (wr / path)
|
|
71
|
+
if cand.exists():
|
|
72
|
+
return cand
|
|
73
|
+
except Exception:
|
|
74
|
+
pass
|
|
75
|
+
|
|
76
|
+
# Fall back to the most likely base for debugging
|
|
77
|
+
if project_root is not None:
|
|
78
|
+
return project_root / path
|
|
79
|
+
return (wr / path)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def infer_source_root() -> Optional[Path]:
|
|
83
|
+
"""Best-effort: detect a source checkout root (repo root) when running from source.
|
|
84
|
+
|
|
85
|
+
This is used only for backwards-compatible PYTHONPATH/editable-install fallbacks.
|
|
86
|
+
"""
|
|
87
|
+
try:
|
|
88
|
+
# workspace.py lives at <root>/src/mlops/core/workspace.py in source checkouts
|
|
89
|
+
mlops_pkg_dir = Path(__file__).resolve().parents[1] # .../mlops
|
|
90
|
+
src_dir = mlops_pkg_dir.parent # .../src (source) or .../site-packages (installed)
|
|
91
|
+
root = src_dir.parent
|
|
92
|
+
if (root / "pyproject.toml").exists() or (root / "setup.py").exists():
|
|
93
|
+
# Heuristic: source checkout root should contain packaging metadata.
|
|
94
|
+
return root
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class EnvironmentManager(ABC):
|
|
8
|
+
"""Abstract base class for environment managers (venv/conda/pyenv/system)."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, config: dict[str, Any]):
|
|
11
|
+
self.config = config
|
|
12
|
+
self.environment_name: str | None = None
|
|
13
|
+
self.python_interpreter: str | None = None
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def setup_environment(self) -> None:
|
|
17
|
+
"""Set up the environment based on configuration."""
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def verify_environment(self) -> bool:
|
|
22
|
+
"""Verify that the environment is properly configured."""
|
|
23
|
+
raise NotImplementedError
|
|
24
|
+
|
|
25
|
+
@abstractmethod
|
|
26
|
+
def get_python_interpreter(self) -> str:
|
|
27
|
+
"""Get the path to the Python interpreter for this environment."""
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
def get_environment_name(self) -> str:
|
|
32
|
+
"""Get the name of the environment."""
|
|
33
|
+
raise NotImplementedError
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
def environment_exists(self) -> bool:
|
|
37
|
+
"""Check if the environment already exists."""
|
|
38
|
+
raise NotImplementedError
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def get_environment_type(self) -> str:
|
|
42
|
+
"""Return the type of environment manager (conda, pyenv, etc.)."""
|
|
43
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from .base import EnvironmentManager
|
|
14
|
+
from .utils import verify_pip_requirements
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CondaEnvironmentManager(EnvironmentManager):
|
|
18
|
+
"""Conda-based environment management."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, config: dict[str, Any]):
|
|
21
|
+
super().__init__(config)
|
|
22
|
+
self.dependencies = self._load_dependencies(config)
|
|
23
|
+
|
|
24
|
+
env_name = config.get("name")
|
|
25
|
+
if not env_name:
|
|
26
|
+
raise ValueError(
|
|
27
|
+
"Conda environment name must be explicitly specified in the configuration. "
|
|
28
|
+
"Please add 'name' field under 'environment.conda' in your config file."
|
|
29
|
+
)
|
|
30
|
+
self.environment_name = env_name
|
|
31
|
+
|
|
32
|
+
def _load_dependencies(self, config: dict[str, Any]) -> list[Any]:
|
|
33
|
+
"""Load dependencies from inline config or environment.yml file."""
|
|
34
|
+
# Option 1: Inline dependencies
|
|
35
|
+
if "dependencies" in config:
|
|
36
|
+
print("[CondaEnvironmentManager] Using inline dependencies from config.")
|
|
37
|
+
return config["dependencies"]
|
|
38
|
+
|
|
39
|
+
# Option 2: From conda environment.yml file
|
|
40
|
+
if "environment_file" in config:
|
|
41
|
+
env_file_path = Path(config["environment_file"])
|
|
42
|
+
if not env_file_path.exists():
|
|
43
|
+
raise FileNotFoundError(f"Environment file not found: {env_file_path}")
|
|
44
|
+
|
|
45
|
+
print(f"[CondaEnvironmentManager] Loading dependencies from environment file: {env_file_path}")
|
|
46
|
+
with open(env_file_path, 'r') as f:
|
|
47
|
+
env_data = yaml.safe_load(f)
|
|
48
|
+
|
|
49
|
+
if "dependencies" not in env_data:
|
|
50
|
+
raise ValueError(f"No 'dependencies' section found in {env_file_path}")
|
|
51
|
+
|
|
52
|
+
return env_data["dependencies"]
|
|
53
|
+
|
|
54
|
+
print("[CondaEnvironmentManager] No dependencies specified.")
|
|
55
|
+
return []
|
|
56
|
+
|
|
57
|
+
def get_environment_type(self) -> str:
|
|
58
|
+
return "conda"
|
|
59
|
+
|
|
60
|
+
def get_environment_name(self) -> str:
|
|
61
|
+
return self.environment_name
|
|
62
|
+
|
|
63
|
+
def get_python_interpreter(self) -> str:
|
|
64
|
+
if not self.python_interpreter:
|
|
65
|
+
raise RuntimeError("Environment not set up yet. Call setup_environment() first.")
|
|
66
|
+
return self.python_interpreter
|
|
67
|
+
|
|
68
|
+
def _get_conda_base_prefix(self) -> str | None:
|
|
69
|
+
"""Attempts to find the Conda base prefix."""
|
|
70
|
+
try:
|
|
71
|
+
result = subprocess.run(["conda", "info", "--json"], capture_output=True, text=True, check=True)
|
|
72
|
+
conda_info = json.loads(result.stdout)
|
|
73
|
+
return conda_info.get("root_prefix") or conda_info.get("conda_prefix")
|
|
74
|
+
except (subprocess.CalledProcessError, FileNotFoundError, json.JSONDecodeError) as e:
|
|
75
|
+
print(f"[CondaEnvironmentManager] Could not get conda info: {e}")
|
|
76
|
+
conda_exe_path = os.environ.get("CONDA_EXE")
|
|
77
|
+
if conda_exe_path:
|
|
78
|
+
return str(Path(conda_exe_path).parent.parent)
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
def _get_conda_executable(self) -> str:
|
|
82
|
+
"""Determines the path to the conda executable."""
|
|
83
|
+
conda_exe = "conda"
|
|
84
|
+
conda_base = self._get_conda_base_prefix()
|
|
85
|
+
if conda_base:
|
|
86
|
+
specific_conda_exe = Path(conda_base) / "bin" / "conda"
|
|
87
|
+
if specific_conda_exe.exists():
|
|
88
|
+
conda_exe = str(specific_conda_exe)
|
|
89
|
+
else: # Try Scripts for Windows
|
|
90
|
+
specific_conda_exe_win = Path(conda_base) / "Scripts" / "conda.exe"
|
|
91
|
+
if specific_conda_exe_win.exists():
|
|
92
|
+
conda_exe = str(specific_conda_exe_win)
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
subprocess.run([conda_exe, "--version"], capture_output=True, check=True, text=True)
|
|
96
|
+
except (FileNotFoundError, subprocess.CalledProcessError) as e:
|
|
97
|
+
raise RuntimeError(
|
|
98
|
+
f"Conda executable ('{conda_exe}') not found or not runnable. "
|
|
99
|
+
"Please ensure Conda is installed and configured correctly in your PATH. "
|
|
100
|
+
f"Error: {e}"
|
|
101
|
+
)
|
|
102
|
+
return conda_exe
|
|
103
|
+
|
|
104
|
+
def environment_exists(self) -> bool:
|
|
105
|
+
"""Check if the conda environment already exists."""
|
|
106
|
+
try:
|
|
107
|
+
conda_exe = self._get_conda_executable()
|
|
108
|
+
result = subprocess.run([conda_exe, "env", "list", "--json"], capture_output=True, text=True, check=True)
|
|
109
|
+
env_list = json.loads(result.stdout).get("envs", [])
|
|
110
|
+
return any(Path(env_path).name == self.environment_name for env_path in env_list)
|
|
111
|
+
except (subprocess.CalledProcessError, json.JSONDecodeError, FileNotFoundError) as e:
|
|
112
|
+
print(f"[CondaEnvironmentManager] Failed to list Conda environments: {e}")
|
|
113
|
+
return False
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def _python_path(env_path: Path) -> Path:
|
|
117
|
+
if os.name == "nt":
|
|
118
|
+
return env_path / "python.exe"
|
|
119
|
+
return env_path / "bin" / "python"
|
|
120
|
+
|
|
121
|
+
def setup_environment(self) -> None:
|
|
122
|
+
"""Set up the conda environment based on configuration."""
|
|
123
|
+
print(f"[CondaEnvironmentManager] Starting Conda environment setup for '{self.environment_name}'...")
|
|
124
|
+
|
|
125
|
+
if not self.dependencies:
|
|
126
|
+
print("[CondaEnvironmentManager] No dependencies specified. Using current environment.")
|
|
127
|
+
self.python_interpreter = sys.executable
|
|
128
|
+
return
|
|
129
|
+
|
|
130
|
+
conda_exe = self._get_conda_executable()
|
|
131
|
+
|
|
132
|
+
if not self.environment_exists():
|
|
133
|
+
print(f"[CondaEnvironmentManager] Environment '{self.environment_name}' not found. Creating it...")
|
|
134
|
+
|
|
135
|
+
env_yaml_content = {"name": self.environment_name, "dependencies": self.dependencies}
|
|
136
|
+
|
|
137
|
+
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as tmp_env_file:
|
|
138
|
+
yaml.dump(env_yaml_content, tmp_env_file)
|
|
139
|
+
tmp_env_file_path = tmp_env_file.name
|
|
140
|
+
|
|
141
|
+
try:
|
|
142
|
+
print(f"[CondaEnvironmentManager] Creating environment from temporary file: {tmp_env_file_path}")
|
|
143
|
+
cmd_libmamba = [conda_exe, "env", "create", "-f", tmp_env_file_path, "-q", "--solver=libmamba"]
|
|
144
|
+
res = subprocess.run(cmd_libmamba, capture_output=True, text=True)
|
|
145
|
+
|
|
146
|
+
cmd_used = cmd_libmamba
|
|
147
|
+
if res.returncode != 0:
|
|
148
|
+
stderr_lower = (res.stderr or "").lower()
|
|
149
|
+
if "libmamba" in stderr_lower or "invalid choice" in stderr_lower:
|
|
150
|
+
print("[CondaEnvironmentManager] libmamba solver not available, falling back to classic solver...")
|
|
151
|
+
cmd_classic = [conda_exe, "env", "create", "-f", tmp_env_file_path, "-q"]
|
|
152
|
+
res = subprocess.run(cmd_classic, capture_output=True, text=True)
|
|
153
|
+
cmd_used = cmd_classic
|
|
154
|
+
|
|
155
|
+
if res.returncode != 0:
|
|
156
|
+
error_message = (
|
|
157
|
+
f"Failed to create Conda environment '{self.environment_name}'. Return code: {res.returncode}\n"
|
|
158
|
+
f"Command: {' '.join(cmd_used)}\n"
|
|
159
|
+
f"Stdout:\n{res.stdout}\n"
|
|
160
|
+
f"Stderr:\n{res.stderr}"
|
|
161
|
+
)
|
|
162
|
+
raise RuntimeError(error_message)
|
|
163
|
+
|
|
164
|
+
print(f"[CondaEnvironmentManager] Environment '{self.environment_name}' created successfully.")
|
|
165
|
+
finally:
|
|
166
|
+
try:
|
|
167
|
+
os.remove(tmp_env_file_path)
|
|
168
|
+
except OSError:
|
|
169
|
+
print(f"[CondaEnvironmentManager] Warning: Could not remove temporary env file {tmp_env_file_path}")
|
|
170
|
+
else:
|
|
171
|
+
print(f"[CondaEnvironmentManager] Using existing environment: '{self.environment_name}'")
|
|
172
|
+
|
|
173
|
+
# Set up Python interpreter path
|
|
174
|
+
try:
|
|
175
|
+
info_result = subprocess.run([conda_exe, "info", "--envs", "--json"], capture_output=True, text=True, check=True)
|
|
176
|
+
envs_info = json.loads(info_result.stdout).get("envs", [])
|
|
177
|
+
env_path_str = next((p for p in envs_info if Path(p).name == self.environment_name), None)
|
|
178
|
+
|
|
179
|
+
if not env_path_str:
|
|
180
|
+
conda_base = self._get_conda_base_prefix()
|
|
181
|
+
if conda_base:
|
|
182
|
+
env_path_str = str(Path(conda_base) / "envs" / self.environment_name)
|
|
183
|
+
else:
|
|
184
|
+
env_path_str = self.environment_name
|
|
185
|
+
|
|
186
|
+
env_path = Path(env_path_str)
|
|
187
|
+
if not env_path.exists():
|
|
188
|
+
raise RuntimeError(
|
|
189
|
+
f"Could not determine path for environment '{self.environment_name}'. "
|
|
190
|
+
f"Path '{env_path_str}' does not exist."
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
self.python_interpreter = str(self._python_path(env_path))
|
|
194
|
+
|
|
195
|
+
if not Path(self.python_interpreter).exists():
|
|
196
|
+
raise FileNotFoundError(f"Python interpreter not found in environment '{self.environment_name}' at expected path: {self.python_interpreter}")
|
|
197
|
+
|
|
198
|
+
except (subprocess.CalledProcessError, json.JSONDecodeError, StopIteration, FileNotFoundError) as e:
|
|
199
|
+
raise RuntimeError(f"Failed to determine Python interpreter for environment '{self.environment_name}': {e}")
|
|
200
|
+
|
|
201
|
+
print(f"[CondaEnvironmentManager] Python interpreter: {self.python_interpreter}")
|
|
202
|
+
print(f"[CondaEnvironmentManager] Environment setup completed for '{self.environment_name}'.")
|
|
203
|
+
|
|
204
|
+
def verify_environment(self) -> bool:
|
|
205
|
+
"""Verify that the environment is properly configured."""
|
|
206
|
+
if not self.python_interpreter:
|
|
207
|
+
print("[CondaEnvironmentManager] Python interpreter not set. Cannot verify environment.")
|
|
208
|
+
return False
|
|
209
|
+
|
|
210
|
+
print(f"[CondaEnvironmentManager] Verifying environment '{self.environment_name}'...")
|
|
211
|
+
|
|
212
|
+
if not self.dependencies:
|
|
213
|
+
print("[CondaEnvironmentManager] No dependencies to verify. Skipping verification.")
|
|
214
|
+
return True
|
|
215
|
+
|
|
216
|
+
# Verify that the Python interpreter exists and works
|
|
217
|
+
try:
|
|
218
|
+
python_version_output = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
|
|
219
|
+
print(f"[CondaEnvironmentManager] Python interpreter is working: {python_version_output}")
|
|
220
|
+
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
|
221
|
+
print(f"[CondaEnvironmentManager] Python interpreter '{self.python_interpreter}' is not working: {e}")
|
|
222
|
+
return False
|
|
223
|
+
|
|
224
|
+
# Verify Python version if specified
|
|
225
|
+
python_version_config = next(
|
|
226
|
+
(d for d in (self.dependencies or []) if isinstance(d, str) and d.startswith("python=")),
|
|
227
|
+
None,
|
|
228
|
+
)
|
|
229
|
+
if isinstance(python_version_config, str) and python_version_config.startswith("python="):
|
|
230
|
+
expected_py_ver = python_version_config.split("=")[1]
|
|
231
|
+
try:
|
|
232
|
+
current_py_ver_full = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
|
|
233
|
+
current_py_ver_parts = current_py_ver_full.split()
|
|
234
|
+
current_py_ver = current_py_ver_parts[-1] if current_py_ver_parts else ""
|
|
235
|
+
|
|
236
|
+
if '.' in expected_py_ver:
|
|
237
|
+
expected_parts = expected_py_ver.split('.')
|
|
238
|
+
current_parts = current_py_ver.split('.')
|
|
239
|
+
if len(expected_parts) <= len(current_parts):
|
|
240
|
+
match = all(expected_parts[i] == current_parts[i] for i in range(len(expected_parts)))
|
|
241
|
+
if not match:
|
|
242
|
+
print(f"[CondaEnvironmentManager] Python version mismatch. Expected prefix: {expected_py_ver}, Found: {current_py_ver}")
|
|
243
|
+
return False
|
|
244
|
+
else:
|
|
245
|
+
print(f"[CondaEnvironmentManager] Python version precision mismatch. Expected: {expected_py_ver}, Found: {current_py_ver}")
|
|
246
|
+
return False
|
|
247
|
+
elif current_py_ver != expected_py_ver:
|
|
248
|
+
print(f"[CondaEnvironmentManager] Python version mismatch. Expected: {expected_py_ver}, Found: {current_py_ver}")
|
|
249
|
+
return False
|
|
250
|
+
print(f"[CondaEnvironmentManager] Python version OK: {current_py_ver} (matches expected: {expected_py_ver})")
|
|
251
|
+
|
|
252
|
+
except subprocess.CalledProcessError as e:
|
|
253
|
+
print(f"[CondaEnvironmentManager] Could not verify Python version: {e.output if hasattr(e, 'output') else e}")
|
|
254
|
+
return False
|
|
255
|
+
|
|
256
|
+
# Basic package verification (simplified for now)
|
|
257
|
+
verification_failed = False
|
|
258
|
+
|
|
259
|
+
for package_entry in self.dependencies:
|
|
260
|
+
if isinstance(package_entry, dict) and "pip" in package_entry:
|
|
261
|
+
pip_specs = package_entry.get("pip") or []
|
|
262
|
+
if isinstance(pip_specs, list):
|
|
263
|
+
ok, missing = verify_pip_requirements(self.python_interpreter, [str(x) for x in pip_specs])
|
|
264
|
+
if not ok:
|
|
265
|
+
print(f"[CondaEnvironmentManager] Missing pip packages: {', '.join(missing)}")
|
|
266
|
+
verification_failed = True
|
|
267
|
+
elif isinstance(package_entry, str) and ("==" in package_entry or "=" in package_entry):
|
|
268
|
+
try:
|
|
269
|
+
# Support both conda-style `pkg=1.2` and pip-style `pkg==1.2`
|
|
270
|
+
if "==" in package_entry:
|
|
271
|
+
name, version = package_entry.split("==", 1)
|
|
272
|
+
else:
|
|
273
|
+
name, version = package_entry.split("=", 1)
|
|
274
|
+
name = name.split("::")[-1].strip()
|
|
275
|
+
if name == "python":
|
|
276
|
+
continue
|
|
277
|
+
|
|
278
|
+
conda_exe = self._get_conda_executable()
|
|
279
|
+
conda_list_cmd = [conda_exe, "list", "-n", self.environment_name, name, "--json"]
|
|
280
|
+
result = subprocess.run(conda_list_cmd, capture_output=True, text=True)
|
|
281
|
+
|
|
282
|
+
if result.returncode == 0:
|
|
283
|
+
package_info_list = json.loads(result.stdout)
|
|
284
|
+
if package_info_list:
|
|
285
|
+
installed_version = package_info_list[0].get("version", "")
|
|
286
|
+
if installed_version == version:
|
|
287
|
+
print(f"[CondaEnvironmentManager] Package {name} (conda) found with correct version {version}.")
|
|
288
|
+
else:
|
|
289
|
+
print(f"[CondaEnvironmentManager] Package {name} (conda) version mismatch. Expected: {version}, Found: {installed_version}.")
|
|
290
|
+
verification_failed = True
|
|
291
|
+
else:
|
|
292
|
+
print(f"[CondaEnvironmentManager] Package {name} (conda) not found.")
|
|
293
|
+
verification_failed = True
|
|
294
|
+
else:
|
|
295
|
+
print(f"[CondaEnvironmentManager] Package {name} verification failed.")
|
|
296
|
+
verification_failed = True
|
|
297
|
+
|
|
298
|
+
except Exception as e:
|
|
299
|
+
print(f"[CondaEnvironmentManager] Error verifying package {name}: {e}")
|
|
300
|
+
verification_failed = True
|
|
301
|
+
|
|
302
|
+
if verification_failed:
|
|
303
|
+
print(f"[CondaEnvironmentManager] Environment verification failed for '{self.environment_name}'.")
|
|
304
|
+
return False
|
|
305
|
+
|
|
306
|
+
print(f"[CondaEnvironmentManager] Environment verification successful for '{self.environment_name}'.")
|
|
307
|
+
return True
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from .base import EnvironmentManager
|
|
6
|
+
from .conda_manager import CondaEnvironmentManager
|
|
7
|
+
from .venv_manager import VenvEnvironmentManager
|
|
8
|
+
from .system_manager import SystemEnvironmentManager
|
|
9
|
+
from .pyenv_manager import PyenvEnvironmentManager
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class EnvironmentManagerFactory:
|
|
13
|
+
"""Factory for creating environment managers based on configuration."""
|
|
14
|
+
|
|
15
|
+
_managers: dict[str, type[EnvironmentManager]] = {
|
|
16
|
+
"conda": CondaEnvironmentManager,
|
|
17
|
+
"venv": VenvEnvironmentManager,
|
|
18
|
+
"virtualenv": VenvEnvironmentManager,
|
|
19
|
+
"pyenv": PyenvEnvironmentManager,
|
|
20
|
+
"system": SystemEnvironmentManager,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def create_environment_manager(cls, platform_config: dict[str, Any]) -> EnvironmentManager:
|
|
25
|
+
"""Create an environment manager based on the platform configuration."""
|
|
26
|
+
env_config = platform_config.get("environment")
|
|
27
|
+
if not isinstance(env_config, dict) or not env_config:
|
|
28
|
+
return SystemEnvironmentManager({})
|
|
29
|
+
|
|
30
|
+
env_type, env_specific_config = cls._select_manager(env_config)
|
|
31
|
+
manager_class = cls._managers.get(env_type)
|
|
32
|
+
if manager_class is None:
|
|
33
|
+
raise ValueError(
|
|
34
|
+
f"Unsupported environment type: {env_type}. Supported types: {sorted(cls._managers.keys())}"
|
|
35
|
+
)
|
|
36
|
+
return manager_class(env_specific_config)
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def _select_manager(cls, env_config: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
|
40
|
+
# Primary form:
|
|
41
|
+
# environment: { venv: {...} } or { conda: {...} } etc.
|
|
42
|
+
for supported_type in cls._managers:
|
|
43
|
+
if supported_type in env_config:
|
|
44
|
+
raw = env_config.get(supported_type)
|
|
45
|
+
return supported_type, raw if isinstance(raw, dict) else {}
|
|
46
|
+
|
|
47
|
+
# Legacy fallbacks where `environment:` contains the manager's config directly.
|
|
48
|
+
if any(k in env_config for k in ("dependencies", "environment_file")):
|
|
49
|
+
return "conda", env_config
|
|
50
|
+
if any(k in env_config for k in ("requirements", "requirements_file")):
|
|
51
|
+
return "venv", env_config
|
|
52
|
+
|
|
53
|
+
return "system", env_config
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def list_supported_types(cls) -> list[str]:
|
|
57
|
+
"""List all supported environment types."""
|
|
58
|
+
return list(cls._managers.keys())
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def register_manager(cls, env_type: str, manager_class: type[EnvironmentManager]) -> None:
|
|
62
|
+
"""Register a new environment manager type."""
|
|
63
|
+
if not issubclass(manager_class, EnvironmentManager):
|
|
64
|
+
raise ValueError(f"Manager class must inherit from EnvironmentManager")
|
|
65
|
+
cls._managers[env_type] = manager_class
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def create_environment_manager(platform_config: dict[str, Any]) -> EnvironmentManager:
|
|
69
|
+
"""Convenience function to create an environment manager."""
|
|
70
|
+
return EnvironmentManagerFactory.create_environment_manager(platform_config)
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .base import EnvironmentManager
|
|
8
|
+
from .utils import load_requirements, verify_pip_requirements
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PyenvEnvironmentManager(EnvironmentManager):
|
|
12
|
+
"""Pyenv-based environment management."""
|
|
13
|
+
|
|
14
|
+
def __init__(self, config: dict[str, Any]):
|
|
15
|
+
super().__init__(config)
|
|
16
|
+
self.requirements = load_requirements(config)
|
|
17
|
+
self.python_version = config.get("python_version", "3.9.0")
|
|
18
|
+
|
|
19
|
+
env_name = config.get("name")
|
|
20
|
+
if not env_name:
|
|
21
|
+
raise ValueError(
|
|
22
|
+
"Pyenv environment name must be explicitly specified in the configuration. "
|
|
23
|
+
"Please add 'name' field under 'environment.pyenv' in your config file."
|
|
24
|
+
)
|
|
25
|
+
self.environment_name = env_name
|
|
26
|
+
|
|
27
|
+
def get_environment_type(self) -> str:
|
|
28
|
+
return "pyenv"
|
|
29
|
+
|
|
30
|
+
def get_environment_name(self) -> str:
|
|
31
|
+
return self.environment_name
|
|
32
|
+
|
|
33
|
+
def get_python_interpreter(self) -> str:
|
|
34
|
+
if not self.python_interpreter:
|
|
35
|
+
raise RuntimeError("Environment not set up yet. Call setup_environment() first.")
|
|
36
|
+
return self.python_interpreter
|
|
37
|
+
|
|
38
|
+
def _get_pyenv_executable(self) -> str:
|
|
39
|
+
"""Get the pyenv executable path."""
|
|
40
|
+
try:
|
|
41
|
+
subprocess.run(["pyenv", "--version"], capture_output=True, check=True, text=True)
|
|
42
|
+
return "pyenv"
|
|
43
|
+
except (FileNotFoundError, subprocess.CalledProcessError) as e:
|
|
44
|
+
raise RuntimeError(
|
|
45
|
+
"pyenv executable not found or not runnable. "
|
|
46
|
+
"Please ensure pyenv is installed and configured correctly in your PATH. "
|
|
47
|
+
f"Error: {e}"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def environment_exists(self) -> bool:
|
|
51
|
+
"""Check if the pyenv virtual environment already exists."""
|
|
52
|
+
try:
|
|
53
|
+
pyenv_exe = self._get_pyenv_executable()
|
|
54
|
+
result = subprocess.run([pyenv_exe, "versions"], capture_output=True, text=True, check=True)
|
|
55
|
+
return self.environment_name in result.stdout
|
|
56
|
+
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
|
57
|
+
print(f"[PyenvEnvironmentManager] Failed to list pyenv versions: {e}")
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
def setup_environment(self) -> None:
|
|
61
|
+
"""Set up the pyenv environment based on configuration."""
|
|
62
|
+
print(f"[PyenvEnvironmentManager] Starting pyenv environment setup for '{self.environment_name}'...")
|
|
63
|
+
|
|
64
|
+
pyenv_exe = self._get_pyenv_executable()
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
result = subprocess.run([pyenv_exe, "versions"], capture_output=True, text=True, check=True)
|
|
68
|
+
if self.python_version not in result.stdout:
|
|
69
|
+
print(f"[PyenvEnvironmentManager] Python version {self.python_version} not found. Installing it...")
|
|
70
|
+
subprocess.run([pyenv_exe, "install", self.python_version], check=True)
|
|
71
|
+
print(f"[PyenvEnvironmentManager] Python version {self.python_version} installed successfully.")
|
|
72
|
+
except subprocess.CalledProcessError as e:
|
|
73
|
+
raise RuntimeError(f"Failed to install Python version {self.python_version}: {e}")
|
|
74
|
+
|
|
75
|
+
if not self.environment_exists():
|
|
76
|
+
print(f"[PyenvEnvironmentManager] Environment '{self.environment_name}' not found. Creating it...")
|
|
77
|
+
|
|
78
|
+
# Create virtual environment
|
|
79
|
+
try:
|
|
80
|
+
subprocess.run([pyenv_exe, "virtualenv", self.python_version, self.environment_name], check=True)
|
|
81
|
+
print(f"[PyenvEnvironmentManager] Environment '{self.environment_name}' created successfully.")
|
|
82
|
+
except subprocess.CalledProcessError as e:
|
|
83
|
+
raise RuntimeError(f"Failed to create pyenv virtual environment '{self.environment_name}': {e}")
|
|
84
|
+
else:
|
|
85
|
+
print(f"[PyenvEnvironmentManager] Using existing environment: '{self.environment_name}'")
|
|
86
|
+
|
|
87
|
+
try:
|
|
88
|
+
pyenv_root_result = subprocess.run([pyenv_exe, "root"], capture_output=True, text=True, check=True)
|
|
89
|
+
pyenv_root = Path(pyenv_root_result.stdout.strip())
|
|
90
|
+
|
|
91
|
+
self.python_interpreter = str(pyenv_root / "versions" / self.environment_name / "bin" / "python")
|
|
92
|
+
|
|
93
|
+
if not Path(self.python_interpreter).exists():
|
|
94
|
+
raise FileNotFoundError(f"Python interpreter not found at expected path: {self.python_interpreter}")
|
|
95
|
+
|
|
96
|
+
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
|
97
|
+
raise RuntimeError(f"Failed to determine Python interpreter for pyenv environment '{self.environment_name}': {e}")
|
|
98
|
+
|
|
99
|
+
if self.requirements:
|
|
100
|
+
print(f"[PyenvEnvironmentManager] Installing requirements...")
|
|
101
|
+
try:
|
|
102
|
+
pip_install_cmd = [self.python_interpreter, "-m", "pip", "install"] + self.requirements
|
|
103
|
+
subprocess.run(pip_install_cmd, check=True)
|
|
104
|
+
print(f"[PyenvEnvironmentManager] Requirements installed successfully.")
|
|
105
|
+
except subprocess.CalledProcessError as e:
|
|
106
|
+
raise RuntimeError(f"Failed to install requirements: {e}")
|
|
107
|
+
|
|
108
|
+
print(f"[PyenvEnvironmentManager] Python interpreter: {self.python_interpreter}")
|
|
109
|
+
print(f"[PyenvEnvironmentManager] Environment setup completed for '{self.environment_name}'.")
|
|
110
|
+
|
|
111
|
+
def verify_environment(self) -> bool:
|
|
112
|
+
"""Verify that the environment is properly configured."""
|
|
113
|
+
if not self.python_interpreter:
|
|
114
|
+
print("[PyenvEnvironmentManager] Python interpreter not set. Cannot verify environment.")
|
|
115
|
+
return False
|
|
116
|
+
|
|
117
|
+
print(f"[PyenvEnvironmentManager] Verifying environment '{self.environment_name}'...")
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
python_version_output = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
|
|
121
|
+
print(f"[PyenvEnvironmentManager] Python interpreter is working: {python_version_output}")
|
|
122
|
+
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
|
123
|
+
print(f"[PyenvEnvironmentManager] Python interpreter '{self.python_interpreter}' is not working: {e}")
|
|
124
|
+
return False
|
|
125
|
+
|
|
126
|
+
try:
|
|
127
|
+
current_py_ver_full = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
|
|
128
|
+
current_py_ver = current_py_ver_full.split()[-1] if current_py_ver_full.split() else ""
|
|
129
|
+
|
|
130
|
+
if not current_py_ver.startswith(self.python_version.split('.')[0]): # Check major version
|
|
131
|
+
print(f"[PyenvEnvironmentManager] Python version mismatch. Expected: {self.python_version}, Found: {current_py_ver}")
|
|
132
|
+
return False
|
|
133
|
+
print(f"[PyenvEnvironmentManager] Python version OK: {current_py_ver}")
|
|
134
|
+
except subprocess.CalledProcessError as e:
|
|
135
|
+
print(f"[PyenvEnvironmentManager] Could not verify Python version: {e}")
|
|
136
|
+
return False
|
|
137
|
+
|
|
138
|
+
if self.requirements:
|
|
139
|
+
ok, missing = verify_pip_requirements(self.python_interpreter, self.requirements)
|
|
140
|
+
if not ok:
|
|
141
|
+
print(f"[PyenvEnvironmentManager] Missing packages: {', '.join(missing)}")
|
|
142
|
+
print(f"[PyenvEnvironmentManager] Environment verification failed for '{self.environment_name}'.")
|
|
143
|
+
return False
|
|
144
|
+
|
|
145
|
+
print(f"[PyenvEnvironmentManager] Environment verification successful for '{self.environment_name}'.")
|
|
146
|
+
return True
|