expops 0.1.3__py3-none-any.whl

mlops/environment/setup_env.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Setup project environment and write interpreter path")
+    p.add_argument("--project-id", required=True)
+    p.add_argument("--project-dir", required=True, help="Path to repository root")
+    p.add_argument("--env-file", required=True, help="Path to write the interpreter path")
+    return p.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    repo_root = Path(args.project_dir).resolve()
+    src_path = repo_root / "src"
+    if str(src_path) not in sys.path:
+        sys.path.insert(0, str(src_path))
+
+    from mlops.core.pipeline_utils import setup_environment_and_write_interpreter
+
+    py = setup_environment_and_write_interpreter(repo_root, args.project_id, args.env_file)
+    print(f"[env] Environment setup completed. Interpreter: {py}")
+
+
+if __name__ == "__main__":
+    main()

mlops/environment/system_manager.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from typing import Any
+
+from .base import EnvironmentManager
+from .utils import load_requirements, verify_pip_requirements
+
+
+class SystemEnvironmentManager(EnvironmentManager):
+    """System Python environment management (no virtual environment)."""
+    
+    def __init__(self, config: dict[str, Any]):
+        super().__init__(config)
+        self.requirements = load_requirements(config)
+        self.environment_name = "system"
+        self.python_interpreter = sys.executable
+
+    def get_environment_type(self) -> str:
+        return "system"
+
+    def get_environment_name(self) -> str:
+        return self.environment_name
+
+    def get_python_interpreter(self) -> str:
+        return self.python_interpreter
+
+    def environment_exists(self) -> bool:
+        """System environment always exists."""
+        return True
+
+    def setup_environment(self) -> None:
+        """Set up using the system Python environment."""
+        print(f"[SystemEnvironmentManager] Using system Python environment...")
+        
+        if self.requirements:
+            print(f"[SystemEnvironmentManager] Installing requirements to system Python...")
+            try:
+                pip_install_cmd = [self.python_interpreter, "-m", "pip", "install"] + self.requirements
+                subprocess.run(pip_install_cmd, check=True)
+            except subprocess.CalledProcessError as e:
+                raise RuntimeError(f"Failed to install requirements to system Python: {e}")
+
+        print(f"[SystemEnvironmentManager] Python interpreter: {self.python_interpreter}")
+
+    def verify_environment(self) -> bool:
+        """Verify that the system environment is properly configured."""
+        print(f"[SystemEnvironmentManager] Verifying system Python environment...")
+
+        try:
+            python_version_output = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
+            print(f"[SystemEnvironmentManager] Python interpreter is working: {python_version_output}")
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            print(f"[SystemEnvironmentManager] Python interpreter '{self.python_interpreter}' is not working: {e}")
+            return False
+
+        if self.requirements:
+            ok, missing = verify_pip_requirements(self.python_interpreter, self.requirements)
+            if not ok:
+                print(f"[SystemEnvironmentManager] Missing packages: {', '.join(missing)}")
+                print(f"[SystemEnvironmentManager] Environment verification failed for system Python.")
+                return False
+        
+        print(f"[SystemEnvironmentManager] Environment verification successful for system Python.")
+        return True 

mlops/environment/utils.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+from typing import Any
+
+
+def _read_requirement_lines(path: Path) -> list[str]:
+    lines: list[str] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for raw in f:
+            line = raw.strip()
+            if not line or line.startswith("#"):
+                continue
+            lines.append(line)
+    return lines
+
+
+def load_requirements(config: dict[str, Any]) -> list[str]:
+    """Load pip-style requirements from a config dict.
+
+    Supported keys:
+    - `requirements`: list[str] or newline-separated str
+    - `requirements_file`: path to a requirements.txt file
+    """
+    if "requirements" in config:
+        reqs = config.get("requirements")
+        if isinstance(reqs, str):
+            out: list[str] = []
+            for raw in reqs.splitlines():
+                line = raw.strip()
+                if not line or line.startswith("#"):
+                    continue
+                out.append(line)
+            return out
+        if isinstance(reqs, list):
+            out: list[str] = []
+            for item in reqs:
+                s = str(item).strip()
+                if s and not s.startswith("#"):
+                    out.append(s)
+            return out
+
+    req_file = config.get("requirements_file")
+    if req_file:
+        path = Path(str(req_file))
+        if not path.exists():
+            raise FileNotFoundError(f"Requirements file not found: {path}")
+        return _read_requirement_lines(path)
+
+    return []
+
+
+_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+")
+
+
+def requirement_to_package_name(requirement: str) -> str:
+    """Best-effort extract a package name for `pip show` from a requirement spec."""
+    s = str(requirement).strip()
+    if not s:
+        return ""
+
+    # Handle "name @ url"
+    if "@" in s and not s.startswith("@"):
+        s = s.split("@", 1)[0].strip()
+
+    # Strip environment markers (PEP 508)
+    if ";" in s:
+        s = s.split(";", 1)[0].strip()
+
+    # Strip extras
+    if "[" in s:
+        s = s.split("[", 1)[0].strip()
+
+    # Strip version specifiers (common cases)
+    for op in ("===", "==", ">=", "<=", "!=", "~=", ">", "<", "="):
+        if op in s:
+            s = s.split(op, 1)[0].strip()
+            break
+
+    m = _NAME_RE.match(s)
+    return m.group(0) if m else s
+
+
+def verify_pip_requirements(python_exec: str, requirements: list[str]) -> tuple[bool, list[str]]:
+    """Return (ok, missing_packages) using `pip show`."""
+    missing: list[str] = []
+    for req in requirements or []:
+        name = requirement_to_package_name(req)
+        if not name:
+            continue
+        try:
+            subprocess.check_output(
+                [python_exec, "-m", "pip", "show", name],
+                text=True,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            missing.append(name)
+        except FileNotFoundError:
+            missing.append(name)
+    return (len(missing) == 0), missing
+
+

mlops/environment/venv_manager.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+from .base import EnvironmentManager
+from .utils import load_requirements, verify_pip_requirements
+
+
+class VenvEnvironmentManager(EnvironmentManager):
+    """Standard Python venv-based environment management."""
+    
+    def __init__(self, config: dict[str, Any]):
+        super().__init__(config)
+        self.requirements = load_requirements(config)
+        
+        env_name = config.get("name")
+        if not env_name:
+            raise ValueError(
+                "Virtual environment name must be explicitly specified in the configuration. "
+                "Please add 'name' field under 'environment.venv' in your config file."
+            )
+        self.environment_name = env_name
+        # Prefer project-local `.venvs/` but preserve existing cache environments if present.
+        preferred_path = Path.cwd() / ".venvs" / self.environment_name
+        cache_path = Path.home() / ".cache" / "mlops-platform" / "venvs" / self.environment_name
+
+        if self._python_path(preferred_path).exists():
+            self.venv_path = preferred_path
+        elif self._python_path(cache_path).exists():
+            self.venv_path = cache_path
+        else:
+            self.venv_path = preferred_path if self._ensure_writable_dir(preferred_path.parent) else cache_path
+
+    @staticmethod
+    def _python_path(venv_path: Path) -> Path:
+        if os.name == "nt":
+            return venv_path / "Scripts" / "python.exe"
+        return venv_path / "bin" / "python"
+
+    @staticmethod
+    def _ensure_writable_dir(dir_path: Path) -> bool:
+        try:
+            dir_path.mkdir(parents=True, exist_ok=True)
+            probe = dir_path / ".mlops_write_probe"
+            probe.write_text("", encoding="utf-8")
+            probe.unlink(missing_ok=True)
+            return True
+        except Exception:
+            return False
+
+    def get_environment_type(self) -> str:
+        return "venv"
+
+    def get_environment_name(self) -> str:
+        return self.environment_name
+
+    def get_python_interpreter(self) -> str:
+        if not self.python_interpreter:
+            raise RuntimeError("Environment not set up yet. Call setup_environment() first.")
+        return self.python_interpreter
+
+    def environment_exists(self) -> bool:
+        """Check if the virtual environment already exists."""
+        return self._python_path(self.venv_path).exists()
+
+    def setup_environment(self) -> None:
+        """Set up the virtual environment based on configuration."""
+        print(f"[VenvEnvironmentManager] Starting venv environment setup for '{self.environment_name}'...")
+        
+        if not self.environment_exists():
+            print(f"[VenvEnvironmentManager] Environment '{self.environment_name}' not found. Creating it...")
+            
+            self.venv_path.parent.mkdir(parents=True, exist_ok=True)
+            
+            try:
+                subprocess.run([sys.executable, "-m", "venv", str(self.venv_path)], check=True)
+                print(f"[VenvEnvironmentManager] Environment '{self.environment_name}' created successfully.")
+            except subprocess.CalledProcessError as e:
+                # One more fallback: use virtualenv if available
+                try:
+                    subprocess.run([sys.executable, "-m", "virtualenv", str(self.venv_path)], check=True)
+                    print(f"[VenvEnvironmentManager] Environment '{self.environment_name}' created via virtualenv.")
+                except Exception:
+                    raise RuntimeError(f"Failed to create virtual environment '{self.environment_name}': {e}")
+        else:
+            print(f"[VenvEnvironmentManager] Using existing environment: '{self.environment_name}'")
+
+        self.python_interpreter = str(self._python_path(self.venv_path))
+        
+        if not Path(self.python_interpreter).exists():
+            raise FileNotFoundError(f"Python interpreter not found in environment '{self.environment_name}' at expected path: {self.python_interpreter}")
+
+        if self.requirements:
+            print(f"[VenvEnvironmentManager] Installing requirements...")
+            try:
+                # Ensure modern build tooling to prefer wheels over source builds
+                subprocess.run([self.python_interpreter, "-m", "pip", "install", "--upgrade", "pip", "setuptools", "wheel"], check=True)
+                pip_install_cmd = [self.python_interpreter, "-m", "pip", "install", "--no-cache-dir"] + self.requirements
+                subprocess.run(pip_install_cmd, check=True)
+                print(f"[VenvEnvironmentManager] Requirements installed successfully.")
+            except subprocess.CalledProcessError as e:
+                raise RuntimeError(f"Failed to install requirements: {e}")
+
+        print(f"[VenvEnvironmentManager] Python interpreter: {self.python_interpreter}")
+        print(f"[VenvEnvironmentManager] Environment setup completed for '{self.environment_name}'.")
+
+    def verify_environment(self) -> bool:
+        """Verify that the environment is properly configured."""
+        if not self.python_interpreter:
+            print("[VenvEnvironmentManager] Python interpreter not set. Cannot verify environment.")
+            return False
+
+        print(f"[VenvEnvironmentManager] Verifying environment '{self.environment_name}'...")
+
+        try:
+            python_version_output = subprocess.check_output([self.python_interpreter, '--version'], text=True, stderr=subprocess.STDOUT).strip()
+            print(f"[VenvEnvironmentManager] Python interpreter is working: {python_version_output}")
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            print(f"[VenvEnvironmentManager] Python interpreter '{self.python_interpreter}' is not working: {e}")
+            return False
+
+        if self.requirements:
+            ok, missing = verify_pip_requirements(self.python_interpreter, self.requirements)
+            if not ok:
+                print(f"[VenvEnvironmentManager] Missing packages: {', '.join(missing)}")
+                print(f"[VenvEnvironmentManager] Environment verification failed for '{self.environment_name}'.")
+                return False
+        
+        print(f"[VenvEnvironmentManager] Environment verification successful for '{self.environment_name}'.")
+        return True