fraclab-sdk 0.1.0__py3-none-any.whl

fraclab_sdk/devkit/compile.py

@@ -0,0 +1,342 @@
+"""Algorithm compilation: generate static artifacts from workspace.
+
+Compile workflow:
+1. Import schema.inputspec:INPUT_SPEC → model_json_schema() → dist/params.schema.json
+2. Import schema.output_contract:OUTPUT_CONTRACT → model_dump() → dist/output_contract.json
+3. Copy drs.json from bundle → dist/drs.json
+4. Update manifest.json with files pointers
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from fraclab_sdk.errors import AlgorithmError
+
+
+@dataclass
+class CompileLog:
+    """Log entry for a compile subprocess."""
+
+    step: str
+    stdout: str
+    stderr: str
+    returncode: int
+    timestamp: str
+
+
+@dataclass
+class CompileResult:
+    """Result of algorithm compilation."""
+
+    params_schema_path: Path
+    output_contract_path: Path
+    drs_path: Path
+    manifest_updated: bool
+    bound_bundle: dict[str, str] | None = None
+
+
+def _run_in_subprocess(
+    workspace: Path,
+    script: str,
+    step_name: str,
+    log_dir: Path | None = None,
+) -> tuple[dict[str, Any], CompileLog]:
+    """Run Python script in isolated subprocess with workspace on PYTHONPATH.
+
+    Args:
+        workspace: Algorithm workspace directory.
+        script: Python script to execute.
+        step_name: Name of the compilation step (for logging).
+        log_dir: Directory to save logs (optional).
+
+    Returns:
+        Tuple of (parsed JSON output, compile log).
+
+    Raises:
+        AlgorithmError: If subprocess fails.
+    """
+    env = {
+        "PYTHONPATH": str(workspace),
+        "PYTHONUNBUFFERED": "1",
+    }
+
+    result = subprocess.run(
+        [sys.executable, "-c", script],
+        cwd=workspace,
+        env={**dict(__import__("os").environ), **env},
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+
+    # Create log entry
+    log = CompileLog(
+        step=step_name,
+        stdout=result.stdout,
+        stderr=result.stderr,
+        returncode=result.returncode,
+        timestamp=datetime.now().isoformat(),
+    )
+
+    # Save log to file if log_dir provided
+    if log_dir is not None:
+        log_file = log_dir / f"{step_name}.log"
+        log_content = (
+            f"Step: {step_name}\n"
+            f"Timestamp: {log.timestamp}\n"
+            f"Return code: {result.returncode}\n"
+            f"\n=== STDOUT ===\n{result.stdout}\n"
+            f"\n=== STDERR ===\n{result.stderr}\n"
+        )
+        log_file.write_text(log_content, encoding="utf-8")
+
+    if result.returncode != 0:
+        error_summary = result.stderr.strip() or result.stdout.strip() or "Unknown error"
+        # Truncate for CLI display, full log is in file
+        if len(error_summary) > 500:
+            error_summary = error_summary[:500] + "..."
+        log_path_hint = f" (full log: {log_dir / f'{step_name}.log'})" if log_dir else ""
+        raise AlgorithmError(
+            f"Compilation step '{step_name}' failed: {error_summary}{log_path_hint}"
+        )
+
+    try:
+        return json.loads(result.stdout), log
+    except json.JSONDecodeError as e:
+        log_path_hint = f" (full log: {log_dir / f'{step_name}.log'})" if log_dir else ""
+        raise AlgorithmError(
+            f"Invalid JSON output from '{step_name}': {e}{log_path_hint}"
+        ) from e
+
+
+def _extract_params_schema(workspace: Path, log_dir: Path | None = None) -> dict[str, Any]:
+    """Extract params JSON Schema from INPUT_SPEC.
+
+    Args:
+        workspace: Algorithm workspace directory.
+        log_dir: Directory to save logs (optional).
+
+    Returns:
+        JSON Schema dict.
+    """
+    script = '''
+import json
+import sys
+
+try:
+    from schema.inputspec import INPUT_SPEC
+    model = INPUT_SPEC
+    schema = model.model_json_schema()
+    print(json.dumps(schema))
+except ImportError as e:
+    print(json.dumps({"error": f"Failed to import INPUT_SPEC: {e}"}))
+    sys.exit(0)
+except Exception as e:
+    print(json.dumps({"error": f"Failed to generate schema: {e}"}))
+    sys.exit(0)
+'''
+
+    result, _ = _run_in_subprocess(workspace, script, "params_schema", log_dir)
+    if "error" in result:
+        raise AlgorithmError(result["error"])
+    return result
+
+
+def _extract_output_contract(workspace: Path, log_dir: Path | None = None) -> dict[str, Any]:
+    """Extract OutputContract from OUTPUT_CONTRACT.
+
+    Args:
+        workspace: Algorithm workspace directory.
+        log_dir: Directory to save logs (optional).
+
+    Returns:
+        OutputContract dict.
+    """
+    script = '''
+import json
+import sys
+
+try:
+    from schema.output_contract import OUTPUT_CONTRACT
+    # Use model_dump with mode="json" for JSON-serializable output
+    if hasattr(OUTPUT_CONTRACT, 'model_dump'):
+        data = OUTPUT_CONTRACT.model_dump(mode="json")
+    else:
+        data = OUTPUT_CONTRACT.dict()
+    print(json.dumps(data))
+except ImportError as e:
+    print(json.dumps({"error": f"Failed to import OUTPUT_CONTRACT: {e}"}))
+    sys.exit(0)
+except Exception as e:
+    print(json.dumps({"error": f"Failed to dump contract: {e}"}))
+    sys.exit(0)
+'''
+
+    result, _ = _run_in_subprocess(workspace, script, "output_contract", log_dir)
+    if "error" in result:
+        raise AlgorithmError(result["error"])
+    return result
+
+
+def _compute_file_hash(path: Path) -> str:
+    """Compute SHA256 hash of file contents (raw bytes).
+
+    Args:
+        path: File path.
+
+    Returns:
+        Hex-encoded SHA256 hash.
+    """
+    hasher = hashlib.sha256()
+    hasher.update(path.read_bytes())
+    return hasher.hexdigest()
+
+
+def compile_algorithm(
+    workspace: Path,
+    bundle_path: Path | None = None,
+    skip_inputspec: bool = False,
+    skip_output_contract: bool = False,
+) -> CompileResult:
+    """Compile algorithm workspace to generate static artifacts.
+
+    This generates:
+    - dist/params.schema.json (from schema.inputspec:CONFIG_MODEL)
+    - dist/output_contract.json (from schema.output_contract:OUTPUT_CONTRACT)
+    - dist/drs.json (copied from bundle)
+
+    And updates manifest.json with files pointers.
+
+    Args:
+        workspace: Path to algorithm workspace directory.
+        bundle_path: Path to data bundle (for drs.json). If None, drs.json must exist.
+        skip_inputspec: Skip InputSpec compilation (use existing params.schema.json).
+        skip_output_contract: Skip OutputContract compilation.
+
+    Returns:
+        CompileResult with paths to generated artifacts.
+
+    Raises:
+        AlgorithmError: If compilation fails.
+    """
+    workspace = Path(workspace).resolve()
+
+    if not workspace.is_dir():
+        raise AlgorithmError(f"Workspace not found: {workspace}")
+
+    # Validate workspace structure
+    manifest_path = workspace / "manifest.json"
+    algorithm_py_path = workspace / "main.py"
+
+    if not manifest_path.exists():
+        raise AlgorithmError(f"manifest.json not found in workspace: {workspace}")
+    if not algorithm_py_path.exists():
+        raise AlgorithmError(f"main.py not found in workspace: {workspace}")
+
+    # Create dist directory and compile logs directory
+    dist_dir = workspace / "dist"
+    dist_dir.mkdir(exist_ok=True)
+
+    log_dir = dist_dir / "_compile_logs"
+    log_dir.mkdir(exist_ok=True)
+
+    # 1. Generate params.schema.json from InputSpec
+    params_schema_path = dist_dir / "params.schema.json"
+    if not skip_inputspec:
+        schema_dir = workspace / "schema"
+        inputspec_path = schema_dir / "inputspec.py"
+        if not inputspec_path.exists():
+            raise AlgorithmError(
+                "schema/inputspec.py not found. Required for params schema generation."
+            )
+
+        params_schema = _extract_params_schema(workspace, log_dir)
+        params_schema_path.write_text(json.dumps(params_schema, indent=2), encoding="utf-8")
+
+    # 2. Generate output_contract.json from OutputContract
+    output_contract_path = dist_dir / "output_contract.json"
+    if not skip_output_contract:
+        schema_dir = workspace / "schema"
+        output_contract_file = schema_dir / "output_contract.py"
+        if not output_contract_file.exists():
+            raise AlgorithmError(
+                "schema/output_contract.py not found. Required for output contract generation."
+            )
+
+        output_contract = _extract_output_contract(workspace, log_dir)
+        output_contract_path.write_text(
+            json.dumps(output_contract, indent=2), encoding="utf-8"
+        )
+
+    # 3. Copy drs.json from bundle (or use existing)
+    drs_path = dist_dir / "drs.json"
+    bound_bundle: dict[str, str] | None = None
+
+    if bundle_path is not None:
+        bundle_path = Path(bundle_path).resolve()
+        if not bundle_path.is_dir():
+            raise AlgorithmError(f"Bundle path not found: {bundle_path}")
+
+        bundle_drs = bundle_path / "drs.json"
+        bundle_ds = bundle_path / "ds.json"
+        bundle_manifest = bundle_path / "manifest.json"
+
+        if not bundle_drs.exists():
+            raise AlgorithmError(f"drs.json not found in bundle: {bundle_path}")
+
+        # Copy drs.json (raw bytes to preserve hash)
+        shutil.copy2(bundle_drs, drs_path)
+
+        # Extract hash info from bundle manifest if available
+        if bundle_manifest.exists():
+            try:
+                manifest = json.loads(bundle_manifest.read_text())
+                spec_files = manifest.get("specFiles", {})
+                bound_bundle = {
+                    "drsSha256": spec_files.get("drsSha256") or _compute_file_hash(bundle_drs),
+                }
+                if bundle_ds.exists():
+                    bound_bundle["dsSha256"] = spec_files.get("dsSha256") or _compute_file_hash(
+                        bundle_ds
+                    )
+            except (json.JSONDecodeError, KeyError):
+                pass
+    elif not drs_path.exists():
+        raise AlgorithmError(
+            "drs.json not found in dist/. Provide --bundle to copy from bundle."
+        )
+
+    # 4. Update manifest.json with files pointers
+    manifest = json.loads(manifest_path.read_text())
+
+    files = manifest.get("files", {})
+    files["paramsSchemaPath"] = "dist/params.schema.json"
+    files["outputContractPath"] = "dist/output_contract.json"
+    files["drsPath"] = "dist/drs.json"
+    manifest["files"] = files
+
+    # Add bound bundle info if available
+    if bound_bundle:
+        manifest["boundBundle"] = bound_bundle
+
+    manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+
+    return CompileResult(
+        params_schema_path=params_schema_path,
+        output_contract_path=output_contract_path,
+        drs_path=drs_path,
+        manifest_updated=True,
+        bound_bundle=bound_bundle,
+    )
+
+
+__all__ = ["compile_algorithm", "CompileResult"]

fraclab_sdk/devkit/export.py

@@ -0,0 +1,354 @@
+"""Algorithm export: package workspace for distribution.
+
+Export creates a distributable algorithm package containing:
+- main.py (required)
+- manifest.json (required)
+- dist/params.schema.json (required)
+- dist/output_contract.json (required)
+- dist/drs.json (required)
+- README.md (optional)
+- schema/** (optional, source code)
+- examples/** (optional)
+"""
+
+from __future__ import annotations
+
+import shutil
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from fraclab_sdk.errors import AlgorithmError
+
+# Files that must exist in dist/
+REQUIRED_DIST_FILES = [
+    "params.schema.json",
+    "output_contract.json",
+    "drs.json",
+]
+
+# Required workspace files
+REQUIRED_WORKSPACE_FILES = [
+    "main.py",
+    "manifest.json",
+]
+
+# Patterns to include (relative to workspace)
+INCLUDE_PATTERNS = [
+    "main.py",
+    "manifest.json",
+    "dist/params.schema.json",
+    "dist/output_contract.json",
+    "dist/drs.json",
+    "README.md",
+    "schema/**",
+    "examples/**",
+]
+
+# Patterns to always reject
+REJECT_PATTERNS = [
+    "__pycache__",
+    ".DS_Store",
+    ".idea",
+    ".git",
+    ".gitignore",
+    ".vscode",
+    "*.pyc",
+    "*.pyo",
+    "*.egg-info",
+    ".eggs",
+    "*.tmp",
+    "*.temp",
+    "*.bak",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+]
+
+
+@dataclass
+class ExportResult:
+    """Result of algorithm export."""
+
+    output_path: Path
+    files_included: list[str]
+    files_rejected: list[str]
+
+
+def _should_reject(path: Path, workspace: Path) -> bool:
+    """Check if a path should be rejected.
+
+    Args:
+        path: Path to check.
+        workspace: Workspace root.
+
+    Returns:
+        True if path should be rejected.
+    """
+    rel_path = path.relative_to(workspace)
+    name = path.name
+
+    for pattern in REJECT_PATTERNS:
+        if pattern.startswith("*."):
+            if name.endswith(pattern[1:]):
+                return True
+        elif name == pattern or pattern in str(rel_path):
+            return True
+
+    # Reject hidden files/directories
+    return name.startswith(".")
+
+
+def _is_path_contained(path: Path, root: Path) -> bool:
+    """Check if a path is contained within root (no directory traversal).
+
+    Args:
+        path: Path to check (will be resolved).
+        root: Root directory (will be resolved).
+
+    Returns:
+        True if path is within root.
+    """
+    try:
+        resolved = path.resolve()
+        root_resolved = root.resolve()
+        resolved.relative_to(root_resolved)
+        return True
+    except ValueError:
+        return False
+
+
+def _check_symlink_safety(path: Path, workspace: Path) -> tuple[bool, str | None]:
+    """Check if a symlink is safe to include.
+
+    Args:
+        path: Path to check.
+        workspace: Workspace root.
+
+    Returns:
+        Tuple of (is_safe, reason_if_unsafe).
+    """
+    if not path.is_symlink():
+        return True, None
+
+    # Resolve the symlink target
+    try:
+        target = path.resolve()
+    except (OSError, RuntimeError) as e:
+        return False, f"Failed to resolve symlink: {e}"
+
+    # Check if target is within workspace
+    if not _is_path_contained(target, workspace):
+        return False, f"Symlink points outside workspace: {target}"
+
+    return True, None
+
+
+def _collect_files(workspace: Path) -> tuple[list[Path], list[Path]]:
+    """Collect files to include in export.
+
+    Args:
+        workspace: Workspace root.
+
+    Returns:
+        Tuple of (included_files, rejected_files).
+
+    Raises:
+        AlgorithmError: If required files are missing or symlinks escape workspace.
+    """
+    included: list[Path] = []
+    rejected: list[Path] = []
+
+    def check_and_add(path: Path, required: bool = False) -> None:
+        """Check a file and add to included/rejected lists.
+
+        Args:
+            path: Path to check.
+            required: If True, raise error if file is unsafe.
+        """
+        if not path.exists():
+            if required:
+                raise AlgorithmError(f"Required file not found: {path.relative_to(workspace)}")
+            return
+
+        # Check symlink safety
+        is_safe, reason = _check_symlink_safety(path, workspace)
+        if not is_safe:
+            if required:
+                raise AlgorithmError(f"Required file is unsafe: {path.relative_to(workspace)} - {reason}")
+            rejected.append(path)
+            return
+
+        # Check path containment (resolved path must be in workspace)
+        if not _is_path_contained(path, workspace):
+            if required:
+                raise AlgorithmError(f"Required file escapes workspace: {path.relative_to(workspace)}")
+            rejected.append(path)
+            return
+
+        included.append(path)
+
+    # Required files
+    for filename in REQUIRED_WORKSPACE_FILES:
+        check_and_add(workspace / filename, required=True)
+
+    # Required dist files
+    dist_dir = workspace / "dist"
+    for filename in REQUIRED_DIST_FILES:
+        path = dist_dir / filename
+        if not path.exists():
+            raise AlgorithmError(
+                f"Required dist file not found: dist/{filename}. "
+                f"Run 'fraclab-sdk algo compile' first."
+            )
+        check_and_add(path, required=True)
+
+    # Optional files
+    readme = workspace / "README.md"
+    if readme.exists():
+        check_and_add(readme, required=False)
+
+    # Optional directories (schema, examples)
+    for dir_name in ["schema", "examples"]:
+        dir_path = workspace / dir_name
+        if dir_path.is_dir():
+            for file_path in dir_path.rglob("*"):
+                if file_path.is_file():
+                    # Check symlink and containment first
+                    is_safe, _ = _check_symlink_safety(file_path, workspace)
+                    if not is_safe:
+                        rejected.append(file_path)
+                        continue
+
+                    if not _is_path_contained(file_path, workspace):
+                        rejected.append(file_path)
+                        continue
+
+                    # Then check standard rejection patterns
+                    if _should_reject(file_path, workspace):
+                        rejected.append(file_path)
+                    else:
+                        included.append(file_path)
+
+    return included, rejected
+
+
+def _validate_workspace(workspace: Path) -> None:
+    """Validate workspace has required structure.
+
+    Args:
+        workspace: Workspace path.
+
+    Raises:
+        AlgorithmError: If validation fails.
+    """
+    if not workspace.is_dir():
+        raise AlgorithmError(f"Workspace not found: {workspace}")
+
+    for filename in REQUIRED_WORKSPACE_FILES:
+        if not (workspace / filename).exists():
+            raise AlgorithmError(f"Required file not found: {filename}")
+
+    dist_dir = workspace / "dist"
+    if not dist_dir.is_dir():
+        raise AlgorithmError(
+            "dist/ directory not found. Run 'fraclab-sdk algo compile' first."
+        )
+
+    for filename in REQUIRED_DIST_FILES:
+        if not (dist_dir / filename).exists():
+            raise AlgorithmError(
+                f"dist/{filename} not found. Run 'fraclab-sdk algo compile' first."
+            )
+
+
+def export_algorithm_package(
+    workspace: Path,
+    output: Path,
+    auto_compile: bool = False,
+    bundle_path: Path | None = None,
+) -> ExportResult:
+    """Export algorithm workspace as a distributable package.
+
+    Args:
+        workspace: Path to algorithm workspace.
+        output: Output path (.zip file or directory).
+        auto_compile: If True, run compile before export if dist/ is missing.
+        bundle_path: Bundle path for auto-compile.
+
+    Returns:
+        ExportResult with export details.
+
+    Raises:
+        AlgorithmError: If export fails.
+    """
+    workspace = Path(workspace).resolve()
+    output = Path(output).resolve()
+
+    # Auto-compile if needed
+    if auto_compile:
+        dist_dir = workspace / "dist"
+        needs_compile = not dist_dir.exists() or not all(
+            (dist_dir / f).exists() for f in REQUIRED_DIST_FILES
+        )
+        if needs_compile:
+            from fraclab_sdk.devkit.compile import compile_algorithm
+
+            compile_algorithm(workspace, bundle_path=bundle_path)
+
+    # Validate workspace
+    _validate_workspace(workspace)
+
+    # Collect files
+    included, rejected = _collect_files(workspace)
+
+    # Export
+    if output.suffix == ".zip":
+        _export_to_zip(workspace, output, included)
+    else:
+        _export_to_dir(workspace, output, included)
+
+    return ExportResult(
+        output_path=output,
+        files_included=[str(f.relative_to(workspace)) for f in included],
+        files_rejected=[str(f.relative_to(workspace)) for f in rejected],
+    )
+
+
+def _export_to_zip(workspace: Path, output: Path, files: list[Path]) -> None:
+    """Export files to a zip archive.
+
+    Args:
+        workspace: Workspace root.
+        output: Output zip path.
+        files: Files to include.
+    """
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zf:
+        for file_path in files:
+            rel_path = file_path.relative_to(workspace)
+            zf.write(file_path, rel_path)
+
+
+def _export_to_dir(workspace: Path, output: Path, files: list[Path]) -> None:
+    """Export files to a directory.
+
+    Args:
+        workspace: Workspace root.
+        output: Output directory path.
+        files: Files to include.
+    """
+    if output.exists():
+        shutil.rmtree(output)
+
+    output.mkdir(parents=True)
+
+    for file_path in files:
+        rel_path = file_path.relative_to(workspace)
+        dest = output / rel_path
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(file_path, dest)
+
+
+__all__ = ["export_algorithm_package", "ExportResult"]