pycontrol-core 3.0.0a1__py3-none-any.whl

pycontrol/__init__.py

@@ -0,0 +1,5 @@
+"""pyControl host library: sans-IO core + extension points."""
+
+__version__ = "3.0.0a1"
+
+__all__ = ["__version__"]

pycontrol/bundles/__init__.py

@@ -0,0 +1,48 @@
+"""Self-contained pyControl protocol and run bundle helpers."""
+
+from typing import Any
+
+from pycontrol.bundles.models import (
+    PROTOCOL_BUNDLE_TYPE,
+    RUN_BUNDLE_TYPE,
+    ProtocolBundleManifest,
+    RunBundleManifest,
+)
+from pycontrol.bundles.zipio import ArchiveError
+
+
+def __getattr__(name: str) -> Any:
+    if name in {"export_legacy_tsv"}:
+        from pycontrol.bundles import legacy
+
+        return getattr(legacy, name)
+    if name in {"create_protocol_bundle", "extract_protocol_bundle", "validate_protocol_bundle"}:
+        from pycontrol.bundles import protocol
+
+        return getattr(protocol, name)
+    if name in {"collect_task_files", "find_task_variables"}:
+        from pycontrol.bundles import collect
+
+        return getattr(collect, name)
+    if name in {"create_run_bundle", "extract_run_bundle", "validate_run_bundle"}:
+        from pycontrol.bundles import run
+
+        return getattr(run, name)
+    raise AttributeError(name)
+
+__all__ = [
+    "PROTOCOL_BUNDLE_TYPE",
+    "RUN_BUNDLE_TYPE",
+    "ArchiveError",
+    "ProtocolBundleManifest",
+    "RunBundleManifest",
+    "collect_task_files",
+    "create_protocol_bundle",
+    "create_run_bundle",
+    "export_legacy_tsv",
+    "extract_protocol_bundle",
+    "extract_run_bundle",
+    "find_task_variables",
+    "validate_protocol_bundle",
+    "validate_run_bundle",
+]

pycontrol/bundles/collect.py

@@ -0,0 +1,233 @@
+"""Collect workspace-local files needed to share a task setup."""
+
+from __future__ import annotations
+
+import ast
+from collections import defaultdict
+from pathlib import Path
+
+from pycontrol.workspace import Workspace
+
+SHARE_FOLDERS: tuple[str, ...] = (
+    "tasks",
+    "hardware_definitions",
+    "devices",
+    "plugins/task_extensions",
+    "plugins/experiment_extensions",
+    "plugins/task_controls",
+    "plugins/task_plotters",
+)
+
+TASK_VARIABLE_TO_FOLDER: dict[str, str] = {
+    "task_extension": "plugins/task_extensions",
+    "task_controls": "plugins/task_controls",
+    "task_plotter": "plugins/task_plotters",
+}
+
+PLUGIN_ROOTS: dict[str, str] = {
+    "experiment_extensions": "plugins/experiment_extensions",
+    "task_extensions": "plugins/task_extensions",
+    "task_controls": "plugins/task_controls",
+    "task_plotters": "plugins/task_plotters",
+}
+
+
+def collect_task_files(
+    workspace: Workspace,
+    task_stem: str,
+    hwdef_stem: str | Path | None = None,
+    experiment_extension_stem: str | Path | None = None,
+) -> dict[str, list[Path]]:
+    """Return files needed to share ``task_stem``, grouped by archive folder."""
+
+    collected: dict[str, set[Path]] = defaultdict(set)
+    task_path = workspace.resolve_task(task_stem)
+    task_folder, task_rel = _workspace_archive_path(workspace, task_path)
+    collected[task_folder].add(task_rel)
+
+    task_variables = find_task_variables(task_path)
+    for variable_name, value in task_variables.items():
+        if value is None:
+            continue
+        _collect_named_plugin(workspace, variable_name, value, collected)
+
+    class_to_device = _device_class_map(workspace.devices_dir)
+    _collect_used_devices(workspace, task_path, class_to_device, collected)
+
+    if hwdef_stem is not None:
+        hwdef_path = workspace.resolve_hwdef(hwdef_stem)
+        if hwdef_path is not None:
+            folder, rel_path = _workspace_archive_path(workspace, hwdef_path)
+            collected[folder].add(rel_path)
+            _collect_used_devices(workspace, hwdef_path, class_to_device, collected)
+
+    if experiment_extension_stem not in (None, ""):
+        exp_ext_path = workspace.resolve_experiment_extension(str(experiment_extension_stem))
+        if exp_ext_path is not None:
+            _collect_plugin_path(workspace, "plugins/experiment_extensions", exp_ext_path, collected)
+
+    return {folder: sorted(paths, key=lambda p: p.as_posix()) for folder, paths in collected.items() if paths}
+
+
+def find_task_variables(task_path: str | Path) -> dict[str, str | None]:
+    """Extract string-valued special ``v.*`` variables from a task file."""
+
+    tree = _parse_file(Path(task_path))
+    out: dict[str, str | None] = {}
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Assign):
+            continue
+        value = node.value.value if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str) else None
+        for target in node.targets:
+            if (
+                isinstance(target, ast.Attribute)
+                and isinstance(target.value, ast.Name)
+                and target.value.id == "v"
+                and target.attr in TASK_VARIABLE_TO_FOLDER
+            ):
+                out[target.attr] = value
+    return out
+
+
+def _collect_named_plugin(
+    workspace: Workspace, variable_name: str, plugin_stem: str, collected: dict[str, set[Path]]
+) -> None:
+    folder = TASK_VARIABLE_TO_FOLDER[variable_name]
+    candidates = _plugin_candidates(workspace, folder, plugin_stem)
+    for path in candidates:
+        if path.exists():
+            _collect_plugin_path(workspace, folder, path, collected)
+            return
+
+
+def _collect_plugin_path(workspace: Workspace, folder: str, path: Path, collected: dict[str, set[Path]]) -> None:
+    folder_key, rel_path = _workspace_archive_path(workspace, path)
+    if rel_path in collected[folder_key]:
+        return
+    collected[folder_key].add(rel_path)
+    if path.suffix != ".py":
+        return
+    for dep_folder, dep_stem in _plugin_imports(path, folder):
+        dep_candidates = _plugin_candidates(workspace, dep_folder, dep_stem)
+        for dep_path in dep_candidates:
+            if dep_path.exists():
+                _collect_plugin_path(workspace, dep_folder, dep_path, collected)
+                break
+
+
+def _plugin_candidates(workspace: Workspace, folder: str, stem: str) -> list[Path]:
+    base = workspace.root / folder
+    if folder == "plugins/task_controls":
+        return [base / f"{stem}.json", base / f"{stem}.py"]
+    return [base / f"{stem}.py"]
+
+
+def _plugin_imports(path: Path, current_folder: str) -> list[tuple[str, str]]:
+    tree = _parse_file(path)
+    deps: list[tuple[str, str]] = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ImportFrom) and node.module:
+            dep = (
+                (current_folder, node.module)
+                if node.level > 0 and "." not in node.module
+                else _plugin_dep_from_module(node.module)
+            )
+            if dep is not None:
+                deps.append(dep)
+        elif isinstance(node, ast.Import):
+            for alias in node.names:
+                dep = _plugin_dep_from_module(alias.name)
+                if dep is None and "." not in alias.name and (path.parent / f"{alias.name}.py").exists():
+                    dep = (current_folder, alias.name)
+                if dep is not None:
+                    deps.append(dep)
+        elif isinstance(node, ast.Call):
+            dep = _plugin_file_reference(path, current_folder, node)
+            if dep is not None:
+                deps.append(dep)
+    return deps
+
+
+def _plugin_dep_from_module(module_name: str) -> tuple[str, str] | None:
+    parts = module_name.split(".")
+    if len(parts) != 3 or parts[0] != "plugins":
+        return None
+    folder = PLUGIN_ROOTS.get(parts[1])
+    if folder is None:
+        return None
+    return folder, parts[2]
+
+
+def _plugin_file_reference(path: Path, current_folder: str, node: ast.Call) -> tuple[str, str] | None:
+    """Detect dynamic sibling loads like ``Path(__file__).with_name("helper.py")``."""
+
+    if not isinstance(node.func, ast.Attribute) or node.func.attr != "with_name":
+        return None
+    if len(node.args) != 1 or not isinstance(node.args[0], ast.Constant) or not isinstance(node.args[0].value, str):
+        return None
+    filename = node.args[0].value
+    if not filename.endswith(".py"):
+        return None
+    sibling = path.parent / filename
+    if not sibling.exists():
+        return None
+    return current_folder, sibling.stem
+
+
+def _device_class_map(devices_dir: Path) -> dict[str, Path]:
+    out: dict[str, Path] = {}
+    if not devices_dir.exists():
+        return out
+    for path in sorted(devices_dir.glob("*.py")):
+        if path.name.startswith("_") or path.name == "__init__.py":
+            continue
+        tree = _parse_file(path)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ClassDef):
+                out[node.name] = path
+    return out
+
+
+def _collect_used_devices(
+    workspace: Workspace, ref_path: Path, class_to_device: dict[str, Path], collected: dict[str, set[Path]]
+) -> None:
+    tree = _parse_file(ref_path)
+    local_classes = {node.name for node in ast.walk(tree) if isinstance(node, ast.ClassDef)}
+    referenced_names = {node.id for node in ast.walk(tree) if isinstance(node, ast.Name)}
+    for class_name in sorted(referenced_names - local_classes):
+        device_path = class_to_device.get(class_name)
+        if device_path is None:
+            continue
+        folder, rel_path = _workspace_archive_path(workspace, device_path)
+        if rel_path in collected[folder]:
+            continue
+        collected[folder].add(rel_path)
+        _collect_used_devices(workspace, device_path, class_to_device, collected)
+
+
+def _workspace_archive_path(workspace: Workspace, path: Path) -> tuple[str, Path]:
+    resolved = path.resolve()
+    rel = resolved.relative_to(workspace.root)
+    folder = _folder_for_relative_path(rel)
+    folder_path = Path(folder)
+    return folder, rel.relative_to(folder_path)
+
+
+def _folder_for_relative_path(rel_path: Path) -> str:
+    parts = rel_path.parts
+    if not parts:
+        raise ValueError("empty workspace-relative path")
+    if parts[0] == "plugins" and len(parts) >= 3:
+        folder = f"plugins/{parts[1]}"
+        if folder in SHARE_FOLDERS:
+            return folder
+    if parts[0] in SHARE_FOLDERS:
+        return parts[0]
+    raise ValueError(f"path is not in a shareable workspace folder: {rel_path}")
+
+
+def _parse_file(path: Path) -> ast.Module:
+    return ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+
+
+__all__ = ["SHARE_FOLDERS", "collect_task_files", "find_task_variables"]

pycontrol/bundles/legacy.py

@@ -0,0 +1,102 @@
+"""Compatibility exports from native run bundles to legacy loose files."""
+
+from __future__ import annotations
+
+import csv
+import json
+import shutil
+import zipfile
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+from pycontrol.recording.formatting import tsv_row
+
+from .run import validate_run_bundle
+
+
+def export_legacy_tsv(bundle_path: str | Path, out_dir: str | Path) -> Path:
+    """Extract a run bundle into old-style TSV and analog filenames."""
+
+    bundle = Path(bundle_path).resolve()
+    meta = validate_run_bundle(bundle)
+    destination = Path(out_dir).expanduser().resolve()
+    destination.mkdir(parents=True, exist_ok=True)
+    stem = _legacy_stem(bundle)
+    tsv_path = destination / f"{stem}.tsv"
+    with zipfile.ZipFile(bundle, "r") as zf:
+        events_text = zf.read(meta.data.events).decode("utf-8")
+        _write_legacy_tsv(events_text, tsv_path)
+        for analog in meta.data.analog:
+            time_path = destination / f"{stem}_{analog.channel}.time.npy"
+            data_path = destination / f"{stem}_{analog.channel}.data.npy"
+            with TemporaryDirectory() as tmp:
+                tmp_root = Path(tmp)
+                extracted_times = tmp_root / "times.npy"
+                extracted_samples = tmp_root / "samples.npy"
+                extracted_times.write_bytes(zf.read(analog.times))
+                extracted_samples.write_bytes(zf.read(analog.samples))
+                shutil.copy2(extracted_times, time_path)
+                shutil.copy2(extracted_samples, data_path)
+    return tsv_path
+
+
+def _write_legacy_tsv(events_text: str, tsv_path: Path) -> None:
+    with tsv_path.open("w", encoding="utf-8", newline="\n") as out:
+        out.write(tsv_row("type", "time", "subtype", "content"))
+        reader = csv.DictReader(events_text.splitlines(), delimiter="\t")
+        for row in reader:
+            out.write(_legacy_row(row))
+
+
+def _legacy_row(row: dict[str, str]) -> str:
+    timestamp_ms = round(float(row["time"]) * 1000)
+    kind = row["kind"]
+    name = row["name"]
+    value = _decode_json(row["value_json"])
+    source = row["source"]
+    if kind == "info":
+        return tsv_row("info", timestamp_ms, name, _legacy_value(value))
+    if kind == "state":
+        return tsv_row("state", timestamp_ms, content=name)
+    if kind == "event":
+        return tsv_row("event", timestamp_ms, content=name)
+    if kind == "variable":
+        payload = {name: value} if name and name not in {"run_start", "run_end"} else value
+        return tsv_row("variable", timestamp_ms, name, json.dumps(payload, ensure_ascii=False))
+    if kind == "print":
+        return tsv_row("print", timestamp_ms, _legacy_print_subtype(source), _legacy_value(value))
+    if kind == "warning":
+        return tsv_row("warning", timestamp_ms, content=_legacy_value(value))
+    if kind == "error":
+        return tsv_row("error", timestamp_ms, content=_legacy_value(value))
+    return tsv_row(kind, timestamp_ms, name, _legacy_value(value))
+
+
+def _decode_json(value: str) -> Any:
+    try:
+        return json.loads(value)
+    except json.JSONDecodeError:
+        return value
+
+
+def _legacy_value(value: Any) -> str:
+    return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+
+
+def _legacy_print_subtype(source: str) -> str:
+    if source == "task_extension":
+        return "api"
+    if source == "operator":
+        return "user"
+    return "task"
+
+
+def _legacy_stem(bundle: Path) -> str:
+    name = bundle.name
+    if name.endswith(".pycontrol.zip"):
+        return name.removesuffix(".pycontrol.zip")
+    return bundle.stem
+
+
+__all__ = ["export_legacy_tsv"]

pycontrol/bundles/models.py

@@ -0,0 +1,200 @@
+"""Pydantic models for self-contained pyControl bundle metadata."""
+
+from __future__ import annotations
+
+from pathlib import PurePosixPath
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from pycontrol.provenance.models import ProtocolManifest, ProvenanceFile, RunManifest
+
+from .zipio import validate_archive_member
+
+PROTOCOL_BUNDLE_TYPE = "pycontrol.protocol_bundle.v1"
+RUN_BUNDLE_TYPE = "pycontrol.run_bundle.v1"
+SCHEMA_VERSION = 1
+
+ALLOWED_PROTOCOL_FILE_FOLDERS: frozenset[str] = frozenset(
+    {
+        "tasks",
+        "hardware_definitions",
+        "devices",
+        "assets",
+        "plugins/task_extensions",
+        "plugins/experiment_extensions",
+        "plugins/task_controls",
+        "plugins/task_plotters",
+    }
+)
+
+ALLOWED_RUN_BUNDLE_ROOTS: frozenset[str] = frozenset({"data", "files"})
+
+_FOLDER_ROLES: dict[str, str] = {
+    "tasks": "task",
+    "hardware_definitions": "hardware_definition",
+    "devices": "workspace_device",
+    "assets": "asset",
+    "plugins/task_extensions": "task_extension",
+    "plugins/experiment_extensions": "experiment_extension",
+    "plugins/task_controls": "task_control",
+    "plugins/task_plotters": "task_plotter",
+}
+
+
+class BundleInfo(BaseModel):
+    """Common metadata for a pyControl bundle."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    type: Literal["pycontrol.protocol_bundle.v1", "pycontrol.run_bundle.v1"]
+    schema_version: Literal[1] = 1
+    pycontrol_core_version: str
+    pycontrol_gui_version: str | None = None
+    created_at: str
+
+
+class ProtocolBundleManifest(BaseModel):
+    """Metadata stored at the root of a shareable protocol bundle."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    bundle: BundleInfo
+    task_name: str
+    description: str = ""
+    files: list[ProvenanceFile] = Field(default_factory=list)
+    task_variables: dict[str, str | None] = Field(default_factory=dict)
+    provenance: ProtocolManifest | None = None
+
+    @field_validator("files")
+    @classmethod
+    def validate_files(cls, files: list[ProvenanceFile]) -> list[ProvenanceFile]:
+        for file in files:
+            validate_archive_member(file.archive_path, allowed_roots=ALLOWED_PROTOCOL_FILE_FOLDERS)
+        return files
+
+    def files_by_folder(self) -> dict[str, list[str]]:
+        grouped: dict[str, list[str]] = {}
+        for file in self.files:
+            folder, rel_path = split_protocol_archive_path(file.archive_path)
+            grouped.setdefault(folder, []).append(rel_path)
+        return {folder: sorted(paths) for folder, paths in sorted(grouped.items())}
+
+    @property
+    def bundle_type(self) -> str:
+        return self.bundle.type
+
+    @property
+    def schema_version(self) -> int:
+        return self.bundle.schema_version
+
+    @property
+    def pycontrol_core_version(self) -> str:
+        return self.bundle.pycontrol_core_version
+
+    @property
+    def pycontrol_gui_version(self) -> str | None:
+        return self.bundle.pycontrol_gui_version
+
+    @property
+    def exported_at(self) -> str:
+        return self.bundle.created_at
+
+
+class AnalogDataFile(BaseModel):
+    """Internal paths for one analog channel payload inside a run bundle."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    channel: str
+    times: str
+    samples: str
+    sampling_rate_hz: float | None = None
+    dtype: str | None = None
+
+    @field_validator("times", "samples")
+    @classmethod
+    def validate_data_path(cls, value: str) -> str:
+        validate_archive_member(value, allowed_roots={"data"})
+        return value
+
+
+class RunBundleData(BaseModel):
+    """Internal paths for data payloads inside a run bundle."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    events: str
+    analog: list[AnalogDataFile] = Field(default_factory=list)
+
+    @field_validator("events")
+    @classmethod
+    def validate_events_path(cls, value: str) -> str:
+        validate_archive_member(value, allowed_roots={"data"})
+        return value
+
+
+class RunBundleManifest(BaseModel):
+    """Root metadata for a completed run bundle."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    bundle: BundleInfo
+    run: RunManifest
+    data: RunBundleData
+    protocol: ProtocolManifest
+
+    @property
+    def bundle_type(self) -> str:
+        return self.bundle.type
+
+    @property
+    def schema_version(self) -> int:
+        return self.bundle.schema_version
+
+    @property
+    def pycontrol_core_version(self) -> str:
+        return self.bundle.pycontrol_core_version
+
+    @property
+    def created_at(self) -> str:
+        return self.bundle.created_at
+
+    @property
+    def run_id(self) -> str:
+        return self.run.run_id
+
+    @property
+    def protocol_hash(self) -> str:
+        return self.protocol.protocol_hash
+
+    @property
+    def task_name(self) -> str:
+        for file in self.protocol.files:
+            if file.role == "task":
+                _folder, rel_path = split_protocol_archive_path(file.archive_path)
+                return PurePosixPath(rel_path).with_suffix("").as_posix()
+        return ""
+
+
+def split_protocol_archive_path(name: str) -> tuple[str, str]:
+    for folder in sorted(ALLOWED_PROTOCOL_FILE_FOLDERS, key=len, reverse=True):
+        prefix = f"{folder}/"
+        if name.startswith(prefix):
+            return folder, name.removeprefix(prefix)
+    raise ValueError(f"archive member is outside allowed folders: {name}")
+
+
+__all__ = [
+    "ALLOWED_PROTOCOL_FILE_FOLDERS",
+    "ALLOWED_RUN_BUNDLE_ROOTS",
+    "PROTOCOL_BUNDLE_TYPE",
+    "RUN_BUNDLE_TYPE",
+    "SCHEMA_VERSION",
+    "AnalogDataFile",
+    "BundleInfo",
+    "ProtocolBundleManifest",
+    "RunBundleData",
+    "RunBundleManifest",
+    "split_protocol_archive_path",
+]