playmolecule 2.7.2__tar.gz → 2.7.4__tar.gz

{playmolecule-2.7.2/playmolecule.egg-info → playmolecule-2.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: playmolecule
-Version: 2.7.2
+Version: 2.7.4
 Summary: PlayMolecule python API
 Author-email: Acellera <info@acellera.com>
 Project-URL: Homepage, https://github.com/Acellera/playmolecule

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule/__init__.py

@@ -15,7 +15,7 @@ from playmolecule._protocols import __find_protocols_by_import, __find_protocols
 from playmolecule._config import _get_config
 from playmolecule._public_api import JobStatus, ExecutableDirectory, describe_apps
 from playmolecule._backends._http import login, logout, _check_login_status
-from playmolecule._dump import dump_manifests
+from playmolecule._dump import dump_api
 
 # from importlib.resources import files
 

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule/_backends/_docker.py

@@ -773,10 +773,33 @@ def _update_docker_apps_from_gcloud(
             tag = image_info["full_tag"]
             try:
                 # Check if image exists locally
-                docker_client.images.get(tag)
-                # If we get here, the image exists locally, so update it
-                logger.info(f"Updating existing image: {tag} ...")
-                docker_client.images.pull(tag)
+                local_image = docker_client.images.get(tag)
+                # Image exists locally - compare local vs remote digest first so
+                # we only pull (and announce) the images that actually changed.
+                logger.debug(f"Checking {tag} for updates ...")
+                try:
+                    remote_digest = backend._get_remote_digest(
+                        image_info["project_id"],
+                        image_info["location"],
+                        image_info["repo"],
+                        image_info["image"],
+                        image_info["tag"],
+                    )
+                    repo_digests = local_image.attrs.get("RepoDigests", [])
+                    up_to_date = bool(remote_digest) and any(
+                        remote_digest in rd for rd in repo_digests
+                    )
+                except Exception as e:
+                    # Couldn't determine the remote digest - fall back to pulling
+                    logger.debug(f"Could not check remote digest for {tag}: {e}")
+                    up_to_date = False
+
+                if up_to_date:
+                    logger.info(f"Skipping {tag} - already up to date")
+                else:
+                    logger.info(f"Updating existing image: {tag} ...")
+                    docker_client.images.pull(tag)
+                    logger.info(f"Successfully updated {tag}")
             except docker.errors.ImageNotFound:
                 # Image doesn't exist locally
                 if pull_new:

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule/_backends/_http.py

@@ -604,7 +604,7 @@ class _HTTPExecutionBackend:
         job_id: Optional[str] = None,
         prefix: Optional[str] = None,
         _logger: bool = True,
-    ) -> str:
+    ) -> tuple[str, Optional[str]]:
         """Run the app via HTTP API.
 
         Parameters
@@ -622,8 +622,8 @@ class _HTTPExecutionBackend:
 
         Returns
         -------
-        str
-            Job ID
+        tuple[str, str | None]
+            The ``(job_id, run_id)`` pair assigned by the server.
         """
         config = _get_config()
         session = self._session.session
@@ -668,7 +668,8 @@ class _HTTPExecutionBackend:
 
             if res.status_code != 200:
                 raise RuntimeError(f"Failed to run job on {self.base_url}: {res.text}")
-            return res.json()["job_id"]
+            data = res.json()
+            return data["job_id"], data.get("run_id")
         finally:
             # Always close file handles
             for fh in file_handles:
@@ -677,7 +678,9 @@ class _HTTPExecutionBackend:
                 except Exception as e:
                     logger.warning(f"Failed to close file handle: {e}")
 
-    def get_status(self, dirname: str, job_id: str) -> JobStatus:
+    def get_status(
+        self, dirname: str, job_id: str, run_id: str | None = None
+    ) -> JobStatus:
         """Get job status from HTTP API.
 
         Parameters
@@ -686,6 +689,10 @@ class _HTTPExecutionBackend:
             Output directory (for downloading results)
         job_id : str
             Job ID
+        run_id : str, optional
+            When given, narrows the query to the single run with this id
+            (the ``(job_id, run_id)`` pair is the unique key); otherwise the
+            prefix-only query is used.
 
         Returns
         -------
@@ -699,8 +706,11 @@ class _HTTPExecutionBackend:
         try:
             headers = _get_headers()
             headers["Connection"] = "close"
+            url = f"{self.base_url}/jobs?prefix={job_id}"
+            if run_id is not None:
+                url += f"&run_id={run_id}"
             res = session.get(
-                f"{self.base_url}/jobs?prefix={job_id}",
+                url,
                 timeout=_DEFAULT_TIMEOUT,
                 headers=headers,
             )

playmolecule-2.7.4/playmolecule/_dump.py

@@ -0,0 +1,198 @@
+# (c) 2015-2023 Acellera Ltd http://www.acellera.com
+# All Rights Reserved
+# Distributed under HTMD Software License Agreement
+# No redistribution in whole or part
+#
+"""Dump app API reference docs for the agent sandbox.
+
+Introspects the live generated functions (build info is never touched).
+Requires discovery — run without PM_DEFER_DISCOVERY.
+"""
+import inspect
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def _fenced(code: str) -> str:
+    """Wrap code in a python-fenced block."""
+    return "```python\n" + code.rstrip() + "\n```"
+
+
+def _render_function_block(func) -> str:
+    """`def <name><signature>:` + the function's __doc__, as a python fence."""
+    sig = str(inspect.signature(func))
+    # Render Path annotations cleanly (Python 3.13 exposes pathlib._local.Path).
+    sig = sig.replace("pathlib._local.Path", "Path").replace("pathlib.Path", "Path")
+    lines = [f"def {func.__name__}{sig}:"]
+    doc = (func.__doc__ or "").rstrip()
+    if doc:
+        lines.append('    """')
+        lines.extend(("    " + dl).rstrip() for dl in doc.splitlines())
+        lines.append('    """')
+    lines.append("    ...")
+    return _fenced("\n".join(lines))
+
+
+def _arg_repr(value) -> str:
+    """Render a test argument. Bundled files become a clear placeholder; other
+    values use repr (so strings get quotes, numbers stay bare). Recurses into
+    dict/list/tuple so nested _File objects never leak their internal .path."""
+    # Deferred import to avoid a circular import at module load time.
+    from playmolecule._appfiles import _File
+
+    if isinstance(value, _File):
+        return f"<file {value.name!r}>"
+    if isinstance(value, dict):
+        inner = ", ".join(
+            f"{_arg_repr(k)}: {_arg_repr(v)}" for k, v in value.items()
+        )
+        return "{" + inner + "}"
+    if isinstance(value, (list, tuple)):
+        inner = ", ".join(_arg_repr(item) for item in value)
+        if isinstance(value, tuple):
+            return "(" + inner + ("," if len(value) == 1 else "") + ")"
+        return "[" + inner + "]"
+    return repr(value)
+
+
+def _render_tests(func) -> str:
+    """A python fence of `# test: <name>` + a concrete call per func.tests entry.
+    Empty string when the function has no tests."""
+    tests = getattr(func, "tests", None)
+    items = vars(tests) if tests is not None else {}
+    if not items:
+        return ""
+    blocks = []
+    for test in items.values():
+        args = (getattr(test, "config", {}) or {}).get("arguments", {})
+        argstr = ", ".join(f"{k}={_arg_repr(v)}" for k, v in args.items())
+        blocks.append(f"# test: {test.name}\n{func.__name__}({argstr})")
+    return _fenced("\n".join(blocks))
+
+
+def _render_datasets(app: str, module) -> str:
+    """Markdown bullets of module.datasets (accessor name + description), with a
+    header showing how to reference them. Empty string when the app declares
+    none."""
+    ds = getattr(module, "datasets", None)
+    items = vars(ds) if ds is not None else {}
+    if not items:
+        return ""
+    lines = [f"Datasets (reference as `{app}.datasets.<name>`):", ""]
+    for name, f in items.items():
+        desc = getattr(f, "description", None)
+        lines.append(f"- `{name}`" + (f" — {desc}" if desc else ""))
+    return "\n".join(lines)
+
+
+def _render_app_md(app: str, version: str, module, funcs) -> str:
+    """Assemble the full <app>/<vN>.md body. Sections are omitted when empty.
+    Each function is rendered independently; a broken one is logged and skipped
+    so sibling functions still appear."""
+    parts = [f"# {app} · {version}", ""]
+    rendered_funcs = []
+    for func in funcs:
+        try:
+            parts.append(_render_function_block(func))
+            parts.append("")
+            rendered_funcs.append(func)
+        except Exception as e:
+            logger.warning(
+                f"Skipping function {getattr(func, '__name__', repr(func))} "
+                f"in {app}.{version}: {e}"
+            )
+    # Usage: unversioned import (resolves to latest), per prompt guidance.
+    # Base the example on the first successfully rendered function, not funcs[0].
+    first = rendered_funcs[0].__name__ if rendered_funcs else app
+    parts.append("Usage:")
+    parts.append(_fenced(
+        f"from playmolecule.apps.{app}.{version} import {first}\n{first}(...).run()"
+    ))
+    parts.append("")
+    test_blocks = []
+    for func in rendered_funcs:
+        try:
+            t = _render_tests(func)
+            if t:
+                test_blocks.append(t)
+        except Exception as e:
+            logger.warning(
+                f"Skipping tests for {getattr(func, '__name__', repr(func))} "
+                f"in {app}.{version}: {e}"
+            )
+    tests = "\n\n".join(test_blocks)
+    if tests:
+        parts.append("Tested invocations:")
+        parts.append(tests)
+        parts.append("")
+    datasets = _render_datasets(app, module)
+    if datasets:
+        parts.append(datasets)
+        parts.append("")
+    parts.append(f"> Tutorials: ./{version}/docs/")
+    return "\n".join(parts).rstrip() + "\n"
+
+
+def dump_api(out_dir: str, registry: str = None) -> None:
+    """Write a markdown API reference for the agent sandbox: one
+    ``<app>/<vN>.md`` per app version (signatures + docstrings + tested
+    invocations + datasets) and ``<app>/<vN>/docs/<name>.md`` tutorial
+    sidecars. Introspects the *generated functions* (build info is never
+    touched). Requires discovery — run without PM_DEFER_DISCOVERY. Never raises
+    on a single bad app; logs and skips."""
+    import sys
+    from playmolecule.apps import _function_dict, _find_apps
+
+    if registry is None:
+        from playmolecule._config import _get_config
+
+        regs = _get_config().registries or []
+    else:
+        regs = [registry]
+
+    # Discovery: only run when the function tree is not yet populated.
+    if not _function_dict and regs:
+        try:
+            _find_apps(regs)
+        except Exception as e:
+            logger.error(f"Discovery failed; nothing dumped: {e}")
+            return
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    # Group generated functions by (app, version).
+    by_version = {}
+    prefix = "playmolecule.apps."
+    for fpath, func in _function_dict.items():
+        rest = fpath[len(prefix):] if fpath.startswith(prefix) else fpath
+        bits = rest.split(".")
+        if len(bits) != 3:
+            continue
+        app, version, _fn = bits
+        by_version.setdefault((app, version), []).append(func)
+
+    for (app, version), funcs in by_version.items():
+        module_path = f"{prefix}{app}.{version}"
+        module = sys.modules.get(module_path)
+        try:
+            md = _render_app_md(app, version, module, funcs)
+            os.makedirs(os.path.join(out_dir, app), exist_ok=True)
+            with open(os.path.join(out_dir, app, f"{version}.md"), "w") as f:
+                f.write(md)
+        except Exception as e:
+            logger.error(f"Failed to write API reference for {app}.{version}: {e}")
+            continue
+
+        # Tutorials sidecars from module.docs ({name: markdown}); names may nest.
+        docs = getattr(module, "docs", None) or {}
+        docs_dir = os.path.join(out_dir, app, version, "docs")
+        for name, content in docs.items():
+            try:
+                path = os.path.join(docs_dir, name)
+                os.makedirs(os.path.dirname(path), exist_ok=True)
+                with open(path, "w") as f:
+                    f.write(content)
+            except Exception as e:
+                logger.error(f"Failed to write doc {name} for {app}.{version}: {e}")

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule/_identity.py

@@ -20,18 +20,32 @@ def app_identity(manifest: dict):
     """Return ``(app_lower, "vN")`` for a manifest, or ``None`` (logged) if it
     has no usable identity. Never raises.
 
-    Container-built manifests carry ``name``/``version`` under
-    ``container_config``; local (flat) manifests carry them at the top level.
-    Read from ``container_config`` when present, otherwise from the top level.
+    ``name``/``version`` may live under ``container_config`` (container-built
+    manifests) or at the top level. Local manifests keep them at the top level
+    and use ``container_config`` only for build directives
+    (``appfunction``/``condaenvs``/``base_image``/...). Each field is read from
+    ``container_config`` when present there, otherwise from the top level.
     """
+    if not isinstance(manifest, dict):
+        logger.error(
+            "Skipping app with unusable identity in manifest: expected dict, got "
+            f"{type(manifest).__name__}"
+        )
+        return None
+    cc = manifest.get("container_config")
+    cc = cc if isinstance(cc, dict) else {}
+    name = cc.get("name", manifest.get("name"))
+    version = cc.get("version", manifest.get("version"))
+    if name is None or version is None:
+        missing = "name" if name is None else "version"
+        logger.error(f"Skipping app with unusable identity in manifest: {missing!r}")
+        return None
     try:
-        cc = manifest["container_config"] if "container_config" in manifest else manifest
-        app = str(cc["name"]).lower()
-        major = int(cc["version"])
-    except (KeyError, TypeError, ValueError) as e:
+        major = int(version)
+    except (TypeError, ValueError) as e:
         logger.error(f"Skipping app with unusable identity in manifest: {e}")
         return None
-    return app, f"v{major}"
+    return str(name).lower(), f"v{major}"
 
 
 def func_name(function_ref: str, appname: str) -> str:

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule/_public_api.py

@@ -45,23 +45,32 @@ class ExecutableDirectory:
         self.dirname = dirname
 
         if inputs_dir is None:
-            # Find all the run_ folders in the directory and use the most recent one
-            run_folders = [
-                f
-                for f in os.listdir(dirname)
-                if f.startswith("run_") and os.path.isdir(os.path.join(dirname, f))
-            ]
-            run_folders.sort(key=lambda x: os.path.getmtime(os.path.join(dirname, x)))
-            inputs_dir = os.path.join(dirname, run_folders[-1])
-
-        self.runsh = os.path.basename(inputs_dir) + ".sh"
+            # Reconstructed from a path: use the most-recent run_ folder if the
+            # directory exists and has one; otherwise leave it unresolved
+            # (None) rather than crashing — status can still fall back to the
+            # prefix-only query.
+            run_folders = []
+            if os.path.isdir(dirname):
+                run_folders = [
+                    f
+                    for f in os.listdir(dirname)
+                    if f.startswith("run_") and os.path.isdir(os.path.join(dirname, f))
+                ]
+                run_folders.sort(key=lambda x: os.path.getmtime(os.path.join(dirname, x)))
+            inputs_dir = os.path.join(dirname, run_folders[-1]) if run_folders else None
+
         self.inputs_dir = inputs_dir
+        self.runsh = (os.path.basename(inputs_dir) + ".sh") if inputs_dir else None
 
         self.execution_resources = _execution_resources
         self.slurmq = None
         self._input_json = _input_json
         self._execution_backend = _execution_backend
+        # job_id and run_id are recovered lazily in `status` for a reconstructed
+        # directory (job_id needs the configured prefix; run_id comes from the
+        # run folder). A fresh `.run()` sets both from the server response.
         self._job_id = None
+        self._run_id = None
 
     def _get_backend(self):
         """Get the execution backend, from stored value or config."""
@@ -93,14 +102,19 @@ class ExecutableDirectory:
         if isinstance(backend, _HTTPExecutionBackend):
             if self._job_id is None:
                 # ExecutableDirectory was reconstructed from a path (e.g. by a
-                # protocol resuming from disk), so the in-memory job_id from
-                # .run() is gone. Recover the full id (prefix + relpath) so
-                # the status prefix-query matches what the server stored at
-                # submission time.
+                # protocol resuming from disk), so the in-memory ids from
+                # .run() are gone. Recover the job_id (prefix + relpath) so the
+                # status query matches what the server stored at submission, and
+                # the run_id from the most-recent run folder (the server lays
+                # results out under {run_id}/) so the query resolves a single
+                # run. If neither can be determined, fall back to the
+                # prefix-only query.
                 self._job_id = _derive_job_id_from_dirname(
                     self.dirname, include_prefix=True
                 )
-            return backend.get_status(self.dirname, self._job_id)
+                run_name = os.path.basename(self.inputs_dir) if self.inputs_dir else ""
+                self._run_id = run_name if run_name.startswith("run_") else None
+            return backend.get_status(self.dirname, self._job_id, self._run_id)
         else:
             # Local backend
             return backend.get_status(self.dirname, self.inputs_dir, self.slurmq)
@@ -149,7 +163,9 @@ class ExecutableDirectory:
 
         # HTTP backend
         if isinstance(backend, _HTTPExecutionBackend):
-            self._job_id = backend.run(self.dirname, self._input_json, **kwargs)
+            self._job_id, self._run_id = backend.run(
+                self.dirname, self._input_json, **kwargs
+            )
             return
 
         # Local execution

{playmolecule-2.7.2 → playmolecule-2.7.4/playmolecule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: playmolecule
-Version: 2.7.2
+Version: 2.7.4
 Summary: PlayMolecule python API
 Author-email: Acellera <info@acellera.com>
 Project-URL: Homepage, https://github.com/Acellera/playmolecule

{playmolecule-2.7.2 → playmolecule-2.7.4}/playmolecule.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ playmolecule/apps/__init__.py
 playmolecule/share/apptainer_run.sh
 playmolecule/share/docker_run.sh
 tests/3ptb.pdb
+tests/bench_dump_api.py
 tests/conftest.py
 tests/test_app_docs_namespace.py
 tests/test_apps_linking.py
@@ -82,7 +83,7 @@ tests/test_defer_discovery.py
 tests/test_describe_apps.py
 tests/test_docker_backend.py
 tests/test_docker_docs_split.py
-tests/test_dump_manifests.py
+tests/test_dump_api.py
 tests/test_find_apps_resilient.py
 tests/test_func_name_lowercase.py
 tests/test_get_app_list_channels.py

playmolecule-2.7.4/tests/bench_dump_api.py

@@ -0,0 +1,92 @@
+"""Benchmark dump_api against the live v3 catalog. Run manually:
+    PM_DEFER_DISCOVERY=1 .venv/bin/python tests/bench_dump_api.py
+Prints http / parse / structure / build / serialize+write / total seconds.
+
+The fetch is broken down into raw HTTP transfer, JSON parse, and the
+identity-keying "structure" step (the latter mirrors what
+``_HTTPManifestBackend.get_apps`` does after fetch+parse) so each cost is
+visible separately instead of hidden inside ``get_apps_and_docs``.
+"""
+import os
+import sys
+import time
+import tempfile
+
+REGISTRY = "https://open.playmolecule.org/v3"
+
+
+def main():
+    os.environ["PM_DEFER_DISCOVERY"] = "1"  # import without discovery; we time it
+    from playmolecule._backends import _get_manifest_backend
+    from playmolecule._backends._http import _get_headers, _DEFAULT_TIMEOUT
+    from playmolecule._identity import app_identity
+    from playmolecule.apps import _manifest_to_func, _function_dict
+    from playmolecule import dump_api
+
+    backend = _get_manifest_backend(REGISTRY)
+    sess = backend._session.session
+    pc = time.perf_counter
+
+    # --- /apps/manifests: raw HTTP transfer, then JSON parse ---
+    t0 = pc()
+    rsp_m = sess.get(
+        f"{backend.base_url}/apps/manifests",
+        headers=_get_headers(),
+        timeout=_DEFAULT_TIMEOUT,
+    )
+    t1 = pc()
+    manifests_json = rsp_m.json()
+    t2 = pc()
+
+    # --- /apps/docs: raw HTTP transfer, then JSON parse ---
+    rsp_d = sess.get(
+        f"{backend.base_url}/apps/docs",
+        headers=_get_headers(),
+        timeout=_DEFAULT_TIMEOUT,
+    )
+    t3 = pc()
+    docs = rsp_d.json() if rsp_d.status_code == 200 else {}
+    t4 = pc()
+
+    # --- structure: app_identity keying (what get_apps does beyond fetch+parse) ---
+    apps = {}
+    for _key, manifest in manifests_json.items():
+        identity = app_identity(manifest)
+        if identity is None:
+            continue
+        app_name, version = identity
+        apps.setdefault(app_name, {})[version] = {
+            "manifest": manifest,
+            "appdir": None,
+            "run.sh": None,
+            "files": backend._get_app_files(manifest),
+        }
+    t5 = pc()
+
+    # --- build the dynamic function tree ---
+    for appname in apps:
+        _manifest_to_func(appname, apps[appname])
+        for version in apps[appname]:
+            vmod = sys.modules.get(f"playmolecule.apps.{appname}.{version}")
+            if vmod is not None:
+                setattr(vmod, "docs", docs.get(f"{appname}.{version}", {}))
+    t6 = pc()
+
+    # --- serialize + write (tree already built; dump_api skips re-discovery) ---
+    with tempfile.TemporaryDirectory() as out:
+        dump_api(out, registry=REGISTRY)
+    t7 = pc()
+
+    http = (t1 - t0) + (t3 - t2)
+    parse = (t2 - t1) + (t4 - t3)
+    print(f"apps: {len(apps)}  functions: {len(_function_dict)}")
+    print(f"http           : {http:.3f}s  (manifests {t1 - t0:.3f}s + docs {t3 - t2:.3f}s)")
+    print(f"parse          : {parse:.3f}s  (manifests {t2 - t1:.3f}s + docs {t4 - t3:.3f}s)")
+    print(f"structure      : {t5 - t4:.3f}s")
+    print(f"build          : {t6 - t5:.3f}s")
+    print(f"serialize+write: {t7 - t6:.3f}s")
+    print(f"total          : {t7 - t0:.3f}s")
+
+
+if __name__ == "__main__":
+    main()