ellf-cli 6.1.2__tar.gz → 6.1.4__tar.gz

{ellf_cli-6.1.2/ellf_cli.egg-info → ellf_cli-6.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ellf-cli
-Version: 6.1.2
+Version: 6.1.4
 Summary: Ellf Command Line Interface
 Home-page: https://prodi.gy
 Author: ExplosionAI GmbH

{ellf_cli-6.1.2 → ellf_cli-6.1.4}/ellf_cli/about.json

@@ -1,7 +1,7 @@
 {
   "title": "Ellf CLI",
   "name": "ellf-cli",
-  "version": "6.1.2",
+  "version": "6.1.4",
   "summary": "Ellf Command Line Interface",
   "uri": "https://prodi.gy",
   "prog": "ellf",

ellf_cli-6.1.4/ellf_cli/commands/publish_data.py

@@ -0,0 +1,147 @@
+import json
+import zipfile
+from pathlib import Path
+from typing import Optional
+from uuid import UUID
+
+from radicli import Arg
+
+from ..cli import cli
+from ..errors import CLIError
+from ..messages import Messages
+from . import assets as assets_commands
+from . import files as files_commands
+from ._state import get_auth_state
+
+
+def _read_spacy_model_meta_from_wheel(wheel_path: Path) -> dict:
+    """Extract a spaCy model's ``meta.json`` from a wheel built by
+    ``spacy package``.
+
+    A spaCy model wheel lays out the model under
+    ``<pkg_name>/<pkg_name>-<version>/meta.json``. We don't hard-code
+    that layout; we just look for any ``meta.json`` inside the wheel
+    that has the keys we need. Returns the parsed dict.
+
+    Raises ``CLIError`` if the wheel isn't a spaCy model package.
+    """
+    try:
+        with zipfile.ZipFile(wheel_path) as zf:
+            candidates = [n for n in zf.namelist() if n.endswith("/meta.json")]
+            # Prefer the deepest meta.json (inside the model data dir, not
+            # next to __init__.py) since older spaCy versions also write a
+            # stub meta.json at the package root.
+            candidates.sort(key=lambda n: n.count("/"), reverse=True)
+            for name in candidates:
+                with zf.open(name) as fp:
+                    meta = json.loads(fp.read())
+                if isinstance(meta, dict) and meta.get("lang") and meta.get("name"):
+                    return meta
+    except (zipfile.BadZipFile, KeyError, json.JSONDecodeError) as e:
+        raise CLIError(
+            f"Could not read spaCy model metadata from {wheel_path}: {e}. "
+            "Is this a wheel produced by `spacy package`?"
+        ) from e
+    raise CLIError(
+        f"{wheel_path} does not look like a spaCy model wheel: no meta.json "
+        "with `lang` and `name` fields found inside."
+    )
+
+
+@cli.subcommand(
+    "publish",
+    "data",
+    src=Arg(help="File or directory to publish"),
+    dest=Arg(help="Destination path to copy the data to"),
+    name=Arg("--name", help=Messages.name.format(noun="asset")),
+    version=Arg("--version", help=Messages.version.format(noun="asset")),
+    kind=Arg(
+        "--kind",
+        help=Messages.asset_kind,
+    ),
+    loader=Arg("--loader", help="Loader to convert data for Prodigy"),
+    meta=Arg("--meta", help=Messages.asset_meta),
+    exists_ok=Arg("--exists-ok", help=Messages.exists_ok),
+    # fmt: on
+)
+def publish_data(
+    src: Path,
+    kind: str,
+    dest: Optional[str] = None,
+    name: Optional[str] = None,
+    version: Optional[str] = None,
+    loader: Optional[str] = None,
+    meta: str = "{}",
+    exists_ok: bool = False,
+) -> UUID | None:
+    """
+    Transfer data to the cluster, and advertise it to Ellf.
+
+    These steps can also be done separately. See `ellf files` to transfer
+    data to the cluster without creating an Asset record for it, and
+    `ellf assets create` to create an Asset without transfer.
+    """
+    auth = get_auth_state()
+    if auth.broker_host is None:
+        raise CLIError(Messages.E035)
+    if auth.cluster_id is None:
+        raise CLIError(Messages.E036)
+
+    is_model_wheel = kind == "model" and src.is_file() and src.suffix == ".whl"
+    wheel_meta: Optional[dict] = None
+    if is_model_wheel:
+        wheel_meta = _read_spacy_model_meta_from_wheel(src)
+        # Importable name is `<lang>_<name>` by spaCy's package convention.
+        # `spacy.load(spacy_model_name)` resolves to it via setuptools entry
+        # points after the wheel is pip-installed inside the recipe pod.
+        if name is None:
+            name = f"{wheel_meta['lang']}_{wheel_meta['name']}"
+        if version is None:
+            version = wheel_meta.get("version", "0.0.0")
+    if name is None:
+        name = src.name
+    if dest is None:
+        if version is None:
+            filename = name
+        else:
+            filename = f"{name}-{version}"
+        # The wheel must keep its on-disk filename so pip can identify it
+        # by the canonical `<distname>-<version>-…whl` pattern at install
+        # time; otherwise pip refuses to install. The Asset record can
+        # still use a custom name/version; only the destination filename
+        # is constrained.
+        if is_model_wheel:
+            dest = f"{{__nfs__}}/data/{src.name}"
+        else:
+            dest = f"{{__nfs__}}/data/{filename}"
+    meta_json = json.loads(meta)
+    if loader is not None:
+        meta_json["loader"] = loader
+    if wheel_meta is not None:
+        meta_json["format"] = "package"
+        meta_json["spacy_model_name"] = f"{wheel_meta['lang']}_{wheel_meta['name']}"
+        meta_json["spacy_version"] = wheel_meta.get("spacy_version", "")
+        meta_json["lang"] = wheel_meta["lang"]
+        meta_json["model_version"] = wheel_meta.get("version", "")
+    meta = json.dumps(meta_json)
+    if version is None:
+        version = "0.0.0"
+
+    files_commands.cp(
+        str(src),
+        dest=dest,
+        overwrite=True,
+        make_dirs=True,
+        recurse=True,
+        cluster_host=auth.broker_host,
+    )
+    assets_commands.create(
+        kind=kind,
+        name=name,
+        path=dest,
+        version=version,
+        meta=meta,
+        exists_ok=exists_ok,
+    )
+    # TODO: We should also have an option to create a dataset
+    # from the asset, by triggering a 'db-in' action.

{ellf_cli-6.1.2 → ellf_cli-6.1.4}/ellf_cli/ellf.json

@@ -1,7 +1,7 @@
 {
   "prog": "ellf",
   "help": "Ellf Command Line Interface.",
-  "version": "6.1.1",
+  "version": "6.1.3",
   "extra_key": "_extra",
   "commands": {
     "actions": {

{ellf_cli-6.1.2 → ellf_cli-6.1.4/ellf_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ellf-cli
-Version: 6.1.2
+Version: 6.1.4
 Summary: Ellf Command Line Interface
 Home-page: https://prodi.gy
 Author: ExplosionAI GmbH

ellf_cli-6.1.2/ellf_cli/commands/publish_data.py

@@ -1,87 +0,0 @@
-import json
-from pathlib import Path
-from typing import Optional
-from uuid import UUID
-
-from radicli import Arg
-
-from ..cli import cli
-from ..errors import CLIError
-from ..messages import Messages
-from . import assets as assets_commands
-from . import files as files_commands
-from ._state import get_auth_state
-
-
-@cli.subcommand(
-    "publish",
-    "data",
-    src=Arg(help="File or directory to publish"),
-    dest=Arg(help="Destination path to copy the data to"),
-    name=Arg("--name", help=Messages.name.format(noun="asset")),
-    version=Arg("--version", help=Messages.version.format(noun="asset")),
-    kind=Arg(
-        "--kind",
-        help=Messages.asset_kind,
-    ),
-    loader=Arg("--loader", help="Loader to convert data for Prodigy"),
-    meta=Arg("--meta", help=Messages.asset_meta),
-    exists_ok=Arg("--exists-ok", help=Messages.exists_ok),
-    # fmt: on
-)
-def publish_data(
-    src: Path,
-    kind: str,
-    dest: Optional[str] = None,
-    name: Optional[str] = None,
-    version: Optional[str] = None,
-    loader: Optional[str] = None,
-    meta: str = "{}",
-    exists_ok: bool = False,
-) -> UUID | None:
-    """
-    Transfer data to the cluster, and advertise it to Ellf.
-
-    These steps can also be done separately. See `ellf files` to transfer
-    data to the cluster without creating an Asset record for it, and
-    `ellf assets create` to create an Asset without transfer.
-    """
-    auth = get_auth_state()
-    if auth.broker_host is None:
-        raise CLIError(Messages.E035)
-    if auth.cluster_id is None:
-        raise CLIError(Messages.E036)
-
-    if name is None:
-        name = src.name
-    if dest is None:
-        if version is None:
-            filename = name
-        else:
-            filename = f"{name}-{version}"
-        dest = f"{{__nfs__}}/data/{filename}"
-    if loader is not None:
-        meta_json = json.loads(meta)
-        meta_json.update({"loader": loader})
-        meta = json.dumps(meta_json)
-    if version is None:
-        version = "0.0.0"
-
-    files_commands.cp(
-        str(src),
-        dest=dest,
-        overwrite=True,
-        make_dirs=True,
-        recurse=True,
-        cluster_host=auth.broker_host,
-    )
-    assets_commands.create(
-        kind=kind,
-        name=name,
-        path=dest,
-        version=version,
-        meta=meta,
-        exists_ok=exists_ok,
-    )
-    # TODO: We should also have an option to create a dataset
-    # from the asset, by triggering a 'db-in' action.