instance-repo 1.0.9.dev6__tar.gz → 1.1.0__tar.gz

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.9.dev6
+Version: 1.1.0
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo/__init__.py

@@ -18,7 +18,7 @@ from .errors import (
     TransportError,
 )
 
-__version__ = "1.0.9.dev6"
+__version__ = "1.1.0"
 
 __all__ = [
     "Repo", "load_instances",

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo/clients/instances.py

@@ -655,16 +655,23 @@ class InstancesClient:
 
     def pull(self, dataset: str, version: str, instance_id: str, dest_dir,
              split: str = "default", *, verify: bool = True) -> Path:
-        """下载实例内容。自动检测格式：harbor 下载 content.tgz，swe 下载 {id}.json。
+        """下载实例内容。自动检测格式：harbor 下载 content.tgz，swe 下载 {id}.json，
+        custom 下载 {id}.json + assets 目录。
 
         格式判断优先级：
-        1. verify=True 且 registry 有元数据 → 从 manifest 判断（有 tgz=harbor，否则 swe）
-        2. 无元数据（verify=False 或实例未注册）→ 先尝试 SWE（HEAD {id}.json），
-           存在则下载 json；不存在则 fallback 到 harbor（下载 content.tgz）
+        1. verify=True 且 registry 有元数据 → 从 manifest 判断
+           (有 tgz=harbor，仅 json=swe/custom，需进一步探测)
+        2. 无元数据或 manifest 无法区分 swe/custom → 下载 metadata JSON 检查
+           metadata_model 字段（custom → 下载 assets 目录；否则 swe）
+        3. metadata JSON 不存在 → fallback 到 harbor（下载 content.tgz）
         """
         validate_instance_id(instance_id, source="pull()")
         expected = None
         fmt = None  # None = 未确定，需要探测
+        lay = self._layout()
+        # 已下载的元数据字节（探测阶段下载一次，swe/custom 路径复用，避免二次网络往返）。
+        meta_bytes: bytes | None = None
+
         if verify:
             meta = self._reg.get(dataset, version, instance_id, split)
             if meta is not None and meta.manifest:
@@ -677,19 +684,35 @@ class InstancesClient:
                 if has_tgz:
                     fmt = "harbor"
                 else:
-                    # manifest 非空但无 tgz → 查找 json entry
+                    # manifest 有 json entry 但无 tgz → swe 或 custom（需进一步探测）。
+                    # custom 推的 manifest 由 ingest 扫描生成，与 swe 同形态；
+                    # 唯一区分手段是读 metadata JSON 检查 metadata_model 字段。
                     for m in meta.manifest:
                         if m.content_format == "json":
-                            fmt = "swe"
+                            fmt = "swe"  # 暂定 swe，后续探测可能修正为 custom
                             expected = m.content_digest
                             break
-                    # manifest 非空但既无 tgz 也无 json → fmt 保持 None，走探测
-        # 无法从 manifest 确定格式 → 探测 OSS。内建 store 的探测属于下载流程，
-        # 必须走私网限定的包内方法；自定义 ContentStore 保持旧接口兼容。
-        lay = self._layout()
+
+        # 无法从 manifest 确定 harbor → 探测 OSS 区分 swe/custom 或 fallback harbor。
+        idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
+        sts_prefix = lay.sts_prefix(dataset)
+
+        # ── 探测辅助：下载 metadata JSON 到临时文件并解析，区分 swe / custom ──
+        def _probe_meta_json() -> bytes | None:
+            """下载 metadata JSON 并返回字节；不存在或解析失败时返回 None。"""
+            tmp_dest = Path(dest_dir) / f".{instance_id}.json.probe"
+            tmp_dest.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                self._content.get_object(
+                    idx_key, tmp_dest, sts_prefix=sts_prefix, dataset=dataset)
+                data = tmp_dest.read_bytes()
+                return data
+            except Exception:  # noqa: BLE001 — 下载失败视为不存在
+                return None
+            finally:
+                tmp_dest.unlink(missing_ok=True)
+
         if fmt is None:
-            idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
-            sts_prefix = lay.sts_prefix(dataset)
             download_probe = getattr(self._content, "_download_probe", None)
             if download_probe is not None:
                 exists = download_probe(
@@ -698,22 +721,97 @@ class InstancesClient:
                 exists = self._content.object_exists(
                     idx_key, sts_prefix=sts_prefix)
             if exists:
-                fmt = "swe"
+                # metadata JSON 存在 → 下载并解析以区分 swe / custom
+                fmt = "swe"  # 默认；下载后可能修正为 custom
+                probed = _probe_meta_json()
+                if probed is not None:
+                    meta_bytes = probed
+                    try:
+                        meta_doc = json.loads(meta_bytes)
+                        if (isinstance(meta_doc, dict)
+                                and meta_doc.get("metadata_model") == "custom"):
+                            fmt = "custom"
+                    except (json.JSONDecodeError, UnicodeDecodeError):
+                        pass  # 无法解析 → 保持 swe
             else:
                 fmt = "harbor"
 
+        # 若 manifest 判定为 swe 但实际可能是 custom（两者 manifest 形态相同），
+        # 下载 metadata JSON 做最终确认。
+        if fmt == "swe" and meta_bytes is None:
+            probed = _probe_meta_json()
+            if probed is not None:
+                meta_bytes = probed
+                try:
+                    meta_doc = json.loads(meta_bytes)
+                    if (isinstance(meta_doc, dict)
+                            and meta_doc.get("metadata_model") == "custom"):
+                        fmt = "custom"
+                except (json.JSONDecodeError, UnicodeDecodeError):
+                    pass
+
+        if fmt == "custom":
+            dest = Path(dest_dir) / f"{instance_id}.json"
+            if meta_bytes is not None:
+                # 已在探测阶段下载，直接写入本地（避免二次网络往返）。
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_bytes(meta_bytes)
+            else:
+                self._content.get_object(
+                    idx_key, dest, expected_digest=expected,
+                    sts_prefix=sts_prefix, dataset=dataset)
+            # 下载资产目录（{split}-assets/{id}/ 前缀下的所有文件）
+            self._download_custom_assets(
+                dataset, instance_id, split, dest_dir, lay, sts_prefix)
+            return dest
+
         if fmt == "swe":
             dest = Path(dest_dir) / f"{instance_id}.json"
-            idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
+            if meta_bytes is not None:
+                # 已在探测阶段下载，直接写入本地。
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_bytes(meta_bytes)
+                return dest
             return self._content.get_object(
                 idx_key, dest, expected_digest=expected,
-                sts_prefix=lay.sts_prefix(dataset),
+                sts_prefix=sts_prefix,
                 dataset=dataset)
+
         # harbor：下载 content.tgz
         dest = Path(dest_dir) / instance_id / "content.tgz"
         return self._content.get(dataset, version, instance_id, split, dest,
                                  expected_digest=expected, layout=lay)
 
+    def _download_custom_assets(self, dataset: str, instance_id: str,
+                                split: str, dest_dir, lay, sts_prefix: str
+                                ) -> list[Path]:
+        """下载 custom 格式的资产目录（{split}-assets/{id}/ 前缀下的所有文件）。
+
+        资产文件按相对路径还原到 ``{dest_dir}/{instance_id}/`` 下。返回已下载
+        的本地文件路径列表。无资产时返回空列表（不报错）。
+        """
+        asset_prefix = (f"{lay.dataset_root(dataset)}"
+                        f"{split}-assets/{instance_id}/")
+        list_fn = getattr(self._content, "list_objects", None)
+        if list_fn is None:
+            return []
+        keys = list_fn(asset_prefix)
+        if not keys:
+            return []
+        downloaded: list[Path] = []
+        dest_root = Path(dest_dir) / instance_id
+        for key in keys:
+            rel = key[len(asset_prefix):]
+            if not rel or rel.endswith("/"):
+                continue  # 跳过空键和目录占位符
+            local_path = dest_root / rel
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            self._content.get_object(
+                key, local_path,
+                sts_prefix=sts_prefix, dataset=dataset)
+            downloaded.append(local_path)
+        return downloaded
+
     def pull_many(self, instance_ids: list[str], dataset: str, version: str,
                   dest_dir, *, split: str = "default",
                   concurrency: int = 8, continue_on_error: bool = False,

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo/clients/workflows.py

@@ -18,7 +18,6 @@ from __future__ import annotations
 
 from urllib.parse import quote
 
-from ..errors import TransportError
 from ..models import PagedResult, Workflow
 
 WORKFLOW_STATUS_VALUES = ("pending", "running", "awaiting_approval",

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo/repo.py

@@ -297,7 +297,6 @@ class Repo:
         """
         from pathlib import Path as _Path
         import shutil as _shutil
-        from . import release as _R
         from .clients.config import load_profile
         from .validate import detect_format
 

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo/store/oss.py

@@ -17,6 +17,7 @@ from .._routing import (
     _plan_oss_route,
     _probe_private_endpoint,
 )
+from ..concurrency import ItemError, map_concurrent
 from ..content import sha256_file, verify_file_digest
 from ..errors import DigestMismatch, PrivateNetworkRequired, TransportError
 from ..retry import retry_transient
@@ -384,6 +385,109 @@ class OssContentStore(BaseContentStore):
         return self._execute_oss(
             _OperationIntent.NON_DOWNLOAD, prefix, None, list_all)
 
+    # ── directory-level helpers（自定义格式实例 = 目录级上传/下载）──
+    def put_directory(self, local_dir: Path, oss_prefix: str, *,
+                      overwrite: bool = False,
+                      sts_prefix: str | None = None,
+                      dataset: str | None = None,
+                      concurrency: int = 8) -> dict:
+        """Upload all files under *local_dir* to OSS under *oss_prefix*.
+
+        Each file is uploaded via :meth:`put_object` so every transfer gets
+        proper STS credentials and endpoint routing.  Files are uploaded in
+        parallel using :func:`~instance_repo.concurrency.map_concurrent`.
+
+        Returns a summary dict::
+
+            {"file_count": int, "total_bytes": int, "keys": list[str]}
+        """
+        local_dir = Path(local_dir)
+        if not local_dir.is_dir():
+            raise FileNotFoundError(f"local directory not found: {local_dir}")
+
+        # Normalize prefix to always end with "/" so keys are proper children.
+        if oss_prefix and not oss_prefix.endswith("/"):
+            oss_prefix += "/"
+
+        # Collect (local_path, oss_key) pairs — sorted for determinism.
+        files: list[tuple[Path, str]] = sorted(
+            ((p, oss_prefix + str(p.relative_to(local_dir)))
+             for p in local_dir.rglob("*") if p.is_file()),
+            key=lambda t: t[1],
+        )
+
+        total_bytes = sum(p.stat().st_size for p, _ in files)
+
+        def _upload_one(item: tuple[Path, str]) -> str:
+            path, key = item
+            self.put_object(key, path, overwrite=overwrite,
+                            sts_prefix=sts_prefix, dataset=dataset)
+            return key
+
+        try:
+            keys = map_concurrent(_upload_one, files, concurrency=concurrency)
+        except ItemError as exc:
+            raise exc.error from exc
+        return {
+            "file_count": len(files),
+            "total_bytes": total_bytes,
+            "keys": keys,
+        }
+
+    def get_directory(self, oss_prefix: str, local_dir: Path, *,
+                      expected_digests: dict[str, str] | None = None,
+                      sts_prefix: str | None = None,
+                      dataset: str | None = None,
+                      concurrency: int = 8) -> Path:
+        """Download all objects under *oss_prefix* to *local_dir*.
+
+        Objects are listed via :meth:`list_objects` and then downloaded in
+        parallel.  Each file is fetched via :meth:`get_object` so every
+        transfer gets proper STS credentials and private-download routing.
+
+        *expected_digests* optionally maps **relative paths** (e.g.
+        ``"subdir/file.bin"``) to their expected content digest strings
+        (``"sha256:<hex>"`` or bare md5).  Any mismatch raises
+        :class:`~instance_repo.errors.DigestMismatch`.
+
+        Returns the *local_dir* path.
+        """
+        local_dir = Path(local_dir)
+        local_dir.mkdir(parents=True, exist_ok=True)
+
+        # Normalize prefix for consistent key slicing.
+        if oss_prefix and not oss_prefix.endswith("/"):
+            oss_prefix += "/"
+
+        keys = self.list_objects(oss_prefix)
+
+        def _download_one(key: str) -> Path:
+            rel = key[len(oss_prefix):]
+            if not rel:  # skip the prefix "directory marker" itself
+                return local_dir
+            dest = local_dir / rel
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            self._execute_oss(
+                _OperationIntent.DOWNLOAD,
+                sts_prefix or oss_prefix,
+                dataset,
+                lambda bucket: bucket.get_object_to_file(key, str(dest)),
+            )
+            # Per-file digest check.
+            if expected_digests is not None:
+                expected = expected_digests.get(rel)
+                if expected is not None and not verify_file_digest(dest, expected):
+                    raise DigestMismatch(
+                        f"{key}: content digest mismatch "
+                        f"(expected {expected}, got {sha256_file(dest)})")
+            return dest
+
+        try:
+            map_concurrent(_download_one, keys, concurrency=concurrency)
+        except ItemError as exc:
+            raise exc.error from exc
+        return local_dir
+
     # ── 实例内容便捷封装（content.tgz + {id}.json） ──
     def exists(self, instance_id: str, dataset: str, version: str,
                split: str = "default", *, layout=None) -> bool:

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/instance_repo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.9.dev6
+Version: 1.1.0
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "instance-repo"
-version = "1.0.9.dev6"
+version = "1.1.0"
 description = "InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）"
 requires-python = ">=3.10"
 readme = "README.md"