instance-repo 1.3.3.dev2__tar.gz → 1.3.5.dev1__tar.gz

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.3.3.dev2
+Version: 1.3.5.dev1
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo/__init__.py

@@ -5,10 +5,12 @@
 """
 from .repo import Repo
 from .loader import load_instances
+from .content import compute_source_hash
 from .models import (
     DatasetSeries, DatasetVersion, DatasetInstance, ManifestEntry,
     OwnerReference, Feedback, Identity, PushResult,
     RolloutReport, RolloutJob, ScaffoldConfig,
+    DatasetObjectDirectory, DatasetObjectFile, DatasetObjectListResult,
     canonical_json, parse_feedbacks, append_feedback,
 )
 from .errors import (
@@ -18,14 +20,16 @@ from .errors import (
     TransportError,
 )
 
-__version__ = "1.3.3.dev2"
+__version__ = "1.3.5.dev1"
 
 __all__ = [
     "Repo", "load_instances",
     "DatasetSeries", "DatasetVersion", "DatasetInstance", "ManifestEntry",
     "OwnerReference", "Feedback", "Identity", "PushResult",
     "RolloutReport", "RolloutJob", "ScaffoldConfig",
+    "DatasetObjectDirectory", "DatasetObjectFile", "DatasetObjectListResult",
     "canonical_json", "parse_feedbacks", "append_feedback",
+    "compute_source_hash",
     "InstanceRepoError", "LayoutError", "SchemaError", "OwnerRefMissing",
     "ImageMissing", "Immutable", "DigestMismatch", "Forbidden", "NotVisible",
     "TargetNotAllowed", "NetworkDenied", "CredExpired",

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo/clients/instances.py

@@ -384,6 +384,168 @@ class InstancesClient:
                 result[parts[0]] = ""
         return result
 
+    # ── 实例目录/文件列举（GET /datasets/objects）──────────────────
+
+    def list_objects(self, dataset: str, instance_id: str,
+                     split: str | None = None, *,
+                     version: str = "",
+                     path: str | None = None,
+                     page_size: int = 100,
+                     max_pages: int = 50,
+                     environment: str | None = None,
+                     metadata_model: str | None = None,
+                     dataset_split_id: str | None = None
+                     ) -> "DatasetObjectListResult":
+        """列举实例资产目录下的文件/子目录（自动翻页）。
+
+        调用控制面 ``GET /apis/v1/datasets/objects``，返回指定实例的
+        ``{split}-assets/{instance_id}/`` 路径下的目录和文件列表。
+        自动翻页合并所有结果，直到 ``truncated=False`` 或达到 ``max_pages``。
+
+        Parameters
+        ----------
+        dataset : str
+            Dataset 名称（``L1/L2`` 格式）。
+        instance_id : str
+            实例 ID。
+        split : str, optional
+            Split 名称；为 None 时使用 ``"default"``。
+        version : str
+            版本号，默认空串（split_first 布局下可为空）。
+        path : str, optional
+            子目录路径过滤（相对于实例资产根目录）。
+            为 None 或空串时列举实例根目录下全部内容；
+            非空时仅列举该路径下的内容（如 ``"solution"`` 或 ``"tests/sealed"``）。
+        page_size : int
+            每页大小，默认 100。
+        max_pages : int
+            最大翻页数（安全阀，防无限循环），默认 50。
+        environment : str, optional
+            ``online`` / ``pre``，None 时用 profile 派生值。
+        metadata_model : str, optional
+            ``split_first`` / ``version_first``，None 时用 profile 派生值。
+        dataset_split_id : str, optional
+            Split ID，可直接指定以避免 split 名称解析。
+
+        Returns
+        -------
+        DatasetObjectListResult
+            合并后的完整结果。``truncated`` 为 True 表示因 ``max_pages`` 截断。
+        """
+        from urllib.parse import quote
+        from ..models import DatasetObjectListResult
+        if not (dataset or "").strip():
+            raise SchemaError("dataset is required for instances.list_objects")
+        if not (instance_id or "").strip():
+            raise SchemaError(
+                "instance_id is required for instances.list_objects")
+        split = (split or "default").strip()
+        # 计算实例资产根前缀：{split}-assets/{instance_id}/
+        instance_asset_prefix = f"{split}-assets/{instance_id.strip()}"
+        if path:
+            full_prefix = f"{instance_asset_prefix}/{path.strip().strip('/')}"
+        else:
+            full_prefix = instance_asset_prefix
+        # environment / metadata_model：优先显式值，其次 profile 派生
+        env = (environment or "").strip()
+        if not env:
+            env = (getattr(self._p, "dataset_environment", "") or "").strip()
+        mm = (metadata_model or "").strip()
+        if not mm:
+            mm = (getattr(self._p, "metadata_model", "") or "").strip()
+        # 自动翻页
+        all_dirs: list = []
+        all_files: list = []
+        tk: str | None = None
+        result = DatasetObjectListResult()
+        for _ in range(max_pages):
+            params: list[tuple[str, str]] = [
+                ("dataset_name", dataset.strip()),
+                ("split", split),
+                ("prefix", full_prefix),
+                ("page_size", str(int(page_size)))]
+            if (version or "").strip():
+                params.append(("version", version.strip()))
+            if (dataset_split_id or "").strip():
+                params.append(("dataset_split_id", dataset_split_id.strip()))
+            if tk:
+                params.append(("token", tk))
+            if env:
+                params.append(("environment", env))
+            if mm:
+                params.append(("metadata_model", mm))
+            qs = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
+            data = self._t.get(f"/apis/v1/datasets/objects?{qs}")
+            if not isinstance(data, dict):
+                break
+            page = DatasetObjectListResult.from_dict(data)
+            all_dirs.extend(page.directories)
+            all_files.extend(page.files)
+            result = page  # 保留最后一页的 dataset/version/prefix 元信息
+            if not page.truncated or not page.next_token:
+                break
+            tk = page.next_token
+        result.directories = all_dirs
+        result.files = all_files
+        return result
+
+    # ── 变更检测辅助 ────────────────────────────────────────────────
+
+    def list_content_hashes(self, dataset: str, version: str = "",
+                            split: str | None = None, *,
+                            environment: str | None = None,
+                            metadata_model: str | None = None
+                            ) -> dict[str, str]:
+        """批量查询远端实例的 ``source_hash``（存在 ``extend_meta`` 中）。
+
+        返回 ``{instance_id: source_hash}`` 字典。用于 ``skip_unchanged``
+        变更检测：与本地 ``compute_source_hash`` 或自定义哈希比对，
+        匹配则跳过 pack + upload。
+        """
+        overrides = _registry_overrides(environment, metadata_model)
+        instances = self.list(dataset, version, split, **overrides)
+        result: dict[str, str] = {}
+        for inst in instances:
+            h = (inst.extend_meta or {}).get("source_hash", "")
+            if h:
+                result[inst.instance_id] = h
+        return result
+
+    # ── 实例删除 ────────────────────────────────────────────────────
+
+    def delete(self, dataset: str, version: str, instance_id: str, *,
+               split: str | None = None,
+               environment: str | None = None,
+               metadata_model: str | None = None) -> None:
+        """删除单个实例的元数据记录（``DELETE /datasets/instances/detail``）。
+
+        仅删除 MongoDB 记录，不删除 OSS 对象。后续 ingest 扫描可能重建该记录。
+        """
+        from urllib.parse import quote
+        if not (dataset or "").strip():
+            raise SchemaError("dataset is required for instances.delete")
+        if not (instance_id or "").strip():
+            raise SchemaError("instance_id is required for instances.delete")
+        params: list[tuple[str, str]] = [
+            ("dataset_name", dataset.strip()),
+            ("instance_id", instance_id.strip())]
+        if (version or "").strip():
+            params.append(("version", version.strip()))
+        if (split or "").strip():
+            params.append(("split", (split or "").strip()))
+        env = (environment or "").strip()
+        if not env:
+            env = (getattr(self._p, "dataset_environment", "") or "").strip()
+        if env:
+            params.append(("environment", env))
+        mm = (metadata_model or "").strip()
+        if not mm:
+            mm = (getattr(self._p, "metadata_model", "") or "").strip()
+        if mm:
+            params.append(("metadata_model", mm))
+        qs = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
+        self._t.delete(f"/apis/v1/datasets/instances/detail?{qs}")
+
     def update(self, dataset: str, version: str, instance_id: str, *,
                split: str | None = None, dataset_split_id: str | None = None,
                manifest: list[dict] | None = None,
@@ -462,7 +624,10 @@ class InstancesClient:
              split: str = "default", push_image: bool = True,
              overwrite: bool = False, register: bool = True,
              verify: bool = True,
-             image_release_dst_acr: str | None = None) -> PushResult:
+             image_release_dst_acr: str | None = None,
+             content_hash: str | None = None,
+             skip_unchanged: bool = False,
+             compress_level: int | None = None) -> PushResult:
         """推一个实例：自动检测格式（目录→harbor，.json→swe），validate → 上传。
 
         harbor: validate → (可选)推镜像 → 上传 OSS(content.tgz + {id}.json)。
@@ -476,6 +641,14 @@ class InstancesClient:
         verify=True（默认）在真正发生上传后回读校验：确认 OSS 对象确实可 HEAD 到。
 
         image_release_dst_acr（可选）：目标（prod）ACR host（仅 harbor 格式有效）。
+
+        content_hash（可选）：调用方提供的内容标识（如 git tree id），
+        存入 ``extend_meta.source_hash``，供后续 ``skip_unchanged`` 比对。
+
+        skip_unchanged（可选）：若为 True 且 content_hash 非空，先查远端
+        ``extend_meta.source_hash``，匹配则跳过 pack + upload。
+
+        compress_level（可选）：gzip 压缩级别（1-9），传 None 用默认值。
         """
         local_path = Path(local_path)
         if register and self._ingester is None:
@@ -550,14 +723,38 @@ class InstancesClient:
         )
         uid = instance_id
 
+        # 变更检测：skip_unchanged=True 时，若 content_hash 与远端一致，跳过 pack+upload。
+        if skip_unchanged and content_hash:
+            try:
+                remote = self._reg.get(dataset, version, instance_id, split)
+                if remote is not None:
+                    remote_hash = (remote.extend_meta or {}).get("source_hash", "")
+                    if remote_hash == content_hash:
+                        logger.info("push %s: skip (source_hash match)", instance_id)
+                        return PushResult(
+                            instance_id=instance_id, uploaded=False,
+                            skipped=True, image_pushed=False,
+                            docker_image=docker_ref or None,
+                            digests={}, ingest=None)
+            except (TransportError, Exception):  # noqa: BLE001
+                pass  # 查询失败不阻断，继续正常 push
+        # 将 content_hash 存入 extend_meta，供下次 skip_unchanged 比对
+        if content_hash:
+            extend_meta["source_hash"] = content_hash
+            meta.extend_meta = extend_meta
+
         # 打包 + 构造 manifest（本地）→ register(pending) → 推 ACR → 上传 OSS。
         # register 需要 manifest：apiserver 的 hasFile 校验要求至少一条
         # content_type=file 项，manifest 缺失时 register 被 92005 拒（曾导致
         # custom/harbor 主路径 push(register=True) 失败）。register 仍先于任何
         # 远程写（元数据先行），manifest 在本地打包时即可算出 digest。
+        import time as _time
+        _t0 = _time.monotonic()
         with tempfile.TemporaryDirectory(prefix="irepo_") as td:
             tgz = Path(td) / "content.tgz"
-            digest = pack(local_path, tgz, exclude_image_tar=True)
+            digest = pack(local_path, tgz, exclude_image_tar=True,
+                          compresslevel=compress_level)
+            _pack_elapsed = _time.monotonic() - _t0
             meta.manifest = _manifest(self._p.oss_bucket, dataset, version,
                                       instance_id, split, digest, lay)
 
@@ -573,6 +770,11 @@ class InstancesClient:
                                     meta.to_canonical_json(), overwrite=overwrite,
                                     layout=lay)
         skipped = bool(put.get("skipped"))
+        _upload_elapsed = _time.monotonic() - _t0 - _pack_elapsed
+        _tgz_size = tgz.stat().st_size if not skipped else 0
+        logger.info("push %s: pack=%.1fs (%d bytes) upload=%.1fs skipped=%s",
+                    instance_id, _pack_elapsed, _tgz_size,
+                    _upload_elapsed, skipped)
         # 回读校验（防静默成功）：只在真正发生上传时校验对象已落库；镜像推送后校验可 inspect。
         if verify:
             if not skipped and not self._content.exists(
@@ -807,7 +1009,13 @@ class InstancesClient:
                   overwrite: bool = False, register: bool = False,
                   verify: bool = True,
                   concurrency: int = 8, continue_on_error: bool = False,
-                  image_release_dst_acr: str | None = None
+                  image_release_dst_acr: str | None = None,
+                  skip_unchanged: bool = False,
+                  hash_func: "Callable | None" = None,
+                  content_hashes: dict[str, str] | None = None,
+                  progress_callback: "Callable | None" = None,
+                  executor=None,
+                  compress_level: int | None = None
                   ) -> list:
         """并发推一批实例（默认 register=False：只传 OSS，元数据随后由 ingest 扫描入库）。
 
@@ -824,38 +1032,113 @@ class InstancesClient:
         ``owner_reference`` 整批共用同一来源引用（批量交付通常同源；需要逐实例不同来源时
         请自行循环 ``push``）。CLI 的 push / push-many / deliver 都把它设为必填——交付
         路径必须能追溯上游来源。
+
+        ``skip_unchanged=True``：自动比对本地源文件哈希与远端 ``extend_meta.source_hash``，
+        匹配则跳过 pack+upload。``hash_func`` 可自定义哈希函数（默认用
+        ``content.compute_source_hash``）；``content_hashes`` 可直接提供
+        ``{instance_id: hash}`` 映射跳过本地计算。
+
+        ``progress_callback(completed, total, result_or_error)``：每个 case 完成时调用。
+
+        ``executor``：外部 ``ThreadPoolExecutor``，传入时复用（不关闭）；
+        为 None 时内部新建。
+
+        ``compress_level``：gzip 压缩级别（1-9），透传给 ``pack()``。
         """
         dirs = list(local_dirs)
+
+        # ── skip_unchanged：哈希比对 + 过滤 ──
+        skipped_results: list[PushResult] = []
+        hashes_map: dict[str, str] = dict(content_hashes) if content_hashes else {}
+        if skip_unchanged and dirs:
+            from ..content import compute_source_hash as _default_hash
+            _hash_fn = hash_func or _default_hash
+            # 1. 计算本地哈希（未提供的）
+            for d in dirs:
+                iid = Path(d).name
+                if iid not in hashes_map:
+                    try:
+                        hashes_map[iid] = _hash_fn(str(d))
+                    except Exception:  # noqa: BLE001
+                        pass  # 哈希失败不阻断，视为"变化"
+            # 2. 获取远端哈希
+            try:
+                remote_hashes = self.list_content_hashes(
+                    dataset, version, split)
+            except Exception:  # noqa: BLE001
+                remote_hashes = {}
+            # 3. 过滤：未变化的生成 skip 结果并从 dirs 移除
+            unchanged_ids: set[str] = set()
+            for d in dirs:
+                iid = Path(d).name
+                local_h = hashes_map.get(iid, "")
+                remote_h = remote_hashes.get(iid, "")
+                if local_h and local_h == remote_h:
+                    unchanged_ids.add(iid)
+                    skipped_results.append(PushResult(
+                        instance_id=iid, uploaded=False, skipped=True,
+                        image_pushed=False, docker_image=None,
+                        digests={}, ingest=None))
+            if unchanged_ids:
+                logger.info("push_many: %d/%d unchanged (skipped)",
+                            len(unchanged_ids), len(dirs))
+                dirs = [d for d in dirs if Path(d).name not in unchanged_ids]
+
         warm = getattr(self._content, "prewarm", None)
         if warm is not None:
-            # prewarm(prefix, dataset) 与 Go Prewarm(prefix, dataset) 对齐；
-            # dataset 用于 Dataset RBAC 签发（缺失时 apiserver 回 94020）。
-            # sts_prefix 是 dataset 级，两模型一致，但仍统一经 layout 取，避免再触 paths.*。
             sts_prefix = self._layout().sts_prefix(dataset)
             try:
                 warm(sts_prefix, dataset)
             except TypeError:
-                # 向后兼容旧签名 prewarm(prefix)（用户自定义 store / 旧版 fake）。
                 warm(sts_prefix)
-        # 推镜像时同样预热 ACR 凭据缓存（否则首轮各 worker 会并发重复签同一 host）
         if push_image and getattr(self._p, "acr_host", None):
-            # 签名恒用 apiserver 登记的 host（acr_host，VPC）；token 实例级，
-            # 对公网 endpoint 同样有效（prefer_public 时连接走公网、签名仍走这里）。
             try:
                 self._t.get_acr_credentials(self._p.acr_host)
-            except Exception:  # noqa: BLE001 — 预热失败不阻断，真正推镜像时再报
+            except Exception:  # noqa: BLE001
                 pass
 
         def _one(d):
-            # 逐实例强制 register=False：批量 ingest 只在整批结束后执行一次
+            iid = Path(d).name
+            ch = hashes_map.get(iid)
             return self.push(d, dataset, version,
                              owner_reference=owner_reference, split=split,
                              push_image=push_image, overwrite=overwrite,
                              register=False, verify=verify,
-                             image_release_dst_acr=image_release_dst_acr)
-
-        results = map_concurrent(_one, dirs, concurrency=concurrency,
-                                 return_exceptions=continue_on_error)
+                             image_release_dst_acr=image_release_dst_acr,
+                             content_hash=ch,
+                             compress_level=compress_level)
+
+        # ── 并发执行 + 进度回调 ──
+        total = len(dirs)
+        completed = [0]  # mutable counter for closure
+        lock = __import__("threading").Lock()
+
+        def _wrapped(d):
+            result = _one(d)
+            with lock:
+                completed[0] += 1
+                if progress_callback:
+                    try:
+                        progress_callback(completed[0], total, result)
+                    except Exception:  # noqa: BLE001
+                        pass
+            return result
+
+        if executor is not None:
+            # 外部线程池：submit 并收集结果（保持输入顺序）
+            futures = [executor.submit(_wrapped, d) for d in dirs]
+            results = []
+            for f in futures:
+                r = f.result()
+                results.append(r)
+            if not continue_on_error:
+                from ..concurrency import ItemError
+                for r in results:
+                    if isinstance(r, Exception):
+                        raise ItemError(r)
+        else:
+            results = map_concurrent(_wrapped, dirs, concurrency=concurrency,
+                                     return_exceptions=continue_on_error)
         # Close retired OSS buckets after the batch to release connection pools.
         # Guard for non-OSS or mock stores that don't have close_retired.
         _close = getattr(self._content, "close_retired", None)
@@ -865,11 +1148,13 @@ class InstancesClient:
         # 这里说明整批成功；True 时对成功子集 ingest（与 deliver 语义一致）。
         if register:
             self._after_upload(True, dataset, version, split=split)
-        return results
+        # 将 skipped 结果合并到返回列表（放在末尾，不影响 input 顺序的 pushed/failed 对应）
+        return list(results) + skipped_results
 
     def pull(self, dataset: str, version: str, instance_id: str, dest_dir,
              split: str = "default", *, verify: bool = True,
-             fallback_to_oss: bool = True) -> Path:
+             fallback_to_oss: bool = True,
+             path: str | None = None) -> Path:
         """下载实例内容。自动检测格式：harbor 下载 content.tgz，swe 下载 {id}.json，
         custom 下载 {id}.json + assets 目录。
 
@@ -881,6 +1166,11 @@ class InstancesClient:
         3. metadata JSON 不存在 → fallback 到 harbor（下载 content.tgz）
         4. dataset 未注册（92001）→ OSS 回退：list-only 枚举文件树 + 逐文件下载
            （仍走 DOWNLOAD 意图：仅私网，公网下载照旧禁止）
+
+        ``path`` 参数用于部分下载（仅 custom 和 OSS 回退格式生效）：
+        为空时下载全部内容；非空时只下载该路径前缀下的文件。
+        harbor（content.tgz）和 swe（单 JSON）格式不支持部分下载，
+        ``path`` 会被忽略。
         """
         validate_instance_id(instance_id, source="pull()")
         expected = None
@@ -897,7 +1187,8 @@ class InstancesClient:
                         and self._content is not None):
                     # dataset 未注册 → OSS 回退下载（私网 + 系统调用方语义）
                     return self._pull_unregistered(
-                        dataset, version, instance_id, dest_dir, split)
+                        dataset, version, instance_id, dest_dir, split,
+                        path=path)
                 raise
             if meta is not None and meta.manifest:
                 has_tgz = False
@@ -1000,7 +1291,7 @@ class InstancesClient:
                     asset_rels = [str(x) for x in meta_doc["assets"]]
             self._download_custom_assets(
                 dataset, instance_id, version, split, dest_dir, lay, sts_prefix,
-                asset_rels=asset_rels)
+                asset_rels=asset_rels, path=path)
             return dest
 
         if fmt == "swe":
@@ -1023,7 +1314,8 @@ class InstancesClient:
     def _download_custom_assets(self, dataset: str, instance_id: str,
                                 version: str, split: str, dest_dir, lay,
                                 sts_prefix: str,
-                                asset_rels: list[str] | None = None
+                                asset_rels: list[str] | None = None,
+                                path: str | None = None
                                 ) -> list[Path]:
         """下载 custom 格式的资产目录（与 content.tgz 同层、同版本规则）。
 
@@ -1033,17 +1325,25 @@ class InstancesClient:
         ``asset_rels`` 为元数据 ``assets`` 字段记录的资产相对路径清单：非 None 时
         逐对象 GetObject 还原（普通用户的 dataset-RBAC STS 不含 oss:ListObjects，
         不能靠 LIST 枚举）；为 None（旧数据无该字段）时回退到 LIST 枚举。
+
+        ``path`` 为子目录过滤（相对于资产根目录）：非空时仅下载该路径前缀下的文件。
         """
         asset_prefix = lay.assets_dir(dataset, instance_id,
                                        version=version, split=split)
         dest_root = Path(dest_dir) / instance_id
         downloaded: list[Path] = []
+        # path 过滤：非空时只下载 rel 以 path 开头的文件
+        path_prefix = ""
+        if path:
+            path_prefix = path.strip().strip("/") + "/"
 
         if asset_rels is not None:
             # 元数据记录了资产清单：逐对象下载，免 LIST。
             for rel in asset_rels:
                 if rel.endswith("/"):
                     continue  # 目录占位符（push 侧不生成；兼容历史数据）
+                if path_prefix and not rel.startswith(path_prefix):
+                    continue  # path 过滤
                 _validate_custom_asset_rel(rel)
                 local_path = dest_root / rel
                 local_path.parent.mkdir(parents=True, exist_ok=True)
@@ -1057,7 +1357,9 @@ class InstancesClient:
         list_fn = getattr(self._content, "list_objects", None)
         if list_fn is None:
             return []
-        keys = list_fn(asset_prefix)
+        # path 非空时，LIST 直接用子前缀缩小范围
+        list_prefix = asset_prefix + path_prefix if path_prefix else asset_prefix
+        keys = list_fn(list_prefix)
         if not keys:
             return []
         for key in keys:
@@ -1074,7 +1376,8 @@ class InstancesClient:
         return downloaded
 
     def _pull_unregistered(self, dataset: str, version: str,
-                           instance_id: str, dest_dir, split: str) -> Path:
+                           instance_id: str, dest_dir, split: str,
+                           path: str | None = None) -> Path:
         """未注册 dataset 的 pull 回退：list-only 枚举文件树 + 逐文件下载。
 
         下载仍走 :meth:`~instance_repo.store.oss.OssContentStore.get_object`
@@ -1083,6 +1386,8 @@ class InstancesClient:
         STS 语义：未注册 dataset 无 RBAC 记录，GetObject STS 仅 system 白名单
         调用方可签发（非 system 调用方在签发阶段收到明确错误），与历史行为一致。
 
+        ``path`` 为子目录过滤（相对于实例资产根目录）：非空时仅下载该路径前缀下的文件。
+
         产物结构（harbor 同层，便于 CLI 整体搬移）::
 
             {dest_dir}/{instance_id}/{instance_id}.json     # metadata
@@ -1108,7 +1413,13 @@ class InstancesClient:
         metadata_dest = dest_root / f"{instance_id}.json"
 
         self._content.get_object(metadata_key, metadata_dest, dataset=dataset)
+        # path 过滤：非空时只下载 rel 以 path 开头的文件
+        path_prefix = ""
+        if path:
+            path_prefix = path.strip().strip("/") + "/"
         for key, rel in files:
+            if path_prefix and not rel.startswith(path_prefix):
+                continue
             local_path = dest_root / rel
             local_path.parent.mkdir(parents=True, exist_ok=True)
             self._content.get_object(key, local_path, dataset=dataset)

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo/content.py

@@ -63,17 +63,24 @@ def _normalize(ti: tarfile.TarInfo) -> tarfile.TarInfo:
 
 
 def pack(local_dir: str | Path, out_tgz: str | Path, *,
-         exclude_image_tar: bool = True) -> str:
+         exclude_image_tar: bool = True,
+         compresslevel: int | None = None) -> str:
     """把 local_dir 打包成 out_tgz（gzip tar），返回 content_digest。
 
     确定性：条目按相对路径排序、元数据（mtime/uid/gid/uname/gname）归零、gzip
     header mtime=0，故同一目录内容恒得同一 digest（跨时间/机器可复现）。
+
+    ``compresslevel``：gzip 压缩级别（1-9），默认 None 使用 gzip 库默认值（9）。
+    较低级别（如 1）可显著减少 CPU 时间但产出稍大的文件。
     """
     src = Path(local_dir)
     out = Path(out_tgz)
     files = sorted(p for p in src.rglob("*") if p.is_file())
+    gz_kw: dict = {"filename": "", "mode": "wb", "mtime": 0}
+    if compresslevel is not None:
+        gz_kw["compresslevel"] = compresslevel
     with open(out, "wb") as raw, \
-            gzip.GzipFile(filename="", fileobj=raw, mode="wb", mtime=0) as gz, \
+            gzip.GzipFile(fileobj=raw, **gz_kw) as gz, \
             tarfile.open(fileobj=gz, mode="w") as tar:
         for p in files:
             rel = p.relative_to(src).as_posix()
@@ -81,3 +88,32 @@ def pack(local_dir: str | Path, out_tgz: str | Path, *,
                 continue
             tar.add(str(p), arcname="./" + rel, filter=_normalize)
     return sha256_file(out)
+
+
+def compute_source_hash(local_dir: str | Path, *,
+                        exclude_image_tar: bool = True) -> str:
+    """快速计算源目录的内容哈希（不打包、不压缩）。
+
+    遍历目录所有文件，按相对路径排序，对每个文件算 SHA-256，
+    最终对所有 ``(path, sha256)`` 对再算一次 SHA-256。
+    返回 ``"src-sha256:<hex>"``。
+
+    与 :func:`pack` 的 digest 不同：本函数不做 tar+gzip，只做文件读取 + 哈希，
+    CPU 开销大幅降低（省掉压缩），适合作为变更检测的快速指纹。
+
+    ``exclude_image_tar`` 语义同 :func:`pack`。
+    """
+    src = Path(local_dir)
+    files = sorted(p for p in src.rglob("*") if p.is_file())
+    h = hashlib.sha256()
+    for p in files:
+        rel = p.relative_to(src).as_posix()
+        if exclude_image_tar and rel == _EXCLUDE_REL:
+            continue
+        fh = hashlib.sha256()
+        with open(p, "rb") as f:
+            for chunk in iter(lambda: f.read(1 << 20), b""):
+                fh.update(chunk)
+        h.update(rel.encode("utf-8"))
+        h.update(fh.digest())
+    return "src-sha256:" + h.hexdigest()

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo/models.py

@@ -871,3 +871,82 @@ class RoleBinding:
             role_id=d.get("role_id", ""),
             added_by=d.get("added_by", ""),
             added_at=d.get("added_at", ""))
+
+
+# ── Dataset Objects（目录/文件列举）──────────────────────────────────
+
+@dataclass
+class DatasetObjectDirectory:
+    """``GET /datasets/objects`` 返回的目录项。"""
+    name: str
+    path: str
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "DatasetObjectDirectory":
+        return cls(name=d.get("name", ""), path=d.get("path", ""))
+
+    def to_dict(self) -> dict:
+        return {"name": self.name, "path": self.path}
+
+
+@dataclass
+class DatasetObjectFile:
+    """``GET /datasets/objects`` 返回的文件项。"""
+    name: str
+    path: str
+    size_bytes: int = 0
+    content_type: str = ""
+    previewable: bool = False
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "DatasetObjectFile":
+        return cls(
+            name=d.get("name", ""),
+            path=d.get("path", ""),
+            size_bytes=int(d.get("size_bytes", 0)),
+            content_type=d.get("content_type", ""),
+            previewable=bool(d.get("previewable", False)))
+
+    def to_dict(self) -> dict:
+        return {"name": self.name, "path": self.path,
+                "size_bytes": self.size_bytes,
+                "content_type": self.content_type,
+                "previewable": self.previewable}
+
+
+@dataclass
+class DatasetObjectListResult:
+    """``GET /datasets/objects`` 分页响应。
+
+    ``directories`` 和 ``files`` 为当前页条目；``truncated`` 为 True 时
+    可用 ``next_token`` 继续翻页。
+    """
+    dataset: str = ""
+    version: str = ""
+    prefix: str = ""
+    directories: list[DatasetObjectDirectory] = field(default_factory=list)
+    files: list[DatasetObjectFile] = field(default_factory=list)
+    next_token: str = ""
+    truncated: bool = False
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "DatasetObjectListResult":
+        return cls(
+            dataset=d.get("dataset", ""),
+            version=d.get("version", ""),
+            prefix=d.get("prefix", ""),
+            directories=[DatasetObjectDirectory.from_dict(x)
+                         for x in d.get("directories", [])],
+            files=[DatasetObjectFile.from_dict(x)
+                   for x in d.get("files", [])],
+            next_token=d.get("next_token", ""),
+            truncated=bool(d.get("truncated", False)))
+
+    def to_dict(self) -> dict:
+        return {
+            "dataset": self.dataset, "version": self.version,
+            "prefix": self.prefix,
+            "directories": [d.to_dict() for d in self.directories],
+            "files": [f.to_dict() for f in self.files],
+            "next_token": self.next_token,
+            "truncated": self.truncated}

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo/repo.py

@@ -130,7 +130,15 @@ class Repo:
                 overwrite: bool = False, concurrency: int = 8,
                 publish: bool = False, ingest: bool = True,
                 continue_on_error: bool = False,
-                image_release_dst_acr: str | None = None, **ingest_kwargs) -> dict:
+                image_release_dst_acr: str | None = None,
+                skip_unchanged: bool = False,
+                hash_func=None,
+                content_hashes: dict | None = None,
+                prune: bool = False,
+                progress_callback=None,
+                executor=None,
+                compress_level: int | None = None,
+                **ingest_kwargs) -> dict:
         """一次性交付一批实例：并发上传 OSS（register=False），再触发一次 dataset-ingest 入库。
 
         相比逐个 ``push``（每实例一次 STS 签发 + 串行传输），本方法整批共用 STS、
@@ -138,7 +146,7 @@ class Repo:
 
         ``continue_on_error=False``（默认，全或无）：任一实例失败即抛 ``ItemError``。
         ``continue_on_error=True``：跳过失败项，只要有成功上传就对成功子集触发 ingest，
-        返回 ``{"pushed":[PushResult...], "failed":[ItemError...], "ingest":<job|None>}``；
+        返回 ``{"pushed":[...], "skipped":[...], "failed":[...], "ingest":<job|None>}``；
         全部失败时不触发 ingest（job=None）。``ingest=False`` 则只上传不入库。
 
         ``owner_reference``（可选）：整批共用的上游来源引用，透传给每个 push。CLI 的
@@ -147,22 +155,48 @@ class Repo:
         ``image_release_dst_acr``（可选）：透传给 push，把「实例↔镜像」上架映射写进
         {id}.json 的 extend_meta，随本方法的 ingest 一并入库（见 push 文档）。
 
+        ``skip_unchanged=True``：自动比对本地源文件哈希与远端 ``extend_meta.source_hash``，
+        匹配则跳过 pack+upload。``hash_func`` 可自定义哈希函数（默认用
+        ``content.compute_source_hash``）；``content_hashes`` 可直接提供
+        ``{instance_id: hash}`` 映射跳过本地计算。
+
+        ``prune=True``：交付完成后，删除远端存在但本次 local_dirs 中不存在的实例
+        （逐个调 ``instances.delete``）。注意：ingest 本身已有 prune 语义
+        （删除 OSS 中不存在的旧记录），此选项额外清理 OSS 中也已删除的情况。
+
+        ``progress_callback(completed, total, result_or_error)``：每个 case 完成时调用。
+
+        ``executor``：外部 ``ThreadPoolExecutor``，传入时复用（不关闭）。
+
+        ``compress_level``：gzip 压缩级别（1-9），透传给 pack。
+
         注意：单实例在"推 ACR→传 content.tgz→写 {id}.json"中途失败可能留下孤儿对象/镜像
         （ingest 按 {id}.json 扫描，孤儿对此无害，但占用存储，需要时另行清理）。
         """
+        import logging as _logging
+        _logger = _logging.getLogger(__name__)
+
         results = self.instances.push_many(
             local_dirs, dataset, version, owner_reference=owner_reference,
             split=split, push_image=push_image,
             overwrite=overwrite, register=False, concurrency=concurrency,
             continue_on_error=continue_on_error,
-            image_release_dst_acr=image_release_dst_acr)
-        pushed = [r for r in results if not isinstance(r, ItemError)]
+            image_release_dst_acr=image_release_dst_acr,
+            skip_unchanged=skip_unchanged,
+            hash_func=hash_func,
+            content_hashes=content_hashes,
+            progress_callback=progress_callback,
+            executor=executor,
+            compress_level=compress_level)
+        pushed = [r for r in results
+                  if not isinstance(r, ItemError)
+                  and not getattr(r, "skipped", False)]
+        skipped = [r for r in results
+                   if not isinstance(r, ItemError)
+                   and getattr(r, "skipped", False)]
         failed = [r for r in results if isinstance(r, ItemError)]
         job = None
         if ingest and pushed:                 # 仅在有成功上传时才 ingest
-            # 透传 split 给 ingest：用户若未通过 ingest_kwargs 显式指定 splits/split，
-            # 且 split 非默认值，则自动透传，避免 ingest 扫描所有 split 目录时
-            # 对未交付的空 split 报 92005（split path has no instance files）。
             ingest_split_kwargs = {}
             if "splits" not in ingest_kwargs and "split" not in ingest_kwargs:
                 if split and split != "default":
@@ -170,7 +204,25 @@ class Repo:
             job = self.versions.ingest(
                 dataset, version, publish=publish,
                 **ingest_split_kwargs, **ingest_kwargs)
-        return {"pushed": pushed, "failed": failed, "ingest": job}
+        # prune：删除远端有但本次不在 local_dirs 中的实例
+        if prune and not failed:
+            local_ids = {Path(d).name for d in local_dirs}
+            try:
+                remote_instances = self.instances.list(dataset, version, split)
+                for inst in remote_instances:
+                    if inst.instance_id not in local_ids:
+                        try:
+                            self.instances.delete(
+                                dataset, version, inst.instance_id,
+                                split=split)
+                            _logger.info("prune: deleted %s", inst.instance_id)
+                        except Exception:  # noqa: BLE001
+                            _logger.warning("prune: failed to delete %s",
+                                            inst.instance_id)
+            except Exception:  # noqa: BLE001
+                _logger.warning("prune: failed to list remote instances")
+        return {"pushed": pushed, "skipped": skipped,
+                "failed": failed, "ingest": job}
 
     # ── 源到目标交付：上传 → 元数据入库 → 发布 workflow ────────────────────────
     def deliver_to_prod(self, dataset: str, version: str, local_dirs, *,

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/instance_repo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.3.3.dev2
+Version: 1.3.5.dev1
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.3.3.dev2 → instance_repo-1.3.5.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "instance-repo"
-version = "1.3.3.dev2"
+version = "1.3.5.dev1"
 description = "InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）"
 requires-python = ">=3.10"
 readme = "README.md"