instance-repo 1.0.9.dev6__tar.gz → 1.1.0.dev0__tar.gz

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.9.dev6
+Version: 1.1.0.dev0
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/__init__.py

@@ -18,7 +18,7 @@ from .errors import (
     TransportError,
 )
 
-__version__ = "1.0.9.dev6"
+__version__ = "1.1.0.dev0"
 
 __all__ = [
     "Repo", "load_instances",

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/_site_defaults.py

@@ -25,3 +25,15 @@ from __future__ import annotations
 #: **var**，内网渠道经 ``-ldflags -X`` 注入）与 Java 侧 ``SiteDefaults.API_BASE``
 #: （三语言须一致，且各自保留构建期注入方式）。
 API_BASE = ""
+
+#: AP job 查询端点（artifact 下载用）的站点默认值。源码里恒为空串；内网构建时注入。
+AP_ENDPOINT = ""
+
+#: artifact 对象的 OSS 前缀根（如 {root}/{job_id}/result.tgz）。源码恒为空；
+#: 内网构建时注入。
+ARTIFACT_PREFIX_ROOT = ""
+
+#: 「AP cluster 名 → artifact OSS bucket」映射的 JSON 串。源码恒为空；内网构建时
+#: 注入。运行时也可经环境变量 AP_ARTIFACT_BUCKET_MAP（JSON）覆盖，优先级：
+#: 环境变量 > 本模块注入值 > 空（空则 artifact 下载报错并引导配置）。
+ARTIFACT_BUCKET_MAP_JSON = ""

instance_repo-1.1.0.dev0/instance_repo/artifacts.py

@@ -0,0 +1,347 @@
+"""artifacts.py — AP job artifact 下载（STS 直连 OSS，绕开 dataplane）。
+
+调用方：普通用户 / UTS SDK，透传 apikey + job_id + cluster_id。
+鉴权：AP workspace ACL（SDK 调 AP GET /api/jobs/{id} 时自动检查）。
+STS：apiserver POST /apis/v1/credentials，resource=oss，artifact 前缀强制只读。
+OSS：{ARTIFACT_PREFIX_ROOT}/{job_id}/result.tgz（前缀根经 _site_defaults 注入）。
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import posixpath
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from .errors import Forbidden, InstanceRepoError
+from . import _site_defaults
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# 常量
+# ---------------------------------------------------------------------------
+
+# ---------------------------------------------------------------------------
+# 常量（业务值一律经 _site_defaults 注入，源码里不硬编码，见 _site_defaults.py）
+# ---------------------------------------------------------------------------
+
+def _artifact_bucket_map() -> dict[str, str]:
+    """解析 artifact bucket 映射：环境变量 > 站点默认 > 空。
+
+    空则抛错并引导配置——业务桶名不进公网包源码（脱敏红线）。
+    """
+    raw = (os.environ.get("AP_ARTIFACT_BUCKET_MAP", "")
+           or _site_defaults.ARTIFACT_BUCKET_MAP_JSON).strip()
+    if not raw:
+        raise InstanceRepoError(
+            "artifact bucket map is not configured; set AP_ARTIFACT_BUCKET_MAP "
+            "(JSON {\"<cluster>\": \"<bucket>\", ...}) or use an internal build "
+            "with the map injected")
+    try:
+        data = json.loads(raw)
+    except ValueError as exc:
+        raise InstanceRepoError(
+            f"AP_ARTIFACT_BUCKET_MAP must be valid JSON: {exc}") from exc
+    if not isinstance(data, dict) or not all(
+            isinstance(k, str) and isinstance(v, str) for k, v in data.items()):
+        raise InstanceRepoError(
+            "AP_ARTIFACT_BUCKET_MAP must be a JSON object of string pairs")
+    return data
+
+
+def _artifact_prefix_root() -> str:
+    root = (os.environ.get("AP_ARTIFACT_PREFIX_ROOT", "")
+            or _site_defaults.ARTIFACT_PREFIX_ROOT).strip()
+    if not root:
+        raise InstanceRepoError(
+            "artifact prefix root is not configured; set AP_ARTIFACT_PREFIX_ROOT "
+            "or use an internal build with the value injected")
+    return root
+
+
+def _ap_endpoint(ap_endpoint: str | None) -> str:
+    endpoint = (ap_endpoint
+                or os.environ.get("INSTANCE_REPO_AP_ENDPOINT")
+                or _site_defaults.AP_ENDPOINT).strip()
+    if not endpoint:
+        raise InstanceRepoError(
+            "AP endpoint is not configured; set INSTANCE_REPO_AP_ENDPOINT "
+            "or use an internal build with the default injected")
+    return endpoint.rstrip("/")
+
+# 批量下载默认并发数
+DEFAULT_MAX_CONCURRENT = 8
+
+# OSS 限流退避参数
+OSS_RETRY_INITIAL_WAIT = 1.0   # 秒
+OSS_RETRY_MAX_WAIT = 30.0      # 秒
+OSS_RETRY_MAX_ATTEMPTS = 5
+
+
+# ---------------------------------------------------------------------------
+# 异常
+# ---------------------------------------------------------------------------
+
+class ArtifactNotFoundError(InstanceRepoError):
+    """job 不存在或无 artifact。"""
+
+
+class ArtifactPermissionError(Forbidden):
+    """用户无权访问该 job（workspace ACL 拒绝）。"""
+
+
+# ---------------------------------------------------------------------------
+# 内部辅助
+# ---------------------------------------------------------------------------
+
+def resolve_artifact_bucket(cluster: str) -> str:
+    """根据数据面名查 artifact bucket（映射来自注入配置，非源码硬编码）。"""
+    cluster = (cluster or "").strip()
+    bucket_map = _artifact_bucket_map()
+    bucket = bucket_map.get(cluster)
+    if not bucket:
+        raise InstanceRepoError(
+            f"unknown cluster {cluster!r}; known clusters: "
+            f"{sorted(bucket_map)}")
+    return bucket
+
+
+def artifact_prefix(job_id: str) -> str:
+    """构造 artifact STS 前缀：{root}/{job_id}/"""
+    return f"{_artifact_prefix_root()}/{job_id}/"
+
+
+def artifact_key(job_id: str) -> str:
+    """构造 artifact OSS 对象 key：{root}/{job_id}/result.tgz"""
+    return f"{_artifact_prefix_root()}/{job_id}/result.tgz"
+
+
+def fetch_job_info(api_key: str, job_id: str, cluster: str,
+                   ap_endpoint: str | None = None) -> dict[str, Any]:
+    """调 AP GET /api/jobs/{job_id}?include_params=true 获取 job 信息。
+
+    AP 内部执行 workspace ACL 检查（enforce_workspace_action_req）。
+    403 → ArtifactPermissionError；404 → ArtifactNotFoundError。
+    """
+    endpoint = _ap_endpoint(ap_endpoint)
+    url = f"{endpoint}/api/jobs/{job_id}?include_params=true"
+    headers = {
+        "X-API-Key": api_key,
+        "Accept": "application/json",
+    }
+    if cluster:
+        headers["X-Cluster"] = cluster
+
+    req = urllib.request.Request(url, headers=headers, method="GET")
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except urllib.error.HTTPError as e:
+        if e.code == 403:
+            raise ArtifactPermissionError(
+                f"workspace ACL denied access to job {job_id!r}"
+            ) from e
+        if e.code == 404:
+            raise ArtifactNotFoundError(
+                f"job {job_id!r} not found"
+            ) from e
+        raise InstanceRepoError(
+            f"AP returned HTTP {e.code} for job {job_id!r}: {e.reason}"
+        ) from e
+    except urllib.error.URLError as e:
+        raise InstanceRepoError(
+            f"failed to reach AP at {endpoint}: {e.reason}"
+        ) from e
+
+
+def fetch_group_job_ids(api_key: str, group_id: str, cluster: str,
+                        ap_endpoint: str | None = None) -> list[str]:
+    """调 AP GET /groups/{group_id}/jobs 分页收集全部 job_id。"""
+    endpoint = (ap_endpoint or os.environ.get("INSTANCE_REPO_AP_ENDPOINT")
+                or DEFAULT_AP_ENDPOINT).rstrip("/")
+    job_ids: list[str] = []
+    skip = 0
+    limit = 500
+
+    while True:
+        url = (f"{endpoint}/groups/{group_id}/jobs"
+               f"?skip={skip}&limit={limit}&include_post_process=false")
+        headers = {
+            "X-API-Key": api_key,
+            "Accept": "application/json",
+        }
+        if cluster:
+            headers["X-Cluster"] = cluster
+
+        req = urllib.request.Request(url, headers=headers, method="GET")
+        try:
+            with urllib.request.urlopen(req, timeout=60) as resp:
+                data = json.loads(resp.read().decode())
+        except urllib.error.HTTPError as e:
+            if e.code == 403:
+                raise ArtifactPermissionError(
+                    f"workspace ACL denied access to group {group_id!r}"
+                ) from e
+            if e.code == 404:
+                raise ArtifactNotFoundError(
+                    f"group {group_id!r} not found"
+                ) from e
+            raise InstanceRepoError(
+                f"AP returned HTTP {e.code} for group {group_id!r}"
+            ) from e
+        except urllib.error.URLError as e:
+            raise InstanceRepoError(
+                f"failed to reach AP at {endpoint}: {e.reason}"
+            ) from e
+
+        jobs = data.get("jobs", [])
+        if not jobs:
+            break
+        for j in jobs:
+            jid = j.get("job_id")
+            if jid:
+                job_ids.append(jid)
+        if len(jobs) < limit:
+            break
+        skip += limit
+
+    return job_ids
+
+
+def _download_one(transport, bucket: str, job_id: str, output_dir: str) -> str:
+    """下载单个 job 的 artifact 到 output_dir/{job_id}/{文件名}。
+
+    artifact 文件名不固定：实测两种命名（result.tgz 与 {job_id}.tgz）。
+    先 list 前缀目录找非空的 .tgz 对象，兼容所有命名；找不到 → ArtifactNotFoundError。
+
+    返回本地文件路径。失败时抛异常。
+    """
+    import oss2  # 延迟导入，避免无 oss2 环境 import 失败
+
+    prefix = artifact_prefix(job_id)
+
+    # 获取 STS（同数据面多 job 共享缓存）
+    creds = transport.get_oss_credentials(bucket, prefix)
+    auth = oss2.StsAuth(
+        creds["access_key_id"],
+        creds["access_key_secret"],
+        creds["security_token"],
+    )
+
+    # 使用 credential 返回的 endpoint（通常是内网）
+    endpoint = creds.get("endpoint") or creds.get("oss_endpoint", "")
+    if not endpoint:
+        raise InstanceRepoError("STS response missing endpoint")
+
+    bkt = oss2.Bucket(auth, endpoint, bucket)
+
+    # 1. list 前缀目录，找 artifact tgz 对象（跳过 size=0 的目录占位符）
+    key = None
+    for obj in oss2.ObjectIterator(bkt, prefix=prefix):
+        if obj.size > 0 and obj.key.endswith(".tgz"):
+            key = obj.key
+            break
+    if not key:
+        raise ArtifactNotFoundError(
+            f"no artifact tgz found under {prefix!r} for job {job_id!r}")
+
+    # 2. 本地路径（保留 OSS 上的真实文件名）
+    job_dir = os.path.join(output_dir, job_id)
+    os.makedirs(job_dir, exist_ok=True)
+    local_path = os.path.join(job_dir, os.path.basename(key.rstrip("/")))
+
+    # 3. 下载（带 OSS 限流退避）
+    wait = OSS_RETRY_INITIAL_WAIT
+    for attempt in range(1, OSS_RETRY_MAX_ATTEMPTS + 1):
+        try:
+            bkt.get_object_to_file(key, local_path)
+            logger.info("downloaded %s → %s", key, local_path)
+            return local_path
+        except oss2.exceptions.ServerError as e:
+            if e.status == 503 and attempt < OSS_RETRY_MAX_ATTEMPTS:
+                logger.warning("OSS 503 for %s, retry %d/%d in %.1fs",
+                               job_id, attempt, OSS_RETRY_MAX_ATTEMPTS, wait)
+                import time
+                time.sleep(wait)
+                wait = min(wait * 2, OSS_RETRY_MAX_WAIT)
+            else:
+                raise
+    return local_path  # unreachable, but keeps type checker happy
+
+
+# ---------------------------------------------------------------------------
+# 公开 API（给 Repo 类调用）
+# ---------------------------------------------------------------------------
+
+def download_artifact(transport, api_key: str, job_id: str,
+                      cluster: str, output_dir: str = ".",
+                      ap_endpoint: str | None = None) -> str:
+    """下载单个 job 的 artifact (result.tgz)。
+
+    Args:
+        transport: SDK Transport 实例（用于 apiserver STS 请求）
+        api_key: 用户 API Key（透传给 AP 做 workspace ACL 检查）
+        job_id: AP job ID
+        cluster: 数据面名
+        output_dir: 本地输出目录
+        ap_endpoint: AP 控制面 URL（默认从环境变量或常量取）
+
+    Returns:
+        下载文件的本地路径
+    """
+    # 1. AP workspace ACL 检查
+    fetch_job_info(api_key, job_id, cluster, ap_endpoint)
+
+    # 2. 确定 bucket
+    bucket = resolve_artifact_bucket(cluster)
+
+    # 3. 下载
+    return _download_one(transport, bucket, job_id, output_dir)
+
+
+def download_artifacts(transport, api_key: str, job_ids: list[str],
+                       cluster: str, output_dir: str = ".",
+                       max_concurrent: int = DEFAULT_MAX_CONCURRENT,
+                       ap_endpoint: str | None = None,
+                       ) -> dict[str, str | Exception]:
+    """批量下载多个 job 的 artifact。
+
+    并发粒度是跨 job（不是单文件内多线程）。同数据面的多个 job 共享 STS 凭据。
+
+    Args:
+        transport: SDK Transport 实例
+        api_key: 用户 API Key
+        job_ids: AP job ID 列表
+        cluster: 数据面名
+        output_dir: 本地输出目录
+        max_concurrent: 最大并发下载数
+        ap_endpoint: AP 控制面 URL
+
+    Returns:
+        {job_id: 本地文件路径 或 Exception}
+    """
+    bucket = resolve_artifact_bucket(cluster)
+    results: dict[str, str | Exception] = {}
+
+    def _do(jid: str) -> tuple[str, str | Exception]:
+        try:
+            # AP workspace ACL 检查
+            fetch_job_info(api_key, jid, cluster, ap_endpoint)
+            # 下载
+            path = _download_one(transport, bucket, jid, output_dir)
+            return jid, path
+        except Exception as e:
+            logger.error("failed to download artifact for job %s: %s", jid, e)
+            return jid, e
+
+    with ThreadPoolExecutor(max_workers=max_concurrent) as pool:
+        futures = {pool.submit(_do, jid): jid for jid in job_ids}
+        for future in as_completed(futures):
+            jid, result = future.result()
+            results[jid] = result
+
+    return results

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/cli.py

@@ -380,6 +380,39 @@ def build_parser() -> argparse.ArgumentParser:
     ds.add_argument("--creator", default=None, help="workflows creator 过滤")
     ds.add_argument("--page", type=int, default=1)
     ds.add_argument("--page-size", type=int, default=20, dest="page_size")
+
+    # ── Artifact 下载（STS 直连 OSS，workspace ACL 鉴权）──
+    art = sub.add_parser("artifact",
+                         help="AP job artifact 下载（STS 直连 OSS）")
+    asub = art.add_subparsers(dest="artifact_cmd", required=True)
+
+    adl = asub.add_parser("download", help="下载单个 job 的 artifact")
+    adl.add_argument("--job-id", required=True, help="AP job ID")
+    adl.add_argument("--cluster", default=None,
+                     help="数据面名（缺省取 SDK profile 绑定的 cluster）")
+    adl.add_argument("--output-dir", default=".", dest="output_dir",
+                     help="本地输出目录")
+    adl.add_argument("--ap-endpoint", default=None, dest="ap_endpoint",
+                     help="AP 控制面 URL（可选）")
+
+    adlm = asub.add_parser("download-many", help="批量下载多个 job 的 artifact")
+    adlm.add_argument("--job-ids", required=True, dest="job_ids",
+                      help="逗号分隔的 job ID 列表")
+    adlm.add_argument("--cluster", default=None)
+    adlm.add_argument("--output-dir", default=".", dest="output_dir")
+    adlm.add_argument("--concurrent", type=int, default=8,
+                      help="最大并发下载数（默认 8）")
+    adlm.add_argument("--ap-endpoint", default=None, dest="ap_endpoint")
+
+    adlg = asub.add_parser("download-group",
+                           help="下载某 group 下全部 job 的 artifact")
+    adlg.add_argument("--group-id", required=True, dest="group_id",
+                      help="AP group ID")
+    adlg.add_argument("--cluster", default=None)
+    adlg.add_argument("--output-dir", default=".", dest="output_dir")
+    adlg.add_argument("--concurrent", type=int, default=8)
+    adlg.add_argument("--ap-endpoint", default=None, dest="ap_endpoint")
+
     return ap
 
 
@@ -1057,6 +1090,48 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
         print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
         return 0
 
+    if args.cmd == "artifact":
+        if args.artifact_cmd == "download":
+            path = r.download_artifact(
+                args.job_id, cluster=args.cluster,
+                output_dir=args.output_dir,
+                ap_endpoint=args.ap_endpoint)
+            print(str(path), file=out)
+            return 0
+        if args.artifact_cmd == "download-many":
+            job_ids = [j.strip() for j in args.job_ids.split(",") if j.strip()]
+            results = r.download_artifacts(
+                job_ids, cluster=args.cluster,
+                output_dir=args.output_dir,
+                max_concurrent=args.concurrent,
+                ap_endpoint=args.ap_endpoint)
+            ok, fail = 0, 0
+            for jid, res in results.items():
+                if isinstance(res, Exception):
+                    print(f"FAIL  {jid}: {res}", file=out)
+                    fail += 1
+                else:
+                    print(f"OK    {jid}: {res}", file=out)
+                    ok += 1
+            print(f"\ntotal={len(results)} ok={ok} failed={fail}", file=out)
+            return 1 if fail else 0
+        if args.artifact_cmd == "download-group":
+            results = r.download_artifacts_from_group(
+                args.group_id, cluster=args.cluster,
+                output_dir=args.output_dir,
+                max_concurrent=args.concurrent,
+                ap_endpoint=args.ap_endpoint)
+            ok, fail = 0, 0
+            for jid, res in results.items():
+                if isinstance(res, Exception):
+                    print(f"FAIL  {jid}: {res}", file=out)
+                    fail += 1
+                else:
+                    print(f"OK    {jid}: {res}", file=out)
+                    ok += 1
+            print(f"\ntotal={len(results)} ok={ok} failed={fail}", file=out)
+            return 1 if fail else 0
+
     return 2  # pragma: no cover - argparse 已保证 cmd 合法
 
 

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/clients/instances.py

@@ -57,10 +57,14 @@ class InstancesClient:
         self._ingester = ingester
 
     def _after_upload(self, register: bool, dataset: str,
-                      version: str) -> dict | None:
+                      version: str, split: str = "default") -> dict | None:
         """register=True：上传成功后触发一次 ingest 写库，返回 ingest job 摘要。
 
         register=False 返回 None（只传 OSS，由调用方决定何时 ingest）。
+
+        ``split`` 必须透传给 ingest：split_first + 无 version 时服务端要求显式
+        splits，否则 ingest 报 SchemaError——上传成功但 split 记录不落库，文件
+        孤悬 OSS（真机锤实：``instance push --split abc`` 无 --version 场景）。
         """
         if not register:
             return None
@@ -69,7 +73,7 @@ class InstancesClient:
                 "register=True 需要 ingest 回调：请通过 Repo() 构造客户端"
                 "（已注入 versions.ingest），或显式传入 ingester=...；"
                 "直接构造 InstancesClient 且未注入时请用 register=False 只传 OSS")
-        return self._ingester(dataset, version)
+        return self._ingester(dataset, version, split=split)
 
     def _layout(self, metadata_model: str | None = None) -> Layout:
         """按（显式或 profile 的）元数据模型构造布局策略。
@@ -387,7 +391,7 @@ class InstancesClient:
         # ⑤ 写库：register=True 时经 ingest 扫描入库（替代缺失的 :commit 三段式收尾）。
         #    ingest 幂等（重扫 OSS），skip 与否都由服务端按 OSS 实际字节重算 digest，
         #    不再有「本地 digest 覆盖别家生产者」的 skip 分支歧义。
-        ingest_result = self._after_upload(register, dataset, version)
+        ingest_result = self._after_upload(register, dataset, version, split=split)
         digests = {"content.tgz": digest}
         return PushResult(instance_id=instance_id, uploaded=not skipped,
                           skipped=skipped, image_pushed=image_pushed,
@@ -442,7 +446,7 @@ class InstancesClient:
                     f"{instance_id}: OSS 回读校验失败——上传后未在 OSS 找到 {idx_key}")
 
         # ④ 写库：register=True 时经 ingest 扫描入库（替代缺失的 :commit 收尾）
-        ingest_result = self._after_upload(register, dataset, version)
+        ingest_result = self._after_upload(register, dataset, version, split=split)
 
         return PushResult(instance_id=instance_id, uploaded=not skipped,
                           skipped=skipped, image_pushed=False,
@@ -533,7 +537,7 @@ class InstancesClient:
                     f"{instance_id}: OSS 回读校验失败——上传后未在 OSS 找到 {idx_key}")
 
         # ④ 写库：register=True 时经 ingest 扫描入库（替代缺失的 :commit 收尾）
-        ingest_result = self._after_upload(register, dataset, version)
+        ingest_result = self._after_upload(register, dataset, version, split=split)
 
         return PushResult(instance_id=instance_id, uploaded=not skipped,
                           skipped=skipped, image_pushed=False,
@@ -650,21 +654,28 @@ class InstancesClient:
         # continue_on_error=False 时 map_concurrent 已对失败项抛 ItemError，能走到
         # 这里说明整批成功；True 时对成功子集 ingest（与 deliver 语义一致）。
         if register:
-            self._after_upload(True, dataset, version)
+            self._after_upload(True, dataset, version, split=split)
         return results
 
     def pull(self, dataset: str, version: str, instance_id: str, dest_dir,
              split: str = "default", *, verify: bool = True) -> Path:
-        """下载实例内容。自动检测格式：harbor 下载 content.tgz，swe 下载 {id}.json。
+        """下载实例内容。自动检测格式：harbor 下载 content.tgz，swe 下载 {id}.json，
+        custom 下载 {id}.json + assets 目录。
 
         格式判断优先级：
-        1. verify=True 且 registry 有元数据 → 从 manifest 判断（有 tgz=harbor，否则 swe）
-        2. 无元数据（verify=False 或实例未注册）→ 先尝试 SWE（HEAD {id}.json），
-           存在则下载 json；不存在则 fallback 到 harbor（下载 content.tgz）
+        1. verify=True 且 registry 有元数据 → 从 manifest 判断
+           (有 tgz=harbor，仅 json=swe/custom，需进一步探测)
+        2. 无元数据或 manifest 无法区分 swe/custom → 下载 metadata JSON 检查
+           metadata_model 字段（custom → 下载 assets 目录；否则 swe）
+        3. metadata JSON 不存在 → fallback 到 harbor（下载 content.tgz）
         """
         validate_instance_id(instance_id, source="pull()")
         expected = None
         fmt = None  # None = 未确定，需要探测
+        lay = self._layout()
+        # 已下载的元数据字节（探测阶段下载一次，swe/custom 路径复用，避免二次网络往返）。
+        meta_bytes: bytes | None = None
+
         if verify:
             meta = self._reg.get(dataset, version, instance_id, split)
             if meta is not None and meta.manifest:
@@ -677,19 +688,35 @@ class InstancesClient:
                 if has_tgz:
                     fmt = "harbor"
                 else:
-                    # manifest 非空但无 tgz → 查找 json entry
+                    # manifest 有 json entry 但无 tgz → swe 或 custom（需进一步探测）。
+                    # custom 推的 manifest 由 ingest 扫描生成，与 swe 同形态；
+                    # 唯一区分手段是读 metadata JSON 检查 metadata_model 字段。
                     for m in meta.manifest:
                         if m.content_format == "json":
-                            fmt = "swe"
+                            fmt = "swe"  # 暂定 swe，后续探测可能修正为 custom
                             expected = m.content_digest
                             break
-                    # manifest 非空但既无 tgz 也无 json → fmt 保持 None，走探测
-        # 无法从 manifest 确定格式 → 探测 OSS。内建 store 的探测属于下载流程，
-        # 必须走私网限定的包内方法；自定义 ContentStore 保持旧接口兼容。
-        lay = self._layout()
+
+        # 无法从 manifest 确定 harbor → 探测 OSS 区分 swe/custom 或 fallback harbor。
+        idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
+        sts_prefix = lay.sts_prefix(dataset)
+
+        # ── 探测辅助：下载 metadata JSON 到临时文件并解析，区分 swe / custom ──
+        def _probe_meta_json() -> bytes | None:
+            """下载 metadata JSON 并返回字节；不存在或解析失败时返回 None。"""
+            tmp_dest = Path(dest_dir) / f".{instance_id}.json.probe"
+            tmp_dest.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                self._content.get_object(
+                    idx_key, tmp_dest, sts_prefix=sts_prefix, dataset=dataset)
+                data = tmp_dest.read_bytes()
+                return data
+            except Exception:  # noqa: BLE001 — 下载失败视为不存在
+                return None
+            finally:
+                tmp_dest.unlink(missing_ok=True)
+
         if fmt is None:
-            idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
-            sts_prefix = lay.sts_prefix(dataset)
             download_probe = getattr(self._content, "_download_probe", None)
             if download_probe is not None:
                 exists = download_probe(
@@ -698,22 +725,97 @@ class InstancesClient:
                 exists = self._content.object_exists(
                     idx_key, sts_prefix=sts_prefix)
             if exists:
-                fmt = "swe"
+                # metadata JSON 存在 → 下载并解析以区分 swe / custom
+                fmt = "swe"  # 默认；下载后可能修正为 custom
+                probed = _probe_meta_json()
+                if probed is not None:
+                    meta_bytes = probed
+                    try:
+                        meta_doc = json.loads(meta_bytes)
+                        if (isinstance(meta_doc, dict)
+                                and meta_doc.get("metadata_model") == "custom"):
+                            fmt = "custom"
+                    except (json.JSONDecodeError, UnicodeDecodeError):
+                        pass  # 无法解析 → 保持 swe
             else:
                 fmt = "harbor"
 
+        # 若 manifest 判定为 swe 但实际可能是 custom（两者 manifest 形态相同），
+        # 下载 metadata JSON 做最终确认。
+        if fmt == "swe" and meta_bytes is None:
+            probed = _probe_meta_json()
+            if probed is not None:
+                meta_bytes = probed
+                try:
+                    meta_doc = json.loads(meta_bytes)
+                    if (isinstance(meta_doc, dict)
+                            and meta_doc.get("metadata_model") == "custom"):
+                        fmt = "custom"
+                except (json.JSONDecodeError, UnicodeDecodeError):
+                    pass
+
+        if fmt == "custom":
+            dest = Path(dest_dir) / f"{instance_id}.json"
+            if meta_bytes is not None:
+                # 已在探测阶段下载，直接写入本地（避免二次网络往返）。
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_bytes(meta_bytes)
+            else:
+                self._content.get_object(
+                    idx_key, dest, expected_digest=expected,
+                    sts_prefix=sts_prefix, dataset=dataset)
+            # 下载资产目录（{split}-assets/{id}/ 前缀下的所有文件）
+            self._download_custom_assets(
+                dataset, instance_id, split, dest_dir, lay, sts_prefix)
+            return dest
+
         if fmt == "swe":
             dest = Path(dest_dir) / f"{instance_id}.json"
-            idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
+            if meta_bytes is not None:
+                # 已在探测阶段下载，直接写入本地。
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_bytes(meta_bytes)
+                return dest
             return self._content.get_object(
                 idx_key, dest, expected_digest=expected,
-                sts_prefix=lay.sts_prefix(dataset),
+                sts_prefix=sts_prefix,
                 dataset=dataset)
+
         # harbor：下载 content.tgz
         dest = Path(dest_dir) / instance_id / "content.tgz"
         return self._content.get(dataset, version, instance_id, split, dest,
                                  expected_digest=expected, layout=lay)
 
+    def _download_custom_assets(self, dataset: str, instance_id: str,
+                                split: str, dest_dir, lay, sts_prefix: str
+                                ) -> list[Path]:
+        """下载 custom 格式的资产目录（{split}-assets/{id}/ 前缀下的所有文件）。
+
+        资产文件按相对路径还原到 ``{dest_dir}/{instance_id}/`` 下。返回已下载
+        的本地文件路径列表。无资产时返回空列表（不报错）。
+        """
+        asset_prefix = (f"{lay.dataset_root(dataset)}"
+                        f"{split}-assets/{instance_id}/")
+        list_fn = getattr(self._content, "list_objects", None)
+        if list_fn is None:
+            return []
+        keys = list_fn(asset_prefix)
+        if not keys:
+            return []
+        downloaded: list[Path] = []
+        dest_root = Path(dest_dir) / instance_id
+        for key in keys:
+            rel = key[len(asset_prefix):]
+            if not rel or rel.endswith("/"):
+                continue  # 跳过空键和目录占位符
+            local_path = dest_root / rel
+            local_path.parent.mkdir(parents=True, exist_ok=True)
+            self._content.get_object(
+                key, local_path,
+                sts_prefix=sts_prefix, dataset=dataset)
+            downloaded.append(local_path)
+        return downloaded
+
     def pull_many(self, instance_ids: list[str], dataset: str, version: str,
                   dest_dir, *, split: str = "default",
                   concurrency: int = 8, continue_on_error: bool = False,

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/clients/workflows.py

@@ -18,7 +18,6 @@ from __future__ import annotations
 
 from urllib.parse import quote
 
-from ..errors import TransportError
 from ..models import PagedResult, Workflow
 
 WORKFLOW_STATUS_VALUES = ("pending", "running", "awaiting_approval",

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/repo.py

@@ -1,9 +1,12 @@
 """repo.py — Repo Facade：组装 transport + store + 三资源客户端。"""
 from __future__ import annotations
 
+import logging
 import os
 import posixpath
 
+logger = logging.getLogger(__name__)
+
 from . import bootstrap
 from . import paths as P
 from .clients.config import (
@@ -297,7 +300,6 @@ class Repo:
         """
         from pathlib import Path as _Path
         import shutil as _shutil
-        from . import release as _R
         from .clients.config import load_profile
         from .validate import detect_format
 
@@ -460,6 +462,85 @@ class Repo:
             pfx += self._clean_user_data_subpath(subpath)
         return self._content.list_objects(pfx)
 
+    # ── Artifact 下载（STS 直连 OSS，workspace ACL 鉴权）──
+
+    def download_artifact(self, job_id: str, cluster: str | None = None,
+                          output_dir: str = ".", *,
+                          ap_endpoint: str | None = None) -> str:
+        """下载单个 job 的 artifact (result.tgz) 到本地。
+
+        鉴权：AP workspace ACL（SDK 调 AP GET /api/jobs/{id} 时自动检查）。
+        并发模型：单 job = 单文件单线程。
+
+        Args:
+            job_id: AP job ID
+            cluster: 数据面名（默认使用 SDK profile 绑定的 cluster）
+            output_dir: 本地输出目录
+            ap_endpoint: AP 控制面 URL（可选，默认从环境变量或内置常量取）
+
+        Returns:
+            下载文件的本地路径
+        """
+        from .artifacts import download_artifact as _dl
+        cluster = cluster or self.profile.cluster or ""
+        return _dl(self.transport, self.transport.token, job_id,
+                   cluster, output_dir, ap_endpoint)
+
+    def download_artifacts(self, job_ids: list[str],
+                           cluster: str | None = None,
+                           output_dir: str = ".", *,
+                           max_concurrent: int = 8,
+                           ap_endpoint: str | None = None,
+                           ) -> dict[str, str | Exception]:
+        """批量下载多个 job 的 artifact。
+
+        并发粒度是跨 job（不是单文件内多线程）。同数据面的多个 job 共享 STS 凭据。
+
+        Args:
+            job_ids: AP job ID 列表
+            cluster: 数据面名
+            output_dir: 本地输出目录
+            max_concurrent: 最大并发下载数（默认 8）
+            ap_endpoint: AP 控制面 URL
+
+        Returns:
+            {job_id: 本地文件路径 或 Exception}
+        """
+        from .artifacts import download_artifacts as _dl_many
+        cluster = cluster or self.profile.cluster or ""
+        return _dl_many(self.transport, self.transport.token, job_ids,
+                        cluster, output_dir, max_concurrent, ap_endpoint)
+
+    def download_artifacts_from_group(self, group_id: str,
+                                      cluster: str | None = None,
+                                      output_dir: str = ".", *,
+                                      max_concurrent: int = 8,
+                                      ap_endpoint: str | None = None,
+                                      ) -> dict[str, str | Exception]:
+        """下载某个 group 下全部 job 的 artifact。
+
+        先调 AP GET /groups/{group_id}/jobs 分页收集 job_id，再批量下载。
+
+        Args:
+            group_id: AP group ID
+            cluster: 数据面名
+            output_dir: 本地输出目录
+            max_concurrent: 最大并发下载数
+            ap_endpoint: AP 控制面 URL
+
+        Returns:
+            {job_id: 本地文件路径 或 Exception}
+        """
+        from .artifacts import fetch_group_job_ids, download_artifacts as _dl_many
+        cluster = cluster or self.profile.cluster or ""
+        job_ids = fetch_group_job_ids(
+            self.transport.token, group_id, cluster, ap_endpoint)
+        if not job_ids:
+            logger.warning("group %s has no jobs", group_id)
+            return {}
+        return _dl_many(self.transport, self.transport.token, job_ids,
+                        cluster, output_dir, max_concurrent, ap_endpoint)
+
     # ── 用户数据 ACL（role-bindings 接口，resource_type=user_data）──
     _USER_DATA_ROLE_MAP = {
         "reader": "built-in:user-data-reader",

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo/store/oss.py

@@ -17,6 +17,7 @@ from .._routing import (
     _plan_oss_route,
     _probe_private_endpoint,
 )
+from ..concurrency import ItemError, map_concurrent
 from ..content import sha256_file, verify_file_digest
 from ..errors import DigestMismatch, PrivateNetworkRequired, TransportError
 from ..retry import retry_transient
@@ -244,6 +245,13 @@ class OssContentStore(BaseContentStore):
             if not route.standard:
                 raise _unconfirmed_private_endpoint_error(original_endpoint)
             endpoint = route.private_endpoint
+            # Fail-fast: 3 秒探测私网可达性，不通立即报错，避免公网环境下
+            # 傻等 TCP 超时（20s × 多轮重试 = 几分钟挂死）。下载仍不 fallback
+            # 公网（安全约束），但能快速告知调用方需要私网环境。
+            if not self._probe_private_endpoint(endpoint):
+                raise _private_endpoint_error(
+                    endpoint, ConnectionRefusedError(
+                        f"TCP probe failed for {endpoint} within 3s"))
             try:
                 bucket = self._open_bucket(
                     sts, prefix, endpoint, private_download=True)
@@ -384,6 +392,109 @@ class OssContentStore(BaseContentStore):
         return self._execute_oss(
             _OperationIntent.NON_DOWNLOAD, prefix, None, list_all)
 
+    # ── directory-level helpers（自定义格式实例 = 目录级上传/下载）──
+    def put_directory(self, local_dir: Path, oss_prefix: str, *,
+                      overwrite: bool = False,
+                      sts_prefix: str | None = None,
+                      dataset: str | None = None,
+                      concurrency: int = 8) -> dict:
+        """Upload all files under *local_dir* to OSS under *oss_prefix*.
+
+        Each file is uploaded via :meth:`put_object` so every transfer gets
+        proper STS credentials and endpoint routing.  Files are uploaded in
+        parallel using :func:`~instance_repo.concurrency.map_concurrent`.
+
+        Returns a summary dict::
+
+            {"file_count": int, "total_bytes": int, "keys": list[str]}
+        """
+        local_dir = Path(local_dir)
+        if not local_dir.is_dir():
+            raise FileNotFoundError(f"local directory not found: {local_dir}")
+
+        # Normalize prefix to always end with "/" so keys are proper children.
+        if oss_prefix and not oss_prefix.endswith("/"):
+            oss_prefix += "/"
+
+        # Collect (local_path, oss_key) pairs — sorted for determinism.
+        files: list[tuple[Path, str]] = sorted(
+            ((p, oss_prefix + str(p.relative_to(local_dir)))
+             for p in local_dir.rglob("*") if p.is_file()),
+            key=lambda t: t[1],
+        )
+
+        total_bytes = sum(p.stat().st_size for p, _ in files)
+
+        def _upload_one(item: tuple[Path, str]) -> str:
+            path, key = item
+            self.put_object(key, path, overwrite=overwrite,
+                            sts_prefix=sts_prefix, dataset=dataset)
+            return key
+
+        try:
+            keys = map_concurrent(_upload_one, files, concurrency=concurrency)
+        except ItemError as exc:
+            raise exc.error from exc
+        return {
+            "file_count": len(files),
+            "total_bytes": total_bytes,
+            "keys": keys,
+        }
+
+    def get_directory(self, oss_prefix: str, local_dir: Path, *,
+                      expected_digests: dict[str, str] | None = None,
+                      sts_prefix: str | None = None,
+                      dataset: str | None = None,
+                      concurrency: int = 8) -> Path:
+        """Download all objects under *oss_prefix* to *local_dir*.
+
+        Objects are listed via :meth:`list_objects` and then downloaded in
+        parallel.  Each file is fetched via :meth:`get_object` so every
+        transfer gets proper STS credentials and private-download routing.
+
+        *expected_digests* optionally maps **relative paths** (e.g.
+        ``"subdir/file.bin"``) to their expected content digest strings
+        (``"sha256:<hex>"`` or bare md5).  Any mismatch raises
+        :class:`~instance_repo.errors.DigestMismatch`.
+
+        Returns the *local_dir* path.
+        """
+        local_dir = Path(local_dir)
+        local_dir.mkdir(parents=True, exist_ok=True)
+
+        # Normalize prefix for consistent key slicing.
+        if oss_prefix and not oss_prefix.endswith("/"):
+            oss_prefix += "/"
+
+        keys = self.list_objects(oss_prefix)
+
+        def _download_one(key: str) -> Path:
+            rel = key[len(oss_prefix):]
+            if not rel:  # skip the prefix "directory marker" itself
+                return local_dir
+            dest = local_dir / rel
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            self._execute_oss(
+                _OperationIntent.DOWNLOAD,
+                sts_prefix or oss_prefix,
+                dataset,
+                lambda bucket: bucket.get_object_to_file(key, str(dest)),
+            )
+            # Per-file digest check.
+            if expected_digests is not None:
+                expected = expected_digests.get(rel)
+                if expected is not None and not verify_file_digest(dest, expected):
+                    raise DigestMismatch(
+                        f"{key}: content digest mismatch "
+                        f"(expected {expected}, got {sha256_file(dest)})")
+            return dest
+
+        try:
+            map_concurrent(_download_one, keys, concurrency=concurrency)
+        except ItemError as exc:
+            raise exc.error from exc
+        return local_dir
+
     # ── 实例内容便捷封装（content.tgz + {id}.json） ──
     def exists(self, instance_id: str, dataset: str, version: str,
                split: str = "default", *, layout=None) -> bool:

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.9.dev6
+Version: 1.1.0.dev0
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/instance_repo.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ pyproject.toml
 instance_repo/__init__.py
 instance_repo/_routing.py
 instance_repo/_site_defaults.py
+instance_repo/artifacts.py
 instance_repo/bootstrap.py
 instance_repo/cache.py
 instance_repo/cli.py

{instance_repo-1.0.9.dev6 → instance_repo-1.1.0.dev0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "instance-repo"
-version = "1.0.9.dev6"
+version = "1.1.0.dev0"
 description = "InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）"
 requires-python = ">=3.10"
 readme = "README.md"