instance-repo 1.0.7.dev0__tar.gz → 1.0.8__tar.gz

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.7.dev0
+Version: 1.0.8
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/instance_repo/__init__.py

@@ -18,7 +18,7 @@ from .errors import (
     TransportError,
 )
 
-__version__ = "1.0.7.dev0"
+__version__ = "1.0.8"
 
 __all__ = [
     "Repo", "load_instances",

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/instance_repo/clients/instances.py

@@ -21,7 +21,9 @@ from ..layout import Layout
 from ..models import (
     DatasetInstance, ManifestEntry, OwnerReference, PushResult, Feedback,
 )
-from ..validate import check_layout, check_swe_json, detect_format, validate_instance_id
+from ..validate import (check_layout, check_swe_json, check_custom,
+                        detect_format, validate_instance_id,
+                        CUSTOM_SCHEMA_VERSION)
 
 logger = logging.getLogger(__name__)
 
@@ -96,6 +98,11 @@ class InstancesClient:
             return self._push_swe(local_path, dataset, version, split=split,
                                   overwrite=overwrite, register=register,
                                   verify=verify)
+        if fmt == "custom":
+            return self._push_custom(local_path, dataset, version, split=split,
+                                     push_image=push_image,
+                                     overwrite=overwrite, register=register,
+                                     verify=verify)
         # ── harbor 路径 ──
         report = check_layout(local_path)          # ① 本地校验（缺文件即抛）
         instance_id = report.instance_id
@@ -147,23 +154,30 @@ class InstancesClient:
             instance_status="draft",
         )
         uid = instance_id
-        if register:
-            registered = self._reg.register(meta)  # 校验 schema（owner_reference 可选）
-            uid = registered.id or instance_id
-
-        image_pushed = False
-        if push_image and has_image:
-            # ③ 推 ACR（final 改写 Dockerfile）
-            self._image.push(image_tar, docker_ref, overwrite=overwrite)
-            rewrite_dockerfile(dockerfile, docker_ref, tar_role="final")
-            image_pushed = True
 
-        # ④ 打包 + 上传 OSS（排除 image.tar.gz）；{id}.json 供 pull 校验 / ingest 扫描
+        # 打包 + 构造 manifest（本地）→ register(pending) → 推 ACR → 上传 OSS。
+        # register 需要 manifest：apiserver 的 hasFile 校验要求至少一条
+        # content_type=file 项，manifest 缺失时 register 被 92005 拒（曾导致
+        # custom/harbor 主路径 push(register=True) 失败）。register 仍先于任何
+        # 远程写（元数据先行），manifest 在本地打包时即可算出 digest。
         with tempfile.TemporaryDirectory(prefix="irepo_") as td:
             tgz = Path(td) / "content.tgz"
             digest = pack(local_path, tgz, exclude_image_tar=True)
             meta.manifest = _manifest(self._p.oss_bucket, dataset, version,
                                       instance_id, split, digest, lay)
+
+            if register:
+                registered = self._reg.register(meta)  # 校验 schema（owner_reference 可选）
+                uid = registered.id or instance_id
+
+            image_pushed = False
+            if push_image and has_image:
+                # ③ 推 ACR（final 改写 Dockerfile）
+                self._image.push(image_tar, docker_ref, overwrite=overwrite)
+                rewrite_dockerfile(dockerfile, docker_ref, tar_role="final")
+                image_pushed = True
+
+            # ④ 上传 OSS（排除 image.tar.gz）；{id}.json 供 pull 校验 / ingest 扫描
             put = self._content.put(instance_id, dataset, version, split, tgz,
                                     meta.to_canonical_json(), overwrite=overwrite,
                                     layout=lay)
@@ -272,6 +286,120 @@ class InstancesClient:
                           docker_image=docker_image or None,
                           digests={"instance.json": digest})
 
+    def _push_custom(self, local_path: Path, dataset: str, version: str, *,
+                     split: str = "default", push_image: bool = True,
+                     overwrite: bool = False, register: bool = True,
+                     verify: bool = True) -> PushResult:
+        """推一个 custom 格式实例：validate → 上传元数据 JSON → (目录输入时) 上传资产。
+
+        custom 格式的两种输入：
+
+        - **目录**：SDK 生成最小元数据 JSON（``instance_id`` / ``metadata_model`` /
+          ``schema_version`` / ``created_at`` / ``updated_at``）上传到 ``{split}/{id}.json``，
+          目录内容**原样**逐文件上传到 ``{split}-assets/{id}/`` 前缀下（保持相对路径）。
+        - **``.json`` 文件**：校验后**原样**上传到 ``{split}/{id}.json``（无资产）。
+
+        custom 格式**不参与镜像流程**：``push_image=True`` 时告警并忽略（与 SWE 同口径）。
+        """
+        import datetime as _dt
+        import warnings as _warnings
+
+        if push_image:
+            _warnings.warn(
+                "custom 格式不支持 ACR 镜像推送（push_image=True 已忽略）；"
+                "如需镜像请用 harbor 格式",
+                UserWarning, stacklevel=3)
+
+        report = check_custom(local_path)           # ① 校验（含 instance_id 安全检查）
+        instance_id = report.instance_id
+        lay = self._layout()
+        idx_key = lay.metadata_key(dataset, instance_id, version=version, split=split)
+        sts_prefix = lay.sts_prefix(dataset)
+
+        if report.source == "json":
+            # JSON 输入：原样上传
+            raw_bytes = local_path.read_bytes()
+            meta_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest()
+            put = self._content.put_object(
+                idx_key, raw_bytes, overwrite=overwrite,
+                sts_prefix=sts_prefix, dataset=dataset)
+            skipped = bool(put.get("skipped"))
+            asset_digests: dict[str, str] = {}
+        else:
+            # 目录输入：生成最小元数据 JSON + 原样上传资产
+            now_iso = _dt.datetime.now(_dt.timezone.utc).isoformat(
+                timespec="seconds").replace("+00:00", "Z")
+            meta_doc = {
+                "instance_id": instance_id,
+                "metadata_model": "custom",
+                "schema_version": CUSTOM_SCHEMA_VERSION,
+                "created_at": now_iso,
+                "updated_at": now_iso,
+            }
+            raw_bytes = json.dumps(meta_doc, ensure_ascii=False).encode("utf-8")
+            meta_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest()
+            put = self._content.put_object(
+                idx_key, raw_bytes, overwrite=overwrite,
+                sts_prefix=sts_prefix, dataset=dataset)
+            skipped = bool(put.get("skipped"))
+
+            # 原样上传目录内所有文件到 {split}-assets/{id}/<relpath>
+            asset_prefix = (f"{lay.dataset_root(dataset)}"
+                            f"{split}-assets/{instance_id}/")
+            asset_digests = {}
+            for child in sorted(local_path.rglob("*")):
+                if not child.is_file():
+                    continue
+                rel = child.relative_to(local_path).as_posix()
+                key = asset_prefix + rel
+                data = child.read_bytes()
+                digest = "sha256:" + hashlib.sha256(data).hexdigest()
+                asset_digests[rel] = digest
+                aput = self._content.put_object(
+                    key, data, overwrite=overwrite,
+                    sts_prefix=sts_prefix, dataset=dataset)
+                if not skipped and bool(aput.get("skipped")):
+                    skipped = True  # 任一被 skip 即整体视为 skip
+
+        # ③ 回读校验（防静默成功）
+        if verify and not skipped:
+            if not self._content.object_exists(idx_key, sts_prefix=sts_prefix,
+                                               dataset=dataset):
+                raise TransportError(
+                    f"{instance_id}: OSS 回读校验失败——上传后未在 OSS 找到 {idx_key}")
+
+        # ④ 可选 register/commit（manifest 列出元数据 + 每个资产文件的相对路径）
+        manifest = [ManifestEntry(
+            file_path=P.oss_uri(self._p.oss_bucket, idx_key),
+            content_type="file", content_format="json",
+            content_digest=meta_digest)]
+        for rel, digest in asset_digests.items():
+            manifest.append(ManifestEntry(
+                file_path=P.oss_uri(self._p.oss_bucket,
+                                    f"{asset_prefix}{rel}"),
+                content_type="file", content_format="raw",
+                content_digest=digest))
+
+        if register:
+            meta = DatasetInstance(
+                instance_id=instance_id, dataset=dataset,
+                dataset_version=version, split=split,
+                manifest=manifest,
+                extend_meta={"metadata_model": "custom",
+                             "schema_version": CUSTOM_SCHEMA_VERSION},
+                instance_status="draft",
+            )
+            registered = self._reg.register(meta)
+            uid = registered.id or instance_id
+            digests_for_commit = {"instance.json": meta_digest}
+            digests_for_commit.update(asset_digests)
+            self._reg.commit(uid, digests_for_commit, manifest=manifest)
+
+        return PushResult(instance_id=instance_id, uploaded=not skipped,
+                          skipped=skipped, image_pushed=False,
+                          docker_image=None,
+                          digests={"instance.json": meta_digest})
+
     # ── 上架前：把生产副本的 task.toml 指向生产镜像（本地重写，不触网）──────────
     def retarget_prod_content(self, local_dir, dataset: str, version: str,
                               split: str, target_ref: str, *,

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/instance_repo/layout.py

@@ -37,9 +37,13 @@ from .errors import SchemaError
 
 MODEL_VERSION_FIRST = "version_first"
 MODEL_SPLIT_FIRST = "split_first"
+#: ``custom`` 是用户自定义格式的元数据模型：OSS 路径沿用 split_first 约定
+#: （{split}/{id}.json 元数据 + {split}-assets/{id}/ 原样目录资产），SDK 不校验具体
+#: 字段（仅强约束 instance_id），元数据 freeform。
+MODEL_CUSTOM = "custom"
 #: v0.8 起默认 split_first（v0.7.x 行为等价于显式传 version_first）。
 DEFAULT_METADATA_MODEL = MODEL_SPLIT_FIRST
-_MODELS = (MODEL_VERSION_FIRST, MODEL_SPLIT_FIRST)
+_MODELS = (MODEL_VERSION_FIRST, MODEL_SPLIT_FIRST, MODEL_CUSTOM)
 
 # ACR tag 合法字符集与长度（OCI/Docker 约定）：首字符须为字母数字或下划线，
 # 其余可含字母数字、下划线、点、连字符，总长 **< 105**（ACR tag 硬上限 128，
@@ -126,7 +130,9 @@ class Layout:
 
     @property
     def split_first(self) -> bool:
-        return self.model == MODEL_SPLIT_FIRST
+        # custom 沿用 split_first 的路径约定（{split}/{id}.json +
+        # {split}-assets/{id}/ 兄弟目录），故 path 构造上一视同仁。
+        return self.model in (MODEL_SPLIT_FIRST, MODEL_CUSTOM)
 
     # ── 目录基座 ──
     def dataset_root(self, dataset: str) -> str:

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/instance_repo/validate.py

@@ -1,8 +1,10 @@
-"""validate.py — Instance 本地校验（Layout / SWE JSON），纯本地无网络。
+"""validate.py — Instance 本地校验（Layout / SWE JSON / Custom），纯本地无网络。
 
-两种格式自动检测（detect_format）：
-    - harbor（目录）：instruction.md · task.toml · environment/Dockerfile · tests/test.sh|test.ps1
+三种格式自动检测（detect_format）：
+    - harbor（目录 + task.toml）：instruction.md · task.toml · environment/Dockerfile · tests/test.sh|test.ps1
     - swe（单 JSON 文件）：instance_id + docker_image 必备；数据和元数据合一
+    - custom（目录无 task.toml / 单 JSON 文件 freeform）：仅强约束 instance_id；
+      元数据 freeform，push 时目录输入由 SDK 生成最小 JSON、JSON 输入原样上传
 
 测试脚本兼容 Linux（tests/test.sh）与 Windows（tests/test.ps1），二者至少其一。
 """
@@ -50,21 +52,25 @@ def validate_instance_id(instance_id: str, source: str = "") -> None:
 
 # ── 格式检测 ─────────────────────────────────────────────────────────────────
 
-def detect_format(local_path: str | Path) -> Literal["harbor", "swe"]:
-    """自动检测输入格式：目录→harbor，.json 文件→swe。
+def detect_format(local_path: str | Path) -> Literal["harbor", "swe", "custom"]:
+    """自动检测输入格式：
 
-    不做内容校验（那是 check_layout / check_swe_json 的责任），只按路径形态判断。
+    - 目录 + 含 task.toml → harbor
+    - 目录 + 不含 task.toml → custom（用户自定义格式，SDK 不校验具体字段）
+    - .json 文件 → swe
+
+    不做内容校验（那是 check_layout / check_swe_json / check_custom 的责任）。
     """
     p = Path(local_path)
     if p.is_dir():
-        return "harbor"
+        return "harbor" if (p / "task.toml").is_file() else "custom"
     if p.is_file() and p.suffix.lower() == ".json":
         return "swe"
     if not p.exists():
         raise LayoutError(f"path does not exist: {p}")
     raise SchemaError(
         f"unrecognized instance format: {p} "
-        f"(expected a directory for harbor or a .json file for swe)")
+        f"(expected a directory for harbor/custom or a .json file for swe)")
 
 
 # ── Harbor 格式校验 ──────────────────────────────────────────────────────────
@@ -204,3 +210,70 @@ def check_swe_json(json_path: str | Path) -> SWEValidationReport:
     return SWEValidationReport(
         instance_id=instance_id, ok=True, warnings=warnings,
         metadata=metadata, raw_data=data)
+
+
+# ── Custom 格式校验（用户自定义，freeform） ─────────────────────────────────
+
+CUSTOM_SCHEMA_VERSION = "1.0"
+
+
+@dataclass
+class CustomValidationReport:
+    """Custom 格式校验结果。"""
+    instance_id: str
+    format: str = "custom"
+    ok: bool = True
+    metadata: dict = field(default_factory=dict)  # 目录输入时 SDK 生成；JSON 输入时原样解析
+    raw_data: dict = field(default_factory=dict)  # JSON 输入时的原始数据；目录输入时为空
+    source: str = "directory"                     # "directory" 或 "json"
+
+
+def check_custom(local_path: str | Path) -> CustomValidationReport:
+    """校验 custom 格式输入。
+
+    两种输入形态：
+
+    - 目录：``instance_id`` = 目录名（强约束 charset），元数据由 SDK push 时生成
+      （最小 schema：``instance_id`` / ``metadata_model`` / ``schema_version`` /
+      ``created_at`` / ``updated_at``）。
+    - ``.json`` 文件：必须含 ``instance_id`` 字段且与文件名 stem 一致；其余字段
+      freeform，push 时原样上传。
+    """
+    p = Path(local_path)
+    if p.is_dir():
+        instance_id = p.name
+        validate_instance_id(instance_id, source=str(p))
+        return CustomValidationReport(
+            instance_id=instance_id, source="directory")
+    if p.is_file() and p.suffix.lower() == ".json":
+        try:
+            with open(p, "r", encoding="utf-8") as f:
+                data = json.load(f)
+        except UnicodeDecodeError as e:
+            raise SchemaError(
+                f"{p.name}: not valid UTF-8: {e}") from e
+        except json.JSONDecodeError as e:
+            raise SchemaError(f"{p.name}: invalid JSON: {e}") from e
+        if not isinstance(data, dict):
+            raise SchemaError(
+                f"{p.name}: JSON root must be an object, got {type(data).__name__}")
+        raw_id = data.get("instance_id")
+        if not raw_id or (isinstance(raw_id, str) and not raw_id.strip()):
+            raise SchemaError(f"{p.name}: required field 'instance_id' missing or empty")
+        if not isinstance(raw_id, str):
+            raise SchemaError(
+                f"{p.name}: instance_id must be a string, got {type(raw_id).__name__}")
+        instance_id = raw_id.strip()
+        validate_instance_id(instance_id, source=p.name)
+        stem = p.stem
+        if stem != instance_id:
+            raise SchemaError(
+                f"{p.name}: filename stem '{stem}' does not match instance_id "
+                f"'{instance_id}' — file must be named '{instance_id}.json'")
+        return CustomValidationReport(
+            instance_id=instance_id, source="json",
+            metadata=dict(data), raw_data=dict(data))
+    if not p.exists():
+        raise LayoutError(f"path does not exist: {p}")
+    raise SchemaError(
+        f"custom format expects a directory or .json file, got: {p}")

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/instance_repo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.0.7.dev0
+Version: 1.0.8
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.0.7.dev0 → instance_repo-1.0.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "instance-repo"
-version = "1.0.7.dev0"
+version = "1.0.8"
 description = "InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）"
 requires-python = ">=3.10"
 readme = "README.md"