instance-repo 1.3.2.dev2__tar.gz → 1.3.2.dev3__tar.gz

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.3.2.dev2
+Version: 1.3.2.dev3
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/__init__.py

@@ -18,7 +18,7 @@ from .errors import (
     TransportError,
 )
 
-__version__ = "1.3.2.dev2"
+__version__ = "1.3.2.dev3"
 
 __all__ = [
     "Repo", "load_instances",

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/_routing.py

@@ -45,14 +45,14 @@ def _split_scheme(endpoint: str) -> tuple[str, str]:
     return "", endpoint
 
 
-def _probe_private_endpoint(endpoint: str, timeout: float = 3.0) -> bool:
+def _probe_private_endpoint(endpoint: str, timeout: float = 0.5) -> bool:
     """Quick TCP connect probe to decide whether a private endpoint is reachable.
 
     Performs a raw TCP connect (no TLS handshake) to the endpoint host/port.
     Returns True if the connect succeeds, False on any error.  The default
-    timeout is 3 seconds — short enough to save 60+ seconds of waiting when
-    the private endpoint is unreachable, yet long enough for a normal TCP
-    handshake on a reachable host.
+    timeout is 0.5 seconds — 内网 TCP 握手通常在几十毫秒内完成，0.5s 足够
+    判定可达性，同时把公网环境下私网不可达的探测开销从 3s 压到 0.5s
+    （结果按 store 缓存 5 分钟，仅端点变更或缓存过期才重探）。
     """
     _, host = _split_scheme(endpoint)
     # Determine port: default 443 for https, 80 for http.

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/clients/datasets.py

@@ -16,10 +16,13 @@ class DatasetsClient:
     _BASE = "/apis/v1/datasets/series"
 
     def __init__(self, transport, *, content_store=None,
-                 oss_prefix: str = "swe/datasets") -> None:
+                 oss_prefix: str = "swe/datasets",
+                 metadata_model: str | None = None) -> None:
         self._t = transport
         self._content = content_store
         self._oss_prefix = oss_prefix
+        # OSS 回退探测的布局语义（split_first 默认；version_first 显式切换）
+        self._metadata_model = metadata_model or "split_first"
 
     def list(self, l1: str | None = None) -> list[DatasetSeries]:
         """列出 dataset（便利语义：自动翻页取全量，见 :meth:`list_paged` 单页对照）。
@@ -398,9 +401,10 @@ class DatasetsClient:
             parse_dataset_level, build_synthetic_dataset_detail,
             _warn_if_truncated)
         info = parse_dataset_level(
-            self._content, name, oss_prefix=self._oss_prefix)
+            self._content, name, oss_prefix=self._oss_prefix,
+            metadata_model=self._metadata_model)
         _warn_if_truncated(info, name, 500)
-        if not info.versions:
+        if not info.versions and not info.unversioned_splits:
             from ..errors import DatasetNotFound
             raise DatasetNotFound(f"dataset {name!r} not found in OSS")
         return build_synthetic_dataset_detail(

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/clients/splits.py

@@ -264,16 +264,23 @@ class SplitsClient:
         注意：回退场景（dataset 未注册）下 ``version_scope`` 是 MongoDB 语义，
         OSS 探测无从区分 versioned/unversioned——**忽略 scope 并返回全部探测结果**
         （G4：否则 ap CLI 无 --version 的 split list 恒空）。``version`` 非空时
-        仍精确匹配该 version。
+        仍精确匹配该 version；version 为空时额外包含 split_first 无版本 split
+        （version="" 归属）。
         """
         from ..store.oss_discovery import (
             parse_dataset_level, build_synthetic_split,
             _in_memory_paged, _warn_if_truncated,
         )
         info = parse_dataset_level(
-            self._content, dataset, oss_prefix=self._oss_prefix)
+            self._content, dataset, oss_prefix=self._oss_prefix,
+            metadata_model=self._discovery_mm())
         _warn_if_truncated(info, dataset, 500)
         splits: list[DatasetSplit] = []
+        # split_first 无版本布局：dataset 根下直接是 split 目录
+        if not version:
+            for us in info.unversioned_splits:
+                splits.append(build_synthetic_split(
+                    dataset, "", us.name, instance_count=us.instance_count))
         for v in info.versions:
             if version and v.version != version:
                 continue
@@ -295,8 +302,14 @@ class SplitsClient:
             parse_dataset_level, build_synthetic_split, _warn_if_truncated,
         )
         info = parse_dataset_level(
-            self._content, dataset, oss_prefix=self._oss_prefix)
+            self._content, dataset, oss_prefix=self._oss_prefix,
+            metadata_model=self._discovery_mm())
         _warn_if_truncated(info, dataset, 500)
+        if not version:
+            for us in info.unversioned_splits:
+                if us.name == split_name:
+                    return build_synthetic_split(
+                        dataset, "", us.name, instance_count=us.instance_count)
         for v in info.versions:
             if version and v.version != version:
                 continue
@@ -309,6 +322,13 @@ class SplitsClient:
                     dataset, v.version, split_name)
         return None
 
+    def _discovery_mm(self) -> str:
+        """OSS 回退探测的布局语义（split_first 默认；version_first 显式切换）。"""
+        from ..layout import normalize_metadata_model
+        return normalize_metadata_model(
+            getattr(self._p, "metadata_model", None)
+            if self._p is not None else None)
+
     # ── 写 ──
 
     def _post_create(self, dataset: str, split: str, version: str,

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/clients/versions.py

@@ -895,9 +895,11 @@ class VersionsClient:
         from ..store.oss_discovery import (
             parse_dataset_level, build_synthetic_version, _warn_if_truncated)
         info = parse_dataset_level(
-            self._content, dataset, oss_prefix=self._oss_prefix)
+            self._content, dataset, oss_prefix=self._oss_prefix,
+            metadata_model=self._discovery_mm())
         _warn_if_truncated(info, dataset, 500)
-        if not info.versions:
+        # split_first 无版本布局（只有 split）也是发现成功，只是没有版本可列
+        if not info.versions and not info.unversioned_splits:
             from ..errors import DatasetNotFound
             raise DatasetNotFound(f"dataset {dataset!r} not found in OSS")
         return [build_synthetic_version(dataset, v) for v in info.versions]
@@ -915,7 +917,8 @@ class VersionsClient:
         from ..store.oss_discovery import (
             parse_dataset_level, build_synthetic_version, _warn_if_truncated)
         info = parse_dataset_level(
-            self._content, dataset, oss_prefix=self._oss_prefix)
+            self._content, dataset, oss_prefix=self._oss_prefix,
+            metadata_model=self._discovery_mm())
         _warn_if_truncated(info, dataset, 500)
         for v in info.versions:
             if v.version == version:
@@ -924,6 +927,13 @@ class VersionsClient:
         raise DatasetNotFound(
             f"version {version!r} not found in OSS for dataset {dataset!r}")
 
+    def _discovery_mm(self) -> str:
+        """OSS 回退探测的布局语义（split_first 默认；version_first 显式切换）。"""
+        from ..layout import normalize_metadata_model
+        return normalize_metadata_model(
+            getattr(self._p, "metadata_model", None)
+            if self._p is not None else None)
+
 
 # 与 apiserver `DatasetVersionListItem`（pkg/server/dataset_service.go:315）对齐:
 # 列表项带 instance_count/split_count/updated_at/last_ingested_at,详情接口才带

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/repo.py

@@ -84,7 +84,8 @@ class Repo:
         self.datasets = DatasetsClient(
             self.transport,
             content_store=self._content,
-            oss_prefix=self.profile.oss_prefix)
+            oss_prefix=self.profile.oss_prefix,
+            metadata_model=self.profile.metadata_model)
         self.versions = VersionsClient(
             self.transport, self.profile,
             content_store=self._content,

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo/store/oss_discovery.py

@@ -63,6 +63,14 @@ class OssVersionInfo:
     default_instance_count: int = 0
 
 
+@dataclass
+class OssUnversionedSplit:
+    """split_first 无版本布局：dataset 根下直接是 split 目录（无 version 层）。"""
+    name: str
+    has_assets: bool          # 同层 {name}-assets/ 存在（split 级 content）
+    instance_count: int       # {name}/ 下直接 .json 数
+
+
 @dataclass
 class OssDatasetLevelInfo:
     """Overall dataset structure discovered from OSS directory layout."""
@@ -70,6 +78,12 @@ class OssDatasetLevelInfo:
     versions: list[OssVersionInfo]
     total_instance_count: int
     truncated: bool = False
+    # split_first 无版本布局下的 split（version="" 语义）
+    unversioned_splits: list[OssUnversionedSplit] = None  # type: ignore[assignment]
+
+    def __post_init__(self):
+        if self.unversioned_splits is None:
+            self.unversioned_splits = []
 
 
 # ---------------------------------------------------------------------------
@@ -185,11 +199,18 @@ def parse_dataset_level(
     *,
     oss_prefix: str = "swe/datasets",
     max_versions: int = 500,
+    metadata_model: str = "split_first",
 ) -> OssDatasetLevelInfo:
     """Discover version/split structure from OSS directory layout.
 
-    Performs at most ``1 + min(N, max_versions)`` LIST calls where *N* is the
-    number of Level 1 entries.
+    Performs ``1 + min(N, max_versions)`` LIST calls where *N* is the number
+    of Level 1 entries; the per-group probes run concurrently (8-way).
+
+    语义消歧（``metadata_model``）：
+    - ``split_first``（SDK 默认）：level-1 目录 `X/` 下**只有 .json、无子目录**
+      时，`X` 是无版本 split（`{split}/{id}.json` 契约形态）；含子目录时是
+      version（子目录为其 split）。
+    - ``version_first``（显式切换）：level-1 目录一律视为 version（旧行为）。
 
     Args:
         content_store: :class:`~instance_repo.store.oss.OssContentStore` instance.
@@ -197,6 +218,8 @@ def parse_dataset_level(
         oss_prefix: Root prefix inside the bucket (default ``"swe/datasets"``).
         max_versions: Upper bound on the number of versions to probe.  When
             exceeded the result is truncated and ``truncated=True`` is set.
+        metadata_model: ``split_first``（默认）or ``version_first``；决定
+            level-1 目录的语义归属（见上）。
 
     Returns:
         :class:`OssDatasetLevelInfo` with the discovered version/split tree.
@@ -231,90 +254,104 @@ def parse_dataset_level(
             f"dataset {dataset!r} not found in OSS under {root!r}"
         )
 
-    # -- Group level 1 names into version candidates -----------------------
+    # -- Group level 1 names into candidates -------------------------------
     l1_groups = _group_level_names(level1_names)
 
-    if not l1_groups and not level1_json_keys:
+    if not l1_groups:
+        if level1_json_keys:
+            # 实例 .json 直接位于 dataset 根下（无 version/split 层）是**非法布局**：
+            # paths.py 契约要求 metadata 至少带 version 层（version_first）或
+            # split 层（split_first）。不兼容识别，明确报错引导上传方修正。
+            raise DatasetNotFound(
+                f"dataset {dataset!r} exists in OSS but has no recognizable "
+                f"version/split structure: {len(level1_json_keys)} .json file(s) "
+                f"directly under {root!r} (expected {root!r}{{version}}/[{{split}}/] "
+                f"layout per paths.py contract)")
         raise DatasetNotFound(
             f"dataset {dataset!r} exists in OSS but has no recognizable "
             f"version/split structure under {root!r}"
         )
 
     # Sort for deterministic output.
-    version_names = sorted(l1_groups.keys())
+    names = sorted(l1_groups.keys())
 
     # -- Truncation protection ---------------------------------------------
     truncated = False
-    if len(version_names) > max_versions:
-        version_names = version_names[:max_versions]
+    if len(names) > max_versions:
+        names = names[:max_versions]
         truncated = True
 
-    # -- Level 2: probe each version ---------------------------------------
-    versions: list[OssVersionInfo] = []
-    total_instance_count = 0
-
-    for v_name in version_names:
-        grp = l1_groups[v_name]
-        version_dir = f"{root}{v_name}/"
-
-        # 3-layer content/report siblings at level 1.
-        has_three_layer_content = grp.has_assets  # {version}-assets/ exists
+    split_first = (metadata_model or "split_first") != "version_first"
 
-        # Level 2 listing (subdirectories of {version}/).
+    # -- Level 2: probe each candidate (concurrent) ------------------------
+    # 每个 group 的完整分析（level-2 列举 + 逐 split 计数）都在 probe 内完成，
+    # 使大 version 数数据集的探测全程并行，避免串行逐 split 计数拖慢整链。
+    def probe(name: str):
+        """分析单个 level-1 group，返回 OssVersionInfo / OssUnversionedSplit。"""
+        grp = l1_groups[name]
+        group_dir = f"{root}{name}/"
         try:
-            level2_names = content_store.list_prefixes(version_dir, list_only=True)
+            level2_names, level2_keys = content_store.list_level(
+                group_dir, list_only=True)
         except Exception:  # noqa: BLE001
-            level2_names = []
-
+            level2_names, level2_keys = [], []
         l2_groups = _group_level_names(level2_names)
-
-        # Splits are level 2 groups that have a real directory (has_dir) or
-        # an assets directory (4-layer content pattern).
-        splits: list[str] = sorted(
-            name for name, g in l2_groups.items()
-            if g.has_dir or g.has_assets
-        )
-
-        # Detect default-split instances: .json files directly under {version}/.
-        try:
-            _level2_dirs, v_all_keys = content_store.list_level(
-                version_dir, list_only=True)
-            v_json_keys = _direct_json_keys(v_all_keys, version_dir)
-            has_default_instances = len(v_json_keys) > 0
-            default_instance_count = len(v_json_keys)
-            instance_count = len(v_json_keys)
-        except Exception:  # noqa: BLE001
-            has_default_instances = False
-            default_instance_count = 0
-            instance_count = 0
-
-        # Count instances in each named split directory.
+        direct_json = _direct_json_keys(level2_keys, group_dir)
+        subdirs = [g for g in l2_groups.values() if g.has_dir or g.has_assets]
+
+        # split_first 无版本布局：`{name}/` 下只有 .json、无子目录 →
+        # 无版本 split（contract: {split}/{id}.json + {split}-assets/）。
+        if split_first and grp.has_dir and not subdirs and direct_json:
+            return OssUnversionedSplit(
+                name=name,
+                has_assets=grp.has_assets,
+                instance_count=len(direct_json),
+            )
+
+        # version 路径（version_first 或含子目录的 split_first 形态）。
+        splits: list[str] = sorted(g.name for g in subdirs)
+        instance_count = len(direct_json)
         for s_name in splits:
-            split_dir = f"{version_dir}{s_name}/"
+            split_dir = f"{group_dir}{s_name}/"
             try:
                 _s_dirs, s_keys = content_store.list_level(
                     split_dir, list_only=True)
-                s_json = _direct_json_keys(s_keys, split_dir)
-                instance_count += len(s_json)
+                instance_count += len(_direct_json_keys(s_keys, split_dir))
             except Exception:  # noqa: BLE001
                 pass  # best-effort
-
-        total_instance_count += instance_count
-
-        versions.append(OssVersionInfo(
-            version=v_name,
-            has_three_layer_content=has_three_layer_content,
+        return OssVersionInfo(
+            version=name,
+            has_three_layer_content=grp.has_assets,
             splits=splits,
-            has_default_instances=has_default_instances,
+            has_default_instances=len(direct_json) > 0,
             instance_count=instance_count,
-            default_instance_count=default_instance_count,
-        ))
+            default_instance_count=len(direct_json),
+        )
+
+    try:
+        from ..concurrency import map_concurrent
+        results = map_concurrent(probe, names, concurrency=16)
+    except Exception:  # noqa: BLE001 - 并发不可用时退化为串行
+        results = [probe(n) for n in names]
+
+    versions: list[OssVersionInfo] = []
+    unversioned_splits: list[OssUnversionedSplit] = []
+    total_instance_count = 0
+
+    for entry in results:
+        if isinstance(entry, OssUnversionedSplit):
+            unversioned_splits.append(entry)
+            total_instance_count += entry.instance_count
+        else:
+            versions.append(entry)
+            total_instance_count += entry.instance_count
 
     return OssDatasetLevelInfo(
         dataset=dataset,
         versions=versions,
         total_instance_count=total_instance_count,
         truncated=truncated,
+        unversioned_splits=unversioned_splits,
     )
 
 
@@ -323,9 +360,10 @@ def _warn_if_truncated(info: OssDatasetLevelInfo, dataset: str, max_versions: in
     if info.truncated:
         import logging
         logging.getLogger(__name__).warning(
-            "dataset %r has more than %d versions; OSS discovery results "
+            "dataset %r has more than %d level-1 entries; OSS discovery results "
             "are truncated (showing %d of >%d)",
-            dataset, max_versions, len(info.versions), max_versions)
+            dataset, max_versions,
+            len(info.versions) + len(info.unversioned_splits), max_versions)
 
 
 # ---------------------------------------------------------------------------
@@ -389,6 +427,10 @@ def build_synthetic_dataset_detail(
         if v.has_default_instances:
             all_splits.add("default")
         total_instances += v.instance_count
+    # split_first 无版本布局：level-1 目录即 split
+    for us in info.unversioned_splits:
+        all_splits.add(us.name)
+        total_instances += us.instance_count
 
     available_envs: list[str] = [storage_env] if storage_env else []
 
@@ -450,10 +492,11 @@ def build_synthetic_split(
     instance_count: int = 0,
 ) -> DatasetSplit:
     """Build a :class:`DatasetSplit` from OSS discovery (``registered=False``)."""
+    base = f"swe/datasets/{dataset}" + (f"/{version}" if version else "")
     if split_name == "default":
-        storage_path = f"swe/datasets/{dataset}/{version}/"
+        storage_path = f"{base}/"
     else:
-        storage_path = f"swe/datasets/{dataset}/{version}/{split_name}/"
+        storage_path = f"{base}/{split_name}/"
 
     return DatasetSplit(
         dataset=dataset,
@@ -468,6 +511,16 @@ def build_synthetic_split(
     )
 
 
+def _metadata_prefix(dataset: str, version: str, split: str | None,
+                     oss_prefix: str) -> str:
+    """拼接 metadata 列举前缀；version 为空时表示 split_first 无版本布局。"""
+    l1, l2 = P.split_dataset(dataset)
+    base = f"{oss_prefix}/{l1}/{l2}" + (f"/{version}" if version else "")
+    if split and split != "default":
+        return f"{base}/{split}/"
+    return f"{base}/"
+
+
 def build_synthetic_instances(
     content_store,
     dataset: str,
@@ -475,20 +528,16 @@ def build_synthetic_instances(
     split: str | None = None,
     *,
     oss_prefix: str = "swe/datasets",
+    read_content: bool = False,
 ) -> list[DatasetInstance]:
     """List and build :class:`DatasetInstance` objects from OSS metadata JSONs.
 
     For each ``{instance_id}.json`` found directly under the target prefix, a
-    ``DatasetInstance`` is constructed.  An attempt is made to read the JSON
-    body to populate ``tags``, ``difficulty``, ``docker_image``, and
-    ``extend_meta``; on failure the instance is returned with those fields
-    left at their defaults (best-effort).
+    ``DatasetInstance`` is constructed.  ``read_content=True`` 时尝试读 JSON
+    回填 tags/difficulty/docker_image；默认 False——发现模式（list-only STS）
+    无 GetObject，逐实例 GET 必然失败，跳过可省 N 次无谓的 STS 签发与下载。
     """
-    l1, l2 = P.split_dataset(dataset)
-    if split and split != "default":
-        metadata_prefix = f"{oss_prefix}/{l1}/{l2}/{version}/{split}/"
-    else:
-        metadata_prefix = f"{oss_prefix}/{l1}/{l2}/{version}/"
+    metadata_prefix = _metadata_prefix(dataset, version, split, oss_prefix)
 
     all_keys = content_store.list_objects(metadata_prefix, list_only=True)
     json_keys = _direct_json_keys(all_keys, metadata_prefix)
@@ -503,15 +552,13 @@ def build_synthetic_instances(
             split=split or "default",
             registered=False,
         )
-        # Best-effort: read JSON to populate optional fields.  With list_only=True
-        # STS, GetObject is not granted, so these reads will fail silently (returning
-        # empty dicts) — the instance will still be listed but with empty optional fields.
-        data = _read_json_object(content_store, key)
-        if data:
-            inst.tags = data.get("tags", {})
-            inst.difficulty = data.get("difficulty")
-            inst.docker_image = data.get("docker_image")
-            inst.extend_meta = data.get("extend_meta", {})
+        if read_content:
+            data = _read_json_object(content_store, key)
+            if data:
+                inst.tags = data.get("tags", {})
+                inst.difficulty = data.get("difficulty")
+                inst.docker_image = data.get("docker_image")
+                inst.extend_meta = data.get("extend_meta", {})
         instances.append(inst)
 
     return instances
@@ -525,13 +572,24 @@ def build_synthetic_instance(
     split: str | None = None,
     *,
     oss_prefix: str = "swe/datasets",
+    read_content: bool = False,
 ) -> DatasetInstance | None:
     """Build a single :class:`DatasetInstance` by exact key lookup in OSS.
 
-    Returns ``None`` if the metadata JSON does not exist.
+    Returns ``None`` if the metadata JSON does not exist.  ``read_content``
+    语义同 :func:`build_synthetic_instances`（发现模式默认跳过 JSON 回填）。
     """
-    key = P.metadata_key(
-        dataset, version, instance_id, split, prefix=oss_prefix)
+    if version:
+        key = P.metadata_key(
+            dataset, version, instance_id, split, prefix=oss_prefix)
+    else:
+        # split_first 无版本布局：{split}/{id}.json（split=None/default 不合
+        # 此形态，调用方须传真实 split 名）
+        l1, l2 = P.split_dataset(dataset)
+        base = f"{oss_prefix}/{l1}/{l2}"
+        key = (f"{base}/{split}/{instance_id}.json"
+               if split and split != "default"
+               else f"{base}/{instance_id}.json")
 
     # 存在性判定用 LIST（list_only=True 仅需 ListObjects）而非 HEAD——
     # list-only STS 无 oss:GetObject，object_exists 的 HEAD 会在 STS 签发或
@@ -551,11 +609,12 @@ def build_synthetic_instance(
         split=split or "default",
         registered=False,
     )
-    data = _read_json_object(content_store, key)
-    if data:
-        inst.tags = data.get("tags", {})
-        inst.difficulty = data.get("difficulty")
-        inst.docker_image = data.get("docker_image")
-        inst.extend_meta = data.get("extend_meta", {})
+    if read_content:
+        data = _read_json_object(content_store, key)
+        if data:
+            inst.tags = data.get("tags", {})
+            inst.difficulty = data.get("difficulty")
+            inst.docker_image = data.get("docker_image")
+            inst.extend_meta = data.get("extend_meta", {})
 
     return inst

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/instance_repo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: instance-repo
-Version: 1.3.2.dev2
+Version: 1.3.2.dev3
 Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{instance_repo-1.3.2.dev2 → instance_repo-1.3.2.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "instance-repo"
-version = "1.3.2.dev2"
+version = "1.3.2.dev3"
 description = "InstanceRepo SDK — 评测 Instance 统一存储客户端（SDK-only；控制面由 apiserver 承载）"
 requires-python = ">=3.10"
 readme = "README.md"