instance-repo 1.4.0.dev6__tar.gz → 1.4.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. instance_repo-1.4.6/PKG-INFO +85 -0
  2. instance_repo-1.4.6/README.md +71 -0
  3. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/__init__.py +1 -1
  4. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/_site_defaults.py +8 -0
  5. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/artifacts.py +18 -87
  6. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/bootstrap.py +6 -6
  7. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/cache.py +32 -1
  8. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/cli.py +27 -12
  9. instance_repo-1.4.6/instance_repo/clients/_rfc3339.py +41 -0
  10. instance_repo-1.4.6/instance_repo/clients/ap_tool.py +265 -0
  11. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/config.py +228 -5
  12. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/instances.py +124 -29
  13. instance_repo-1.4.6/instance_repo/clients/shared.py +1021 -0
  14. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/splits.py +35 -17
  15. instance_repo-1.4.6/instance_repo/clients/user_data_acl.py +306 -0
  16. instance_repo-1.4.6/instance_repo/clients/user_data_directories.py +203 -0
  17. instance_repo-1.4.6/instance_repo/clients/user_data_entries.py +325 -0
  18. instance_repo-1.4.6/instance_repo/clients/user_data_reads.py +991 -0
  19. instance_repo-1.4.6/instance_repo/clients/user_data_upload.py +910 -0
  20. instance_repo-1.4.6/instance_repo/clients/user_data_writes.py +1830 -0
  21. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/errors.py +44 -20
  22. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/paths.py +19 -10
  23. instance_repo-1.4.6/instance_repo/repo.py +1437 -0
  24. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/retry.py +30 -0
  25. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/oss.py +107 -7
  26. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/oss_discovery.py +14 -29
  27. instance_repo-1.4.6/instance_repo/transport.py +901 -0
  28. instance_repo-1.4.6/instance_repo.egg-info/PKG-INFO +85 -0
  29. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo.egg-info/SOURCES.txt +9 -0
  30. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/pyproject.toml +1 -1
  31. instance_repo-1.4.0.dev6/PKG-INFO +0 -242
  32. instance_repo-1.4.0.dev6/README.md +0 -228
  33. instance_repo-1.4.0.dev6/instance_repo/repo.py +0 -831
  34. instance_repo-1.4.0.dev6/instance_repo/transport.py +0 -438
  35. instance_repo-1.4.0.dev6/instance_repo.egg-info/PKG-INFO +0 -242
  36. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/_routing.py +0 -0
  37. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/__init__.py +0 -0
  38. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/benchmarks.py +0 -0
  39. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/datasets.py +0 -0
  40. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/images.py +0 -0
  41. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/reports.py +0 -0
  42. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/scaffold.py +0 -0
  43. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/versions.py +0 -0
  44. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/clients/workflows.py +0 -0
  45. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/concurrency.py +0 -0
  46. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/content.py +0 -0
  47. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/endpoints.py +0 -0
  48. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/image.py +0 -0
  49. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/layout.py +0 -0
  50. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/loader.py +0 -0
  51. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/models.py +0 -0
  52. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/release.py +0 -0
  53. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/__init__.py +0 -0
  54. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/acr.py +0 -0
  55. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/base.py +0 -0
  56. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/store/registry.py +0 -0
  57. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/tasktoml.py +0 -0
  58. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo/validate.py +0 -0
  59. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo.egg-info/dependency_links.txt +0 -0
  60. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo.egg-info/entry_points.txt +0 -0
  61. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo.egg-info/requires.txt +0 -0
  62. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/instance_repo.egg-info/top_level.txt +0 -0
  63. {instance_repo-1.4.0.dev6 → instance_repo-1.4.6}/setup.cfg +0 -0
@@ -0,0 +1,85 @@
1
+ Metadata-Version: 2.4
2
+ Name: instance-repo
3
+ Version: 1.4.6
4
+ Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: irepo-artifact<0.3.0,>=0.1.1
8
+ Requires-Dist: tomli>=1.1.0; python_version < "3.11"
9
+ Provides-Extra: oss
10
+ Requires-Dist: oss2>=2.19; extra == "oss"
11
+ Provides-Extra: dev
12
+ Requires-Dist: pytest>=8; extra == "dev"
13
+ Requires-Dist: coverage>=7; extra == "dev"
14
+
15
+ # InstanceRepository Python SDK
16
+
17
+ Python、Go、Java SDK 源码版本均为 `1.4.6`。Split create 可将初始 `run_type` 交给支持该能力的服务端原子写入,重复创建保留已有标签。具体兼容条件见下文。实际安装版本以获准制品库的发布记录为准。[三语言功能矩阵](../../docs/sdk/parity.md)。
18
+
19
+ ## 安装与配置
20
+
21
+ Python3.10+。从批准的索引中选择已发布版本,真实OSS使用`oss` extra;镜像操作需要`skopeo`,不用镜像时显式`push_image=False`。源码隔离环境已有依赖时可使用:
22
+
23
+ ```bash
24
+ cd sdk/python
25
+ python -m pip install -e .
26
+ ```
27
+
28
+ 规范主配置:`AP_API_KEY`、`AP_BASE_URL`、`IR_API_ENV`、`AP_CLUSTER`或`INSTANCE_REPO_STORAGE_ENV`。旧token/base/storage变量仅保留兼容。内部渠道包可能注入站点默认地址,外部包不能据此假设同样默认值。profile仅作标签;控制面Env不等于数据面环境,也不授予权限。
29
+
30
+ ## Dataset 实例操作
31
+
32
+ ```python
33
+ from instance_repo import Repo
34
+ from instance_repo.models import OwnerReference
35
+
36
+
37
+ def publish_input(repo: Repo, dataset: str, local_dir: str, owner: OwnerReference):
38
+ repo.splits.create(dataset, 'test')
39
+ result = repo.instances.push(local_dir, dataset, '', split='test',
40
+ owner_reference=owner, push_image=False, register=False)
41
+ repo.versions.ingest(dataset, '', splits=['test'])
42
+ return result
43
+
44
+
45
+ def download(repo: Repo, dataset: str, instance_id: str, output_dir: str):
46
+ return repo.instances.pull(dataset, '', instance_id, output_dir, split='test')
47
+
48
+
49
+ def remove_metadata(repo: Repo, dataset: str, instance_id: str):
50
+ return repo.instances.delete(dataset, '', instance_id, split='test')
51
+ ```
52
+
53
+ 业务Repo使用split_first;version位置参数传空串代表真实无版本,不是latest。Split create不接受非空storage_path。`register=True`通过ingest完成元数据注册,不能依赖历史`:commit`。已有目录格式与SWE单JSON格式的差异、批量结果和失败行为见[当前指南](../../docs/sdk/README.md)。
54
+
55
+ ### Split 创建与服务端兼容
56
+
57
+ `repo.splits.create(dataset, split, version=..., run_type=...)` 在 POST 中传入初始标签。服务端返回 `run_type_on_create_supported=true` 时,SDK 直接读取详情,不再 PATCH 已有记录。重复创建会保留已有 `run_type`,包括空标签;新建记录的标签由服务端写入。
58
+
59
+ 旧服务端没有返回该能力标识时,SDK 保留 POST 后 PATCH 的兼容流程。此时重复创建仍可能更新标签。需要严格保持已有标签的调用方,必须先部署支持该契约的服务端。SDK 不通过 legacy 列表或 OSS 扫描判断本次是否新建。
60
+
61
+ Python 版本升级通过 CR 合入后,可按 `.aoneci/release_pysdk.yaml` 使用 `pysdk/v1.4.6` tag 的现有发布流程。本次源码版本不表示制品已经发布。
62
+
63
+ `instances.list_objects`返回文件/目录及`truncated`;不完整结果不能当完整目录。`compute_source_hash`与`list_content_hashes`用于变更检测;`push/push_many`的`skip_unchanged`不能替代内容校验或权限判定。
64
+
65
+ 显式`delete_files=True`只处理匹配注册记录的canonical主JSON、专属assets/report;默认仍仅删除元数据。必须停止同实例的上传/ingest;OSS与DB无跨系统原子性,文件删除后DB可能失败。它不是尚在设计的operation_id删除协议,不会删除镜像、历史对象版本或任意manifest引用。
66
+
67
+ ## canonical user-data 与 shared
68
+
69
+ - `Repo(expected_replica_policy=policy, cluster=..., bootstrap_dataset=False)`可用于只进行canonical用户操作的调用方。policy由部署方可信配置构造`ReplicasPolicy`,含恰好CN/SG的storage_id、bucket与OSS region;不能来自同一未验证的discovery响应。
70
+ - 上传使用`upload_user_data_verified`;下载使用`download_user_data_verified`;兼容 façade `upload_user_data/download_user_data`同样走canonical流程。结果须按receipt和最终状态判定。
71
+ - metadata列表使用`user_data_entries_page/iter_user_data_entries/list_user_data`,默认总限制10000行、32MiB、512页;旧直接OSS LIST与其语义不同。
72
+ - 目录操作是低层`UserDataDirectoriesClient.mkdir`,必须提供可信policy及幂等UUID,不存在`Repo.mkdir`。
73
+ - 共享发布是独立`SharedReleaseClient`,不存在`Repo.shared`;generic shared读取另需真实`shared_library_id`。精确ap-tool profile仅限当前注册的RewardKit格式。
74
+ - ACL façade的`grant/revoke/list/permissions`保留,明确主体、role、subpath、storage_env和deadline;展示权限不是客户端授权证明。
75
+
76
+ 准确构造示例、低层入口、错误族与流式/预算要求见[当前指南](../../docs/sdk/README.md#3-canonical-user-data)。Dataset、user-data、shared、artifact配置及授权不得互相借用或静默降级。
77
+
78
+ ## 验证
79
+
80
+ ```bash
81
+ cd sdk/python
82
+ python -m pytest tests -m 'not integration'
83
+ ```
84
+
85
+ 真实环境只在具备批准权限与受控资源后执行;单元通过不等于生产双副本、真实STS或跨地域网络通过。日志不得保存API Key、AK/SK、STS、签名URL或含这些值的异常链。
@@ -0,0 +1,71 @@
1
+ # InstanceRepository Python SDK
2
+
3
+ Python、Go、Java SDK 源码版本均为 `1.4.6`。Split create 可将初始 `run_type` 交给支持该能力的服务端原子写入,重复创建保留已有标签。具体兼容条件见下文。实际安装版本以获准制品库的发布记录为准。[三语言功能矩阵](../../docs/sdk/parity.md)。
4
+
5
+ ## 安装与配置
6
+
7
+ Python3.10+。从批准的索引中选择已发布版本,真实OSS使用`oss` extra;镜像操作需要`skopeo`,不用镜像时显式`push_image=False`。源码隔离环境已有依赖时可使用:
8
+
9
+ ```bash
10
+ cd sdk/python
11
+ python -m pip install -e .
12
+ ```
13
+
14
+ 规范主配置:`AP_API_KEY`、`AP_BASE_URL`、`IR_API_ENV`、`AP_CLUSTER`或`INSTANCE_REPO_STORAGE_ENV`。旧token/base/storage变量仅保留兼容。内部渠道包可能注入站点默认地址,外部包不能据此假设同样默认值。profile仅作标签;控制面Env不等于数据面环境,也不授予权限。
15
+
16
+ ## Dataset 实例操作
17
+
18
+ ```python
19
+ from instance_repo import Repo
20
+ from instance_repo.models import OwnerReference
21
+
22
+
23
+ def publish_input(repo: Repo, dataset: str, local_dir: str, owner: OwnerReference):
24
+ repo.splits.create(dataset, 'test')
25
+ result = repo.instances.push(local_dir, dataset, '', split='test',
26
+ owner_reference=owner, push_image=False, register=False)
27
+ repo.versions.ingest(dataset, '', splits=['test'])
28
+ return result
29
+
30
+
31
+ def download(repo: Repo, dataset: str, instance_id: str, output_dir: str):
32
+ return repo.instances.pull(dataset, '', instance_id, output_dir, split='test')
33
+
34
+
35
+ def remove_metadata(repo: Repo, dataset: str, instance_id: str):
36
+ return repo.instances.delete(dataset, '', instance_id, split='test')
37
+ ```
38
+
39
+ 业务Repo使用split_first;version位置参数传空串代表真实无版本,不是latest。Split create不接受非空storage_path。`register=True`通过ingest完成元数据注册,不能依赖历史`:commit`。已有目录格式与SWE单JSON格式的差异、批量结果和失败行为见[当前指南](../../docs/sdk/README.md)。
40
+
41
+ ### Split 创建与服务端兼容
42
+
43
+ `repo.splits.create(dataset, split, version=..., run_type=...)` 在 POST 中传入初始标签。服务端返回 `run_type_on_create_supported=true` 时,SDK 直接读取详情,不再 PATCH 已有记录。重复创建会保留已有 `run_type`,包括空标签;新建记录的标签由服务端写入。
44
+
45
+ 旧服务端没有返回该能力标识时,SDK 保留 POST 后 PATCH 的兼容流程。此时重复创建仍可能更新标签。需要严格保持已有标签的调用方,必须先部署支持该契约的服务端。SDK 不通过 legacy 列表或 OSS 扫描判断本次是否新建。
46
+
47
+ Python 版本升级通过 CR 合入后,可按 `.aoneci/release_pysdk.yaml` 使用 `pysdk/v1.4.6` tag 的现有发布流程。本次源码版本不表示制品已经发布。
48
+
49
+ `instances.list_objects`返回文件/目录及`truncated`;不完整结果不能当完整目录。`compute_source_hash`与`list_content_hashes`用于变更检测;`push/push_many`的`skip_unchanged`不能替代内容校验或权限判定。
50
+
51
+ 显式`delete_files=True`只处理匹配注册记录的canonical主JSON、专属assets/report;默认仍仅删除元数据。必须停止同实例的上传/ingest;OSS与DB无跨系统原子性,文件删除后DB可能失败。它不是尚在设计的operation_id删除协议,不会删除镜像、历史对象版本或任意manifest引用。
52
+
53
+ ## canonical user-data 与 shared
54
+
55
+ - `Repo(expected_replica_policy=policy, cluster=..., bootstrap_dataset=False)`可用于只进行canonical用户操作的调用方。policy由部署方可信配置构造`ReplicasPolicy`,含恰好CN/SG的storage_id、bucket与OSS region;不能来自同一未验证的discovery响应。
56
+ - 上传使用`upload_user_data_verified`;下载使用`download_user_data_verified`;兼容 façade `upload_user_data/download_user_data`同样走canonical流程。结果须按receipt和最终状态判定。
57
+ - metadata列表使用`user_data_entries_page/iter_user_data_entries/list_user_data`,默认总限制10000行、32MiB、512页;旧直接OSS LIST与其语义不同。
58
+ - 目录操作是低层`UserDataDirectoriesClient.mkdir`,必须提供可信policy及幂等UUID,不存在`Repo.mkdir`。
59
+ - 共享发布是独立`SharedReleaseClient`,不存在`Repo.shared`;generic shared读取另需真实`shared_library_id`。精确ap-tool profile仅限当前注册的RewardKit格式。
60
+ - ACL façade的`grant/revoke/list/permissions`保留,明确主体、role、subpath、storage_env和deadline;展示权限不是客户端授权证明。
61
+
62
+ 准确构造示例、低层入口、错误族与流式/预算要求见[当前指南](../../docs/sdk/README.md#3-canonical-user-data)。Dataset、user-data、shared、artifact配置及授权不得互相借用或静默降级。
63
+
64
+ ## 验证
65
+
66
+ ```bash
67
+ cd sdk/python
68
+ python -m pytest tests -m 'not integration'
69
+ ```
70
+
71
+ 真实环境只在具备批准权限与受控资源后执行;单元通过不等于生产双副本、真实STS或跨地域网络通过。日志不得保存API Key、AK/SK、STS、签名URL或含这些值的异常链。
@@ -20,7 +20,7 @@ from .errors import (
20
20
  RegistrationFailed, TransportError,
21
21
  )
22
22
 
23
- __version__ = "1.4.0.dev6"
23
+ __version__ = "1.4.6"
24
24
 
25
25
  __all__ = [
26
26
  "Repo", "load_instances",
@@ -37,3 +37,11 @@ ARTIFACT_PREFIX_ROOT = ""
37
37
  #: 注入。运行时也可经环境变量 AP_ARTIFACT_BUCKET_MAP(JSON)覆盖,优先级:
38
38
  #: 环境变量 > 本模块注入值 > 空(空则 artifact 下载报错并引导配置)。
39
39
  ARTIFACT_BUCKET_MAP_JSON = ""
40
+
41
+ #: 期望 replica policy(canonical 双桶对)的 JSON 串。**源码恒为空**;
42
+ #: 内网构建时由发布脚本注入(与 API_BASE 同模式——静态可审计、无
43
+ #: envvar 通道)。运行时解析为 ``ReplicasPolicy``(严格 JSON/拒重复
44
+ #: keys/构造验证);公共包通过显式构造参数传入有效 policy 亦可运行。
45
+ #: 优先级:显式构造参数 > 本模块注入值 > 缺失(缺失=新客户端零 IO
46
+ #: typed error)。
47
+ EXPECTED_REPLICA_POLICY_JSON = ""
@@ -119,98 +119,29 @@ def artifact_key(job_id: str) -> str:
119
119
 
120
120
  def fetch_job_info(api_key: str, job_id: str, cluster: str,
121
121
  ap_endpoint: str | None = None) -> dict[str, Any]:
122
- """调 AP GET /api/jobs/{job_id}?include_params=true 获取 job 信息。
123
-
124
- AP 内部执行 workspace ACL 检查(enforce_workspace_action_req)。
125
- 403 → ArtifactPermissionError;404 → ArtifactNotFoundError。
126
- """
127
- endpoint = _ap_endpoint(ap_endpoint)
128
- url = f"{endpoint}/api/jobs/{job_id}?include_params=true"
129
- headers = {
130
- "X-API-Key": api_key,
131
- "Accept": "application/json",
132
- }
133
- if cluster:
134
- headers["X-Cluster"] = cluster
135
-
136
- req = urllib.request.Request(url, headers=headers, method="GET")
122
+ """Explicit legacy detail helper; normal downloads do not precheck Jobs."""
123
+ from irepo_artifact import ArtifactError
124
+ from irepo_artifact.api import validate_id
125
+ failure = None
137
126
  try:
138
- with urllib.request.urlopen(req, timeout=30) as resp:
139
- return json.loads(resp.read().decode())
140
- except urllib.error.HTTPError as e:
141
- if e.code == 403:
142
- raise ArtifactPermissionError(
143
- f"workspace ACL denied access to job {job_id!r}"
144
- ) from e
145
- if e.code == 404:
146
- raise ArtifactNotFoundError(
147
- f"job {job_id!r} not found"
148
- ) from e
149
- raise InstanceRepoError(
150
- f"AP returned HTTP {e.code} for job {job_id!r}: {e.reason}"
151
- ) from e
152
- except urllib.error.URLError as e:
153
- raise InstanceRepoError(
154
- f"failed to reach AP at {endpoint}: {e.reason}"
155
- ) from e
127
+ validate_id(job_id)
128
+ return _client(api_key, cluster, ap_endpoint)._api._call(
129
+ f"/jobs/{job_id}?include_params=true")
130
+ except ArtifactError as exc:
131
+ failure = _legacy_error(exc.clone())
132
+ raise failure
156
133
 
157
134
 
158
135
  def fetch_group_job_ids(api_key: str, group_id: str, cluster: str,
159
136
  ap_endpoint: str | None = None) -> list[str]:
160
- """调 AP GET /api/groups/{group_id}/jobs 分页收集全部 job_id。
161
-
162
- endpoint 解析与 fetch_job_info 完全一致(复用 _ap_endpoint),
163
- 且 URL 必须带 /api 前缀:AP 网关的裸 /groups/{id}/jobs 路径返回
164
- 控制台 HTML 页(HTTP 200),json 解析会报错(真机实测)。
165
- """
166
- endpoint = _ap_endpoint(ap_endpoint)
167
- job_ids: list[str] = []
168
- skip = 0
169
- limit = 500
170
-
171
- while True:
172
- url = (f"{endpoint}/api/groups/{group_id}/jobs"
173
- f"?skip={skip}&limit={limit}&include_post_process=false")
174
- headers = {
175
- "X-API-Key": api_key,
176
- "Accept": "application/json",
177
- }
178
- if cluster:
179
- headers["X-Cluster"] = cluster
180
-
181
- req = urllib.request.Request(url, headers=headers, method="GET")
182
- try:
183
- with urllib.request.urlopen(req, timeout=60) as resp:
184
- data = json.loads(resp.read().decode())
185
- except urllib.error.HTTPError as e:
186
- if e.code == 403:
187
- raise ArtifactPermissionError(
188
- f"workspace ACL denied access to group {group_id!r}"
189
- ) from e
190
- if e.code == 404:
191
- raise ArtifactNotFoundError(
192
- f"group {group_id!r} not found"
193
- ) from e
194
- raise InstanceRepoError(
195
- f"AP returned HTTP {e.code} for group {group_id!r}"
196
- ) from e
197
- except urllib.error.URLError as e:
198
- raise InstanceRepoError(
199
- f"failed to reach AP at {endpoint}: {e.reason}"
200
- ) from e
201
-
202
- jobs = data.get("jobs", [])
203
- if not jobs:
204
- break
205
- for j in jobs:
206
- jid = j.get("job_id")
207
- if jid:
208
- job_ids.append(jid)
209
- if len(jobs) < limit:
210
- break
211
- skip += limit
212
-
213
- return job_ids
137
+ """Bounded, deduplicated Group enumeration on credential-isolated AP HTTP."""
138
+ from irepo_artifact import ArtifactError
139
+ failure = None
140
+ try:
141
+ return _client(api_key, cluster, ap_endpoint)._api.list_group_job_ids(group_id)
142
+ except ArtifactError as exc:
143
+ failure = _legacy_error(exc.clone())
144
+ raise failure
214
145
 
215
146
 
216
147
  def _client(api_key: str, cluster: str, ap_endpoint: str | None):
@@ -14,7 +14,7 @@ registry/namespace),校验并缓存。
14
14
  **二、优先级固定为 显式 > 遗留环境变量 > 自动发现。** 已经显式配好的调用方不会因为
15
15
  引入自动发现而行为漂移;反过来,自动发现也不会覆盖调用方的明确意图。
16
16
 
17
- **三、缓存按 ``(api_base, api_env, selector_type, selector_value)`` 四元组。** cluster
17
+ **三、缓存按 ``(api_base, api_env, selector_type, selector_value)`` 及 transport 实例隔离。** cluster
18
18
  与 storage_env 即使值相同也不会串味;默认 TTL 内复用,``refresh=True`` 可强制重新拉取。
19
19
  """
20
20
  from __future__ import annotations
@@ -23,7 +23,7 @@ import logging
23
23
  import threading
24
24
  import time
25
25
 
26
- from .cache import TTLCache
26
+ from .cache import TTLCache, transport_cache_identity
27
27
  from .clients.config import (
28
28
  Profile, fetch_repo_config, normalize_storage_env,
29
29
  )
@@ -39,7 +39,7 @@ DISCOVERY_TTL_SECONDS = 3600.0
39
39
  DISCOVERABLE_FIELDS = ("oss_bucket", "oss_prefix", "acr_host",
40
40
  "acr_public_host", "acr_namespace")
41
41
 
42
- _cache = TTLCache(safety=0.0)
42
+ _cache = TTLCache(safety=0.0, max_entries=512)
43
43
  _lock = threading.Lock()
44
44
 
45
45
 
@@ -56,9 +56,9 @@ def _selector(profile: Profile) -> tuple[str, str]:
56
56
  return "", ""
57
57
 
58
58
 
59
- def _cache_key(profile: Profile) -> tuple:
59
+ def _cache_key(profile: Profile, transport) -> tuple:
60
60
  selector_type, selector_value = _selector(profile)
61
- return (profile.api_base, profile.api_env, selector_type, selector_value)
61
+ return (transport_cache_identity(transport), profile.api_base, profile.api_env, selector_type, selector_value)
62
62
 
63
63
 
64
64
  def discover(transport, profile: Profile, *, refresh: bool = False) -> dict:
@@ -71,7 +71,7 @@ def discover(transport, profile: Profile, *, refresh: bool = False) -> dict:
71
71
  selector_type, selector_value = _selector(profile)
72
72
  if not selector_type:
73
73
  return {}
74
- key = _cache_key(profile)
74
+ key = _cache_key(profile, transport)
75
75
  if not refresh:
76
76
  cached = _cache.get(key)
77
77
  if cached is not None:
@@ -6,6 +6,7 @@ oss2 Bucket 按 (bucket,prefix,security_token) 缓存以复用底层连接池。
6
6
  """
7
7
  from __future__ import annotations
8
8
 
9
+ import hashlib
9
10
  import threading
10
11
  import time
11
12
  from datetime import datetime, timezone
@@ -29,13 +30,36 @@ def parse_expiry(rfc3339: str | None) -> float | None:
29
30
  return dt.timestamp()
30
31
 
31
32
 
33
+ _identity_lock = threading.Lock()
34
+
35
+
36
+ def transport_cache_identity(transport):
37
+ """Opaque per-instance token: no credentials or recyclable object addresses."""
38
+ with _identity_lock:
39
+ token = getattr(transport, "_config_cache_identity", None)
40
+ if token is None:
41
+ token = object()
42
+ try:
43
+ setattr(transport, "_config_cache_identity", token)
44
+ except (AttributeError, TypeError):
45
+ pass # Immutable adapters safely disable reuse instead of sharing.
46
+ subject = getattr(transport, "token", "")
47
+ digest = hashlib.sha256(subject.encode("utf-8", "surrogatepass")).digest() if isinstance(subject, str) else b""
48
+ return token, digest
49
+
50
+
32
51
  class TTLCache:
33
52
  """线程安全 TTL 缓存。"""
34
53
 
35
- def __init__(self, *, safety: float = 60.0) -> None:
54
+ def __init__(self, *, safety: float = 60.0, max_entries: int | None = None) -> None:
36
55
  self._d: dict[Any, tuple[Any, float | None]] = {}
37
56
  self._lock = threading.Lock()
38
57
  self._safety = safety
58
+ if max_entries is not None and (isinstance(max_entries, bool)
59
+ or not isinstance(max_entries, int)
60
+ or max_entries < 1):
61
+ raise ValueError("max_entries must be a positive integer")
62
+ self._max_entries = max_entries
39
63
 
40
64
  def get(self, key: Any) -> Any | None:
41
65
  with self._lock:
@@ -50,6 +74,13 @@ class TTLCache:
50
74
 
51
75
  def put(self, key: Any, value: Any, expiry: float | None) -> None:
52
76
  with self._lock:
77
+ if self._max_entries is not None:
78
+ now = time.time()
79
+ for stale, (_, exp) in list(self._d.items()):
80
+ if exp is not None and now >= exp - self._safety:
81
+ self._d.pop(stale, None)
82
+ if key not in self._d and len(self._d) >= self._max_entries:
83
+ self._d.pop(next(iter(self._d)))
53
84
  self._d[key] = (value, expiry)
54
85
 
55
86
  def clear(self) -> None:
@@ -451,9 +451,16 @@ def build_parser() -> argparse.ArgumentParser:
451
451
 
452
452
 
453
453
  def _split_ref(ref: str, parts: int) -> list[str]:
454
+ """切分位置参数 ref:尾部 ``parts-2`` 段是 version/instance-id 槽位,其余(≥1 段)
455
+ 全部是 dataset 名——多段 dataset 名 (``A/B/C``) 因此可以原样出现在 ref 里。
456
+
457
+ ``parts=2`` 时 ref 整体就是 dataset 名(无尾部槽位)。
458
+ """
454
459
  segs = ref.split("/")
455
- if len(segs) != parts:
456
- raise SystemExit(f"ref must have {parts} segments, got {ref!r}")
460
+ minimum = parts - 1
461
+ if len(segs) < minimum:
462
+ raise SystemExit(
463
+ f"ref must have at least {minimum} segments, got {ref!r}")
457
464
  return segs
458
465
 
459
466
 
@@ -475,15 +482,18 @@ def _split_instance_count(split) -> int:
475
482
 
476
483
 
477
484
  def _resolve_split_ref(args) -> tuple[str, str]:
478
- """定位 dataset + split:位置 ref({L1}/{L2}/{split})或 --dataset/--split。"""
485
+ """定位 dataset + split:位置 ref(``{dataset}/{split}``)或 --dataset/--split。
486
+
487
+ 多段 dataset 名时 split 取最后一节,前面所有段拼回 dataset 名。
488
+ """
479
489
  if args.dataset:
480
490
  if not args.split:
481
491
  raise SystemExit("给定 --dataset 时 split 需要 --split")
482
492
  return args.dataset, args.split
483
493
  if args.ref:
484
494
  segs = _split_ref(args.ref, 3)
485
- return f"{segs[0]}/{segs[1]}", segs[2]
486
- raise SystemExit("split get/create/set-run-type 需要 {L1}/{L2}/{split} 或 "
495
+ return "/".join(segs[:-1]), segs[-1]
496
+ raise SystemExit("split get/create/set-run-type 需要 {dataset}/{split} 或 "
487
497
  "--dataset/--split")
488
498
 
489
499
 
@@ -493,15 +503,15 @@ def _split_dataset(dataset: str) -> tuple[str, str]:
493
503
  name = P.DatasetName.parse(dataset)
494
504
  except InstanceRepoError as exc:
495
505
  raise SystemExit(
496
- f"--dataset must be one or two canonical segments, got "
506
+ f"--dataset must be one or more canonical segments, got "
497
507
  f"{dataset!r}") from exc
498
508
  return name.l1, name.l2
499
509
 
500
510
 
501
511
  _TARGET_SHAPE = {
502
- 2: ("{L1}/{L2}", "--dataset"),
503
- 3: ("{L1}/{L2}/{version}", "--dataset --version"),
504
- 4: ("{L1}/{L2}/{version}/{id}", "--dataset --version --instance-id"),
512
+ 2: ("{dataset}", "--dataset"),
513
+ 3: ("{dataset}/{version}", "--dataset --version"),
514
+ 4: ("{dataset}/{version}/{id}", "--dataset --version --instance-id"),
505
515
  }
506
516
 
507
517
 
@@ -532,9 +542,14 @@ def _resolve_target(args, parts: int) -> tuple[str, str | None, str | None]:
532
542
  f"(ref={ref!r})")
533
543
  if ref:
534
544
  segs = _split_ref(ref, parts)
535
- return ("/".join(segs[:2]),
536
- segs[2] if parts >= 3 else None,
537
- segs[3] if parts >= 4 else None)
545
+ # 尾部 parts-2 段是 version/instance-id 槽位,其余全部是 dataset 名
546
+ # (多段 dataset 名按原样保留)。
547
+ slots = parts - 2
548
+ if slots == 0:
549
+ return ("/".join(segs), None, None)
550
+ if slots == 1:
551
+ return ("/".join(segs[:-1]), segs[-1], None)
552
+ return ("/".join(segs[:-2]), segs[-2], segs[-1])
538
553
  if not given:
539
554
  raise SystemExit(
540
555
  f"需要位置参数 ref({shape})或 flag 形式({flag_hint})")
@@ -0,0 +1,41 @@
1
+ """Private RFC3339 calendar parser with Python 3.10+ fraction behavior.
2
+
3
+ Wire values remain untouched at the call sites. Python datetime arithmetic uses
4
+ microseconds, matching 3.11+ fromisoformat truncation of finer fractions; expiry
5
+ is never rounded up. The original wire precision is not an integrity anchor.
6
+ """
7
+ from datetime import datetime, timedelta, timezone
8
+ import re
9
+
10
+ _UTC = re.compile(
11
+ r'([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})'
12
+ r'(?:\.([0-9]+))?(Z|[+-][0-9]{2}:[0-9]{2})\Z')
13
+
14
+
15
+ def parse_utc(value: str, *, allow_offset: bool = False) -> datetime:
16
+ """Validate calendar fields without version-specific ISO parsing.
17
+
18
+ Fraction processing is bounded to six digits, including arbitrarily precise
19
+ values permitted by existing read/write callers. Entries keeps its existing
20
+ one-to-nine-digit and byte bounds before invoking this helper.
21
+ """
22
+ match = _UTC.fullmatch(value) if type(value) is str and type(allow_offset) is bool else None
23
+ if match is None:
24
+ raise ValueError('invalid RFC3339 UTC datetime')
25
+ fraction = match.group(7) or ''
26
+ microsecond = int((fraction[:6] + '000000')[:6])
27
+ suffix = match.group(8)
28
+ hours, minutes = (0, 0) if suffix == 'Z' else (int(suffix[1:3]), int(suffix[4:6]))
29
+ if hours > 23 or minutes > 59 or (not allow_offset and (hours or minutes)):
30
+ raise ValueError('invalid RFC3339 datetime offset')
31
+ offset = (hours * 60 + minutes) * (-1 if suffix.startswith('-') else 1)
32
+ parsed = None
33
+ try:
34
+ parsed = datetime(*(int(match.group(n)) for n in range(1, 7)),
35
+ microsecond=microsecond, tzinfo=timezone(timedelta(minutes=offset)))
36
+ except ValueError:
37
+ pass
38
+ if parsed is None:
39
+ # Do not attach a calendar exception or raw wire string to the error.
40
+ raise ValueError('invalid RFC3339 UTC calendar datetime')
41
+ return parsed