sci-protocol 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pysci/__init__.py +77 -0
- pysci/__main__.py +8 -0
- pysci/_cli/__init__.py +5 -0
- pysci/_cli/checks.py +173 -0
- pysci/_cli/clients/__init__.py +0 -0
- pysci/_cli/clients/archive.py +456 -0
- pysci/_cli/clients/credentials.py +54 -0
- pysci/_cli/clients/github.py +734 -0
- pysci/_cli/clients/hosting.py +238 -0
- pysci/_cli/clients/http.py +481 -0
- pysci/_cli/clients/huggingface.py +492 -0
- pysci/_cli/clients/index.py +435 -0
- pysci/_cli/clients/tsa.py +223 -0
- pysci/_cli/commands/__init__.py +44 -0
- pysci/_cli/commands/check.py +115 -0
- pysci/_cli/commands/claim.py +179 -0
- pysci/_cli/commands/common.py +91 -0
- pysci/_cli/commands/folders.py +83 -0
- pysci/_cli/commands/index.py +51 -0
- pysci/_cli/commands/keygen.py +58 -0
- pysci/_cli/commands/license.py +75 -0
- pysci/_cli/commands/protocol.py +72 -0
- pysci/_cli/commands/publish.py +282 -0
- pysci/_cli/commands/pull.py +337 -0
- pysci/_cli/commands/read.py +364 -0
- pysci/_cli/commands/revoke.py +207 -0
- pysci/_cli/commands/search.py +400 -0
- pysci/_cli/commands/sign.py +284 -0
- pysci/_cli/commands/submit.py +172 -0
- pysci/_cli/commands/timestamp.py +196 -0
- pysci/_cli/commands/verify.py +163 -0
- pysci/_cli/config.py +233 -0
- pysci/_cli/context.py +79 -0
- pysci/_cli/destinations.py +163 -0
- pysci/_cli/errors.py +168 -0
- pysci/_cli/main.py +229 -0
- pysci/_cli/output.py +80 -0
- pysci/_cli/retrieval.py +51 -0
- pysci/_cli/spec.py +113 -0
- pysci/_cli/storage/__init__.py +0 -0
- pysci/_cli/storage/catalog.py +371 -0
- pysci/_cli/storage/files.py +252 -0
- pysci/_cli/storage/inputs.py +205 -0
- pysci/_cli/storage/keys.py +132 -0
- pysci/_cli/storage/local_publish.py +88 -0
- pysci/accounts.py +135 -0
- pysci/baseline.py +142 -0
- pysci/bundle.py +878 -0
- pysci/citations.py +233 -0
- pysci/data/freetsa-cacert.pem +45 -0
- pysci/data/sci-baseline-LICENSE.txt +457 -0
- pysci/data/sci-protocol.md +613 -0
- pysci/der.py +146 -0
- pysci/discovery.py +461 -0
- pysci/draft.py +374 -0
- pysci/ed25519.py +153 -0
- pysci/errors.py +204 -0
- pysci/ids.py +61 -0
- pysci/jsonstrict.py +225 -0
- pysci/keys.py +247 -0
- pysci/manifest.py +302 -0
- pysci/metadata.py +406 -0
- pysci/paths.py +136 -0
- pysci/png.py +217 -0
- pysci/py.typed +0 -0
- pysci/records.py +384 -0
- pysci/rfc3339.py +66 -0
- pysci/schemas.py +75 -0
- pysci/source.py +166 -0
- pysci/specification.py +58 -0
- pysci/submission.py +225 -0
- pysci/templates.py +129 -0
- pysci/text.py +226 -0
- pysci/timestamps.py +867 -0
- pysci/zipformat.py +400 -0
- sci_protocol-0.2.0.dist-info/METADATA +112 -0
- sci_protocol-0.2.0.dist-info/RECORD +80 -0
- sci_protocol-0.2.0.dist-info/WHEEL +4 -0
- sci_protocol-0.2.0.dist-info/entry_points.txt +2 -0
- sci_protocol-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
"""``sci search``: search local campaigns, remote catalogs, or both."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import base64
|
|
7
|
+
import binascii
|
|
8
|
+
import json
|
|
9
|
+
import sqlite3
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from ..clients.index import IndexClient
|
|
14
|
+
from ..config import CATALOG_FILE, normalize_origin
|
|
15
|
+
from ..context import Context
|
|
16
|
+
from ..errors import CliError, ErrorRecord, Exit
|
|
17
|
+
from ..output import Result
|
|
18
|
+
from ..spec import Command, arg
|
|
19
|
+
from ..storage.catalog import SEARCHED_FIELDS, Catalog, LocalHit, ScanReport, is_corrupt
|
|
20
|
+
from ..storage.local_publish import remember_folder
|
|
21
|
+
|
|
22
|
+
CURSOR_VERSION = 1
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _encode_cursor(state: dict[str, Any]) -> str:
|
|
26
|
+
raw = json.dumps(state, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
|
27
|
+
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _decode_cursor(token: str) -> dict[str, Any]:
|
|
31
|
+
try:
|
|
32
|
+
raw = base64.urlsafe_b64decode(token + "=" * (-len(token) % 4))
|
|
33
|
+
state = json.loads(raw)
|
|
34
|
+
except (binascii.Error, ValueError):
|
|
35
|
+
raise CliError("invalid_cursor", "--cursor is not a continuation token") from None
|
|
36
|
+
if not isinstance(state, dict) or state.get("v") != CURSOR_VERSION:
|
|
37
|
+
raise CliError("invalid_cursor", "--cursor is not a continuation token from this version")
|
|
38
|
+
return state
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _resolve_sources(
|
|
42
|
+
ctx: Context, values: list[str] | None
|
|
43
|
+
) -> tuple[list[str], list[str], list[str]]:
|
|
44
|
+
"""Return (folders, index origins, explicitly named folders to remember)."""
|
|
45
|
+
folders: list[str] = []
|
|
46
|
+
origins: list[str] = []
|
|
47
|
+
explicit: list[str] = []
|
|
48
|
+
if not values:
|
|
49
|
+
return [str(ctx.cwd.resolve())], [], []
|
|
50
|
+
for value in values:
|
|
51
|
+
if value == "@local":
|
|
52
|
+
for folder in ctx.config.folders:
|
|
53
|
+
if folder not in folders:
|
|
54
|
+
folders.append(folder)
|
|
55
|
+
elif value == "@index":
|
|
56
|
+
if not ctx.config.index:
|
|
57
|
+
raise CliError(
|
|
58
|
+
"missing_index", "@index requires a configured index (`sci index URL`)"
|
|
59
|
+
)
|
|
60
|
+
if ctx.config.index not in origins:
|
|
61
|
+
origins.append(ctx.config.index)
|
|
62
|
+
elif "://" in value:
|
|
63
|
+
origin = normalize_origin(value)
|
|
64
|
+
if origin not in origins:
|
|
65
|
+
origins.append(origin)
|
|
66
|
+
elif value.startswith("@"):
|
|
67
|
+
raise CliError("invalid_source", f"unknown source {value!r}; use @local or @index")
|
|
68
|
+
else:
|
|
69
|
+
path = ctx.resolve(value)
|
|
70
|
+
folder = str(path.resolve()) if path.exists() else str(path.absolute())
|
|
71
|
+
if folder not in folders:
|
|
72
|
+
folders.append(folder)
|
|
73
|
+
if path.is_dir():
|
|
74
|
+
explicit.append(folder)
|
|
75
|
+
return folders, origins, explicit
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def run(args: argparse.Namespace, ctx: Context) -> Result:
|
|
79
|
+
category = tuple(args.category) if args.category else None
|
|
80
|
+
request: dict[str, Any] = {
|
|
81
|
+
"q": args.query,
|
|
82
|
+
"category": list(category) if category else None,
|
|
83
|
+
"key_x": args.key_x,
|
|
84
|
+
"domain": args.domain,
|
|
85
|
+
"limit": args.limit,
|
|
86
|
+
}
|
|
87
|
+
positions: dict[str, Any] = {}
|
|
88
|
+
explicit: list[str] = []
|
|
89
|
+
if args.cursor:
|
|
90
|
+
state = _decode_cursor(args.cursor)
|
|
91
|
+
for name, value in request.items():
|
|
92
|
+
limit_given = getattr(args, "limit_given", False)
|
|
93
|
+
given = value if name != "limit" else (value if limit_given else None)
|
|
94
|
+
if given is not None and given != state.get(name):
|
|
95
|
+
raise CliError(
|
|
96
|
+
"cursor_conflict",
|
|
97
|
+
f"--cursor continues an earlier search; {name} differs from that search",
|
|
98
|
+
)
|
|
99
|
+
request[name] = state.get(name)
|
|
100
|
+
if args.in_:
|
|
101
|
+
folders_now, origins_now, _ = _resolve_sources(ctx, args.in_)
|
|
102
|
+
if folders_now != state["folders"] or origins_now != state["origins"]:
|
|
103
|
+
raise CliError("cursor_conflict", "--cursor continues different sources")
|
|
104
|
+
folders, origins = list(state["folders"]), list(state["origins"])
|
|
105
|
+
positions = dict(state.get("positions", {}))
|
|
106
|
+
else:
|
|
107
|
+
folders, origins, explicit = _resolve_sources(ctx, args.in_)
|
|
108
|
+
query: str | None = request["q"]
|
|
109
|
+
limit: int = request["limit"]
|
|
110
|
+
if limit < 1:
|
|
111
|
+
raise CliError("invalid_arguments", "--limit must be positive")
|
|
112
|
+
if origins and not (query or request["category"] or request["key_x"] or request["domain"]):
|
|
113
|
+
raise CliError(
|
|
114
|
+
"missing_query",
|
|
115
|
+
"searching an index needs QUERY or a filter (--category, --key-x, --domain); "
|
|
116
|
+
"only local folders can be listed without one",
|
|
117
|
+
)
|
|
118
|
+
if request["domain"] and folders:
|
|
119
|
+
raise CliError("invalid_arguments", "--domain requires remote-only scope (index sources)")
|
|
120
|
+
if not folders and not origins:
|
|
121
|
+
raise CliError("no_sources", "no sources selected (no known folders for @local?)")
|
|
122
|
+
category_pair = tuple(request["category"]) if request["category"] else None
|
|
123
|
+
|
|
124
|
+
result = Result("search")
|
|
125
|
+
groups: list[dict[str, Any]] = []
|
|
126
|
+
next_positions: dict[str, Any] = {}
|
|
127
|
+
partial = False
|
|
128
|
+
|
|
129
|
+
if folders and positions.get("local", "") != "done":
|
|
130
|
+
group, after, more, group_partial = _local(
|
|
131
|
+
ctx, folders, query, category_pair, request["key_x"], positions.get("local"), limit
|
|
132
|
+
)
|
|
133
|
+
groups.append(group)
|
|
134
|
+
partial = partial or group_partial
|
|
135
|
+
if more:
|
|
136
|
+
next_positions["local"] = after
|
|
137
|
+
for folder in explicit:
|
|
138
|
+
if remember_folder(ctx.config, Path(folder)):
|
|
139
|
+
ctx.save_config()
|
|
140
|
+
for origin in origins:
|
|
141
|
+
key = "index:" + origin
|
|
142
|
+
if positions.get(key, "") == "done":
|
|
143
|
+
continue
|
|
144
|
+
group, after_remote, more = _remote(
|
|
145
|
+
ctx, args, origin, query, category_pair, request, positions.get(key), limit
|
|
146
|
+
)
|
|
147
|
+
groups.append(group)
|
|
148
|
+
if group.get("error"):
|
|
149
|
+
partial = True
|
|
150
|
+
error = group["error"]
|
|
151
|
+
result.errors.append(ErrorRecord(error["code"], error["message"], error["retryable"]))
|
|
152
|
+
if more:
|
|
153
|
+
next_positions[key] = after_remote
|
|
154
|
+
next_cursor = None
|
|
155
|
+
if next_positions:
|
|
156
|
+
state = {
|
|
157
|
+
"v": CURSOR_VERSION,
|
|
158
|
+
**request,
|
|
159
|
+
"folders": folders,
|
|
160
|
+
"origins": origins,
|
|
161
|
+
"positions": {
|
|
162
|
+
**dict.fromkeys(["local", *("index:" + o for o in origins)], "done"),
|
|
163
|
+
**next_positions,
|
|
164
|
+
},
|
|
165
|
+
}
|
|
166
|
+
if not folders:
|
|
167
|
+
state["positions"].pop("local", None)
|
|
168
|
+
next_cursor = _encode_cursor(state)
|
|
169
|
+
result.data = {
|
|
170
|
+
"query": query,
|
|
171
|
+
"filters": {
|
|
172
|
+
"category": request["category"],
|
|
173
|
+
"key_x": request["key_x"],
|
|
174
|
+
"domain": request["domain"],
|
|
175
|
+
},
|
|
176
|
+
"limit": limit,
|
|
177
|
+
"groups": groups,
|
|
178
|
+
"next_cursor": next_cursor,
|
|
179
|
+
"partial": partial,
|
|
180
|
+
}
|
|
181
|
+
_describe(result, groups, next_cursor)
|
|
182
|
+
if partial:
|
|
183
|
+
failed = [g for g in groups if g.get("error")]
|
|
184
|
+
if failed and len(failed) == len(groups) and len({g["error_exit"] for g in failed}) == 1:
|
|
185
|
+
result.exit_code = failed[0]["error_exit"]
|
|
186
|
+
else:
|
|
187
|
+
result.exit_code = Exit.INCOMPLETE
|
|
188
|
+
result.partial = True
|
|
189
|
+
for group in groups:
|
|
190
|
+
group.pop("error_exit", None)
|
|
191
|
+
return result
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _local(
|
|
195
|
+
ctx: Context,
|
|
196
|
+
folders: list[str],
|
|
197
|
+
query: str | None,
|
|
198
|
+
category: tuple[str, ...] | None,
|
|
199
|
+
key_x: str | None,
|
|
200
|
+
after: str | None,
|
|
201
|
+
limit: int,
|
|
202
|
+
) -> tuple[dict[str, Any], str | None, bool, bool]:
|
|
203
|
+
terms = query.casefold().split() if query else []
|
|
204
|
+
pair = (category[0], category[1]) if category else None
|
|
205
|
+
|
|
206
|
+
def scan_and_search(catalog: Catalog) -> tuple[ScanReport, list[LocalHit], bool]:
|
|
207
|
+
scan = catalog.scan([Path(f) for f in folders])
|
|
208
|
+
hits, more = catalog.search(
|
|
209
|
+
scan.available, terms, category=pair, key_x=key_x, after=after, limit=limit
|
|
210
|
+
)
|
|
211
|
+
return scan, hits, more
|
|
212
|
+
|
|
213
|
+
try:
|
|
214
|
+
catalog = Catalog(ctx.home / CATALOG_FILE)
|
|
215
|
+
try:
|
|
216
|
+
try:
|
|
217
|
+
scan, hits, more = scan_and_search(catalog)
|
|
218
|
+
except sqlite3.DatabaseError as exc:
|
|
219
|
+
if not is_corrupt(exc):
|
|
220
|
+
raise
|
|
221
|
+
catalog.reset()
|
|
222
|
+
scan, hits, more = scan_and_search(catalog)
|
|
223
|
+
finally:
|
|
224
|
+
catalog.close()
|
|
225
|
+
except sqlite3.Error as exc:
|
|
226
|
+
raise CliError(
|
|
227
|
+
"catalog_unavailable",
|
|
228
|
+
f"the local search cache could not be used: {exc}",
|
|
229
|
+
Exit.INCOMPLETE,
|
|
230
|
+
retryable=True,
|
|
231
|
+
file=str(ctx.home / CATALOG_FILE),
|
|
232
|
+
) from exc
|
|
233
|
+
group: dict[str, Any] = {
|
|
234
|
+
"source": "local",
|
|
235
|
+
"kind": "local",
|
|
236
|
+
"folders": folders,
|
|
237
|
+
"searched_fields": list(SEARCHED_FIELDS),
|
|
238
|
+
"verification": "complete bundle checks (§6.4) on each ZIP before indexing",
|
|
239
|
+
"results": [hit.to_json() for hit in hits],
|
|
240
|
+
"more": more,
|
|
241
|
+
"invalid_candidates": scan.invalid,
|
|
242
|
+
"unavailable_folders": scan.unavailable,
|
|
243
|
+
"cache": {"verified": scan.verified, "reused": scan.reused, "rebuilt": catalog.rebuilt},
|
|
244
|
+
"partial": bool(scan.unavailable),
|
|
245
|
+
}
|
|
246
|
+
last = hits[-1].publication_id if hits else after
|
|
247
|
+
return group, last, more, bool(scan.unavailable)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _remote(
|
|
251
|
+
ctx: Context,
|
|
252
|
+
args: argparse.Namespace,
|
|
253
|
+
origin: str,
|
|
254
|
+
query: str | None,
|
|
255
|
+
category: tuple[str, ...] | None,
|
|
256
|
+
request: dict[str, Any],
|
|
257
|
+
after: Any,
|
|
258
|
+
limit: int,
|
|
259
|
+
) -> tuple[dict[str, Any], Any, bool]:
|
|
260
|
+
group: dict[str, Any] = {
|
|
261
|
+
"source": origin,
|
|
262
|
+
"kind": "index",
|
|
263
|
+
"searched_fields": ["title", "abstract", "keywords"],
|
|
264
|
+
"verification": "catalog metadata from the index; files were not retrieved or verified",
|
|
265
|
+
"results": [],
|
|
266
|
+
"more": False,
|
|
267
|
+
}
|
|
268
|
+
client = IndexClient(ctx.http, origin)
|
|
269
|
+
try:
|
|
270
|
+
rows, more, _ = client.search(
|
|
271
|
+
ctx.deadline(args.timeout),
|
|
272
|
+
query=query,
|
|
273
|
+
category=(category[0], category[1]) if category else None,
|
|
274
|
+
domain=request["domain"],
|
|
275
|
+
key_x=request["key_x"],
|
|
276
|
+
after=(after[0], after[1]) if isinstance(after, list) else None,
|
|
277
|
+
limit=limit,
|
|
278
|
+
)
|
|
279
|
+
except CliError as exc:
|
|
280
|
+
group["error"] = exc.records[0].to_json()
|
|
281
|
+
group["error_exit"] = int(exc.exit_code)
|
|
282
|
+
group["partial"] = True
|
|
283
|
+
return group, None, False
|
|
284
|
+
group["results"] = rows
|
|
285
|
+
group["more"] = more
|
|
286
|
+
last = [rows[-1]["accepted_at"], rows[-1]["id"].removeprefix("sha256:")] if rows else None
|
|
287
|
+
return group, last, more
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _describe(result: Result, groups: list[dict[str, Any]], cursor: str | None) -> None:
|
|
291
|
+
for group in groups:
|
|
292
|
+
if group["kind"] == "local":
|
|
293
|
+
result.say(f"== local: {', '.join(group['folders'])}")
|
|
294
|
+
else:
|
|
295
|
+
result.say(f"== {group['source']} (catalog metadata, not verified locally)")
|
|
296
|
+
for hit in group["results"]:
|
|
297
|
+
result.say(f"{hit['id']} {hit['title']}")
|
|
298
|
+
if group["kind"] == "local":
|
|
299
|
+
for path in hit["paths"]:
|
|
300
|
+
result.say(f" {path}")
|
|
301
|
+
else:
|
|
302
|
+
when = hit.get("accepted_at")
|
|
303
|
+
page = hit.get("publication_url") or ""
|
|
304
|
+
result.say(f" accepted {when} {page}".rstrip())
|
|
305
|
+
if not group["results"]:
|
|
306
|
+
result.say("(no results)")
|
|
307
|
+
for candidate in group.get("invalid_candidates", []):
|
|
308
|
+
first = candidate["issues"][0]["message"] if candidate["issues"] else ""
|
|
309
|
+
result.say(f"invalid: {candidate['path']} ({candidate['status']}) {first}".rstrip())
|
|
310
|
+
for folder in group.get("unavailable_folders", []):
|
|
311
|
+
result.say(f"unavailable: {folder['folder']} ({folder['error']})")
|
|
312
|
+
if group.get("error"):
|
|
313
|
+
result.say(f"error: {group['error']['message']}")
|
|
314
|
+
if group.get("more"):
|
|
315
|
+
result.say("(more results available)")
|
|
316
|
+
if cursor:
|
|
317
|
+
result.say(f"continue with: --cursor {cursor}")
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class _LimitAction(argparse.Action):
|
|
321
|
+
def __call__(self, parser: Any, namespace: Any, values: Any, option: Any = None) -> None:
|
|
322
|
+
setattr(namespace, self.dest, values)
|
|
323
|
+
namespace.limit_given = True
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
COMMAND = Command(
|
|
327
|
+
name="search",
|
|
328
|
+
usage_hint=(
|
|
329
|
+
"search [QUERY] [--in SOURCE ...] [--category SCHEME CODE] [--key-x X] [--domain DOMAIN] "
|
|
330
|
+
"[--limit N] [--cursor TOKEN] [--json]"
|
|
331
|
+
),
|
|
332
|
+
summary="Search local campaigns, remote catalogs, or both.",
|
|
333
|
+
description="""
|
|
334
|
+
Sources (repeat --in to combine; duplicates are merged):
|
|
335
|
+
DIR immediate *.zip children of that folder
|
|
336
|
+
https://... catalog discovered through that index's descriptor
|
|
337
|
+
@local known folders (see `sci folders`)
|
|
338
|
+
@index the configured index
|
|
339
|
+
Without --in, the current folder is searched offline. Without QUERY or a
|
|
340
|
+
filter, local folders are listed; use --limit and --cursor to page through
|
|
341
|
+
them. Index sources need QUERY or a filter.
|
|
342
|
+
|
|
343
|
+
Local search verifies new or changed ZIPs completely (SCI protocol §6.4),
|
|
344
|
+
caches their metadata and README text in catalog.sqlite (keyed by verified
|
|
345
|
+
publication ID), and matches all case-folded query terms in title,
|
|
346
|
+
abstract, keywords, and README. Results are ordered by publication ID;
|
|
347
|
+
invalid candidates are reported separately. Remote search queries the index
|
|
348
|
+
catalog (Sciscape catalog_url, sciscape_catalog_v1) newest admissions first
|
|
349
|
+
and labels results as catalog metadata; nothing is downloaded or verified.
|
|
350
|
+
An archive without a catalog is not searchable. If a source fails, available
|
|
351
|
+
results are returned and marked partial.
|
|
352
|
+
""",
|
|
353
|
+
args=(
|
|
354
|
+
arg("query", metavar="QUERY", nargs="?", help="search terms (omit to list local folders)"),
|
|
355
|
+
arg(
|
|
356
|
+
"--in",
|
|
357
|
+
dest="in_",
|
|
358
|
+
metavar="SOURCE",
|
|
359
|
+
action="append",
|
|
360
|
+
help="source to search (repeatable): folder, https:// index origin, @local, or @index",
|
|
361
|
+
),
|
|
362
|
+
arg(
|
|
363
|
+
"--category",
|
|
364
|
+
nargs=2,
|
|
365
|
+
metavar=("SCHEME", "CODE"),
|
|
366
|
+
help="exact category filter, e.g. arxiv cs.LG",
|
|
367
|
+
),
|
|
368
|
+
arg("--key-x", metavar="X", help="signer public key x filter"),
|
|
369
|
+
arg("--domain", metavar="DOMAIN", help="admission-time domain association (remote only)"),
|
|
370
|
+
arg(
|
|
371
|
+
"--limit",
|
|
372
|
+
type=int,
|
|
373
|
+
default=20,
|
|
374
|
+
metavar="N",
|
|
375
|
+
action=_LimitAction,
|
|
376
|
+
help="results per group",
|
|
377
|
+
),
|
|
378
|
+
arg(
|
|
379
|
+
"--cursor",
|
|
380
|
+
metavar="TOKEN",
|
|
381
|
+
help="continue a previous search (same sources, query, and filters)",
|
|
382
|
+
),
|
|
383
|
+
),
|
|
384
|
+
outputs="""One local group and one group per index: source, searched fields, verification
|
|
385
|
+
status, results (ID, title, abstract, signer; local paths and citations, or
|
|
386
|
+
admission time and publication URL), and a continuation token.""",
|
|
387
|
+
files="""Updates the search cache <state>/catalog.sqlite; remembers folders named with
|
|
388
|
+
--in in config.json.""",
|
|
389
|
+
network="None for local sources. Index sources: descriptor and catalog GET requests.",
|
|
390
|
+
exit_codes={
|
|
391
|
+
4: "an index has no supported catalog (reported in its group)",
|
|
392
|
+
5: "partial result: a source failed or a folder was unavailable",
|
|
393
|
+
},
|
|
394
|
+
examples=(
|
|
395
|
+
'sci search "neural decoding" --in ./campaign-a --in ./campaign-b',
|
|
396
|
+
'sci search "neural decoding" --in @local --in https://sciscape.io',
|
|
397
|
+
),
|
|
398
|
+
run=run,
|
|
399
|
+
timeout=True,
|
|
400
|
+
)
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""``sci sign``: sign a draft, timestamp it, and optionally store its ZIP in a campaign folder."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from pysci.bundle import verify_directory, verify_loose
|
|
10
|
+
from pysci.der import frame_values
|
|
11
|
+
from pysci.draft import BaselineRefused, check_draft, load_draft, sign_draft
|
|
12
|
+
from pysci.errors import SciError
|
|
13
|
+
from pysci.keys import public_key_of
|
|
14
|
+
from pysci.manifest import MANIFEST_PATH, SIGNATURE_PATH, TIMESTAMP_SIDECAR_PATH
|
|
15
|
+
from pysci.source import ByteSource
|
|
16
|
+
|
|
17
|
+
from ..checks import proof_json
|
|
18
|
+
from ..clients.tsa import FREETSA_URL, append_sidecar, read_sidecar, request_proof
|
|
19
|
+
from ..config import normalize_url
|
|
20
|
+
from ..context import Context
|
|
21
|
+
from ..errors import CliError, ErrorRecord, Exit, exit_for_outcome, issue_record
|
|
22
|
+
from ..output import Result
|
|
23
|
+
from ..spec import Command, arg
|
|
24
|
+
from ..storage.files import recover_replacements, replace_together, temp_names
|
|
25
|
+
from ..storage.inputs import SignedInput
|
|
26
|
+
from ..storage.keys import require_outside
|
|
27
|
+
from ..storage.local_publish import publish_to_folder, remember_folder
|
|
28
|
+
from .check import describe, draft_report_json, warn_display
|
|
29
|
+
from .common import KEY_ARG_HELP, signing_key, trust
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def tsa_url(ctx: Context, explicit: str | None) -> str:
|
|
33
|
+
if explicit:
|
|
34
|
+
return normalize_url(explicit, what="TSA URL")
|
|
35
|
+
return ctx.config.tsa_url or FREETSA_URL
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _verify_in_place(draft: Path, manifest: bytes, signature: bytes, expected: str) -> None:
|
|
39
|
+
sources, _, _ = load_draft(draft, temp_names(draft))
|
|
40
|
+
files: dict[str, ByteSource | bytes] = {
|
|
41
|
+
**sources,
|
|
42
|
+
MANIFEST_PATH: manifest,
|
|
43
|
+
SIGNATURE_PATH: signature,
|
|
44
|
+
}
|
|
45
|
+
result = verify_loose(files, expected_id=expected)
|
|
46
|
+
if not result.ok:
|
|
47
|
+
raise CliError(
|
|
48
|
+
"draft_changed",
|
|
49
|
+
"draft files changed during signing; the previous files were restored",
|
|
50
|
+
Exit.INCOMPLETE,
|
|
51
|
+
retryable=True,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def timestamp_step(
|
|
56
|
+
ctx: Context, result: Result, sidecar: Path, signature: bytes, url: str, timeout: float
|
|
57
|
+
) -> bool:
|
|
58
|
+
"""Request one proof and append it; on failure record a partial result."""
|
|
59
|
+
deadline = ctx.deadline(timeout)
|
|
60
|
+
try:
|
|
61
|
+
trust_config = trust(ctx)
|
|
62
|
+
proof = request_proof(ctx.http, url, signature, trust_config, deadline)
|
|
63
|
+
appended = append_sidecar(sidecar, proof.response)
|
|
64
|
+
except CliError as exc:
|
|
65
|
+
result.data["timestamp"] = {
|
|
66
|
+
"status": "failed",
|
|
67
|
+
"tsa": url,
|
|
68
|
+
"error": exc.records[0].to_json(),
|
|
69
|
+
}
|
|
70
|
+
result.errors.extend(exc.records)
|
|
71
|
+
return False
|
|
72
|
+
result.data["timestamp"] = {
|
|
73
|
+
"status": "appended" if appended else "duplicate",
|
|
74
|
+
"tsa": url,
|
|
75
|
+
"sidecar": str(sidecar),
|
|
76
|
+
"proof": proof_json(proof.verification),
|
|
77
|
+
}
|
|
78
|
+
return True
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def run(args: argparse.Namespace, ctx: Context) -> Result:
|
|
82
|
+
draft = ctx.resolve(args.dir)
|
|
83
|
+
if not draft.is_dir():
|
|
84
|
+
raise CliError("draft_missing", f"{draft} is not a directory", Exit.USAGE, file=str(draft))
|
|
85
|
+
if args.no_timestamp and args.tsa:
|
|
86
|
+
raise CliError("conflicting_options", "--tsa cannot be combined with --no-timestamp")
|
|
87
|
+
url = None if args.no_timestamp else tsa_url(ctx, args.tsa)
|
|
88
|
+
output = ctx.resolve(args.output) if args.output else None
|
|
89
|
+
restored = recover_replacements(draft)
|
|
90
|
+
key_path, key, warnings = signing_key(ctx, args)
|
|
91
|
+
require_outside(key_path, draft, *([output] if output else []))
|
|
92
|
+
|
|
93
|
+
result = Result("sign")
|
|
94
|
+
result.warnings.extend(warnings)
|
|
95
|
+
if restored:
|
|
96
|
+
result.warn(f"restored files from an interrupted signing operation: {restored}")
|
|
97
|
+
report = check_draft(draft, ignore=temp_names(draft))
|
|
98
|
+
result.data = {"path": str(draft), "check": draft_report_json(report)}
|
|
99
|
+
warn_display(result, report)
|
|
100
|
+
if report.issues:
|
|
101
|
+
describe(result, report)
|
|
102
|
+
result.exit_code = exit_for_outcome(report.outcome)
|
|
103
|
+
result.errors.extend(issue_record(i) for i in report.issues)
|
|
104
|
+
return result
|
|
105
|
+
assert report.baseline is not None
|
|
106
|
+
if not report.baseline.eligible and not args.allow_nonbaseline:
|
|
107
|
+
describe(result, report)
|
|
108
|
+
result.exit_code = Exit.REFUSED
|
|
109
|
+
result.errors.append(
|
|
110
|
+
ErrorRecord(
|
|
111
|
+
"baseline_refused",
|
|
112
|
+
"the draft does not meet the baseline; pass --allow-nonbaseline to sign a "
|
|
113
|
+
"protocol-valid non-baseline publication",
|
|
114
|
+
)
|
|
115
|
+
)
|
|
116
|
+
result.errors.extend(issue_record(i) for i in report.baseline.reasons)
|
|
117
|
+
return result
|
|
118
|
+
|
|
119
|
+
selected = public_key_of(key)
|
|
120
|
+
signed_before = (draft / MANIFEST_PATH).exists() or (draft / SIGNATURE_PATH).exists()
|
|
121
|
+
status = "signed"
|
|
122
|
+
if signed_before:
|
|
123
|
+
existing = verify_directory(draft)
|
|
124
|
+
if (
|
|
125
|
+
existing.ok
|
|
126
|
+
and existing.manifest is not None
|
|
127
|
+
and existing.manifest.public_key == selected
|
|
128
|
+
):
|
|
129
|
+
status = "unchanged"
|
|
130
|
+
publication_id = existing.manifest.publication_id
|
|
131
|
+
signature = (draft / SIGNATURE_PATH).read_bytes()
|
|
132
|
+
elif not args.replace:
|
|
133
|
+
raise CliError(
|
|
134
|
+
"signature_exists",
|
|
135
|
+
f"{draft} has a signature that no longer matches its files or the selected key; "
|
|
136
|
+
"pass --replace to re-sign (this discards timestamp proofs for the old signature)",
|
|
137
|
+
Exit.USAGE,
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
status = "replaced"
|
|
141
|
+
if status != "unchanged":
|
|
142
|
+
try:
|
|
143
|
+
signed = sign_draft(
|
|
144
|
+
draft, key, allow_nonbaseline=args.allow_nonbaseline, ignore=temp_names(draft)
|
|
145
|
+
)
|
|
146
|
+
except BaselineRefused as exc: # pragma: no cover - checked above
|
|
147
|
+
raise CliError("baseline_refused", str(exc), Exit.REFUSED) from None
|
|
148
|
+
except SciError as exc:
|
|
149
|
+
raise CliError.from_sci(exc, "signing failed") from None
|
|
150
|
+
replacements: dict[str, bytes | None] = {
|
|
151
|
+
MANIFEST_PATH: signed.manifest_bytes,
|
|
152
|
+
SIGNATURE_PATH: signed.signature,
|
|
153
|
+
}
|
|
154
|
+
if status == "replaced":
|
|
155
|
+
replacements[TIMESTAMP_SIDECAR_PATH] = None
|
|
156
|
+
with replace_together(draft, replacements):
|
|
157
|
+
_verify_in_place(draft, signed.manifest_bytes, signed.signature, signed.publication_id)
|
|
158
|
+
publication_id = signed.publication_id
|
|
159
|
+
signature = signed.signature
|
|
160
|
+
result.data.update(
|
|
161
|
+
{
|
|
162
|
+
"status": status,
|
|
163
|
+
"publication_id": publication_id,
|
|
164
|
+
"signer": selected.to_jwk(),
|
|
165
|
+
"baseline": report.baseline.eligible,
|
|
166
|
+
}
|
|
167
|
+
)
|
|
168
|
+
verb = {"signed": "Signed", "replaced": "Re-signed", "unchanged": "Unchanged signature"}[status]
|
|
169
|
+
result.say(f"{verb}: {publication_id}")
|
|
170
|
+
if status == "replaced":
|
|
171
|
+
result.say("previous timestamp proofs were discarded with the replaced signature")
|
|
172
|
+
if not report.baseline.eligible:
|
|
173
|
+
result.say("baseline: not eligible (signed with --allow-nonbaseline)")
|
|
174
|
+
|
|
175
|
+
sidecar = draft / TIMESTAMP_SIDECAR_PATH
|
|
176
|
+
timestamp_ok = True
|
|
177
|
+
if url is None:
|
|
178
|
+
result.data["timestamp"] = {"status": "skipped", "reason": "--no-timestamp"}
|
|
179
|
+
result.say("timestamp: skipped (--no-timestamp)")
|
|
180
|
+
else:
|
|
181
|
+
existing_proofs = read_sidecar(sidecar)
|
|
182
|
+
framed = frame_values(existing_proofs) if existing_proofs else None
|
|
183
|
+
if status == "unchanged" and framed is not None and framed.values and framed.complete:
|
|
184
|
+
result.data["timestamp"] = {"status": "existing", "sidecar": str(sidecar)}
|
|
185
|
+
result.say(f"timestamp: existing sidecar kept ({len(framed.values)} proof(s))")
|
|
186
|
+
else:
|
|
187
|
+
timestamp_ok = timestamp_step(ctx, result, sidecar, signature, url, args.timeout)
|
|
188
|
+
ts = result.data["timestamp"]
|
|
189
|
+
if timestamp_ok:
|
|
190
|
+
trust_result = ts["proof"]["result"]
|
|
191
|
+
result.say(f"timestamp: {ts['status']} from {url} (trust: {trust_result})")
|
|
192
|
+
else:
|
|
193
|
+
result.say(f"timestamp: failed ({ts['error']['message']})")
|
|
194
|
+
result.say(f"retry later with: sci timestamp {draft}")
|
|
195
|
+
|
|
196
|
+
pack_exit: int = Exit.OK
|
|
197
|
+
if output is not None:
|
|
198
|
+
verified = verify_directory(draft, expected_id=publication_id, compute_zip_sha256=True)
|
|
199
|
+
if not verified.ok:
|
|
200
|
+
raise CliError(
|
|
201
|
+
"draft_changed",
|
|
202
|
+
"signed files changed before packing",
|
|
203
|
+
Exit.INCOMPLETE,
|
|
204
|
+
retryable=True,
|
|
205
|
+
)
|
|
206
|
+
item = SignedInput(draft, "directory", verified)
|
|
207
|
+
stored, pack_exit = publish_to_folder(item, output)
|
|
208
|
+
result.data["output"] = stored
|
|
209
|
+
if pack_exit == Exit.OK:
|
|
210
|
+
if remember_folder(ctx.config, output):
|
|
211
|
+
ctx.save_config()
|
|
212
|
+
result.say(f"stored: {stored['path']} ({stored['bundle']})")
|
|
213
|
+
else:
|
|
214
|
+
error: dict[str, Any] = stored["error"]
|
|
215
|
+
result.errors.append(ErrorRecord(error["code"], error["message"], error["retryable"]))
|
|
216
|
+
result.say(f"packing failed; the signed directory {draft} is preserved")
|
|
217
|
+
if not timestamp_ok:
|
|
218
|
+
result.exit_code = Exit.INCOMPLETE
|
|
219
|
+
result.partial = True
|
|
220
|
+
if pack_exit != Exit.OK:
|
|
221
|
+
result.exit_code = pack_exit
|
|
222
|
+
result.partial = True
|
|
223
|
+
return result
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
COMMAND = Command(
|
|
227
|
+
name="sign",
|
|
228
|
+
usage_hint=(
|
|
229
|
+
"sign DIR [--output DIR] [--key PATH] [--no-timestamp | --tsa URL] [--replace] "
|
|
230
|
+
"[--allow-nonbaseline] [--json]"
|
|
231
|
+
),
|
|
232
|
+
summary="Sign a draft and optionally save its ZIP in a campaign folder.",
|
|
233
|
+
description="""
|
|
234
|
+
Applies the `check` draft checks (rejecting symbolic links, special files, and
|
|
235
|
+
private-key material), builds manifest.json with sorted paths and a stable
|
|
236
|
+
serializer, signs it (SCI protocol §5.1), and verifies the result (§6.4)
|
|
237
|
+
before keeping it. manifest.json and manifest.json.sig are replaced together
|
|
238
|
+
through a recoverable operation; if the files change while signing, the
|
|
239
|
+
previous files are restored.
|
|
240
|
+
|
|
241
|
+
An unchanged draft signed by the same key keeps its manifest bytes and ID.
|
|
242
|
+
Otherwise --replace is required, and the timestamp proofs of the replaced
|
|
243
|
+
signature are discarded.
|
|
244
|
+
|
|
245
|
+
Unless --no-timestamp is given, the signature is timestamped by the
|
|
246
|
+
configured TSA (default FreeTSA, https://freetsa.org/tsr; no automatic
|
|
247
|
+
fallback). A timestamp failure keeps the signed publication and returns a
|
|
248
|
+
partial result; retry with `sci timestamp DIR`.
|
|
249
|
+
|
|
250
|
+
--output DIR also stores <digest>.zip (and <digest>.tsr) in a campaign folder
|
|
251
|
+
under the local publishing rules of `sci publish`.
|
|
252
|
+
""",
|
|
253
|
+
args=(
|
|
254
|
+
arg("dir", metavar="DIR", help="draft directory to sign in place"),
|
|
255
|
+
arg("--output", metavar="DIR", help="campaign folder that receives <digest>.zip"),
|
|
256
|
+
arg("--key", metavar="PATH", help=KEY_ARG_HELP),
|
|
257
|
+
arg("--no-timestamp", action="store_true", help="skip timestamping (signing is offline)"),
|
|
258
|
+
arg(
|
|
259
|
+
"--tsa",
|
|
260
|
+
metavar="URL",
|
|
261
|
+
help="timestamp authority URL (default: config tsa_url, else https://freetsa.org/tsr)",
|
|
262
|
+
),
|
|
263
|
+
arg("--replace", action="store_true", help="replace an existing, different signature"),
|
|
264
|
+
arg(
|
|
265
|
+
"--allow-nonbaseline",
|
|
266
|
+
action="store_true",
|
|
267
|
+
help="sign a protocol-valid publication that is outside the baseline",
|
|
268
|
+
),
|
|
269
|
+
),
|
|
270
|
+
outputs="Publication ID, signer, baseline eligibility, timestamp result, and stored ZIP path.",
|
|
271
|
+
files="""Writes DIR/manifest.json and DIR/manifest.json.sig; appends to
|
|
272
|
+
DIR/manifest.json.sig.tsr; with --output writes <digest>.zip and <digest>.tsr in
|
|
273
|
+
that folder and remembers the folder in config.json.""",
|
|
274
|
+
network="""One request to the TSA unless --no-timestamp (paced to at most one per
|
|
275
|
+
second together with index writes). Otherwise offline.""",
|
|
276
|
+
exit_codes={
|
|
277
|
+
3: "the draft fails protocol checks, or --output holds a conflicting ZIP",
|
|
278
|
+
5: "signed, but timestamping or packing did not complete (ID is reported)",
|
|
279
|
+
6: "baseline refusal (use --allow-nonbaseline for protocol-valid drafts)",
|
|
280
|
+
},
|
|
281
|
+
examples=("sci license ./draft && sci sign ./draft --no-timestamp --output ./campaign",),
|
|
282
|
+
run=run,
|
|
283
|
+
timeout=True,
|
|
284
|
+
)
|