formist-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
formist_cli/files.py ADDED
@@ -0,0 +1,670 @@
1
+ """``fm files``: the file-row operations surface (``merge``, ``reevaluate``, ``download``).
2
+
3
+ Three different jobs on the same noun, kept as real argparse
4
+ subcommands rather than one flat parser, because ``merge`` and
5
+ ``reevaluate`` disagree on what ``--apply`` means: ``reevaluate
6
+ --apply`` is a flag, ``merge --apply`` takes the ``report_token`` a
7
+ prior dry run printed. One parser cannot carry both spellings of one
8
+ option name.
9
+
10
+ --- fm files merge ------------------------------------------------------
11
+
12
+ Merges two duplicate File rows that share one remote id. The decision
13
+ of which row survives, and what it carries, lives on the server; this
14
+ module is a client and holds no copy of it, since two independent
15
+ answers to "which of these rows survives" is precisely the kind of
16
+ disagreement that leaves a half-merged row behind.
17
+
18
+ This talks to ``/projects/{space}/admin/...`` rather than the
19
+ admin-only helpers, because files are region-local: the admin helpers
20
+ refuse a non-Global prod target before a socket opens, which is
21
+ correct for the platform-admin tier, but would refuse the only region
22
+ that actually holds the data. ``fm participants dedupe`` and
23
+ ``fm airtable`` make the same call for the same reason.
24
+
25
+ The safety choreography:
26
+
27
+ * a dry run always runs, including under ``--apply``: it is printed
28
+ first, and what is submitted is the ``report_token`` of the report
29
+ just printed.
30
+ * ``--apply`` takes that token, not a bare flag, since the pair can
31
+ change between the dry run and the apply (the connector refreshes
32
+ every 30 minutes); a stale token gets a 409 and nothing is written.
33
+ * ``--apply`` never inherits an environment from ``fm use``.
34
+ * one pair per call, and ``--reason`` is required, enforced
35
+ server-side too: there is deliberately no ``--all`` and no batch
36
+ file of pairs, since deduplication is a manual, judgement-based
37
+ decision, and naming two ids is the judgement the rule asks for.
38
+ * the dry run describes: both rows' identity columns side by side with
39
+ conflicts marked, every reference that would move, which row's
40
+ content the merged row would carry, and which blob keys are shared
41
+ (protected) versus exclusive (destroyed with the dropped row).
42
+ Nothing in it is a command to run on its own.
43
+
44
+ --- fm files reevaluate -------------------------------------------------
45
+
46
+ Re-derives whether a File row's stored skip verdict still matches the
47
+ current processing criteria (the size cap, the supported-format list,
48
+ and so on), through the routes the product already exposes:
49
+
50
+ * no ``--file``: ``POST /projects/{id}/files/reevaluate`` (a
51
+ project-wide sweep, admin-only, bounded per call, resumable via
52
+ ``--after-id``)
53
+ * ``--file <id>``: ``POST /projects/{id}/files/{file_id}/reevaluate``
54
+ (one row, any writer)
55
+
56
+ Dry run by default on both forms: the sweep route already defaults to
57
+ a dry run server-side, but the single-file route does not, so this
58
+ client always sends its own ``dry_run`` rather than trusting whichever
59
+ default the server happens to pick. ``--apply`` flips it.
60
+
61
+ ``only_skipped`` / ``--limit`` / ``--after-id`` are sweep-only; the
62
+ single-file body rejects unknown fields, so they are never sent when
63
+ ``--file`` is given (sending them would 422 instead of being silently
64
+ ignored).
65
+
66
+ --- fm files download ---------------------------------------------------
67
+
68
+ A pure read: every file under ``--folder``, recursively, downloaded
69
+ into ``--out`` with its subfolder structure preserved.
70
+
71
+ This does not use the server's own ``folder=`` filter for that,
72
+ because that filter matches only a file's immediate parent folder; a
73
+ file one level deeper is invisible to it. So this command instead
74
+ pages through the space's whole (unfiltered) file listing and filters
75
+ ``File.path`` client-side for everything under ``--folder``, the only
76
+ way to get "under a folder, including its subfolders" out of a
77
+ listing whose server-side filter deliberately does not cascade.
78
+
79
+ Each match is downloaded through the same per-file download route the
80
+ product's own file preview uses, one request per file; a
81
+ connector-backed row without a durable local copy is fetched from the
82
+ source on demand by that same route, so this command never talks to a
83
+ connector directly.
84
+
85
+ Every download is checked against the size the listing already
86
+ reported for that row. A size mismatch is a per-file failure: the
87
+ file is still written to disk, since the bytes already arrived, but
88
+ it is named in the failure list rather than counted as a quiet
89
+ success.
90
+
91
+ One file failing does not stop the rest, the same reasoning
92
+ `fm upload DIR` takes. Exit code is non-zero iff at least one file
93
+ failed.
94
+ """
95
+
96
+ from __future__ import annotations
97
+
98
+ import argparse
99
+ import json
100
+ import sys
101
+ from typing import Any
102
+
103
+ from formist_cli import api, context
104
+ from formist_cli.cli_support import PLATFORM_ADMIN_GATE_NOTE, add_admin_flag, require_object
105
+ from formist_cli.cli_support import build_standalone_parser as _build_standalone_parser
106
+
107
+ #: The subcommands ``fm files`` dispatches to.
108
+ SUBJECTS = ("merge", "reevaluate", "download")
109
+
110
+
111
+ class FilesError(Exception):
112
+ """A refusal about the file surface, written for an operator."""
113
+
114
+
115
+ _DESCRIPTION = """Re-decide the skip verdict for this project's file rows under
116
+ today's processing criteria: the mechanism that notices a raised size cap or a
117
+ newly-supported format actually reached already-synced rows, which nothing
118
+ else does.
119
+
120
+ fm files reevaluate --space <id> project-wide sweep (dry run)
121
+ fm files reevaluate --space <id> --apply sweep, for real
122
+ fm files reevaluate --space <id> --file <file-id> one row (dry run)
123
+ fm files reevaluate --space <id> --file <file-id> --apply
124
+
125
+ DRY RUN by default on both forms. --apply is what writes.
126
+ """
127
+
128
+
129
+ def add_parser(subparsers) -> argparse.ArgumentParser:
130
+ from formist_cli.cli import _add_env, _add_space
131
+
132
+ parser = subparsers.add_parser(
133
+ "files",
134
+ help="merge two duplicate File rows, re-apply today's processing criteria, or download a folder",
135
+ description=(
136
+ "fm files merge merge two duplicate File rows in a space (WRITE)\n"
137
+ "fm files reevaluate re-apply today's file-processing criteria (WRITE)\n"
138
+ "fm files download download every file under a folder, recursively (READ)\n\n"
139
+ "merge/reevaluate are DRY RUN by default; download is a plain read with no "
140
+ "--apply of its own. Run `fm files <subcommand> --help` for the full "
141
+ "contract of each; they are different jobs."
142
+ ),
143
+ formatter_class=argparse.RawDescriptionHelpFormatter,
144
+ )
145
+ subjects = parser.add_subparsers(dest="subject", metavar="{merge,reevaluate,download}")
146
+ subjects.required = True
147
+
148
+ merge = subjects.add_parser(
149
+ "merge",
150
+ help="merge two duplicate File rows in a space",
151
+ description=(
152
+ "Merges the File row named by --drop into the one named by --keep: "
153
+ "every live reference (task links, the search index row, chip lists, "
154
+ "status points, chat attachments, non-terminal proposals) moves onto "
155
+ "--keep, --keep fills its empty columns from --drop, and --drop is "
156
+ "hard-deleted, not trashed. Audit history is never rewritten; "
157
+ "--drop's columns survive in the delete audit row's before-snapshot.\n\n"
158
+ "DRY RUN BY DEFAULT. --apply takes the report_token this run printed; "
159
+ "if either row has changed since (the connector refreshes every 30 "
160
+ "minutes), the server answers 409 and writes nothing.\n\n"
161
+ "ONE PAIR PER CALL, and --reason is REQUIRED under --apply: one "
162
+ "sentence saying why these two rows are the same file. There is no "
163
+ "'merge them all' shape, since deduplication is a manual, "
164
+ "judgement-based decision, and a request without a reason is "
165
+ "refused 422 by the server.\n\n"
166
+ "BLOB SAFETY. A markdown blob the two rows SHARE is removed from the "
167
+ "dropped row before it is destroyed, so the hard-delete cannot take the "
168
+ "survivor's copy with it. A blob that is exclusively the dropped row's "
169
+ "own, including an UPLOAD's stored bytes, IS deleted immediately, "
170
+ "along with the row; that is no longer a separate, later decision."
171
+ ),
172
+ epilog=PLATFORM_ADMIN_GATE_NOTE,
173
+ formatter_class=argparse.RawDescriptionHelpFormatter,
174
+ )
175
+ _add_env(merge)
176
+ _add_space(merge)
177
+ merge.add_argument(
178
+ "--keep",
179
+ required=True,
180
+ metavar="FILE_ID",
181
+ help="the File row that SURVIVES. Read the dry run before choosing: the "
182
+ "survivor keeps its own markdown surfaces unless it has none at all",
183
+ )
184
+ merge.add_argument(
185
+ "--drop",
186
+ required=True,
187
+ metavar="FILE_ID",
188
+ help="the File row merged into --keep and then HARD-deleted, not "
189
+ "trashed. Its references move first; its columns survive only in the "
190
+ "delete audit row's snapshot",
191
+ )
192
+ merge.add_argument(
193
+ "--reason",
194
+ default=None,
195
+ metavar="TEXT",
196
+ help="REQUIRED under --apply: one sentence saying why these two rows are the "
197
+ "same file. Becomes the audit row's explanation",
198
+ )
199
+ merge.add_argument(
200
+ "--apply",
201
+ metavar="REPORT_TOKEN",
202
+ default=None,
203
+ help="merge, if both rows still match the report_token this run printed",
204
+ )
205
+ merge.add_argument("--json", action="store_true", help="machine-readable output")
206
+ add_admin_flag(merge)
207
+
208
+ reevaluate = subjects.add_parser(
209
+ "reevaluate",
210
+ help="re-apply today's file-processing criteria to synced rows (WRITE)",
211
+ description=_DESCRIPTION,
212
+ formatter_class=argparse.RawDescriptionHelpFormatter,
213
+ )
214
+ _add_env(reevaluate, for_write=True)
215
+ _add_space(reevaluate)
216
+ reevaluate.add_argument("--file", default=None, help="reevaluate ONE file id instead of the whole project")
217
+ reevaluate.add_argument("--apply", action="store_true", help="write (default: dry run, counts only)")
218
+ reevaluate.add_argument(
219
+ "--only-skipped", dest="only_skipped", action="store_true", default=True,
220
+ help="sweep only: rows that currently carry a skip reason (default)",
221
+ )
222
+ reevaluate.add_argument(
223
+ "--include-processed", dest="only_skipped", action="store_false",
224
+ help="sweep only: also examine rows with NO skip reason (after TIGHTENING a criterion)",
225
+ )
226
+ reevaluate.add_argument(
227
+ "--no-rederive", dest="no_rederive", action="store_true",
228
+ help="mark rows for the next sync instead of re-deriving them now",
229
+ )
230
+ reevaluate.add_argument("--limit", type=int, default=None, help="sweep only: cap this call's batch size")
231
+ reevaluate.add_argument("--after-id", dest="after_id", default=None, help="sweep only: resume after this row id")
232
+ reevaluate.add_argument("--json", action="store_true", help="raw JSON instead of a report")
233
+
234
+ download = subjects.add_parser(
235
+ "download",
236
+ help="download every file under a folder, recursively (READ)",
237
+ description=(
238
+ "Lists the space's files under --folder, RECURSIVELY, and downloads "
239
+ "each one into --out, preserving the subfolder structure it found "
240
+ "them in.\n\n"
241
+ "The server's own folder= filter (GET /projects/{id}/files) is "
242
+ "EXACT-MATCH ONLY: it deliberately does not cascade into "
243
+ "subfolders, so this command pages through the space's WHOLE file "
244
+ "listing instead and filters by path client-side. See the module "
245
+ "docstring.\n\n"
246
+ "Each file's downloaded size is checked against what the listing "
247
+ "reported; a mismatch is a per-file FAILURE, not a silently short "
248
+ "file. One file failing does not stop the rest; the exit code is "
249
+ "non-zero iff at least one failed."
250
+ ),
251
+ formatter_class=argparse.RawDescriptionHelpFormatter,
252
+ )
253
+ _add_env(download)
254
+ _add_space(download)
255
+ download.add_argument(
256
+ "--folder", required=True, metavar="PATH",
257
+ help="absolute folder path, e.g. /Photos/Day 1 (or / for the whole space)", # system-folder-ok: an example in --help text, not a path the code uses
258
+ )
259
+ download.add_argument(
260
+ "--out", required=True, metavar="DIR", help="local directory to download into",
261
+ )
262
+ download.add_argument("--json", action="store_true", help="machine-readable summary")
263
+ return parser
264
+
265
+
266
+ def build_standalone_parser() -> argparse.ArgumentParser:
267
+ return _build_standalone_parser(add_parser)
268
+
269
+
270
+ def cmd_files(args) -> int:
271
+ """Dispatch ``fm files <subject>``. Nothing is shared between the branches
272
+ except this line; see the module docstring for why they are separate parsers."""
273
+ subject = getattr(args, "subject", None)
274
+ if subject == "reevaluate":
275
+ return _run_reevaluate(args)
276
+ if subject == "download":
277
+ return _run_download(args)
278
+ return _run_merge(args)
279
+
280
+
281
+ FILE_MERGE_PATH = "/projects/{space}/admin/files/merge"
282
+
283
+
284
+ def _report(body: object) -> dict:
285
+ """The merge report from a response body, or a refusal naming what came back."""
286
+ envelope = require_object(body, what="the file merge endpoint", error=FilesError)
287
+ if "report_token" not in envelope:
288
+ raise FilesError(
289
+ "the file merge endpoint returned an object with no `report_token` key: "
290
+ f"{str(envelope)[:200]}"
291
+ )
292
+ return envelope
293
+
294
+
295
+ def fetch_report(target, space: str, keep: str, drop: str, *, token: str | None, get=None) -> dict:
296
+ """The dry run. Writes nothing."""
297
+ from urllib.parse import urlencode
298
+
299
+ from formist_cli import api
300
+
301
+ get = get or api.get
302
+ path = f"{FILE_MERGE_PATH.format(space=space)}?{urlencode({'keep': keep, 'drop': drop})}"
303
+ return _report(get(target, path, token=token))
304
+
305
+
306
+ def submit(
307
+ target,
308
+ space: str,
309
+ keep: str,
310
+ drop: str,
311
+ report_token: str,
312
+ *,
313
+ token: str | None,
314
+ reason: str | None,
315
+ post=None,
316
+ ) -> dict:
317
+ """Merge this pair, if it still matches ``report_token``.
318
+
319
+ Sends ``reason`` as-is and lets the server refuse (422) a blank
320
+ one, rather than duplicating that validation here.
321
+ """
322
+ from urllib.parse import urlencode
323
+
324
+ from formist_cli import api
325
+
326
+ post = post or api.post
327
+ path = f"{FILE_MERGE_PATH.format(space=space)}?{urlencode({'keep': keep, 'drop': drop})}"
328
+ body: dict[str, Any] = {"report_token": report_token, "reason": reason}
329
+ return _report(post(target, path, body, token=token))
330
+
331
+
332
+ def render(report: dict, *, out=None) -> None:
333
+ """DESCRIBE only. Nothing printed here is a decision the tool has taken."""
334
+ out = out or sys.stdout
335
+ refusals = report.get("refusals") or []
336
+ if refusals:
337
+ print("REFUSED: nothing can be merged:", file=out)
338
+ for refusal in refusals:
339
+ print(f" - {refusal}", file=out)
340
+ return
341
+
342
+ if report.get("idempotent"):
343
+ print(f"ALREADY MERGED: {report.get('idempotent_reason')}", file=out)
344
+ return
345
+
346
+ print(
347
+ f"space {report.get('project_id')} | keep {report.get('keep_id')} | "
348
+ f"drop {report.get('drop_id')}",
349
+ file=out,
350
+ )
351
+ print("\n## both rows, side by side (x marks a conflict)\n", file=out)
352
+ print("| field | keep | drop | |", file=out)
353
+ print("|---|---|---|---|", file=out)
354
+ for row in report.get("fields") or []:
355
+ mark = "x" if row.get("conflict") else ("+" if row.get("fills_a_gap") else "")
356
+ print(
357
+ f"| {row.get('field')} | {row.get('keep')!r} | {row.get('drop')!r} | {mark} |",
358
+ file=out,
359
+ )
360
+
361
+ updates = report.get("updates") or {}
362
+ if updates:
363
+ print(
364
+ f"\nwould write onto the survivor: {', '.join(sorted(updates))}",
365
+ file=out,
366
+ )
367
+ for conflict in report.get("conflicts") or []:
368
+ # NEVER folded into a count. A conflict is the reason a human is reading
369
+ # this, and a report that said "2 conflicts" while silently picking a side
370
+ # would be a check that cannot fail.
371
+ print(
372
+ f" CONFLICT {conflict.get('field')}: keeping {conflict.get('keeper')!r}, "
373
+ f"discarding {conflict.get('loser')!r}",
374
+ file=out,
375
+ )
376
+
377
+ group = report.get("content_group") or {}
378
+ if group:
379
+ print(
380
+ f"\n## markdown surfaces: taken from the {group.get('winner')} row\n"
381
+ f"{group.get('reason')}",
382
+ file=out,
383
+ )
384
+ for name, value in sorted((group.get("discarded") or {}).items()):
385
+ print(f" discarding {name} = {value!r} (from {group.get('discarded_from')})", file=out)
386
+
387
+ refs = report.get("references") or {}
388
+ print("\n## references\n", file=out)
389
+ if refs:
390
+ for key, count in sorted(refs.items()):
391
+ print(f" {key}: {count}", file=out)
392
+ else:
393
+ print(" no live references on the dropped row", file=out)
394
+
395
+ print(
396
+ "\n## blob artifacts (a SHARED one is protected; an EXCLUSIVE one is "
397
+ "destroyed with the row)\n",
398
+ file=out,
399
+ )
400
+ for art in report.get("artifacts") or []:
401
+ shared = "SHARED with the survivor" if art.get("shared_key") else "not shared"
402
+ print(
403
+ f" {art.get('surface')}: {shared}"
404
+ + (f", releasable ({art.get('bytes') or '?'} bytes)" if art.get("releasable") else ""),
405
+ file=out,
406
+ )
407
+ print(f" {art.get('action')}", file=out)
408
+
409
+
410
+ def _run_merge(args) -> int:
411
+ from formist_cli import privileges
412
+ from formist_cli.cli import _effective, _token_for
413
+
414
+ # No member-level equivalent: hard-deleting the dropped row is
415
+ # irreversible, which is also why this is the one subject of `fm files`
416
+ # that needs --admin at all (`reevaluate`/`download` do not).
417
+ privileges.require_admin_flag("fm files merge", bool(getattr(args, "admin", False)))
418
+
419
+ eff = _effective(args, for_write=bool(args.apply))
420
+ target, space = eff.target, eff.space
421
+ token = _token_for(eff)
422
+
423
+ report = fetch_report(target, space, args.keep, args.drop, token=token)
424
+ if not args.json:
425
+ render(report)
426
+ if not args.apply:
427
+ if args.json:
428
+ print(json.dumps(report, indent=2))
429
+ elif not report.get("refusals") and not report.get("idempotent"):
430
+ print(
431
+ f"\nRe-run with --reason \"<why these are one file>\" --apply "
432
+ f"{report.get('report_token')} --host <host> to merge this pair."
433
+ )
434
+ return 0
435
+
436
+ if report.get("refusals"):
437
+ raise FilesError(
438
+ "REFUSING TO APPLY: the dry run above says this pair cannot be merged. "
439
+ "Nothing was sent."
440
+ )
441
+ if report.get("idempotent"):
442
+ print("\nalready merged, no request was sent.")
443
+ return 0
444
+ if args.apply != report.get("report_token"):
445
+ raise FilesError(
446
+ "REFUSING TO APPLY: this is not the report you were shown.\n"
447
+ f" you passed : {args.apply}\n"
448
+ f" report now : {report.get('report_token')}\n"
449
+ "One of the two rows changed between your dry run and this call (the "
450
+ "connector refreshes every 30 minutes). Read the report above and apply "
451
+ "that token instead."
452
+ )
453
+
454
+ applied = submit(
455
+ target, space, args.keep, args.drop, args.apply, token=token, reason=args.reason
456
+ )
457
+ if args.json:
458
+ print(json.dumps(applied, indent=2))
459
+ return 0
460
+ render(applied)
461
+ if applied.get("applied"):
462
+ print(
463
+ f"\nmerged. {applied.get('drop_id')} was permanently deleted, "
464
+ f"every reference had already moved onto {applied.get('keep_id')}. It is "
465
+ "NOT recoverable; its columns survive only in the delete audit row's "
466
+ "before-snapshot."
467
+ )
468
+ return 0
469
+ print("\nthe server did not report the merge as applied; see above.", file=sys.stderr)
470
+ return 1
471
+
472
+
473
+ def _render(payload: dict, *, dry_run: bool) -> str:
474
+ lines = [f"mode {'DRY RUN (writes nothing)' if dry_run else 'APPLIED'}"]
475
+ for key in ("to_fill", "applied", "examined"):
476
+ if key in payload:
477
+ lines.append(f"{key:<10} {payload[key]}")
478
+ counts = payload.get("counts")
479
+ if isinstance(counts, dict) and counts:
480
+ lines.append("")
481
+ for action, n in sorted(counts.items()):
482
+ lines.append(f" {action:<20} {n}")
483
+ if "item" in payload:
484
+ item = payload.get("item") or {}
485
+ lines.append(f"file {item.get('id')} {item.get('name')}")
486
+ if "reevaluation" in payload:
487
+ lines.append(f"outcome {payload.get('reevaluation')}")
488
+ if payload.get("next_cursor"):
489
+ lines.append(f"\nmore rows exist, resume with --after-id {payload['next_cursor']}")
490
+ return "\n".join(lines)
491
+
492
+
493
+ def _run_reevaluate(args) -> int:
494
+ ctx = context.resolve(args, for_write=True, need_space=True, need_token=True)
495
+ dry_run = not bool(args.apply)
496
+
497
+ if args.file:
498
+ body: dict = {"dry_run": dry_run, "rederive": not bool(args.no_rederive)}
499
+ path = f"/projects/{ctx.space}/files/{args.file}/reevaluate"
500
+ else:
501
+ body = {
502
+ "dry_run": dry_run,
503
+ "only_skipped": bool(args.only_skipped),
504
+ "rederive": not bool(args.no_rederive),
505
+ }
506
+ if args.limit is not None:
507
+ body["limit"] = args.limit
508
+ if args.after_id is not None:
509
+ body["after_id"] = args.after_id
510
+ path = f"/projects/{ctx.space}/files/reevaluate"
511
+
512
+ payload = api.post(ctx.target, path, body, token=ctx.token)
513
+ if not isinstance(payload, dict):
514
+ raise FilesError(
515
+ f"expected a JSON object from {path}, got {type(payload).__name__}. "
516
+ "That is a server contract change, not an empty result."
517
+ )
518
+ if args.json:
519
+ print(json.dumps(payload, indent=2, ensure_ascii=False))
520
+ else:
521
+ print(_render(payload, dry_run=dry_run))
522
+ return 0
523
+
524
+
525
+ # ===========================================================================
526
+ # fm files download: every file under a folder, recursively (READ)
527
+ # ===========================================================================
528
+
529
+ #: The server's own list-envelope page size cap. Paging at this size
530
+ #: minimises round trips without depending on the server ever raising
531
+ #: its own cap. Shared with the one other module that pages the same
532
+ #: listing, so the two cannot drift apart silently.
533
+ from formist_cli.meetings import _LIST_PAGE_SIZE # noqa: E402
534
+
535
+
536
+ def _normalize_folder(folder: str) -> str:
537
+ """``folder`` as an absolute path with no trailing slash (``/`` for
538
+ the space root). Matches the server's own normalisation, even
539
+ though this module does not use the server's folder filter (see
540
+ the module docstring): a file's own ``path`` column is stored the
541
+ same way, so the comparison has to start from the same shape.
542
+ """
543
+ stripped = folder.strip("/")
544
+ return f"/{stripped}" if stripped else "/"
545
+
546
+
547
+ def is_under_folder(folder: str, file_path: str) -> bool:
548
+ """True iff ``file_path`` sits at or below ``folder``, including
549
+ every subfolder, which is the one thing the server's own
550
+ ``folder=`` filter refuses to do (see the module docstring).
551
+ ``folder="/"`` matches every path in the space."""
552
+ norm = _normalize_folder(folder)
553
+ if norm == "/":
554
+ return True
555
+ path = file_path or ""
556
+ return path == norm or path.startswith(f"{norm}/")
557
+
558
+
559
+ def relative_download_path(folder: str, file_path: str) -> str:
560
+ """``file_path``'s path relative to ``folder``, safe to join onto ``--out``.
561
+
562
+ Refuses (rather than silently sanitising) anything that would
563
+ resolve outside ``--out``, a ``..`` segment or an empty result,
564
+ since a path taken from a server response and joined onto a local
565
+ directory without that check can write somewhere else entirely.
566
+ """
567
+ norm = _normalize_folder(folder)
568
+ path = (file_path or "").replace("\\", "/")
569
+ if norm != "/" and (path == norm or path.startswith(f"{norm}/")):
570
+ rel = path[len(norm):].lstrip("/")
571
+ else:
572
+ rel = path.lstrip("/")
573
+ parts = [p for p in rel.split("/") if p not in ("", ".")]
574
+ if not parts or any(p == ".." for p in parts):
575
+ raise FilesError(
576
+ f"refusing to download {file_path!r}; its path does not resolve to "
577
+ f"a location under --out ({rel!r})"
578
+ )
579
+ return "/".join(parts)
580
+
581
+
582
+ def list_files_under(target, space: str, folder: str, *, token: str | None, get=None) -> list[dict]:
583
+ """Every file in ``space`` whose path is under ``folder``, recursively:
584
+ paged through the space's whole (unfiltered) file listing
585
+ (``GET /projects/{id}/files``) and filtered client-side by path. See
586
+ the module docstring for why the server's own ``folder=`` param
587
+ cannot be used for this (it is exact-match only).
588
+ """
589
+ get = get or api.get
590
+ out: list[dict] = []
591
+ offset = 0
592
+ while True:
593
+ payload = get(
594
+ target, f"/projects/{space}/files", token=token,
595
+ params={"limit": str(_LIST_PAGE_SIZE), "offset": str(offset)},
596
+ )
597
+ envelope = require_object(payload, what="the files listing", error=FilesError)
598
+ items = [i for i in (envelope.get("items") or []) if isinstance(i, dict)]
599
+ out.extend(items)
600
+ total = envelope.get("total")
601
+ offset += len(items)
602
+ if not items or total is None or offset >= int(total):
603
+ break
604
+ return [f for f in out if is_under_folder(folder, f.get("path") or "")]
605
+
606
+
607
+ def _run_download(args) -> int:
608
+ from pathlib import Path
609
+
610
+ ctx = context.resolve(args, need_space=True, need_token=True)
611
+ target, space = ctx.target, ctx.space
612
+ out_dir = Path(args.out)
613
+
614
+ items = list_files_under(target, space, args.folder, token=ctx.token)
615
+ out_dir.mkdir(parents=True, exist_ok=True)
616
+
617
+ downloaded: list[str] = []
618
+ failed: list[tuple[str, str]] = []
619
+ for item in items:
620
+ file_id = item.get("id")
621
+ path = item.get("path") or ""
622
+ try:
623
+ rel = relative_download_path(args.folder, path)
624
+ except FilesError as exc:
625
+ print(f"FAILED {path} {exc}", file=sys.stderr)
626
+ failed.append((path, str(exc)))
627
+ continue
628
+
629
+ dest = out_dir / rel
630
+ try:
631
+ data, _disposition = api.download(
632
+ target, f"/projects/{space}/files/{file_id}/download", token=ctx.token
633
+ )
634
+ expected_size = item.get("size")
635
+ if isinstance(expected_size, int) and len(data) != expected_size:
636
+ raise FilesError(
637
+ f"downloaded {len(data)} bytes, expected {expected_size}; "
638
+ "do not trust this file"
639
+ )
640
+ dest.parent.mkdir(parents=True, exist_ok=True)
641
+ dest.write_bytes(data)
642
+ except Exception as exc: # noqa: BLE001 - one file must not end the run
643
+ print(f"FAILED {rel} {exc}", file=sys.stderr)
644
+ failed.append((rel, str(exc)))
645
+ continue
646
+ print(f"downloaded {rel} ({len(data)} bytes)")
647
+ downloaded.append(rel)
648
+
649
+ if args.json:
650
+ print(json.dumps(
651
+ {
652
+ "folder": args.folder,
653
+ "out": str(out_dir),
654
+ "downloaded": len(downloaded),
655
+ "failed": [{"path": rel, "reason": reason} for rel, reason in failed],
656
+ "total": len(items),
657
+ },
658
+ indent=2, ensure_ascii=False,
659
+ ))
660
+ print(
661
+ f"\n{len(downloaded)} downloaded, {len(failed)} failed, {len(items)} total "
662
+ f"under {args.folder!r}",
663
+ file=sys.stderr,
664
+ )
665
+ if failed:
666
+ print("failed:", file=sys.stderr)
667
+ for rel, reason in failed:
668
+ print(f" {rel}: {reason}", file=sys.stderr)
669
+ return 1
670
+ return 0