sdb-reader 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdb_reader/cli.py ADDED
@@ -0,0 +1,1079 @@
1
+ """sdb-reader: read safety data sheets, or hand out the data embedded in their PDFs.
2
+
3
+ ``read`` prints what each file says, as JSON; ``xml`` prints the eSDScom
4
+ file a supplier embedded in the PDF, byte for byte (SDW-13); ``ask`` asks a
5
+ language model about each file and keeps its answer (SDW-25); ``diff`` sets
6
+ two runs of ``read`` or ``ask`` beside each other (SDW-27); ``show`` reads
7
+ like ``read`` and prints a report for a person instead (SDW-37).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import argparse
13
+ import json
14
+ import os
15
+ import sys
16
+ import traceback
17
+ from collections.abc import Callable, Sequence
18
+ from dataclasses import dataclass
19
+ from datetime import date, datetime
20
+ from functools import cache
21
+ from importlib import metadata
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+ import structlog
26
+
27
+ from . import ask, diff, presets, scan, show
28
+ from .embedded import read_embedded
29
+ from .extraction import ExtractionError, ExtractionResult, extract_pdf_text
30
+ from .messages import (
31
+ DEFAULT_LANGUAGE,
32
+ Catalog,
33
+ Message,
34
+ load_catalog,
35
+ shipped_catalog,
36
+ shipped_languages,
37
+ )
38
+ from .reading import read_sheet
39
+ from .scan import Outcome
40
+ from .serialise import reading_to_dict
41
+ from .verification import DocumentKind, SDSVerifier, Verification
42
+
43
+ ENDPOINT_VARIABLE = "SDB_READER_ENDPOINT"
44
+ MODEL_VARIABLE = "SDB_READER_MODEL"
45
+ API_KEY_VARIABLE = "SDB_READER_API_KEY"
46
+ REQUEST_VARIABLE = "SDB_READER_REQUEST_JSON"
47
+ PRESET_VARIABLE = "SDB_READER_PRESET"
48
+ LANGUAGE_VARIABLE = "SDB_READER_LANG"
49
+
50
+
51
+ def build_argument_parser() -> argparse.ArgumentParser:
52
+ parser = argparse.ArgumentParser(
53
+ prog="sdb-reader",
54
+ description="Read safety data sheet PDFs, or the data embedded in them.",
55
+ )
56
+ parser.add_argument(
57
+ "--version",
58
+ action="version",
59
+ version=f"sdb-reader {metadata.version('sdb-reader')}",
60
+ help="Print the version of sdb-reader and exit.",
61
+ )
62
+ commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
63
+
64
+ read = commands.add_parser(
65
+ "read",
66
+ help="Whether each file is a safety data sheet, and what it says, as JSON.",
67
+ )
68
+ read.add_argument(
69
+ "paths",
70
+ nargs="+",
71
+ type=Path,
72
+ help="PDF files to read; a directory stands for every PDF under it.",
73
+ )
74
+ read.add_argument(
75
+ "--max-pages",
76
+ type=int,
77
+ default=None,
78
+ help="Read at most this many pages of each file (default: all).",
79
+ )
80
+ read.add_argument(
81
+ "--indent",
82
+ type=int,
83
+ default=2,
84
+ help="Indent level for JSON output (default: 2). Use 0 for compact output.",
85
+ )
86
+ read.add_argument(
87
+ "--include-raw-text",
88
+ action="store_true",
89
+ help="Include the raw extracted text in the output.",
90
+ )
91
+
92
+ _add_run_options(read)
93
+ _add_language_options(read)
94
+
95
+ showing = commands.add_parser(
96
+ "show",
97
+ help="What each file is and says, as a report to read; `read` gives JSON.",
98
+ description=(
99
+ "Reads the files like `read` and prints a report for a person: the "
100
+ "full report for a file named singly, one line for each file of a "
101
+ "directory. The text can change with any release; a program takes "
102
+ "`read`."
103
+ ),
104
+ )
105
+ showing.add_argument(
106
+ "paths",
107
+ nargs="+",
108
+ type=Path,
109
+ help="PDF files to show; a directory stands for every PDF under it.",
110
+ )
111
+ showing.add_argument(
112
+ "--max-pages",
113
+ type=int,
114
+ default=None,
115
+ help="Read at most this many pages of each file (default: all).",
116
+ )
117
+ _add_selection_options(showing)
118
+ _add_language_options(showing)
119
+
120
+ xml = commands.add_parser(
121
+ "xml",
122
+ help="The eSDScom file embedded in a PDF, unchanged, on stdout.",
123
+ )
124
+ xml.add_argument("paths", nargs="+", type=Path, help="PDF files.")
125
+ xml.add_argument(
126
+ "--save",
127
+ type=Path,
128
+ metavar="DIRECTORY",
129
+ default=None,
130
+ help=(
131
+ "Write each PDF's embedded file to DIRECTORY as <PDF name>.xml "
132
+ "instead of printing it; takes several PDFs. An existing file is "
133
+ "not overwritten."
134
+ ),
135
+ )
136
+
137
+ _add_language_options(xml)
138
+
139
+ asking = commands.add_parser(
140
+ "ask",
141
+ help="What a language model says each sheet holds, a record per file, as JSON.",
142
+ description=(
143
+ "Asks a language model about every German safety data sheet and keeps "
144
+ "its answer: a record per file, to set beside a reading with `diff`. "
145
+ "The file's text up to section 3 is sent to the endpoint. An API key "
146
+ f"is taken from {API_KEY_VARIABLE} only."
147
+ ),
148
+ )
149
+ asking.add_argument(
150
+ "paths",
151
+ nargs="+",
152
+ type=Path,
153
+ help="PDF files to ask about; a directory stands for every PDF under it.",
154
+ )
155
+ _add_model_options(asking)
156
+ asking.add_argument(
157
+ "--indent",
158
+ type=int,
159
+ default=2,
160
+ help="Indent level for JSON output (default: 2). Use 0 for compact output.",
161
+ )
162
+ _add_run_options(asking)
163
+ _add_language_options(asking)
164
+
165
+ runs = commands.add_parser(
166
+ "diff",
167
+ help="Where two runs of `read` or `ask` differ, field by field.",
168
+ description=(
169
+ "Sets two runs beside each other: files of records as `read` or `ask` "
170
+ "wrote them, JSON Lines or one JSON list. Reads no PDF and asks no "
171
+ "model. Exit code 0 where no file differs, 1 where one does."
172
+ ),
173
+ )
174
+ runs.add_argument("a", type=Path, metavar="A", help="The first run's records.")
175
+ runs.add_argument("b", type=Path, metavar="B", help="The second run's records.")
176
+ runs.add_argument(
177
+ "--match",
178
+ choices=("path", "name"),
179
+ default="path",
180
+ help=(
181
+ "What tells that two records are of the same file: its path "
182
+ "(default), or its name where the runs read copies in different "
183
+ "places."
184
+ ),
185
+ )
186
+ runs.add_argument(
187
+ "--examples",
188
+ type=int,
189
+ default=3,
190
+ metavar="N",
191
+ help="Show up to N files per field (default: 3).",
192
+ )
193
+ runs.add_argument(
194
+ "--json",
195
+ action="store_true",
196
+ help="Print the counts and examples as JSON instead of text.",
197
+ )
198
+ runs.add_argument(
199
+ "--details",
200
+ type=Path,
201
+ default=None,
202
+ metavar="FILE",
203
+ help=(
204
+ "Write every differing file with both values to FILE, as JSON "
205
+ "Lines; an existing FILE is replaced."
206
+ ),
207
+ )
208
+ runs.add_argument(
209
+ "--count-only-from",
210
+ type=Path,
211
+ default=None,
212
+ metavar="FILE",
213
+ help=(
214
+ "Count the files FILE names in a group of their own and show them "
215
+ "in no example and no detail; one per line: a file name, a path, "
216
+ "or the end of a path."
217
+ ),
218
+ )
219
+ runs.add_argument(
220
+ "--indent",
221
+ type=int,
222
+ default=2,
223
+ help="Indent level for --json (default: 2). Use 0 for compact output.",
224
+ )
225
+ _add_language_options(runs)
226
+ return parser
227
+
228
+
229
+ def _add_model_options(command: argparse.ArgumentParser) -> None:
230
+ """Whom to ask, and how (SDW-25)."""
231
+ command.add_argument(
232
+ "--endpoint",
233
+ default=os.environ.get(ENDPOINT_VARIABLE),
234
+ help=(
235
+ "Address of an OpenAI-compatible API, such as http://localhost:11434/v1 "
236
+ f"(default: {ENDPOINT_VARIABLE})."
237
+ ),
238
+ )
239
+ command.add_argument(
240
+ "--model",
241
+ default=os.environ.get(MODEL_VARIABLE),
242
+ help=f"The model's name at the endpoint (default: {MODEL_VARIABLE}).",
243
+ )
244
+ command.add_argument(
245
+ "--pages",
246
+ type=int,
247
+ default=4,
248
+ help=(
249
+ "Pages of the PDF the model gets where the file's text has no "
250
+ "heading of section 3 to end at (default: 4)."
251
+ ),
252
+ )
253
+ command.add_argument(
254
+ "--timeout",
255
+ type=float,
256
+ default=300.0,
257
+ help="Seconds to wait for the model's answer (default: 300).",
258
+ )
259
+ command.add_argument(
260
+ "--retries",
261
+ type=int,
262
+ default=0,
263
+ metavar="N",
264
+ help=(
265
+ "Send a request again up to N times when the endpoint is busy or "
266
+ "not reachable (HTTP 429, 500, 502, 503, 504, a timeout, no "
267
+ "connection), after a growing wait (default: 0)."
268
+ ),
269
+ )
270
+ command.add_argument(
271
+ "--preset",
272
+ default=os.environ.get(PRESET_VARIABLE),
273
+ metavar="NAME",
274
+ help=(
275
+ "Request fields for a known model family, such as switching its "
276
+ f"reasoning off; see --list-presets (default: {PRESET_VARIABLE})."
277
+ ),
278
+ )
279
+ command.add_argument(
280
+ "--presets",
281
+ type=Path,
282
+ default=None,
283
+ metavar="FILE",
284
+ help="A TOML file of presets like the shipped one, to take --preset from.",
285
+ )
286
+ command.add_argument(
287
+ "--list-presets",
288
+ action=_ListPresets,
289
+ help="Print the presets --preset can name, and exit.",
290
+ )
291
+ command.add_argument(
292
+ "--request-json",
293
+ default=os.environ.get(REQUEST_VARIABLE),
294
+ metavar="JSON",
295
+ help=(
296
+ "A JSON object merged into every request, on top of a preset's "
297
+ "fields, for what a service needs beyond them; it may not set "
298
+ f"model or messages (default: {REQUEST_VARIABLE})."
299
+ ),
300
+ )
301
+
302
+
303
+ def _known_presets(file: Path | None) -> dict[str, presets.Preset]:
304
+ """The presets a call can name: the shipped ones, or those of *file*."""
305
+ return presets.load_presets(file) if file is not None else presets.shipped_presets()
306
+
307
+
308
+ class _ListPresets(argparse.Action):
309
+ """Print the presets and end, like --version: no file is needed (SDW-31).
310
+
311
+ What stands before it on the line counts: --lang for the language of the
312
+ lines, --presets for a file of one's own.
313
+ """
314
+
315
+ def __init__(self, option_strings: Sequence[str], dest: str, **kwargs: Any) -> None:
316
+ super().__init__(
317
+ option_strings, dest, nargs=0, default=argparse.SUPPRESS, **kwargs
318
+ )
319
+
320
+ def __call__(
321
+ self,
322
+ parser: argparse.ArgumentParser,
323
+ namespace: argparse.Namespace,
324
+ _values: Any,
325
+ _option_string: str | None = None,
326
+ ) -> None:
327
+ language = getattr(namespace, "lang", None) or DEFAULT_LANGUAGE
328
+ try:
329
+ known = _known_presets(getattr(namespace, "presets", None))
330
+ except ValueError as exc:
331
+ parser.error(str(exc))
332
+ for preset in known.values():
333
+ print(f"{preset.name}: {preset.describe(language)}")
334
+ if preset.request:
335
+ print(f" request: {json.dumps(dict(preset.request))}")
336
+ if preset.omit:
337
+ print(f" omit: {', '.join(sorted(preset.omit))}")
338
+ if preset.tested_with and preset.tested_on:
339
+ print(f" tested: {preset.tested_with}, {preset.tested_on.isoformat()}")
340
+ parser.exit()
341
+
342
+
343
+ def _add_run_options(command: argparse.ArgumentParser) -> None:
344
+ """What a run over many files needs (SDW-21 to SDW-23)."""
345
+ command.add_argument(
346
+ "--output",
347
+ type=Path,
348
+ default=None,
349
+ metavar="FILE",
350
+ help=(
351
+ "Write the records to FILE as JSON Lines, one line per file as it "
352
+ "is done, instead of printing one JSON list. An existing FILE is "
353
+ "refused unless --resume is given."
354
+ ),
355
+ )
356
+ command.add_argument(
357
+ "--resume",
358
+ action="store_true",
359
+ help=(
360
+ "Continue the run that wrote --output FILE: files whose record is "
361
+ "in it already are left out."
362
+ ),
363
+ )
364
+ _add_selection_options(command)
365
+
366
+
367
+ def _add_selection_options(command: argparse.ArgumentParser) -> None:
368
+ """Which files a call works on, and how many at once (SDW-21, SDW-23)."""
369
+ command.add_argument(
370
+ "--jobs",
371
+ type=int,
372
+ default=1,
373
+ metavar="N",
374
+ help="Work on N files at once (default: 1).",
375
+ )
376
+ command.add_argument(
377
+ "--exclude-from",
378
+ type=Path,
379
+ default=None,
380
+ metavar="FILE",
381
+ help=(
382
+ "Leave out the files FILE names, one per line: a file name, a "
383
+ "path relative to a directory given, or an absolute path."
384
+ ),
385
+ )
386
+
387
+
388
+ def _add_language_options(command: argparse.ArgumentParser) -> None:
389
+ """The language of the messages, and a catalog of the caller's own (SDW-17)."""
390
+ command.add_argument(
391
+ "--lang",
392
+ default=os.environ.get(LANGUAGE_VARIABLE) or DEFAULT_LANGUAGE,
393
+ help=(
394
+ "Language of the messages: "
395
+ f"{', '.join(shipped_languages())} (default: {LANGUAGE_VARIABLE}, "
396
+ f"else {DEFAULT_LANGUAGE}). The machine's language plays no part."
397
+ ),
398
+ )
399
+ command.add_argument(
400
+ "--messages",
401
+ type=Path,
402
+ default=None,
403
+ metavar="FILE",
404
+ help=(
405
+ "A message catalog of your own, a TOML file like the shipped ones; "
406
+ "it replaces --lang, and what it lacks is said in English."
407
+ ),
408
+ )
409
+
410
+
411
+ def _catalog(parser: argparse.ArgumentParser, args: argparse.Namespace) -> Catalog:
412
+ """The catalog the run words its messages with; exit code 2 without one."""
413
+ try:
414
+ messages = getattr(args, "messages", None)
415
+ if messages is not None:
416
+ return load_catalog(messages)
417
+ return shipped_catalog(getattr(args, "lang", DEFAULT_LANGUAGE))
418
+ except (ValueError, OSError) as exc:
419
+ parser.error(str(exc))
420
+
421
+
422
+ def main(argv: Sequence[str] | None = None) -> int:
423
+ parser = build_argument_parser()
424
+ args = parser.parse_args(argv)
425
+ # stdout carries the subcommand's output and nothing else: the parser's
426
+ # log lines, which structlog prints to stdout unless told otherwise, go to
427
+ # stderr, in structlog's default form whatever a host configured before.
428
+ structlog.reset_defaults()
429
+ structlog.configure(logger_factory=structlog.PrintLoggerFactory(sys.stderr))
430
+
431
+ # Before any file is read: a language without a catalog ends here.
432
+ catalog = _catalog(parser, args)
433
+ if args.command == "xml":
434
+ missing = [str(path) for path in args.paths if not path.is_file()]
435
+ if missing:
436
+ parser.error(f"not a file: {', '.join(missing)}")
437
+ if args.save is None and len(args.paths) > 1:
438
+ parser.error("xml prints one file; several PDFs need --save DIRECTORY")
439
+ if args.save is not None and not args.save.is_dir():
440
+ parser.error(f"not a directory: {args.save}")
441
+ return _xml(args.paths, args.save, catalog)
442
+
443
+ if args.command == "diff":
444
+ if args.examples < 0:
445
+ parser.error("--examples takes a number from 0")
446
+ return _diff(parser, args, catalog)
447
+
448
+ if args.jobs < 1:
449
+ parser.error("--jobs takes a number from 1")
450
+ if args.command == "show":
451
+ return _show(
452
+ parser,
453
+ args,
454
+ catalog,
455
+ # The day of the run, once for all its files (SDW-34).
456
+ _ReadOptions(args.max_pages, False, args.lang, args.messages, _today()),
457
+ )
458
+ if args.resume and args.output is None:
459
+ parser.error("--resume continues an output file: give --output FILE")
460
+ if args.command == "ask":
461
+ options = _AskOptions(
462
+ _endpoint(parser, args), args.pages, args.retries, args.lang, args.messages
463
+ )
464
+ return _run(parser, args, catalog, _ask_one, options, processes=False)
465
+ return _run(
466
+ parser,
467
+ args,
468
+ catalog,
469
+ _read_one,
470
+ _ReadOptions(
471
+ args.max_pages,
472
+ args.include_raw_text,
473
+ args.lang,
474
+ args.messages,
475
+ # The day of the run, once for all its files (SDW-34).
476
+ _today(),
477
+ ),
478
+ processes=True,
479
+ )
480
+
481
+
482
+ def _today() -> date:
483
+ """The machine's date: the one place the command reads a clock (SDW-34)."""
484
+ return datetime.now().astimezone().date()
485
+
486
+
487
+ def _endpoint(
488
+ parser: argparse.ArgumentParser, args: argparse.Namespace
489
+ ) -> ask.Endpoint:
490
+ """Whom the call asks; exit code 2 where it does not say, or says it wrongly."""
491
+ if args.retries < 0:
492
+ parser.error("--retries takes a number from 0")
493
+ if args.pages < 1:
494
+ parser.error("--pages takes a number from 1")
495
+ if not args.endpoint or not args.model:
496
+ parser.error(
497
+ f"{args.command} needs an endpoint and a model: --endpoint and "
498
+ f"--model, or {ENDPOINT_VARIABLE} and {MODEL_VARIABLE}"
499
+ )
500
+ chosen: presets.Preset | None = None
501
+ if args.presets is not None and not args.preset:
502
+ parser.error("--presets is where --preset is taken from: give --preset NAME")
503
+ if args.preset:
504
+ try:
505
+ known = _known_presets(args.presets)
506
+ except ValueError as exc:
507
+ parser.error(str(exc))
508
+ chosen = known.get(args.preset)
509
+ if chosen is None:
510
+ parser.error(
511
+ f"no preset {args.preset!r}; there are: {', '.join(sorted(known))}"
512
+ )
513
+ extra: Any = {}
514
+ if args.request_json:
515
+ try:
516
+ extra = json.loads(args.request_json)
517
+ except ValueError as exc:
518
+ parser.error(f"--request-json is no JSON: {exc}")
519
+ if not isinstance(extra, dict):
520
+ parser.error("--request-json takes a JSON object")
521
+ try:
522
+ return ask.Endpoint(
523
+ args.endpoint,
524
+ args.model,
525
+ api_key=os.environ.get(API_KEY_VARIABLE) or None,
526
+ timeout=args.timeout,
527
+ # The preset first, then what the call adds or replaces (SDW-31).
528
+ request={**(chosen.request if chosen else {}), **extra},
529
+ omit=(chosen.omit if chosen else frozenset()) - extra.keys(),
530
+ preset=chosen.name if chosen else None,
531
+ )
532
+ except ValueError as exc:
533
+ parser.error(str(exc))
534
+
535
+
536
+ @cache
537
+ def _catalog_for(lang: str, messages: Path | None) -> Catalog:
538
+ """The run's catalog again, in a worker process."""
539
+ return load_catalog(messages) if messages is not None else shipped_catalog(lang)
540
+
541
+
542
+ @dataclass(frozen=True, slots=True)
543
+ class _ReadOptions:
544
+ max_pages: int | None
545
+ include_raw_text: bool
546
+ lang: str
547
+ messages: Path | None
548
+ today: date
549
+
550
+
551
+ @dataclass(frozen=True, slots=True)
552
+ class _AskOptions:
553
+ endpoint: ask.Endpoint
554
+ pages: int
555
+ retries: int
556
+ lang: str
557
+ messages: Path | None
558
+
559
+
560
+ def _run[T](
561
+ parser: argparse.ArgumentParser,
562
+ args: argparse.Namespace,
563
+ catalog: Catalog,
564
+ work: Callable[[Path, T], Outcome],
565
+ options: T,
566
+ *,
567
+ processes: bool,
568
+ ) -> int:
569
+ """Work on every file of the call and put out the records (SDW-21 to SDW-24)."""
570
+ try:
571
+ selection = scan.select(args.paths, args.exclude_from)
572
+ lines = (
573
+ scan.Lines.open(args.output, resume=args.resume)
574
+ if args.output is not None
575
+ else None
576
+ )
577
+ except scan.ScanError as exc:
578
+ parser.error(str(exc))
579
+
580
+ files = list(selection.files)
581
+ skipped = 0
582
+ if lines is not None:
583
+ if lines.repaired:
584
+ _say(catalog, "cli-half-line-removed", output=str(args.output))
585
+ todo = [path for path in files if not lines.has(path)]
586
+ skipped = len(files) - len(todo)
587
+ files = todo
588
+
589
+ counted = len(files) + skipped
590
+ several = counted > 1
591
+ progress = scan.Progress(
592
+ len(files),
593
+ lambda done, total, failed: _say(
594
+ catalog, "cli-progress", done=done, total=total, failed=failed
595
+ ),
596
+ )
597
+ records: dict[int, dict[str, Any]] = {}
598
+ asked = prompt_tokens = completion_tokens = 0
599
+ interrupted = False
600
+ try:
601
+ for index, outcome in scan.outcomes(
602
+ files, work, options, jobs=args.jobs, processes=processes
603
+ ):
604
+ for line in outcome.said:
605
+ print(line, file=sys.stderr)
606
+ if outcome.record is not None:
607
+ if lines is not None:
608
+ lines.write(outcome.record)
609
+ else:
610
+ records[index] = outcome.record
611
+ asked += outcome.asked
612
+ prompt_tokens += outcome.prompt_tokens
613
+ completion_tokens += outcome.completion_tokens
614
+ progress.step(failed=outcome.failed)
615
+ except KeyboardInterrupt:
616
+ interrupted = True
617
+ finally:
618
+ if lines is not None:
619
+ lines.close()
620
+
621
+ if lines is None and not interrupted:
622
+ indent = args.indent if args.indent > 0 else None
623
+ print(
624
+ json.dumps(
625
+ [records[index] for index in sorted(records)],
626
+ indent=indent,
627
+ ensure_ascii=False,
628
+ )
629
+ )
630
+ if interrupted:
631
+ _say(
632
+ catalog,
633
+ "cli-interrupted",
634
+ done=progress.done,
635
+ total=progress.total,
636
+ )
637
+ return 130
638
+ if several:
639
+ if args.command == "ask":
640
+ _say(
641
+ catalog,
642
+ "cli-summary-ask",
643
+ files=counted,
644
+ skipped=skipped,
645
+ excluded=selection.excluded,
646
+ asked=asked,
647
+ failed=progress.failed,
648
+ prompt_tokens=prompt_tokens,
649
+ completion_tokens=completion_tokens,
650
+ )
651
+ else:
652
+ _say(
653
+ catalog,
654
+ "cli-summary-read",
655
+ files=counted,
656
+ skipped=skipped,
657
+ excluded=selection.excluded,
658
+ failed=progress.failed,
659
+ )
660
+ return 1 if progress.failed else 0
661
+
662
+
663
+ def _show(
664
+ parser: argparse.ArgumentParser,
665
+ args: argparse.Namespace,
666
+ catalog: Catalog,
667
+ options: _ReadOptions,
668
+ ) -> int:
669
+ """Read every file of the call and print its report (SDW-37 to SDW-41)."""
670
+ try:
671
+ selection = scan.select(args.paths, args.exclude_from)
672
+ except scan.ScanError as exc:
673
+ parser.error(str(exc))
674
+ files = list(selection.files)
675
+ # A file named on the line gets the full report, one that stands for its
676
+ # directory a line (SDW-40).
677
+ named = {path.resolve() for path in args.paths if path.is_file()}
678
+ progress = scan.Progress(
679
+ len(files),
680
+ lambda done, total, failed: _say(
681
+ catalog, "cli-progress", done=done, total=total, failed=failed
682
+ ),
683
+ )
684
+ counts = {"sheets": 0, "others": 0, "with_notes": 0}
685
+ waiting: dict[int, Outcome] = {}
686
+ printed = 0
687
+ # Whether the last thing printed was a full report: it stands apart.
688
+ after_report = False
689
+ try:
690
+ for index, outcome in scan.outcomes(
691
+ files, _read_one, options, jobs=args.jobs, processes=True
692
+ ):
693
+ waiting[index] = outcome
694
+ progress.step(failed=outcome.failed)
695
+ # In the order of the call, whichever file ends first (SDW-23).
696
+ while printed in waiting:
697
+ done = waiting.pop(printed)
698
+ path = files[printed]
699
+ full = path.resolve() in named
700
+ if done.record is None:
701
+ # It broke the reader: the line and the trace on stderr.
702
+ for said in done.said:
703
+ print(said, file=sys.stderr)
704
+ lines = show.broke(str(path), catalog, full=full)
705
+ elif full:
706
+ lines = show.report(done.record, str(path), catalog)
707
+ else:
708
+ lines = [show.line(done.record, str(path), catalog)]
709
+ if done.record is not None and show.is_sheet(done.record):
710
+ counts["sheets"] += 1
711
+ counts["with_notes"] += show.notes_count(done.record) > 0
712
+ elif done.record is not None and not show.is_unreadable(done.record):
713
+ counts["others"] += 1
714
+ if printed and (full or after_report):
715
+ print()
716
+ print("\n".join(lines), flush=True)
717
+ after_report = full
718
+ printed += 1
719
+ except KeyboardInterrupt:
720
+ _say(catalog, "show-interrupted", done=progress.done, total=progress.total)
721
+ return 130
722
+ if len(files) > 1:
723
+ print()
724
+ print(
725
+ _line(
726
+ catalog,
727
+ "show-summary",
728
+ files=len(files),
729
+ failed=progress.failed,
730
+ excluded=selection.excluded,
731
+ **counts,
732
+ )
733
+ )
734
+ return 1 if progress.failed else 0
735
+
736
+
737
+ def _read_one(path: Path, options: _ReadOptions) -> Outcome:
738
+ """Read one file: its record, and what is to be said about it (SDW-2, SDW-4)."""
739
+ catalog = _catalog_for(options.lang, options.messages)
740
+ said: list[str] = []
741
+ try:
742
+ reading = read_sheet(path, max_pages=options.max_pages)
743
+ # A file no text could be read from is in the output, named on
744
+ # stderr, and fails the exit code (SDW-4). It may be no PDF at
745
+ # all, so nothing is looked for inside it.
746
+ unreadable = reading.extraction is None
747
+ embedded = None if unreadable else read_embedded(path)
748
+ if unreadable:
749
+ reason = " ".join(catalog.render(error) for error in reading.errors)
750
+ said.append(
751
+ _line(catalog, "cli-file-failed", path=str(path), reason=reason)
752
+ )
753
+ record = {
754
+ "file": _file(path),
755
+ **reading_to_dict(
756
+ reading,
757
+ include_raw_text=options.include_raw_text,
758
+ embedded=embedded,
759
+ catalog=catalog,
760
+ today=options.today,
761
+ ),
762
+ }
763
+ # One file that breaks the parser must not cost the output for the
764
+ # others; it is named on stderr and fails the exit code.
765
+ except Exception as exc: # noqa: BLE001
766
+ return Outcome(None, tuple(_broke(catalog, path, exc)), failed=True)
767
+ return Outcome(record, tuple(said), failed=unreadable)
768
+
769
+
770
+ @dataclass(frozen=True, slots=True)
771
+ class _Asked:
772
+ """What asking about one file gave: its record, and what is to be said."""
773
+
774
+ record: dict[str, Any]
775
+ said: tuple[str, ...] = ()
776
+ failed: bool = False
777
+ asked: bool = False
778
+ prompt_tokens: int = 0
779
+ completion_tokens: int = 0
780
+
781
+
782
+ def _ask_about(
783
+ path: Path,
784
+ extraction: ExtractionResult | None,
785
+ verification: Verification,
786
+ unreadable: Sequence[Message],
787
+ options: _AskOptions,
788
+ catalog: Catalog,
789
+ ) -> _Asked:
790
+ """The record of ``ask`` for one file whose text was read (SDW-25, SDW-26).
791
+
792
+ Raises ask.ModelError where the endpoint gives no answer that can be read.
793
+ """
794
+ record: dict[str, Any] = {"file": _file(path), "kind": verification.kind.value}
795
+ if extraction is None:
796
+ # No text, so nothing to send; named on stderr like in read (SDW-4).
797
+ reason = " ".join(catalog.render(error) for error in unreadable)
798
+ record["asked"] = False
799
+ record["reason"] = catalog.to_dict(unreadable[0])
800
+ line = _line(catalog, "cli-file-failed", path=str(path), reason=reason)
801
+ return _Asked(record, (line,), failed=True)
802
+ if verification.kind is not DocumentKind.SAFETY_DATA_SHEET:
803
+ record["asked"] = False
804
+ record["reason"] = catalog.to_dict(Message("model-not-asked"))
805
+ return _Asked(record)
806
+ # One request per sheet of the file (SDW-30); one that fails, fails the file.
807
+ requests = ask.build_requests(extraction, path.name, pages=options.pages)
808
+ replies = [
809
+ ask.send(options.endpoint, request.prompt, retries=options.retries)
810
+ for request in requests
811
+ ]
812
+ answers = [ask.read_model_answer(reply.content) for reply in replies]
813
+ record.update(ask.asked_record(options.endpoint, requests, replies, answers))
814
+ return _Asked(
815
+ record,
816
+ asked=True,
817
+ prompt_tokens=sum(reply.prompt_tokens or 0 for reply in replies),
818
+ completion_tokens=sum(reply.completion_tokens or 0 for reply in replies),
819
+ )
820
+
821
+
822
+ def _model_failed(catalog: Catalog, path: Path, exc: ask.ModelError) -> Outcome:
823
+ line = _line(
824
+ catalog,
825
+ "cli-file-failed",
826
+ path=str(path),
827
+ reason=catalog.render(exc.message),
828
+ )
829
+ return Outcome(None, (line,), failed=True)
830
+
831
+
832
+ def _ask_one(path: Path, options: _AskOptions) -> Outcome:
833
+ """Ask the model about one file and keep what it says (SDW-25)."""
834
+ catalog = _catalog_for(options.lang, options.messages)
835
+ try:
836
+ # The verdict decides whether the model is asked; the parser plays no
837
+ # part, so its errors cannot keep a sheet from a second reading.
838
+ try:
839
+ extraction = extract_pdf_text(path)
840
+ except ExtractionError as exc:
841
+ errors = [exc.message]
842
+ done = _ask_about(
843
+ path,
844
+ None,
845
+ SDSVerifier().refuse_unreadable(errors),
846
+ errors,
847
+ options,
848
+ catalog,
849
+ )
850
+ else:
851
+ done = _ask_about(
852
+ path,
853
+ extraction,
854
+ SDSVerifier().verify(extraction),
855
+ (),
856
+ options,
857
+ catalog,
858
+ )
859
+ except ask.ModelError as exc:
860
+ return _model_failed(catalog, path, exc)
861
+ # As in read: one file that breaks costs only itself.
862
+ except Exception as exc: # noqa: BLE001
863
+ return Outcome(None, tuple(_broke(catalog, path, exc)), failed=True)
864
+ return _outcome(done)
865
+
866
+
867
+ def _outcome(done: _Asked) -> Outcome:
868
+ return Outcome(
869
+ done.record,
870
+ done.said,
871
+ failed=done.failed,
872
+ asked=done.asked,
873
+ prompt_tokens=done.prompt_tokens,
874
+ completion_tokens=done.completion_tokens,
875
+ )
876
+
877
+
878
+ def _diff(
879
+ parser: argparse.ArgumentParser, args: argparse.Namespace, catalog: Catalog
880
+ ) -> int:
881
+ """Set two runs beside each other and say where they differ (SDW-27, SDW-28)."""
882
+ try:
883
+ by_name = args.match == "name"
884
+ a = diff.load(args.a, by_name=by_name)
885
+ b = diff.load(args.b, by_name=by_name)
886
+ count_only = diff.count_only_lines(args.count_only_from)
887
+ except diff.DiffError as exc:
888
+ parser.error(str(exc))
889
+ result = diff.compare_runs(a, b, count_only=count_only, examples=args.examples)
890
+
891
+ for code, one, other in (
892
+ ("diff-stages-differ", a.stages, b.stages),
893
+ ("diff-models-differ", a.models, b.models),
894
+ ("diff-questions-differ", a.questions, b.questions),
895
+ ):
896
+ # Said only where both runs name it: a record of old names no stage.
897
+ if one and other and one != other:
898
+ _say(
899
+ catalog,
900
+ code,
901
+ a=", ".join(sorted(one)),
902
+ b=", ".join(sorted(other)),
903
+ )
904
+
905
+ if args.details is not None:
906
+ try:
907
+ with args.details.open("w", encoding="utf-8") as handle:
908
+ handle.writelines(line + "\n" for line in diff.detail_lines(result))
909
+ except OSError as exc:
910
+ parser.error(f"cannot write {args.details}: {exc}")
911
+ if args.json:
912
+ indent = args.indent if args.indent > 0 else None
913
+ print(json.dumps(diff.to_json(result), indent=indent, ensure_ascii=False))
914
+ else:
915
+ for line in _diff_text(result, catalog):
916
+ print(line)
917
+ return 1 if result.differs else 0
918
+
919
+
920
+ def _diff_text(result: diff.Result, catalog: Catalog) -> list[str]:
921
+ """The comparison for a reader, in the run's language."""
922
+ lines = [
923
+ _line(
924
+ catalog,
925
+ "diff-run",
926
+ side=side,
927
+ path=str(run.path),
928
+ kind=run.kind,
929
+ files=len(run.records),
930
+ )
931
+ for side, run in (("A", result.a), ("B", result.b))
932
+ ]
933
+ lines.append(
934
+ _line(
935
+ catalog,
936
+ "diff-files",
937
+ both=result.both,
938
+ only_a=result.only_a,
939
+ only_b=result.only_b,
940
+ not_compared=result.not_compared,
941
+ )
942
+ )
943
+ lines.append(
944
+ _line(
945
+ catalog,
946
+ "diff-differing",
947
+ differing=result.differing,
948
+ counted_only=result.differing_counted_only,
949
+ )
950
+ )
951
+ if not result.differs:
952
+ lines.append(_line(catalog, "diff-none"))
953
+ for name, count in result.fields.items():
954
+ for code, counts in (
955
+ ("diff-field", count.shown),
956
+ ("diff-field-counted-only", count.counted_only),
957
+ ):
958
+ if counts:
959
+ lines.append(
960
+ _line(
961
+ catalog,
962
+ code,
963
+ field=name,
964
+ gained=counts["gained"],
965
+ lost=counts["lost"],
966
+ changed=counts["changed"],
967
+ )
968
+ )
969
+ if code == "diff-field":
970
+ for path, difference in count.examples:
971
+ statement = f" [{difference.code}]" if difference.code else ""
972
+ lines.append(f" {path}{statement}")
973
+ lines.append(f" A: {_short(difference.a)}")
974
+ lines.append(f" B: {_short(difference.b)}")
975
+ return lines
976
+
977
+
978
+ # An example shows where to look; --details and --json hold the whole value.
979
+ LONGEST_EXAMPLE = 220
980
+
981
+
982
+ def _short(value: Any) -> str:
983
+ text = json.dumps(value, ensure_ascii=False)
984
+ if len(text) <= LONGEST_EXAMPLE:
985
+ return text
986
+ return text[: LONGEST_EXAMPLE - 1] + "…"
987
+
988
+
989
+ def _xml(paths: Sequence[Path], directory: Path | None, catalog: Catalog) -> int:
990
+ """Hand out each PDF's embedded eSDScom file (SDW-13).
991
+
992
+ Byte for byte, because the file's declaration names its encoding. A file
993
+ this tool refuses to read is not handed on either.
994
+ """
995
+ failed = 0
996
+ handed_out = 0
997
+ for path in paths:
998
+ try:
999
+ embedded = read_embedded(path)
1000
+ if embedded is None:
1001
+ # Among many PDFs most carry none; asked for one, it is the answer.
1002
+ if directory is None:
1003
+ failed += 1
1004
+ _say(catalog, "cli-no-embedded-file", path=str(path))
1005
+ continue
1006
+ if embedded.content is None:
1007
+ failed += 1
1008
+ _say(
1009
+ catalog,
1010
+ "cli-embedded-not-handed-out",
1011
+ path=str(path),
1012
+ filename=embedded.filename,
1013
+ reason=catalog.render(embedded.unreadable)
1014
+ if embedded.unreadable
1015
+ else "",
1016
+ )
1017
+ continue
1018
+ if directory is None:
1019
+ sys.stdout.buffer.write(embedded.content)
1020
+ sys.stdout.buffer.flush()
1021
+ handed_out += 1
1022
+ continue
1023
+ target = directory / f"{path.stem}.xml"
1024
+ with target.open("xb") as handle:
1025
+ handle.write(embedded.content)
1026
+ handed_out += 1
1027
+ _say(catalog, "cli-saved", path=str(path), target=str(target))
1028
+ # One PDF that cannot be opened, or whose target exists, costs only
1029
+ # itself; it is named on stderr and fails the exit code.
1030
+ except Exception as exc: # noqa: BLE001
1031
+ failed += 1
1032
+ _report(catalog, path, exc)
1033
+ # With --save a PDF without an embedded file is passed over; a run that
1034
+ # handed out nothing at all has not done what it was asked (SDW-13).
1035
+ return 1 if failed or not handed_out else 0
1036
+
1037
+
1038
+ def _file(path: Path) -> dict[str, Any]:
1039
+ return {
1040
+ "path": str(path.resolve()),
1041
+ "filename": path.name,
1042
+ "size_bytes": path.stat().st_size,
1043
+ }
1044
+
1045
+
1046
+ def _line(catalog: Catalog, code: str, **params: Any) -> str:
1047
+ return catalog.render(Message(code, params))
1048
+
1049
+
1050
+ def _say(catalog: Catalog, code: str, **params: Any) -> None:
1051
+ """One line on stderr, in the run's language (SDW-17)."""
1052
+ print(_line(catalog, code, **params), file=sys.stderr)
1053
+
1054
+
1055
+ def _broke(catalog: Catalog, path: Path, exc: Exception) -> list[str]:
1056
+ """What is said about a file that broke the reader: the line and the trace."""
1057
+ return [
1058
+ _line(
1059
+ catalog,
1060
+ "cli-file-broke",
1061
+ path=str(path),
1062
+ error=f"{type(exc).__name__}: {exc}",
1063
+ ),
1064
+ "".join(traceback.format_exception(exc)).rstrip("\n"),
1065
+ ]
1066
+
1067
+
1068
+ def _report(catalog: Catalog, path: Path, exc: Exception) -> None:
1069
+ _say(
1070
+ catalog,
1071
+ "cli-file-broke",
1072
+ path=str(path),
1073
+ error=f"{type(exc).__name__}: {exc}",
1074
+ )
1075
+ traceback.print_exc(file=sys.stderr)
1076
+
1077
+
1078
+ if __name__ == "__main__": # pragma: no cover - import-time behaviour
1079
+ raise SystemExit(main())