anyformat 2.4.0__tar.gz → 2.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {anyformat-2.4.0 → anyformat-2.5.0}/PKG-INFO +3 -3
  2. {anyformat-2.4.0 → anyformat-2.5.0}/README.md +2 -2
  3. {anyformat-2.4.0 → anyformat-2.5.0}/pyproject.toml +1 -1
  4. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/builder.py +4 -0
  5. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/builder.py +10 -1
  6. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/checks.py +13 -0
  7. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/nodes.py +43 -1
  8. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/results.py +85 -43
  9. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/topology.py +122 -1
  10. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/__init__.py +0 -0
  11. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/_banner.py +0 -0
  12. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/_http.py +0 -0
  13. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/_source.py +0 -0
  14. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/_viz.py +0 -0
  15. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/cli.py +0 -0
  16. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/client.py +0 -0
  17. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/errors.py +0 -0
  18. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/handles.py +0 -0
  19. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/result.py +0 -0
  20. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/sdk/views.py +0 -0
  21. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/__init__.py +0 -0
  22. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/definition.py +0 -0
  23. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/fields.py +0 -0
  24. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/schema.py +0 -0
  25. {anyformat-2.4.0 → anyformat-2.5.0}/src/anyformat/workflow/template_tokens.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: anyformat
3
- Version: 2.4.0
3
+ Version: 2.5.0
4
4
  Summary: Python SDK for the AnyFormat API
5
5
  Project-URL: Homepage, https://anyformat.ai
6
6
  Project-URL: Documentation, https://docs.anyformat.ai
@@ -567,14 +567,14 @@ afx parse invoice.pdf
567
567
  Every knob is a first-class `.parse()` / `.extract()` keyword argument.
568
568
 
569
569
  **`.parse(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`/`"flash"`), `prompt_hint`, `figure_enhancement`,
570
- `cache`; **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
570
+ `cache`, `confidence` (`False` runs no confidence scoring, so every confidence is null); **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
571
571
  own — one OCR-only recipe that never reads `prompt_hint`; **flash-only** `scanned_pages`
572
572
  (`"skip"`/`"fail"`) — no model call, no OCR.
573
573
  Mode-discriminated via per-mode `@overload`s — a wrong-mode knob is a type error.
574
574
 
575
575
  **`.extract(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`),
576
576
  `lookup_suggestion`, `lookup_reasoning_effort` (`"minimal"`/`"low"`/`"medium"`/`"high"`),
577
- `lookup_file_uploads` (inline base64).
577
+ `lookup_file_uploads` (inline base64), `confidence` (`False` runs no confidence scoring, so every field confidence is null).
578
578
 
579
579
  **`Schema.<type>(...)`** — `source="smart_lookup"` (lookup overwrites) or `source="lookup_if_missing"` (extract too; lookup only fills empty slots), `persistent_id=` (pin field identity
580
580
  across updates).
@@ -549,14 +549,14 @@ afx parse invoice.pdf
549
549
  Every knob is a first-class `.parse()` / `.extract()` keyword argument.
550
550
 
551
551
  **`.parse(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`/`"flash"`), `prompt_hint`, `figure_enhancement`,
552
- `cache`; **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
552
+ `cache`, `confidence` (`False` runs no confidence scoring, so every confidence is null); **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
553
553
  own — one OCR-only recipe that never reads `prompt_hint`; **flash-only** `scanned_pages`
554
554
  (`"skip"`/`"fail"`) — no model call, no OCR.
555
555
  Mode-discriminated via per-mode `@overload`s — a wrong-mode knob is a type error.
556
556
 
557
557
  **`.extract(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`),
558
558
  `lookup_suggestion`, `lookup_reasoning_effort` (`"minimal"`/`"low"`/`"medium"`/`"high"`),
559
- `lookup_file_uploads` (inline base64).
559
+ `lookup_file_uploads` (inline base64), `confidence` (`False` runs no confidence scoring, so every field confidence is null).
560
560
 
561
561
  **`Schema.<type>(...)`** — `source="smart_lookup"` (lookup overwrites) or `source="lookup_if_missing"` (extract too; lookup only fills empty slots), `persistent_id=` (pin field identity
562
562
  across updates).
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "anyformat"
3
- version = "2.4.0"
3
+ version = "2.5.0"
4
4
  description = "Python SDK for the AnyFormat API"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -42,6 +42,7 @@ class ClientBoundWorkflowBuilder(WorkflowBuilder):
42
42
  lookup_reasoning_effort: ReasoningEffort | None = None,
43
43
  lookup_files: list[str] | None = None,
44
44
  lookup_file_uploads: list[LookupFileUpload] | None = None,
45
+ confidence: bool = True,
45
46
  ) -> Self:
46
47
  uploads = [
47
48
  LookupFileUpload(filename=Path(p).name, content=base64.b64encode(Path(p).read_bytes()).decode())
@@ -55,6 +56,7 @@ class ClientBoundWorkflowBuilder(WorkflowBuilder):
55
56
  lookup_suggestion=lookup_suggestion,
56
57
  lookup_reasoning_effort=lookup_reasoning_effort,
57
58
  lookup_file_uploads=(uploads + (lookup_file_uploads or [])) or None,
59
+ confidence=confidence,
58
60
  )
59
61
 
60
62
  def create(self) -> Workflow:
@@ -111,6 +113,7 @@ class AsyncClientBoundWorkflowBuilder(WorkflowBuilder):
111
113
  lookup_reasoning_effort: ReasoningEffort | None = None,
112
114
  lookup_files: list[str] | None = None,
113
115
  lookup_file_uploads: list[LookupFileUpload] | None = None,
116
+ confidence: bool = True,
114
117
  ) -> Self:
115
118
  uploads = [
116
119
  LookupFileUpload(filename=Path(p).name, content=base64.b64encode(Path(p).read_bytes()).decode())
@@ -124,6 +127,7 @@ class AsyncClientBoundWorkflowBuilder(WorkflowBuilder):
124
127
  lookup_suggestion=lookup_suggestion,
125
128
  lookup_reasoning_effort=lookup_reasoning_effort,
126
129
  lookup_file_uploads=(uploads + (lookup_file_uploads or [])) or None,
130
+ confidence=confidence,
127
131
  )
128
132
 
129
133
  async def update(self, workflow_id: str) -> AsyncWorkflow:
@@ -118,6 +118,7 @@ class WorkflowBuilder:
118
118
  figure_enhancement: bool = False,
119
119
  annotation_reading: bool = False,
120
120
  cache: bool = True,
121
+ confidence: bool = True,
121
122
  id: str | None = None,
122
123
  ) -> Self: ...
123
124
  @overload
@@ -129,6 +130,7 @@ class WorkflowBuilder:
129
130
  prompt_hint: str | None = None,
130
131
  figure_enhancement: bool = False,
131
132
  cache: bool = True,
133
+ confidence: bool = True,
132
134
  id: str | None = None,
133
135
  ) -> Self: ...
134
136
  @overload
@@ -137,6 +139,7 @@ class WorkflowBuilder:
137
139
  *,
138
140
  mode: Literal["lite"],
139
141
  cache: bool = True,
142
+ confidence: bool = True,
140
143
  id: str | None = None,
141
144
  ) -> Self: ...
142
145
  @overload
@@ -148,6 +151,7 @@ class WorkflowBuilder:
148
151
  prompt_hint: str | None = None,
149
152
  figure_enhancement: bool = False,
150
153
  cache: bool = True,
154
+ confidence: bool = True,
151
155
  id: str | None = None,
152
156
  ) -> Self: ...
153
157
 
@@ -159,6 +163,7 @@ class WorkflowBuilder:
159
163
  figure_enhancement: bool = False,
160
164
  annotation_reading: bool = False,
161
165
  cache: bool = True,
166
+ confidence: bool = True,
162
167
  effort: Literal["low", "mid", "accurate"] | None = None,
163
168
  scanned_pages: Literal["ocr", "skip", "fail"] | None = None,
164
169
  id: str | None = None,
@@ -176,6 +181,7 @@ class WorkflowBuilder:
176
181
  "figure_enhancement": figure_enhancement,
177
182
  "annotation_reading": annotation_reading,
178
183
  "cache": cache,
184
+ "confidence": confidence,
179
185
  }
180
186
  # Map only the mode-specific knobs the caller set; ParseNode supplies per-field defaults.
181
187
  for key, value in (
@@ -230,6 +236,7 @@ class WorkflowBuilder:
230
236
  lookup_suggestion: str | None = None,
231
237
  lookup_reasoning_effort: ReasoningEffort | None = None,
232
238
  lookup_file_uploads: list[LookupFileUpload] | None = None,
239
+ confidence: bool = True,
233
240
  ) -> Self:
234
241
  """Add an extract node.
235
242
 
@@ -243,7 +250,8 @@ class WorkflowBuilder:
243
250
  ``lookup_reasoning_effort`` tune the smart-lookup matcher (effort applies
244
251
  only when ≥1 field has ``source="smart_lookup"``). ``lookup_file_uploads``
245
252
  carries inline file content (base64) that the backend uploads to S3 during
246
- workflow create — pure, no filesystem IO.
253
+ workflow create — pure, no filesystem IO. ``confidence=False`` runs no confidence
254
+ scoring: every field confidence is null.
247
255
  """
248
256
  self._require(self._parse is not None, "extract() requires parse() first")
249
257
  self._require(fields, "extract() requires at least one field")
@@ -267,6 +275,7 @@ class WorkflowBuilder:
267
275
  lookup_suggestion=lookup_suggestion,
268
276
  lookup_reasoning_effort=lookup_reasoning_effort,
269
277
  lookup_file_uploads=lookup_file_uploads or [],
278
+ confidence=confidence,
270
279
  )
271
280
  self._extracts.append((extract_node, branch_id))
272
281
  return self
@@ -227,6 +227,18 @@ def contains_validation_check(check: Check) -> bool:
227
227
  return False
228
228
 
229
229
 
230
+ def confidence_check_fields(check: Check) -> list[str]:
231
+ """The ``field`` of every ``ConfidenceCheck`` in ``check``, walking combinators.
232
+
233
+ A check still a raw dict (a node built without validation) yields nothing.
234
+ """
235
+ if isinstance(check, ConfidenceCheck):
236
+ return [check.field]
237
+ if isinstance(check, (AllOfCheck, AnyOfCheck)):
238
+ return [field for c in check.checks for field in confidence_check_fields(c)]
239
+ return []
240
+
241
+
230
242
  def field_ids(check: Check) -> list[str]:
231
243
  """persistent_ids a check references — what the executor must resolve to field paths."""
232
244
  if isinstance(check, ArithmeticCheck):
@@ -263,6 +275,7 @@ __all__ = [
263
275
  "RegexCheck",
264
276
  "RequiredCheck",
265
277
  "ValidationCheck",
278
+ "confidence_check_fields",
266
279
  "contains_validation_check",
267
280
  "field_ids",
268
281
  ]
@@ -157,6 +157,11 @@ class ParseNode(BaseNode):
157
157
  description="Reuse parsed output when the same file was already parsed with the same mode "
158
158
  "and settings — a hit skips the whole parse. Disable to force a fresh parse.",
159
159
  )
160
+ confidence: bool = Field(
161
+ default=True,
162
+ description="Score how reliable each parsed block is. Disable to skip the confidence "
163
+ "judge: every parse confidence is then null.",
164
+ )
160
165
  # "agentic"-mode knob — ignored for standard/lite.
161
166
  effort: Literal["low", "mid", "accurate"] = Field(
162
167
  default="mid",
@@ -366,6 +371,11 @@ class ExtractNode(BaseNode):
366
371
  description="After row extraction, independently re-check each page's rows against "
367
372
  "document totals and re-extract pages that don't reconcile ('max' mode implies this).",
368
373
  )
374
+ confidence: bool = Field(
375
+ default=True,
376
+ description="Score how reliable each extracted field is. Disable to skip the confidence "
377
+ "judge: every field confidence is then null.",
378
+ )
369
379
  # Internal pipeline-routing knob. Not on the public API surface — the
370
380
  # gateway declares its own ``PublicExtractNode`` without it.
371
381
  context_tracker: str = "group_headers"
@@ -379,6 +389,24 @@ class ExtractNode(BaseNode):
379
389
  serialization_alias="extract_model_alias",
380
390
  )
381
391
 
392
+ @property
393
+ def no_confidence_reason(self) -> str | None:
394
+ """Why this extract scores no confidence, or None when it does. An agentic extract
395
+ compiles before the row extractor, so it scores either way."""
396
+ if not self.confidence:
397
+ return "confidence is off"
398
+ if self.mode == "agentic":
399
+ return None
400
+ if self.mode == "max":
401
+ return "max mode extracts table rows page by page"
402
+ if self.row_extractor_enabled:
403
+ return "it extracts table rows page by page"
404
+ return None
405
+
406
+ @property
407
+ def produces_confidence(self) -> bool:
408
+ return self.no_confidence_reason is None
409
+
382
410
  @classmethod
383
411
  def accepts(cls, predecessor: type["BaseNode"]) -> bool:
384
412
  # Structural predecessors — the shapes an Extract can sit downstream of.
@@ -888,6 +916,13 @@ PUBLIC_DEPRECATED_EMPTY_KEYS: dict[str, frozenset[str]] = {
888
916
  "code": frozenset({"instructions", "template"}),
889
917
  }
890
918
 
919
+ # Keys dropped from the public wire while they equal the default, so an older
920
+ # client's strict node shape still accepts a default node.
921
+ PUBLIC_DEFAULT_OMITTED_KEYS: dict[str, frozenset[str]] = {
922
+ "parse": frozenset({"confidence"}),
923
+ "extract": frozenset({"confidence"}),
924
+ }
925
+
891
926
 
892
927
  def strip_public_hidden_keys(nodes: list[dict]) -> list[dict]:
893
928
  """Return ``nodes`` (list of dumped node dicts) with hidden keys removed.
@@ -900,7 +935,14 @@ def strip_public_hidden_keys(nodes: list[dict]) -> list[dict]:
900
935
  for n in nodes:
901
936
  hidden = PUBLIC_HIDDEN_NODE_KEYS.get(n.get("type"), frozenset())
902
937
  deprecated = PUBLIC_DEPRECATED_EMPTY_KEYS.get(n.get("type"), frozenset())
903
- out.append({k: v for k, v in n.items() if k not in hidden and not (k in deprecated and not v)})
938
+ default_omitted = PUBLIC_DEFAULT_OMITTED_KEYS.get(n.get("type"), frozenset())
939
+ out.append(
940
+ {
941
+ k: v
942
+ for k, v in n.items()
943
+ if k not in hidden and not (k in deprecated and not v) and not (k in default_omitted and v is True)
944
+ }
945
+ )
904
946
  return out
905
947
 
906
948
 
@@ -9,6 +9,7 @@ from typing import Annotated, Any, Generic, Literal, Optional, TypeVar, Union
9
9
 
10
10
  from pydantic import (
11
11
  BaseModel,
12
+ ConfigDict,
12
13
  Discriminator,
13
14
  Field,
14
15
  SerializerFunctionWrapHandler,
@@ -270,7 +271,8 @@ class Block(_OmitsUnsetFileRefs):
270
271
  description=(
271
272
  "The layout detections this block is made of, each with the words printed inside it. "
272
273
  "A block the detector reported as one box has one region, that box. Empty only for a "
273
- "block parsed before detections were reported, or one that has no box at all."
274
+ "block parsed before detections were reported, one that has no box at all, or one whose parse "
275
+ "stored no detections object or one that could not be read back."
274
276
  ),
275
277
  )
276
278
  content: str = Field(
@@ -393,67 +395,103 @@ class RunParseLayerAbsence(BaseModel):
393
395
 
394
396
  absent: str = Field(
395
397
  description=(
396
- "The stored reason, such as `build_failed`, or why a read could not serve a stored layer: `too_large` "
397
- "above the inline limit, `unreadable` when its object cannot be read back. The public API words each as "
398
- "its own."
398
+ "The stored reason, such as `build_failed`, or why a read could not serve a stored layer: `build_failed` "
399
+ "when its copy failed to build on each recent attempt, `unreadable` when its object cannot be read back. "
400
+ "The public API words each as its own."
399
401
  )
400
402
  )
401
403
 
402
404
 
403
- class RunParseSequenceConfidence(BaseModel):
404
- mean: Percent = Field(description="The mean confidence of the sequence's elements, a whole number from 0 to 100.")
405
- min: Percent = Field(description="The lowest confidence of the sequence's elements, a whole number from 0 to 100.")
405
+ ElementSource = Literal["embedded", "ocr", "transcribed"]
406
406
 
407
407
 
408
- class RunParseSequence(_OmitsUnsetFileRefs):
409
- """A run of elements the parse grouped by horizontal proximity inside one block: often a line, never guaranteed."""
408
+ def _open_in_the_spec(schema: dict[str, Any]) -> None:
409
+ schema.pop("additionalProperties", None)
410
410
 
411
- id: str = Field(description="`p<page>_s<index>`, numbered by the packet page `page`.")
411
+
412
+ # Strict, so a record holding a key the contract does not name fails validation; open in the spec, where a new key is
413
+ # additive.
414
+ _PUBLIC_RECORD = ConfigDict(extra="forbid", json_schema_extra=_open_in_the_spec)
415
+
416
+ _PACKET_PAGE = Field(description="1-indexed page number, counted across the packet's files in packet order.")
417
+ _FILE_OF_A_RECORD = Field(description="The file this belongs to, also in a packet of one file.")
418
+ _PAGE_OF_A_RECORD = Field(description="1-indexed page of that file. Equal to `page_number` in a packet of one file.")
419
+
420
+
421
+ class BoxV3(BaseModel):
422
+ """A box on its page, in [0, 1] from the top-left corner, so `y` grows down the page; each coordinate rounded half
423
+ up to 6 decimals, as it is stored.
424
+
425
+ The page is in the frame its `width` and `height` measure: as a PDF viewer shows the file, without turning it by
426
+ the page's `rotation`. On a page whose content is turned, a box frames the turned content, and turns with it.
427
+ """
428
+
429
+ model_config = _PUBLIC_RECORD
430
+
431
+ x0: float
432
+ y0: float
433
+ x1: float
434
+ y1: float
435
+
436
+
437
+ class SequenceConfidenceV3(BaseModel):
438
+ model_config = _PUBLIC_RECORD
439
+
440
+ mean: Percent = Field(description="The mean confidence of its elements, a whole number from 0 to 100.")
441
+ min: Percent = Field(description="The lowest confidence of its elements, a whole number from 0 to 100.")
442
+
443
+
444
+ class SequenceV3(BaseModel):
445
+ """A run of elements grouped by horizontal proximity inside one block: often a line, never guaranteed."""
446
+
447
+ model_config = _PUBLIC_RECORD
448
+
449
+ id: str = Field(description="`p<page_number>_s<index>`, numbered by the packet page.", examples=["p3_s4"])
412
450
  block_id: str = Field(description="The `blocks[].id` of the block that holds it.")
413
- page: int = Field(description="1-indexed page number, counted across the packet's files in packet order.")
414
- file_id: Optional[str] = Field(default=None, description="The file this page belongs to.")
415
- file_page: Optional[int] = Field(default=None, description="1-indexed page of that file.")
416
- text: str = Field(description="The sequence's text, as core read it.")
417
- bbox: dict[str, float] = Field(
418
- description="Normalised bounding box in [0, 1] page coordinates with keys `x0`/`y0`/`x1`/`y1`.",
419
- )
420
- confidence: Optional[RunParseSequenceConfidence] = Field(
451
+ page_number: int = _PACKET_PAGE
452
+ file_id: str = _FILE_OF_A_RECORD
453
+ file_page: int = _PAGE_OF_A_RECORD
454
+ text: str
455
+ bbox: BoxV3
456
+ confidence: Optional[SequenceConfidenceV3] = Field(
421
457
  description="`null` when nothing scored it: a model's reading of handwriting."
422
458
  )
423
459
  cell_id: Optional[str] = Field(
424
- default=None,
425
460
  description=(
426
- "The `r<row>c<col>` cell of its block that table grounding gave every one of its elements; `null` when "
427
- "they were grounded to no cell, or to two."
461
+ "The cell every one of its elements was placed in: its `blocks[].table.cells[].id`, as an evidence item's "
462
+ "`cell_ids` name cells. `null` when none or two."
428
463
  ),
464
+ examples=["p3_b0.r0c0"],
429
465
  )
430
466
 
431
467
 
432
- class RunParseElement(_OmitsUnsetFileRefs):
468
+ class ElementV3(BaseModel):
433
469
  """One piece of text as read: a word, or a whole line when the OCR engine boxed it as one."""
434
470
 
435
- id: str = Field(description="`p<page>_e<index>`, numbered by the packet page `page`.")
471
+ model_config = _PUBLIC_RECORD
472
+
473
+ id: str = Field(description="`p<page_number>_e<index>`, numbered by the packet page.", examples=["p3_e17"])
436
474
  block_id: str = Field(description="The `blocks[].id` of the block that owns it.")
437
- sequence_id: Optional[str] = Field(description="The sequence of its block that holds it; `null` when none does.")
438
- page: int = Field(description="1-indexed page number, counted across the packet's files in packet order.")
439
- file_id: Optional[str] = Field(default=None, description="The file this page belongs to.")
440
- file_page: Optional[int] = Field(default=None, description="1-indexed page of that file.")
441
- text: str = Field(description="The element's text, as core read it.")
442
- bbox: dict[str, float] = Field(
443
- description="Normalised bounding box in [0, 1] page coordinates with keys `x0`/`y0`/`x1`/`y1`.",
444
- )
475
+ sequence_id: Optional[str] = Field(description="The sequence that holds it; `null` when none does.")
476
+ page_number: int = _PACKET_PAGE
477
+ file_id: str = _FILE_OF_A_RECORD
478
+ file_page: int = _PAGE_OF_A_RECORD
479
+ text: str
480
+ bbox: BoxV3
445
481
  confidence: Optional[Percent] = Field(
446
- description="A whole number from 0 to 100; `null` when nothing scored it."
482
+ description="A whole number from 0 to 100; `null` when nothing scored it, as a model's reading of handwriting."
483
+ )
484
+ source: ElementSource = Field(
485
+ description="Where its text came from: the file's own text, OCR, or a model's reading of handwriting."
447
486
  )
448
- source: str = Field(description="Where its text came from: `embedded`, `ocr` or `transcribed`.")
449
487
 
450
488
 
451
489
  class RunParseLayerLink(BaseModel):
452
- """A stored layer object too large to serve inline, handed out as stored: its confidences are on 0-1."""
490
+ """A layer too large to serve inline, as a copy built in the public shape: ``{"<layer>": [record, …]}``, each
491
+ record as it is served inline."""
453
492
 
454
- url: str = Field(description="A presigned GET of the object, valid for 15 minutes; it is served gzip-encoded.")
455
- version: int = Field(description="The published format version the object is stored in.")
456
- size_bytes: int = Field(description="The object's size as stored, gzipped.")
493
+ url: str = Field(description="A presigned GET of the copy, valid for 15 minutes; it is served gzip-encoded.")
494
+ size_bytes: int = Field(description="The copy's size as stored, gzipped.")
457
495
 
458
496
 
459
497
  class RunParseTableCell(BaseModel):
@@ -534,8 +572,9 @@ class RunParseV3(ParseResult):
534
572
  layers: Optional[dict[str, RunParseLayerAbsence]] = Field(
535
573
  default=None,
536
574
  description=(
537
- "Each layer the parse stored without its object, by name, with the reason, and each layer asked for that "
538
- "the read could not serve; any other stored layer is not listed. A parse with a file parsed by a tier that "
575
+ "Each layer the parse stored without its object, by name, with the reason, and each layer the read could "
576
+ "not serve: one asked for, or `cells` or `regions`, which every read serves; any other stored layer is not "
577
+ "listed. A parse with a file parsed by a tier that "
539
578
  "measures neither `sequences` nor `elements` (`lite`) lists both as `not_for_tier`, whatever it stored. "
540
579
  "Otherwise `null` when the run stored no listing of this node's packet parse, as for a parse stored before "
541
580
  "listings were."
@@ -545,11 +584,11 @@ class RunParseV3(ParseResult):
545
584
  default_factory=list,
546
585
  description="Each block, in packet page and reading order, with its table cells.",
547
586
  )
548
- sequences: Optional[list[RunParseSequence]] = Field(
587
+ sequences: Optional[list[SequenceV3]] = Field(
549
588
  default=None,
550
589
  description="Each sequence, in packet page and reading order, when asked for and served; else `null`.",
551
590
  )
552
- elements: Optional[list[RunParseElement]] = Field(
591
+ elements: Optional[list[ElementV3]] = Field(
553
592
  default=None,
554
593
  description="Each element, in packet page and reading order, when asked for and served; else `null`.",
555
594
  )
@@ -1003,10 +1042,13 @@ class ResultsResponseV3(ResultsResponse):
1003
1042
 
1004
1043
  __all__ = [
1005
1044
  "Block",
1045
+ "BoxV3",
1006
1046
  "ClassificationResult",
1007
1047
  "Detection",
1008
1048
  "Edit",
1009
1049
  "EditFormField",
1050
+ "ElementSource",
1051
+ "ElementV3",
1010
1052
  "Evidence",
1011
1053
  "ExtractedField",
1012
1054
  "Extraction",
@@ -1023,13 +1065,13 @@ __all__ = [
1023
1065
  "ResultsResponse",
1024
1066
  "ResultsResponseV3",
1025
1067
  "CodeOutput",
1026
- "RunParseElement",
1027
1068
  "RunParseLayerAbsence",
1028
1069
  "RunParseLayerLink",
1029
1070
  "RunParsePage",
1030
1071
  "RunParsePageStatus",
1031
- "RunParseSequence",
1032
1072
  "RunParseV3",
1073
+ "SequenceConfidenceV3",
1074
+ "SequenceV3",
1033
1075
  "SplitPartition",
1034
1076
  "SplitResult",
1035
1077
  "Usage",
@@ -14,17 +14,19 @@ the backend to run :func:`validate` on the graph it is building, so neither
14
14
  half is restated in the frontend.
15
15
  """
16
16
 
17
+ import uuid
17
18
  from collections.abc import Callable, Mapping, Sequence
18
19
  from dataclasses import dataclass, field
19
20
  from typing import TYPE_CHECKING, Protocol, runtime_checkable
20
21
 
21
- from anyformat.workflow.checks import AllOfCheck, AnyOfCheck, Check, ValidationCheck
22
+ from anyformat.workflow.checks import AllOfCheck, AnyOfCheck, Check, ValidationCheck, confidence_check_fields
22
23
  from anyformat.workflow.fields import AnyField, ObjectField, sanitize_field_name
23
24
  from anyformat.workflow.nodes import (
24
25
  NODE_CLASSES_BY_TYPE,
25
26
  BaseNode,
26
27
  ClassifyNode,
27
28
  CodeNode,
29
+ ExtractionSchema,
28
30
  ExtractNode,
29
31
  FilterNode,
30
32
  IfElseNode,
@@ -33,6 +35,7 @@ from anyformat.workflow.nodes import (
33
35
  SlackAlertNode,
34
36
  SplitterNode,
35
37
  ValidateNode,
38
+ ValidationRule,
36
39
  )
37
40
  from anyformat.workflow.template_tokens import referenced_rule_ids
38
41
 
@@ -856,6 +859,122 @@ class ValidationRuleIdReferencesExist:
856
859
  return violations
857
860
 
858
861
 
862
+ def _upstream_extracts(
863
+ consumer_id: str,
864
+ nodes_by_id: dict[str, "BaseNode"],
865
+ edges_by_target: dict[str, list[str]],
866
+ ) -> list[ExtractNode]:
867
+ """The Extracts a check on ``consumer_id`` reads: those upstream, looking through Validate nodes."""
868
+ extracts: list[ExtractNode] = []
869
+ seen: set[str] = set()
870
+ frontier = list(edges_by_target.get(consumer_id, []))
871
+ while frontier:
872
+ current = frontier.pop()
873
+ if current in seen:
874
+ continue
875
+ seen.add(current)
876
+ node = nodes_by_id.get(current)
877
+ if isinstance(node, ExtractNode):
878
+ extracts.append(node)
879
+ elif isinstance(node, ValidateNode):
880
+ frontier.extend(edges_by_target.get(current, []))
881
+ return extracts
882
+
883
+
884
+ def _smart_lookup_fields(extract: ExtractNode) -> list[tuple[AnyField, bool]]:
885
+ """The fields a smart lookup always fills, which carry no confidence, each flagged nested or not."""
886
+ schema = extract.extraction_schema
887
+ declared = schema.fields if isinstance(schema, ExtractionSchema) else []
888
+ return [
889
+ (f, nested)
890
+ for f, nested in _with_nested([*extract.lookup_schema, *declared])
891
+ if getattr(f, "source", None) == "smart_lookup"
892
+ ]
893
+
894
+
895
+ def _with_nested(fields: "Sequence[AnyField | dict]", *, nested: bool = False) -> list[tuple[AnyField, bool]]:
896
+ """Every field and its nested children. A field relaxed on read can still be a dict, so it is skipped."""
897
+ flat: list[tuple[AnyField, bool]] = []
898
+ for f in fields:
899
+ if isinstance(f, dict):
900
+ continue
901
+ flat.append((f, nested))
902
+ if isinstance(f, ObjectField):
903
+ flat.extend(_with_nested(f.nested_fields, nested=True))
904
+ return flat
905
+
906
+
907
+ def _normalized_id(value: str | None) -> str | None:
908
+ """Hyphenated, hex and upper-case spellings of one UUID compare equal, as in the executor."""
909
+ try:
910
+ return uuid.UUID(str(value)).hex
911
+ except ValueError:
912
+ return value
913
+
914
+
915
+ def _names_field(field: AnyField, ref: str, *, nested: bool) -> bool:
916
+ """The executor resolves a bare name to the top-level key, so a nested field is named by its id only."""
917
+ if field.persistent_id is not None and _normalized_id(field.persistent_id) == _normalized_id(ref):
918
+ return True
919
+ return not nested and ref in (field.name, sanitize_field_name(field.name))
920
+
921
+
922
+ class ConfidenceCheckCanPass:
923
+ """A ``ConfidenceCheck`` on a Validate rule or an If-Else condition must read a
924
+ confidence its Extract produces.
925
+
926
+ An Extract with confidence off, in max mode, or with the row extractor on returns
927
+ every confidence as null, and so does a smart-lookup field. The check then never
928
+ passes, and routing silently takes the fallback branch. Fail at save instead.
929
+ """
930
+
931
+ name = "confidence_check_can_pass"
932
+
933
+ def check(self, graph: "WorkflowDefinition") -> list[Violation]:
934
+ nodes_by_id: dict[str, BaseNode] = {n.id: n for n in graph.nodes}
935
+ edges_by_target: dict[str, list[str]] = {}
936
+ for edge in graph.edges:
937
+ edges_by_target.setdefault(edge.target, []).append(edge.source)
938
+ violations: list[Violation] = []
939
+ for node in graph.nodes:
940
+ refs = self._confidence_refs(node)
941
+ if not refs:
942
+ continue
943
+ for extract in _upstream_extracts(node.id, nodes_by_id, edges_by_target):
944
+ reason = self._unscored_reason(extract, refs)
945
+ if reason:
946
+ violations.append(
947
+ Violation(
948
+ rule=self.name,
949
+ message=f'This confidence check can never pass: extract "{extract.id}" {reason}.',
950
+ node_ids=[node.id],
951
+ )
952
+ )
953
+ return violations
954
+
955
+ @staticmethod
956
+ def _confidence_refs(node: "BaseNode") -> list[str]:
957
+ if isinstance(node, IfElseNode):
958
+ return confidence_check_fields(node.condition)
959
+ if isinstance(node, ValidateNode):
960
+ return [
961
+ ref
962
+ for rule in node.rules
963
+ if isinstance(rule, ValidationRule) and rule.check is not None
964
+ for ref in confidence_check_fields(rule.check)
965
+ ]
966
+ return []
967
+
968
+ @staticmethod
969
+ def _unscored_reason(extract: ExtractNode, refs: list[str]) -> str | None:
970
+ if extract.no_confidence_reason:
971
+ return f"does not score confidence because {extract.no_confidence_reason}"
972
+ for looked_up, nested in _smart_lookup_fields(extract):
973
+ if any(_names_field(looked_up, ref, nested=nested) for ref in refs):
974
+ return f'does not score confidence because field "{looked_up.name}" is set to always look up'
975
+ return None
976
+
977
+
859
978
  class NoCodeInstructions:
860
979
  """Reject a code node that holds the retired ``instructions`` or ``template``.
861
980
 
@@ -901,6 +1020,7 @@ RULES: list[TopologyRule] = [
901
1020
  NoRouterUnderSplitter(),
902
1021
  NoCodeInstructions(),
903
1022
  ValidationRuleIdReferencesExist(),
1023
+ ConfidenceCheckCanPass(),
904
1024
  NoCycles(),
905
1025
  ]
906
1026
 
@@ -1054,6 +1174,7 @@ __all__ = [
1054
1174
  "BranchForbiddenFromNonBranchingSource",
1055
1175
  "BranchMatchesSourceId",
1056
1176
  "BranchRequiredFromBranchingSource",
1177
+ "ConfidenceCheckCanPass",
1057
1178
  "EdgeAcceptsPredecessor",
1058
1179
  "EdgesReferenceExistingNodes",
1059
1180
  "EveryExtractNodeHasFields",