anyformat 2.3.0__tar.gz → 2.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. {anyformat-2.3.0 → anyformat-2.5.0}/PKG-INFO +3 -3
  2. {anyformat-2.3.0 → anyformat-2.5.0}/README.md +2 -2
  3. {anyformat-2.3.0 → anyformat-2.5.0}/pyproject.toml +1 -1
  4. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/builder.py +4 -0
  5. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/cli.py +1 -1
  6. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/views.py +5 -5
  7. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/builder.py +10 -1
  8. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/checks.py +13 -0
  9. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/nodes.py +158 -3
  10. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/results.py +112 -66
  11. anyformat-2.5.0/src/anyformat/workflow/template_tokens.py +46 -0
  12. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/topology.py +146 -11
  13. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/__init__.py +0 -0
  14. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/_banner.py +0 -0
  15. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/_http.py +0 -0
  16. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/_source.py +0 -0
  17. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/_viz.py +0 -0
  18. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/client.py +0 -0
  19. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/errors.py +0 -0
  20. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/handles.py +0 -0
  21. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/sdk/result.py +0 -0
  22. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/__init__.py +0 -0
  23. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/definition.py +0 -0
  24. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/fields.py +0 -0
  25. {anyformat-2.3.0 → anyformat-2.5.0}/src/anyformat/workflow/schema.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: anyformat
3
- Version: 2.3.0
3
+ Version: 2.5.0
4
4
  Summary: Python SDK for the AnyFormat API
5
5
  Project-URL: Homepage, https://anyformat.ai
6
6
  Project-URL: Documentation, https://docs.anyformat.ai
@@ -567,14 +567,14 @@ afx parse invoice.pdf
567
567
  Every knob is a first-class `.parse()` / `.extract()` keyword argument.
568
568
 
569
569
  **`.parse(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`/`"flash"`), `prompt_hint`, `figure_enhancement`,
570
- `cache`; **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
570
+ `cache`, `confidence` (`False` runs no confidence scoring, so every confidence is null); **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
571
571
  own — one OCR-only recipe that never reads `prompt_hint`; **flash-only** `scanned_pages`
572
572
  (`"skip"`/`"fail"`) — no model call, no OCR.
573
573
  Mode-discriminated via per-mode `@overload`s — a wrong-mode knob is a type error.
574
574
 
575
575
  **`.extract(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`),
576
576
  `lookup_suggestion`, `lookup_reasoning_effort` (`"minimal"`/`"low"`/`"medium"`/`"high"`),
577
- `lookup_file_uploads` (inline base64).
577
+ `lookup_file_uploads` (inline base64), `confidence` (`False` runs no confidence scoring, so every field confidence is null).
578
578
 
579
579
  **`Schema.<type>(...)`** — `source="smart_lookup"` (lookup overwrites) or `source="lookup_if_missing"` (extract too; lookup only fills empty slots), `persistent_id=` (pin field identity
580
580
  across updates).
@@ -549,14 +549,14 @@ afx parse invoice.pdf
549
549
  Every knob is a first-class `.parse()` / `.extract()` keyword argument.
550
550
 
551
551
  **`.parse(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`/`"flash"`), `prompt_hint`, `figure_enhancement`,
552
- `cache`; **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
552
+ `cache`, `confidence` (`False` runs no confidence scoring, so every confidence is null); **standard-only** `annotation_reading` (experimental: re-reads handwritten numbers); **agentic-only** `effort` (`"low"`/`"mid"`/`"accurate"`); **lite**: no knobs of its
553
553
  own — one OCR-only recipe that never reads `prompt_hint`; **flash-only** `scanned_pages`
554
554
  (`"skip"`/`"fail"`) — no model call, no OCR.
555
555
  Mode-discriminated via per-mode `@overload`s — a wrong-mode knob is a type error.
556
556
 
557
557
  **`.extract(...)`** — `mode` (`"standard"`/`"agentic"`/`"lite"`),
558
558
  `lookup_suggestion`, `lookup_reasoning_effort` (`"minimal"`/`"low"`/`"medium"`/`"high"`),
559
- `lookup_file_uploads` (inline base64).
559
+ `lookup_file_uploads` (inline base64), `confidence` (`False` runs no confidence scoring, so every field confidence is null).
560
560
 
561
561
  **`Schema.<type>(...)`** — `source="smart_lookup"` (lookup overwrites) or `source="lookup_if_missing"` (extract too; lookup only fills empty slots), `persistent_id=` (pin field identity
562
562
  across updates).
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "anyformat"
3
- version = "2.3.0"
3
+ version = "2.5.0"
4
4
  description = "Python SDK for the AnyFormat API"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -42,6 +42,7 @@ class ClientBoundWorkflowBuilder(WorkflowBuilder):
42
42
  lookup_reasoning_effort: ReasoningEffort | None = None,
43
43
  lookup_files: list[str] | None = None,
44
44
  lookup_file_uploads: list[LookupFileUpload] | None = None,
45
+ confidence: bool = True,
45
46
  ) -> Self:
46
47
  uploads = [
47
48
  LookupFileUpload(filename=Path(p).name, content=base64.b64encode(Path(p).read_bytes()).decode())
@@ -55,6 +56,7 @@ class ClientBoundWorkflowBuilder(WorkflowBuilder):
55
56
  lookup_suggestion=lookup_suggestion,
56
57
  lookup_reasoning_effort=lookup_reasoning_effort,
57
58
  lookup_file_uploads=(uploads + (lookup_file_uploads or [])) or None,
59
+ confidence=confidence,
58
60
  )
59
61
 
60
62
  def create(self) -> Workflow:
@@ -111,6 +113,7 @@ class AsyncClientBoundWorkflowBuilder(WorkflowBuilder):
111
113
  lookup_reasoning_effort: ReasoningEffort | None = None,
112
114
  lookup_files: list[str] | None = None,
113
115
  lookup_file_uploads: list[LookupFileUpload] | None = None,
116
+ confidence: bool = True,
114
117
  ) -> Self:
115
118
  uploads = [
116
119
  LookupFileUpload(filename=Path(p).name, content=base64.b64encode(Path(p).read_bytes()).decode())
@@ -124,6 +127,7 @@ class AsyncClientBoundWorkflowBuilder(WorkflowBuilder):
124
127
  lookup_suggestion=lookup_suggestion,
125
128
  lookup_reasoning_effort=lookup_reasoning_effort,
126
129
  lookup_file_uploads=(uploads + (lookup_file_uploads or [])) or None,
130
+ confidence=confidence,
127
131
  )
128
132
 
129
133
  async def update(self, workflow_id: str) -> AsyncWorkflow:
@@ -655,7 +655,7 @@ def _print_rich_table(out: Console, scalar_fields: dict, raw_fields: dict) -> No
655
655
  table.add_column("Confidence")
656
656
 
657
657
  for field_name, f in scalar_fields.items():
658
- confidence = f"{f.confidence:.2f}" if f.confidence is not None else "-"
658
+ confidence = str(f.confidence) if f.confidence is not None else "-"
659
659
  table.add_row(field_name, str(f.value), confidence)
660
660
 
661
661
  for name, items in object_items:
@@ -47,14 +47,14 @@ class ParseView:
47
47
  return None if isinstance(self._parse, ParseBlocksResultV3) else self._parse.text
48
48
 
49
49
  @property
50
- def parse_confidence(self) -> float | None:
51
- """Document-level confidence (0-100) in the parsed content. May be
52
- ``None`` for some parse modes — fall back to :attr:`layout_confidence`."""
50
+ def parse_confidence(self) -> int | None:
51
+ """Document-level confidence in the parsed content, a whole number from 0 to 100.
52
+ May be ``None`` for some parse modes — fall back to :attr:`layout_confidence`."""
53
53
  return self._parse.parse_confidence
54
54
 
55
55
  @property
56
- def layout_confidence(self) -> float | None:
57
- """Document-level layout model score (0-1) for the block positions.
56
+ def layout_confidence(self) -> int | None:
57
+ """Document-level confidence in the block positions, a whole number from 0 to 100.
58
58
  ``None`` when not available."""
59
59
  return self._parse.layout_confidence
60
60
 
@@ -118,6 +118,7 @@ class WorkflowBuilder:
118
118
  figure_enhancement: bool = False,
119
119
  annotation_reading: bool = False,
120
120
  cache: bool = True,
121
+ confidence: bool = True,
121
122
  id: str | None = None,
122
123
  ) -> Self: ...
123
124
  @overload
@@ -129,6 +130,7 @@ class WorkflowBuilder:
129
130
  prompt_hint: str | None = None,
130
131
  figure_enhancement: bool = False,
131
132
  cache: bool = True,
133
+ confidence: bool = True,
132
134
  id: str | None = None,
133
135
  ) -> Self: ...
134
136
  @overload
@@ -137,6 +139,7 @@ class WorkflowBuilder:
137
139
  *,
138
140
  mode: Literal["lite"],
139
141
  cache: bool = True,
142
+ confidence: bool = True,
140
143
  id: str | None = None,
141
144
  ) -> Self: ...
142
145
  @overload
@@ -148,6 +151,7 @@ class WorkflowBuilder:
148
151
  prompt_hint: str | None = None,
149
152
  figure_enhancement: bool = False,
150
153
  cache: bool = True,
154
+ confidence: bool = True,
151
155
  id: str | None = None,
152
156
  ) -> Self: ...
153
157
 
@@ -159,6 +163,7 @@ class WorkflowBuilder:
159
163
  figure_enhancement: bool = False,
160
164
  annotation_reading: bool = False,
161
165
  cache: bool = True,
166
+ confidence: bool = True,
162
167
  effort: Literal["low", "mid", "accurate"] | None = None,
163
168
  scanned_pages: Literal["ocr", "skip", "fail"] | None = None,
164
169
  id: str | None = None,
@@ -176,6 +181,7 @@ class WorkflowBuilder:
176
181
  "figure_enhancement": figure_enhancement,
177
182
  "annotation_reading": annotation_reading,
178
183
  "cache": cache,
184
+ "confidence": confidence,
179
185
  }
180
186
  # Map only the mode-specific knobs the caller set; ParseNode supplies per-field defaults.
181
187
  for key, value in (
@@ -230,6 +236,7 @@ class WorkflowBuilder:
230
236
  lookup_suggestion: str | None = None,
231
237
  lookup_reasoning_effort: ReasoningEffort | None = None,
232
238
  lookup_file_uploads: list[LookupFileUpload] | None = None,
239
+ confidence: bool = True,
233
240
  ) -> Self:
234
241
  """Add an extract node.
235
242
 
@@ -243,7 +250,8 @@ class WorkflowBuilder:
243
250
  ``lookup_reasoning_effort`` tune the smart-lookup matcher (effort applies
244
251
  only when ≥1 field has ``source="smart_lookup"``). ``lookup_file_uploads``
245
252
  carries inline file content (base64) that the backend uploads to S3 during
246
- workflow create — pure, no filesystem IO.
253
+ workflow create — pure, no filesystem IO. ``confidence=False`` runs no confidence
254
+ scoring: every field confidence is null.
247
255
  """
248
256
  self._require(self._parse is not None, "extract() requires parse() first")
249
257
  self._require(fields, "extract() requires at least one field")
@@ -267,6 +275,7 @@ class WorkflowBuilder:
267
275
  lookup_suggestion=lookup_suggestion,
268
276
  lookup_reasoning_effort=lookup_reasoning_effort,
269
277
  lookup_file_uploads=lookup_file_uploads or [],
278
+ confidence=confidence,
270
279
  )
271
280
  self._extracts.append((extract_node, branch_id))
272
281
  return self
@@ -227,6 +227,18 @@ def contains_validation_check(check: Check) -> bool:
227
227
  return False
228
228
 
229
229
 
230
+ def confidence_check_fields(check: Check) -> list[str]:
231
+ """The ``field`` of every ``ConfidenceCheck`` in ``check``, walking combinators.
232
+
233
+ A check still a raw dict (a node built without validation) yields nothing.
234
+ """
235
+ if isinstance(check, ConfidenceCheck):
236
+ return [check.field]
237
+ if isinstance(check, (AllOfCheck, AnyOfCheck)):
238
+ return [field for c in check.checks for field in confidence_check_fields(c)]
239
+ return []
240
+
241
+
230
242
  def field_ids(check: Check) -> list[str]:
231
243
  """persistent_ids a check references — what the executor must resolve to field paths."""
232
244
  if isinstance(check, ArithmeticCheck):
@@ -263,6 +275,7 @@ __all__ = [
263
275
  "RegexCheck",
264
276
  "RequiredCheck",
265
277
  "ValidationCheck",
278
+ "confidence_check_fields",
266
279
  "contains_validation_check",
267
280
  "field_ids",
268
281
  ]
@@ -13,6 +13,7 @@ from pydantic import (
13
13
  BeforeValidator,
14
14
  ConfigDict,
15
15
  Field,
16
+ StringConstraints,
16
17
  model_validator,
17
18
  )
18
19
 
@@ -81,6 +82,11 @@ class BaseNode(BaseModel):
81
82
  def builder_verb(cls) -> str | None:
82
83
  return None
83
84
 
85
+ @classmethod
86
+ def required_flag(cls) -> str | None:
87
+ """The feature flag a caller must hold to write this node; a stored node still reads without it."""
88
+ return None
89
+
84
90
 
85
91
  ParseMode = Literal["standard", "agentic", "lite", "flash"]
86
92
  # How much reasoning a caller asks a model for. Every stage that turns the knob reads
@@ -151,6 +157,11 @@ class ParseNode(BaseNode):
151
157
  description="Reuse parsed output when the same file was already parsed with the same mode "
152
158
  "and settings — a hit skips the whole parse. Disable to force a fresh parse.",
153
159
  )
160
+ confidence: bool = Field(
161
+ default=True,
162
+ description="Score how reliable each parsed block is. Disable to skip the confidence "
163
+ "judge: every parse confidence is then null.",
164
+ )
154
165
  # "agentic"-mode knob — ignored for standard/lite.
155
166
  effort: Literal["low", "mid", "accurate"] = Field(
156
167
  default="mid",
@@ -360,6 +371,11 @@ class ExtractNode(BaseNode):
360
371
  description="After row extraction, independently re-check each page's rows against "
361
372
  "document totals and re-extract pages that don't reconcile ('max' mode implies this).",
362
373
  )
374
+ confidence: bool = Field(
375
+ default=True,
376
+ description="Score how reliable each extracted field is. Disable to skip the confidence "
377
+ "judge: every field confidence is then null.",
378
+ )
363
379
  # Internal pipeline-routing knob. Not on the public API surface — the
364
380
  # gateway declares its own ``PublicExtractNode`` without it.
365
381
  context_tracker: str = "group_headers"
@@ -373,6 +389,24 @@ class ExtractNode(BaseNode):
373
389
  serialization_alias="extract_model_alias",
374
390
  )
375
391
 
392
+ @property
393
+ def no_confidence_reason(self) -> str | None:
394
+ """Why this extract scores no confidence, or None when it does. An agentic extract
395
+ compiles before the row extractor, so it scores either way."""
396
+ if not self.confidence:
397
+ return "confidence is off"
398
+ if self.mode == "agentic":
399
+ return None
400
+ if self.mode == "max":
401
+ return "max mode extracts table rows page by page"
402
+ if self.row_extractor_enabled:
403
+ return "it extracts table rows page by page"
404
+ return None
405
+
406
+ @property
407
+ def produces_confidence(self) -> bool:
408
+ return self.no_confidence_reason is None
409
+
376
410
  @classmethod
377
411
  def accepts(cls, predecessor: type["BaseNode"]) -> bool:
378
412
  # Structural predecessors — the shapes an Extract can sit downstream of.
@@ -503,10 +537,12 @@ class SlackAlertNode(BaseNode):
503
537
 
504
538
  Sits at the end of a branch (typically the ``true`` branch of an
505
539
  ``IfElseNode``) and emits nothing downstream — the topology's terminal
506
- role. The template supports two token grammars:
540
+ role. The template supports three token grammars:
507
541
 
508
542
  - ``${field.<persistent_id>}`` — extraction-field value, resolved via
509
543
  ``field_index`` (backend-injected at message-build time).
544
+ - ``${table.<persistent_id>}`` — a list-of-objects field as a plain-text
545
+ table, every column in schema order.
510
546
  - ``${validation.<rule_id>.<attr>}`` — outcome of an upstream
511
547
  ``ValidateNode`` rule; ``<attr>`` is one of ``status`` / ``detail`` /
512
548
  ``severity``. Rendered against ``state.validation_results`` filtered
@@ -534,7 +570,8 @@ class SlackAlertNode(BaseNode):
534
570
  message_template: str = Field(
535
571
  min_length=1,
536
572
  description=(
537
- "Message body with ``${field.<persistent_id>}`` placeholders and, when "
573
+ "Message body with ``${field.<persistent_id>}`` placeholders, "
574
+ "``${table.<persistent_id>}`` placeholders for a list-of-objects field and, when "
538
575
  "an upstream ``ValidateNode`` is present, ``${validation.<rule_id>.<attr>}`` "
539
576
  "placeholders (``<attr>`` = ``status`` | ``detail`` | ``severity``). "
540
577
  "Missing / renamed fields or unknown rule ids render as ``<missing: <name>>`` "
@@ -563,6 +600,104 @@ class SlackAlertNode(BaseNode):
563
600
  return True
564
601
 
565
602
 
603
+ # Loose on purpose: one address, no display name, no list separators. Deliverability is the gateway's call.
604
+ EmailAddress = Annotated[
605
+ str,
606
+ StringConstraints(strip_whitespace=True, max_length=254, pattern=r"^[^@\s,;<>]+@[^@\s,;<>]+\.[^@\s,;<>]+$"),
607
+ ]
608
+
609
+
610
+ # Caps each of to and cc, so the email a node sends per split has a bounded fan-out.
611
+ MAX_EMAIL_RECIPIENTS = 50
612
+
613
+
614
+ class EmailBody(BaseModel):
615
+ """Where an Email node's body comes from: its own template, or an upstream report."""
616
+
617
+ model_config = ConfigDict(extra="forbid")
618
+
619
+ kind: Literal["template", "report"] = Field(description="The body source.")
620
+ template: OptionalText = Field(
621
+ default=None,
622
+ description="Body with ``${field.*}`` and ``${validation.*}`` tokens. Required when ``kind`` is ``template``.",
623
+ )
624
+ report_node_id: OptionalText = Field(
625
+ default=None, description="The Report generator before the node. Required when ``kind`` is ``report``."
626
+ )
627
+ wrapper: OptionalText = Field(
628
+ default=None, description="Text around the report; ``${report}`` marks where it goes."
629
+ )
630
+ placement: Literal["inline", "attachment", "both"] = Field(
631
+ default="inline", description="Where the report goes: in the body, as an attachment, or both."
632
+ )
633
+
634
+ @model_validator(mode="after")
635
+ def _source_matches_kind(self) -> "EmailBody":
636
+ if self.kind == "report":
637
+ raise ValueError("a report body is not supported; use kind 'template'")
638
+ if self.template is None or not self.template.strip():
639
+ raise ValueError("a template body needs a template")
640
+ if self.report_node_id is not None or self.wrapper is not None:
641
+ raise ValueError("report_node_id and wrapper apply only to a report body")
642
+ return self
643
+
644
+
645
+ class EmailAlertNode(BaseNode):
646
+ """Terminal sink that sends one email per split that reaches it.
647
+
648
+ Recipients come only from this config, never from the extraction. Takes the
649
+ same upstreams as ``SlackAlertNode``, for the same reason: each carries an
650
+ extraction the templates render against.
651
+ """
652
+
653
+ type: Literal["email_alert"]
654
+ to: list[EmailAddress] = Field(min_length=1, max_length=MAX_EMAIL_RECIPIENTS, description="Recipients.")
655
+ cc: list[EmailAddress] = Field(
656
+ default_factory=list, max_length=MAX_EMAIL_RECIPIENTS, description="Copied recipients."
657
+ )
658
+ reply_to: Annotated[EmailAddress | None, BeforeValidator(_empty_str_to_none)] = Field(
659
+ default=None, description="Where replies go."
660
+ )
661
+ subject_template: str = Field(
662
+ min_length=1, description="Subject with ``${field.*}`` tokens, on one line."
663
+ )
664
+ body: EmailBody
665
+ on_missing: Literal["hold", "send"] = Field(
666
+ default="hold",
667
+ description="When a template field is missing: ``hold`` sends nothing and records the field; "
668
+ "``send`` sends with the field marked missing.",
669
+ )
670
+ field_index: dict[str, str] | None = Field(
671
+ default=None,
672
+ description="Executor-injected at run time (persistent_id → field name); never set by callers or persisted.",
673
+ )
674
+
675
+ @model_validator(mode="after")
676
+ def _subject_on_one_line(self) -> "EmailAlertNode":
677
+ if "\n" in self.subject_template or "\r" in self.subject_template:
678
+ raise ValueError("subject_template must not contain line breaks")
679
+ return self
680
+
681
+ @model_validator(mode="after")
682
+ def _each_recipient_once(self) -> "EmailAlertNode":
683
+ recipients = [address.lower() for address in [*self.to, *self.cc]]
684
+ if len(recipients) != len(set(recipients)):
685
+ raise ValueError("an address appears more than once across to and cc")
686
+ return self
687
+
688
+ @classmethod
689
+ def accepts(cls, predecessor: type["BaseNode"]) -> bool:
690
+ return predecessor in (ExtractNode, ValidateNode, IfElseNode)
691
+
692
+ @classmethod
693
+ def resolves_single_upstream(cls) -> bool:
694
+ return True
695
+
696
+ @classmethod
697
+ def required_flag(cls) -> str:
698
+ return "email_alert"
699
+
700
+
566
701
  class KnowledgeNode(BaseNode):
567
702
  """Terminal knowledge sink — its presence opts the workflow into the
568
703
  knowledge base (priced separately, so absent by default).
@@ -720,6 +855,7 @@ AnyNode = Annotated[
720
855
  IfElseNode,
721
856
  FilterNode,
722
857
  SlackAlertNode,
858
+ EmailAlertNode,
723
859
  KnowledgeNode,
724
860
  EditNode,
725
861
  CodeNode,
@@ -741,6 +877,7 @@ NODE_CLASSES_BY_TYPE: dict[str, type[BaseNode]] = {
741
877
  "if_else": IfElseNode,
742
878
  "filter": FilterNode,
743
879
  "slack_alert": SlackAlertNode,
880
+ "email_alert": EmailAlertNode,
744
881
  "knowledge": KnowledgeNode,
745
882
  "edit": EditNode,
746
883
  "code": CodeNode,
@@ -769,6 +906,7 @@ PUBLIC_HIDDEN_NODE_KEYS: dict[str, frozenset[str]] = {
769
906
  "validate": frozenset({"field_index"}),
770
907
  "if_else": frozenset({"field_index"}),
771
908
  "slack_alert": frozenset({"field_index"}),
909
+ "email_alert": frozenset({"field_index"}),
772
910
  "edit": frozenset({"reference_uris"}),
773
911
  }
774
912
 
@@ -778,6 +916,13 @@ PUBLIC_DEPRECATED_EMPTY_KEYS: dict[str, frozenset[str]] = {
778
916
  "code": frozenset({"instructions", "template"}),
779
917
  }
780
918
 
919
+ # Keys dropped from the public wire while they equal the default, so an older
920
+ # client's strict node shape still accepts a default node.
921
+ PUBLIC_DEFAULT_OMITTED_KEYS: dict[str, frozenset[str]] = {
922
+ "parse": frozenset({"confidence"}),
923
+ "extract": frozenset({"confidence"}),
924
+ }
925
+
781
926
 
782
927
  def strip_public_hidden_keys(nodes: list[dict]) -> list[dict]:
783
928
  """Return ``nodes`` (list of dumped node dicts) with hidden keys removed.
@@ -790,7 +935,14 @@ def strip_public_hidden_keys(nodes: list[dict]) -> list[dict]:
790
935
  for n in nodes:
791
936
  hidden = PUBLIC_HIDDEN_NODE_KEYS.get(n.get("type"), frozenset())
792
937
  deprecated = PUBLIC_DEPRECATED_EMPTY_KEYS.get(n.get("type"), frozenset())
793
- out.append({k: v for k, v in n.items() if k not in hidden and not (k in deprecated and not v)})
938
+ default_omitted = PUBLIC_DEFAULT_OMITTED_KEYS.get(n.get("type"), frozenset())
939
+ out.append(
940
+ {
941
+ k: v
942
+ for k, v in n.items()
943
+ if k not in hidden and not (k in deprecated and not v) and not (k in default_omitted and v is True)
944
+ }
945
+ )
794
946
  return out
795
947
 
796
948
 
@@ -801,8 +953,11 @@ __all__ = [
801
953
  "ClassifyCategory",
802
954
  "ClassifyNode",
803
955
  "EditNode",
956
+ "EmailAlertNode",
957
+ "EmailBody",
804
958
  "ExtractNode",
805
959
  "MAX_CODE_BYTES",
960
+ "MAX_EMAIL_RECIPIENTS",
806
961
  "MAX_EXAMPLE_REPORT_BYTES",
807
962
  "PUBLIC_HIDDEN_NODE_KEYS",
808
963
  "strip_public_hidden_keys",
@@ -9,6 +9,7 @@ from typing import Annotated, Any, Generic, Literal, Optional, TypeVar, Union
9
9
 
10
10
  from pydantic import (
11
11
  BaseModel,
12
+ ConfigDict,
12
13
  Discriminator,
13
14
  Field,
14
15
  SerializerFunctionWrapHandler,
@@ -24,6 +25,9 @@ from pydantic import (
24
25
  # is referenced unparameterised (T resolves to ``Any``).
25
26
  ValueT = TypeVar("ValueT")
26
27
 
28
+ # Every confidence this API serves: a whole number of percent, whatever scale it was computed on.
29
+ Percent = Annotated[int, Field(ge=0, le=100)]
30
+
27
31
 
28
32
  def _drop_unset_file_refs(data: dict[str, Any]) -> dict[str, Any]:
29
33
  """``data`` without ``file_id``, ``file_name`` and ``file_page`` when unset: an absent field is omitted, never ``null``."""
@@ -155,9 +159,9 @@ class ExtractedField(BaseModel, Generic[ValueT]):
155
159
  default=None,
156
160
  description="Verification state for this datapoint (e.g. `not_verified`, `verified`). `null` when not yet reviewed.",
157
161
  )
158
- confidence: Optional[float] = Field(
162
+ confidence: Optional[Percent] = Field(
159
163
  default=None,
160
- description="Model confidence in the extracted value, on a 0-100 scale. `null` when the backend did not produce a confidence (e.g. manual entry).",
164
+ description="Model confidence in the extracted value, a whole number from 0 to 100. `null` when the backend did not produce a confidence (e.g. manual entry).",
161
165
  )
162
166
  evidence: list[Evidence] = Field(
163
167
  default_factory=list,
@@ -250,24 +254,25 @@ class Block(_OmitsUnsetFileRefs):
250
254
  bbox: dict[str, float] = Field(
251
255
  description="Normalised bounding box in [0, 1] page coordinates with keys `x0`/`y0`/`x1`/`y1`.",
252
256
  )
253
- layout_confidence: Optional[float] = Field(
257
+ layout_confidence: Optional[Percent] = Field(
254
258
  default=None,
255
259
  description=(
256
- "0-1 layout detection confidence for this block. `null` when no layout detector ran "
260
+ "Layout detection confidence for this block, a whole number from 0 to 100. `null` when no layout detector ran "
257
261
  "— the `lite` tier gets its layout from the OCR response, "
258
262
  "which reports no confidence in the type it assigned."
259
263
  ),
260
264
  )
261
- parse_confidence: Optional[float] = Field(
265
+ parse_confidence: Optional[Percent] = Field(
262
266
  default=None,
263
- description="0-100 parse confidence calibrated from LLM logprobs. `null` when logprobs were unavailable (e.g. text-bytes strategy).",
267
+ description="Parse confidence calibrated from LLM logprobs, a whole number from 0 to 100. `null` when logprobs were unavailable (e.g. text-bytes strategy).",
264
268
  )
265
269
  regions: list[Detection] = Field(
266
270
  default_factory=list,
267
271
  description=(
268
272
  "The layout detections this block is made of, each with the words printed inside it. "
269
273
  "A block the detector reported as one box has one region, that box. Empty only for a "
270
- "block parsed before detections were reported, or one that has no box at all."
274
+ "block parsed before detections were reported, one that has no box at all, or one whose parse "
275
+ "stored no detections object or one that could not be read back."
271
276
  ),
272
277
  )
273
278
  content: str = Field(
@@ -315,17 +320,17 @@ class ParseResult(BaseModel):
315
320
  "read one or the other, not both. `null` if `markdown` is null."
316
321
  ),
317
322
  )
318
- layout_confidence: Optional[float] = Field(
323
+ layout_confidence: Optional[Percent] = Field(
319
324
  default=None,
320
325
  description=(
321
- "Document-level YOLO layout confidence on a 0-1 scale, "
322
- "char-weighted mean across all blocks. `null` if no annotated sections."
326
+ "Document-level layout confidence, a whole number from 0 to 100: "
327
+ "the char-weighted mean across all blocks. `null` if no annotated sections."
323
328
  ),
324
329
  )
325
- parse_confidence: Optional[float] = Field(
330
+ parse_confidence: Optional[Percent] = Field(
326
331
  default=None,
327
332
  description=(
328
- "Document-level parse confidence on a 0-100 scale, char-weighted mean "
333
+ "Document-level parse confidence, a whole number from 0 to 100: the char-weighted mean "
329
334
  "of per-block LLM logprob scores. `null` when no blocks have logprob-based confidence."
330
335
  ),
331
336
  )
@@ -390,65 +395,103 @@ class RunParseLayerAbsence(BaseModel):
390
395
 
391
396
  absent: str = Field(
392
397
  description=(
393
- "The stored reason, such as `build_failed`, or why a read could not serve a stored layer: `too_large` "
394
- "above the inline limit, `unreadable` when its object cannot be read back. The public API words each as "
395
- "its own."
398
+ "The stored reason, such as `build_failed`, or why a read could not serve a stored layer: `build_failed` "
399
+ "when its copy failed to build on each recent attempt, `unreadable` when its object cannot be read back. "
400
+ "The public API words each as its own."
396
401
  )
397
402
  )
398
403
 
399
404
 
400
- class RunParseSequenceConfidence(BaseModel):
401
- mean: float = Field(description="The mean confidence of the sequence's elements, 0-1, as core measured it.")
402
- min: float = Field(description="The lowest confidence of the sequence's elements, 0-1, as core measured it.")
405
+ ElementSource = Literal["embedded", "ocr", "transcribed"]
406
+
407
+
408
+ def _open_in_the_spec(schema: dict[str, Any]) -> None:
409
+ schema.pop("additionalProperties", None)
410
+
411
+
412
+ # Strict, so a record holding a key the contract does not name fails validation; open in the spec, where a new key is
413
+ # additive.
414
+ _PUBLIC_RECORD = ConfigDict(extra="forbid", json_schema_extra=_open_in_the_spec)
415
+
416
+ _PACKET_PAGE = Field(description="1-indexed page number, counted across the packet's files in packet order.")
417
+ _FILE_OF_A_RECORD = Field(description="The file this belongs to, also in a packet of one file.")
418
+ _PAGE_OF_A_RECORD = Field(description="1-indexed page of that file. Equal to `page_number` in a packet of one file.")
419
+
420
+
421
+ class BoxV3(BaseModel):
422
+ """A box on its page, in [0, 1] from the top-left corner, so `y` grows down the page; each coordinate rounded half
423
+ up to 6 decimals, as it is stored.
424
+
425
+ The page is in the frame its `width` and `height` measure: as a PDF viewer shows the file, without turning it by
426
+ the page's `rotation`. On a page whose content is turned, a box frames the turned content, and turns with it.
427
+ """
428
+
429
+ model_config = _PUBLIC_RECORD
430
+
431
+ x0: float
432
+ y0: float
433
+ x1: float
434
+ y1: float
435
+
436
+
437
+ class SequenceConfidenceV3(BaseModel):
438
+ model_config = _PUBLIC_RECORD
439
+
440
+ mean: Percent = Field(description="The mean confidence of its elements, a whole number from 0 to 100.")
441
+ min: Percent = Field(description="The lowest confidence of its elements, a whole number from 0 to 100.")
403
442
 
404
443
 
405
- class RunParseSequence(_OmitsUnsetFileRefs):
406
- """A run of elements the parse grouped by horizontal proximity inside one block: often a line, never guaranteed."""
444
+ class SequenceV3(BaseModel):
445
+ """A run of elements grouped by horizontal proximity inside one block: often a line, never guaranteed."""
407
446
 
408
- id: str = Field(description="`p<page>_s<index>`, numbered by the packet page `page`.")
447
+ model_config = _PUBLIC_RECORD
448
+
449
+ id: str = Field(description="`p<page_number>_s<index>`, numbered by the packet page.", examples=["p3_s4"])
409
450
  block_id: str = Field(description="The `blocks[].id` of the block that holds it.")
410
- page: int = Field(description="1-indexed page number, counted across the packet's files in packet order.")
411
- file_id: Optional[str] = Field(default=None, description="The file this page belongs to.")
412
- file_page: Optional[int] = Field(default=None, description="1-indexed page of that file.")
413
- text: str = Field(description="The sequence's text, as core read it.")
414
- bbox: dict[str, float] = Field(
415
- description="Normalised bounding box in [0, 1] page coordinates with keys `x0`/`y0`/`x1`/`y1`.",
416
- )
417
- confidence: Optional[RunParseSequenceConfidence] = Field(
451
+ page_number: int = _PACKET_PAGE
452
+ file_id: str = _FILE_OF_A_RECORD
453
+ file_page: int = _PAGE_OF_A_RECORD
454
+ text: str
455
+ bbox: BoxV3
456
+ confidence: Optional[SequenceConfidenceV3] = Field(
418
457
  description="`null` when nothing scored it: a model's reading of handwriting."
419
458
  )
420
459
  cell_id: Optional[str] = Field(
421
- default=None,
422
460
  description=(
423
- "The `r<row>c<col>` cell of its block that table grounding gave every one of its elements; `null` when "
424
- "they were grounded to no cell, or to two."
461
+ "The cell every one of its elements was placed in: its `blocks[].table.cells[].id`, as an evidence item's "
462
+ "`cell_ids` name cells. `null` when none or two."
425
463
  ),
464
+ examples=["p3_b0.r0c0"],
426
465
  )
427
466
 
428
467
 
429
- class RunParseElement(_OmitsUnsetFileRefs):
468
+ class ElementV3(BaseModel):
430
469
  """One piece of text as read: a word, or a whole line when the OCR engine boxed it as one."""
431
470
 
432
- id: str = Field(description="`p<page>_e<index>`, numbered by the packet page `page`.")
471
+ model_config = _PUBLIC_RECORD
472
+
473
+ id: str = Field(description="`p<page_number>_e<index>`, numbered by the packet page.", examples=["p3_e17"])
433
474
  block_id: str = Field(description="The `blocks[].id` of the block that owns it.")
434
- sequence_id: Optional[str] = Field(description="The sequence of its block that holds it; `null` when none does.")
435
- page: int = Field(description="1-indexed page number, counted across the packet's files in packet order.")
436
- file_id: Optional[str] = Field(default=None, description="The file this page belongs to.")
437
- file_page: Optional[int] = Field(default=None, description="1-indexed page of that file.")
438
- text: str = Field(description="The element's text, as core read it.")
439
- bbox: dict[str, float] = Field(
440
- description="Normalised bounding box in [0, 1] page coordinates with keys `x0`/`y0`/`x1`/`y1`.",
475
+ sequence_id: Optional[str] = Field(description="The sequence that holds it; `null` when none does.")
476
+ page_number: int = _PACKET_PAGE
477
+ file_id: str = _FILE_OF_A_RECORD
478
+ file_page: int = _PAGE_OF_A_RECORD
479
+ text: str
480
+ bbox: BoxV3
481
+ confidence: Optional[Percent] = Field(
482
+ description="A whole number from 0 to 100; `null` when nothing scored it, as a model's reading of handwriting."
483
+ )
484
+ source: ElementSource = Field(
485
+ description="Where its text came from: the file's own text, OCR, or a model's reading of handwriting."
441
486
  )
442
- confidence: Optional[float] = Field(description="0-1, as core measured it; `null` when nothing scored it.")
443
- source: str = Field(description="Where its text came from: `embedded`, `ocr` or `transcribed`.")
444
487
 
445
488
 
446
489
  class RunParseLayerLink(BaseModel):
447
- """A stored layer object too large to serve inline, handed out as stored, in its published format."""
490
+ """A layer too large to serve inline, as a copy built in the public shape: ``{"<layer>": [record, …]}``, each
491
+ record as it is served inline."""
448
492
 
449
- url: str = Field(description="A presigned GET of the object, valid for 15 minutes; it is served gzip-encoded.")
450
- version: int = Field(description="The published format version the object is stored in.")
451
- size_bytes: int = Field(description="The object's size as stored, gzipped.")
493
+ url: str = Field(description="A presigned GET of the copy, valid for 15 minutes; it is served gzip-encoded.")
494
+ size_bytes: int = Field(description="The copy's size as stored, gzipped.")
452
495
 
453
496
 
454
497
  class RunParseTableCell(BaseModel):
@@ -467,7 +510,6 @@ class RunParseTableCell(BaseModel):
467
510
  match: Optional[str] = Field(
468
511
  description="How grounding placed it, such as `anchor` or `orphan_spatial`; `null` when it matched nothing."
469
512
  )
470
- parse_confidence: Optional[float] = Field(description="0-100, on its block's scale; `null` when nothing scored it.")
471
513
 
472
514
 
473
515
  class RunParseLink(BaseModel):
@@ -530,8 +572,9 @@ class RunParseV3(ParseResult):
530
572
  layers: Optional[dict[str, RunParseLayerAbsence]] = Field(
531
573
  default=None,
532
574
  description=(
533
- "Each layer the parse stored without its object, by name, with the reason, and each layer asked for that "
534
- "the read could not serve; any other stored layer is not listed. A parse with a file parsed by a tier that "
575
+ "Each layer the parse stored without its object, by name, with the reason, and each layer the read could "
576
+ "not serve: one asked for, or `cells` or `regions`, which every read serves; any other stored layer is not "
577
+ "listed. A parse with a file parsed by a tier that "
535
578
  "measures neither `sequences` nor `elements` (`lite`) lists both as `not_for_tier`, whatever it stored. "
536
579
  "Otherwise `null` when the run stored no listing of this node's packet parse, as for a parse stored before "
537
580
  "listings were."
@@ -541,11 +584,11 @@ class RunParseV3(ParseResult):
541
584
  default_factory=list,
542
585
  description="Each block, in packet page and reading order, with its table cells.",
543
586
  )
544
- sequences: Optional[list[RunParseSequence]] = Field(
587
+ sequences: Optional[list[SequenceV3]] = Field(
545
588
  default=None,
546
589
  description="Each sequence, in packet page and reading order, when asked for and served; else `null`.",
547
590
  )
548
- elements: Optional[list[RunParseElement]] = Field(
591
+ elements: Optional[list[ElementV3]] = Field(
549
592
  default=None,
550
593
  description="Each element, in packet page and reading order, when asked for and served; else `null`.",
551
594
  )
@@ -582,13 +625,13 @@ class ParseBlocksResultV3(BaseModel):
582
625
  """
583
626
 
584
627
  format: Literal["blocks"] = Field(description="`blocks`: the parse as `pages` of blocks.")
585
- layout_confidence: Optional[float] = Field(
628
+ layout_confidence: Optional[Percent] = Field(
586
629
  default=None,
587
- description="Document-level layout confidence on a 0-1 scale, char-weighted mean across all blocks.",
630
+ description="Document-level layout confidence, a whole number from 0 to 100: the char-weighted mean across all blocks.",
588
631
  )
589
- parse_confidence: Optional[float] = Field(
632
+ parse_confidence: Optional[Percent] = Field(
590
633
  default=None,
591
- description="Document-level parse confidence on a 0-100 scale, char-weighted mean across all blocks.",
634
+ description="Document-level parse confidence, a whole number from 0 to 100: the char-weighted mean across all blocks.",
592
635
  )
593
636
  pages: list[ParsePage] = Field(default_factory=list, description="The pages that hold a block, in page order.")
594
637
 
@@ -624,8 +667,8 @@ class SplitPartition(BaseModel):
624
667
  )
625
668
  name: str = Field(description="The partition value (free-form string).")
626
669
  files: list[FilePages]
627
- confidence: Optional[int] = Field(
628
- description="0-100 minimum confidence across the partition's ranges. "
670
+ confidence: Optional[Percent] = Field(
671
+ description="The minimum confidence across the partition's ranges, a whole number from 0 to 100. "
629
672
  "`null` when the partition holds no page, because nothing was measured.",
630
673
  )
631
674
 
@@ -647,8 +690,8 @@ class SplitResult(BaseModel):
647
690
  )
648
691
  name: str = Field(description="The split's category name.")
649
692
  files: list[FilePages] = Field(description="Per-file page lists, union of all partitions.")
650
- confidence: Optional[int] = Field(
651
- description="0-100 aggregate confidence (min across partitions). "
693
+ confidence: Optional[Percent] = Field(
694
+ description="Aggregate confidence, a whole number from 0 to 100 (min across partitions). "
652
695
  "`null` when the category matched no page, because nothing was measured.",
653
696
  )
654
697
  partitions: list[SplitPartition] = Field(default_factory=list)
@@ -658,8 +701,8 @@ class ClassificationResult(BaseModel):
658
701
  """One classifier verdict for the collection."""
659
702
 
660
703
  category: str = Field(description="The category the document was classified as.")
661
- confidence: Optional[float] = Field(
662
- description="0-100 model confidence in the verdict. "
704
+ confidence: Optional[Percent] = Field(
705
+ description="Model confidence in the verdict, a whole number from 0 to 100. "
663
706
  "`null` when the verdict carries no score, because nothing was measured.",
664
707
  )
665
708
  evidence: Optional[str] = Field(
@@ -773,9 +816,9 @@ class EditFormField(BaseModel):
773
816
  "`checkbox`. `null` when no instruction addressed this field."
774
817
  ),
775
818
  )
776
- confidence: Optional[int] = Field(
819
+ confidence: Optional[Percent] = Field(
777
820
  description=(
778
- "Raw, uncalibrated 0-100 match score for pairing an instruction with this "
821
+ "Raw, uncalibrated match score, a whole number from 0 to 100, for pairing an instruction with this "
779
822
  "field. It grades how surely the instruction addresses the field, not whether "
780
823
  "the written value is correct, and it is not a probability. `null` when no "
781
824
  "instruction addressed this field, because nothing was measured."
@@ -999,10 +1042,13 @@ class ResultsResponseV3(ResultsResponse):
999
1042
 
1000
1043
  __all__ = [
1001
1044
  "Block",
1045
+ "BoxV3",
1002
1046
  "ClassificationResult",
1003
1047
  "Detection",
1004
1048
  "Edit",
1005
1049
  "EditFormField",
1050
+ "ElementSource",
1051
+ "ElementV3",
1006
1052
  "Evidence",
1007
1053
  "ExtractedField",
1008
1054
  "Extraction",
@@ -1019,13 +1065,13 @@ __all__ = [
1019
1065
  "ResultsResponse",
1020
1066
  "ResultsResponseV3",
1021
1067
  "CodeOutput",
1022
- "RunParseElement",
1023
1068
  "RunParseLayerAbsence",
1024
1069
  "RunParseLayerLink",
1025
1070
  "RunParsePage",
1026
1071
  "RunParsePageStatus",
1027
- "RunParseSequence",
1028
1072
  "RunParseV3",
1073
+ "SequenceConfidenceV3",
1074
+ "SequenceV3",
1029
1075
  "SplitPartition",
1030
1076
  "SplitResult",
1031
1077
  "Usage",
@@ -0,0 +1,46 @@
1
+ """The ``${…}`` token grammar of an alert node's message template.
2
+
3
+ It lives with the node definitions so every package that reads a template, from the
4
+ topology checks to the renderer in ``anyformat.domain.alert_template``, parses it
5
+ with this one pattern.
6
+ """
7
+
8
+ import re
9
+ from collections.abc import Callable
10
+
11
+ __all__ = ["TOKEN_RE", "referenced_field_ids", "referenced_rule_ids", "remap_field_ids"]
12
+
13
+ _ID = r"[A-Za-z0-9_\-]+"
14
+ TOKEN_RE = re.compile(
15
+ r"\$\{(?:"
16
+ rf"field\.(?P<field>{_ID})"
17
+ rf"|table\.(?P<table>{_ID})"
18
+ rf"|validation\.(?P<rule>{_ID})\.(?P<attr>status|detail|severity)"
19
+ r"|document\.(?P<document>category|filename|link)"
20
+ r")\}"
21
+ )
22
+
23
+
24
+ def referenced_field_ids(template: str) -> list[str]:
25
+ """The persistent ids a template's ``${field.*}`` and ``${table.*}`` tokens name."""
26
+ return [
27
+ match["field"] or match["table"] for match in TOKEN_RE.finditer(template) if match["field"] or match["table"]
28
+ ]
29
+
30
+
31
+ def referenced_rule_ids(template: str) -> list[str]:
32
+ """The rule ids a template's ``${validation.*}`` tokens name."""
33
+ return [match["rule"] for match in TOKEN_RE.finditer(template) if match["rule"]]
34
+
35
+
36
+ def remap_field_ids(template: str, remap: Callable[[str], str]) -> str:
37
+ """``template`` with each ``${field.*}`` and ``${table.*}`` persistent id passed through ``remap``."""
38
+
39
+ def _remapped(match: re.Match[str]) -> str:
40
+ if match["field"]:
41
+ return f"${{field.{remap(match['field'])}}}"
42
+ if match["table"]:
43
+ return f"${{table.{remap(match['table'])}}}"
44
+ return match[0]
45
+
46
+ return TOKEN_RE.sub(_remapped, template)
@@ -14,18 +14,19 @@ the backend to run :func:`validate` on the graph it is building, so neither
14
14
  half is restated in the frontend.
15
15
  """
16
16
 
17
- import re
18
- from collections.abc import Callable, Sequence
17
+ import uuid
18
+ from collections.abc import Callable, Mapping, Sequence
19
19
  from dataclasses import dataclass, field
20
20
  from typing import TYPE_CHECKING, Protocol, runtime_checkable
21
21
 
22
- from anyformat.workflow.checks import AllOfCheck, AnyOfCheck, Check, ValidationCheck
22
+ from anyformat.workflow.checks import AllOfCheck, AnyOfCheck, Check, ValidationCheck, confidence_check_fields
23
23
  from anyformat.workflow.fields import AnyField, ObjectField, sanitize_field_name
24
24
  from anyformat.workflow.nodes import (
25
25
  NODE_CLASSES_BY_TYPE,
26
26
  BaseNode,
27
27
  ClassifyNode,
28
28
  CodeNode,
29
+ ExtractionSchema,
29
30
  ExtractNode,
30
31
  FilterNode,
31
32
  IfElseNode,
@@ -34,7 +35,9 @@ from anyformat.workflow.nodes import (
34
35
  SlackAlertNode,
35
36
  SplitterNode,
36
37
  ValidateNode,
38
+ ValidationRule,
37
39
  )
40
+ from anyformat.workflow.template_tokens import referenced_rule_ids
38
41
 
39
42
  if TYPE_CHECKING:
40
43
  from anyformat.workflow.definition import WorkflowDefinition
@@ -744,13 +747,6 @@ class AtMostOneKnowledgeNode:
744
747
  ]
745
748
 
746
749
 
747
- # Mirrors the runtime regex in
748
- # ``anyformat_core.operations.alerting.base._VALIDATION_REF_RE`` and the backend
749
- # extractor in ``core_tasks``. Kept here so the topology rule below
750
- # doesn't import from either side (they don't import each other).
751
- _TEMPLATE_VALIDATION_REF_RE = re.compile(r"\$\{validation\.([A-Za-z0-9_\-]+)\.(?:status|detail|severity)\}")
752
-
753
-
754
750
  def _validation_rule_ids_in_check(check: Check) -> list[str]:
755
751
  """Every ``rule_id`` a check tree references — walks combinators."""
756
752
  if isinstance(check, ValidationCheck):
@@ -843,7 +839,7 @@ class ValidationRuleIdReferencesExist:
843
839
  )
844
840
  )
845
841
  elif isinstance(node, SlackAlertNode):
846
- referenced_template = _TEMPLATE_VALIDATION_REF_RE.findall(node.message_template)
842
+ referenced_template = referenced_rule_ids(node.message_template)
847
843
  if not referenced_template:
848
844
  continue
849
845
  reachable = _upstream_validate_rule_ids(node.id, nodes_by_id, edges_by_target)
@@ -863,6 +859,122 @@ class ValidationRuleIdReferencesExist:
863
859
  return violations
864
860
 
865
861
 
862
+ def _upstream_extracts(
863
+ consumer_id: str,
864
+ nodes_by_id: dict[str, "BaseNode"],
865
+ edges_by_target: dict[str, list[str]],
866
+ ) -> list[ExtractNode]:
867
+ """The Extracts a check on ``consumer_id`` reads: those upstream, looking through Validate nodes."""
868
+ extracts: list[ExtractNode] = []
869
+ seen: set[str] = set()
870
+ frontier = list(edges_by_target.get(consumer_id, []))
871
+ while frontier:
872
+ current = frontier.pop()
873
+ if current in seen:
874
+ continue
875
+ seen.add(current)
876
+ node = nodes_by_id.get(current)
877
+ if isinstance(node, ExtractNode):
878
+ extracts.append(node)
879
+ elif isinstance(node, ValidateNode):
880
+ frontier.extend(edges_by_target.get(current, []))
881
+ return extracts
882
+
883
+
884
+ def _smart_lookup_fields(extract: ExtractNode) -> list[tuple[AnyField, bool]]:
885
+ """The fields a smart lookup always fills, which carry no confidence, each flagged nested or not."""
886
+ schema = extract.extraction_schema
887
+ declared = schema.fields if isinstance(schema, ExtractionSchema) else []
888
+ return [
889
+ (f, nested)
890
+ for f, nested in _with_nested([*extract.lookup_schema, *declared])
891
+ if getattr(f, "source", None) == "smart_lookup"
892
+ ]
893
+
894
+
895
+ def _with_nested(fields: "Sequence[AnyField | dict]", *, nested: bool = False) -> list[tuple[AnyField, bool]]:
896
+ """Every field and its nested children. A field relaxed on read can still be a dict, so it is skipped."""
897
+ flat: list[tuple[AnyField, bool]] = []
898
+ for f in fields:
899
+ if isinstance(f, dict):
900
+ continue
901
+ flat.append((f, nested))
902
+ if isinstance(f, ObjectField):
903
+ flat.extend(_with_nested(f.nested_fields, nested=True))
904
+ return flat
905
+
906
+
907
+ def _normalized_id(value: str | None) -> str | None:
908
+ """Hyphenated, hex and upper-case spellings of one UUID compare equal, as in the executor."""
909
+ try:
910
+ return uuid.UUID(str(value)).hex
911
+ except ValueError:
912
+ return value
913
+
914
+
915
+ def _names_field(field: AnyField, ref: str, *, nested: bool) -> bool:
916
+ """The executor resolves a bare name to the top-level key, so a nested field is named by its id only."""
917
+ if field.persistent_id is not None and _normalized_id(field.persistent_id) == _normalized_id(ref):
918
+ return True
919
+ return not nested and ref in (field.name, sanitize_field_name(field.name))
920
+
921
+
922
+ class ConfidenceCheckCanPass:
923
+ """A ``ConfidenceCheck`` on a Validate rule or an If-Else condition must read a
924
+ confidence its Extract produces.
925
+
926
+ An Extract with confidence off, in max mode, or with the row extractor on returns
927
+ every confidence as null, and so does a smart-lookup field. The check then never
928
+ passes, and routing silently takes the fallback branch. Fail at save instead.
929
+ """
930
+
931
+ name = "confidence_check_can_pass"
932
+
933
+ def check(self, graph: "WorkflowDefinition") -> list[Violation]:
934
+ nodes_by_id: dict[str, BaseNode] = {n.id: n for n in graph.nodes}
935
+ edges_by_target: dict[str, list[str]] = {}
936
+ for edge in graph.edges:
937
+ edges_by_target.setdefault(edge.target, []).append(edge.source)
938
+ violations: list[Violation] = []
939
+ for node in graph.nodes:
940
+ refs = self._confidence_refs(node)
941
+ if not refs:
942
+ continue
943
+ for extract in _upstream_extracts(node.id, nodes_by_id, edges_by_target):
944
+ reason = self._unscored_reason(extract, refs)
945
+ if reason:
946
+ violations.append(
947
+ Violation(
948
+ rule=self.name,
949
+ message=f'This confidence check can never pass: extract "{extract.id}" {reason}.',
950
+ node_ids=[node.id],
951
+ )
952
+ )
953
+ return violations
954
+
955
+ @staticmethod
956
+ def _confidence_refs(node: "BaseNode") -> list[str]:
957
+ if isinstance(node, IfElseNode):
958
+ return confidence_check_fields(node.condition)
959
+ if isinstance(node, ValidateNode):
960
+ return [
961
+ ref
962
+ for rule in node.rules
963
+ if isinstance(rule, ValidationRule) and rule.check is not None
964
+ for ref in confidence_check_fields(rule.check)
965
+ ]
966
+ return []
967
+
968
+ @staticmethod
969
+ def _unscored_reason(extract: ExtractNode, refs: list[str]) -> str | None:
970
+ if extract.no_confidence_reason:
971
+ return f"does not score confidence because {extract.no_confidence_reason}"
972
+ for looked_up, nested in _smart_lookup_fields(extract):
973
+ if any(_names_field(looked_up, ref, nested=nested) for ref in refs):
974
+ return f'does not score confidence because field "{looked_up.name}" is set to always look up'
975
+ return None
976
+
977
+
866
978
  class NoCodeInstructions:
867
979
  """Reject a code node that holds the retired ``instructions`` or ``template``.
868
980
 
@@ -908,6 +1020,7 @@ RULES: list[TopologyRule] = [
908
1020
  NoRouterUnderSplitter(),
909
1021
  NoCodeInstructions(),
910
1022
  ValidationRuleIdReferencesExist(),
1023
+ ConfidenceCheckCanPass(),
911
1024
  NoCycles(),
912
1025
  ]
913
1026
 
@@ -929,6 +1042,26 @@ def skippable_topology_rules(has_flag: FlagCheck) -> frozenset[str]:
929
1042
  return frozenset(rule for flag, rules in _RULES_WAIVED_BY_FLAG.items() if has_flag(flag) for rule in rules)
930
1043
 
931
1044
 
1045
+ NODE_NEEDS_FLAG = "node_needs_flag"
1046
+
1047
+
1048
+ def unavailable_nodes(node_types: Mapping[str, str], has_flag: FlagCheck) -> list[Violation]:
1049
+ """The nodes, given as id to type, whose type needs a flag ``has_flag`` does not hold.
1050
+
1051
+ Separate from :func:`validate`, so a stored node keeps loading wherever its flag is off."""
1052
+ return [
1053
+ Violation(
1054
+ rule=NODE_NEEDS_FLAG,
1055
+ message=f"node {node_id!r}: {node_type} nodes are not enabled for this organization.",
1056
+ node_ids=[node_id],
1057
+ )
1058
+ for node_id, node_type in node_types.items()
1059
+ if (node_cls := NODE_CLASSES_BY_TYPE.get(node_type)) is not None
1060
+ and (flag := node_cls.required_flag()) is not None
1061
+ and not has_flag(flag)
1062
+ ]
1063
+
1064
+
932
1065
  def validate(graph: "WorkflowDefinition", *, skip_rules: "frozenset[str]" = frozenset()) -> list[Violation]:
933
1066
  """Run every registered rule and return the combined list of violations.
934
1067
 
@@ -1041,6 +1174,7 @@ __all__ = [
1041
1174
  "BranchForbiddenFromNonBranchingSource",
1042
1175
  "BranchMatchesSourceId",
1043
1176
  "BranchRequiredFromBranchingSource",
1177
+ "ConfidenceCheckCanPass",
1044
1178
  "EdgeAcceptsPredecessor",
1045
1179
  "EdgesReferenceExistingNodes",
1046
1180
  "EveryExtractNodeHasFields",
@@ -1066,5 +1200,6 @@ __all__ = [
1066
1200
  "Violation",
1067
1201
  "format_violations",
1068
1202
  "node_adjacency",
1203
+ "unavailable_nodes",
1069
1204
  "validate",
1070
1205
  ]