groundx 3.8.2__tar.gz → 3.8.4__tar.gz

{groundx-3.8.2 → groundx-3.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: groundx
-Version: 3.8.2
+Version: 3.8.4
 Summary: 
 License: MIT
 Requires-Python: >=3.10,<4.0

{groundx-3.8.2 → groundx-3.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ dynamic = ["version"]
 
 [tool.poetry]
 name = "groundx"
-version = "3.8.2"
+version = "3.8.4"
 description = ""
 readme = "README.md"
 authors = []

{groundx-3.8.2 → groundx-3.8.4}/src/groundx/core/client_wrapper.py

@@ -27,12 +27,12 @@ class BaseClientWrapper:
         import platform
 
         headers: typing.Dict[str, str] = {
-            "User-Agent": "groundx/3.8.2",
+            "User-Agent": "groundx/3.8.4",
             "X-Fern-Language": "Python",
             "X-Fern-Runtime": f"python/{platform.python_version()}",
             "X-Fern-Platform": f"{platform.system().lower()}/{platform.release()}",
             "X-Fern-SDK-Name": "groundx",
-            "X-Fern-SDK-Version": "3.8.2",
+            "X-Fern-SDK-Version": "3.8.4",
             **(self.get_custom_headers() or {}),
         }
         headers["X-API-Key"] = self.api_key

{groundx-3.8.2 → groundx-3.8.4}/src/groundx/extract/classes/document.py

@@ -1,20 +1,19 @@
 import json
 import os
 import shutil
-import time
 import typing
 from datetime import datetime, timezone
 from io import BytesIO
 from pathlib import Path
 from urllib.parse import urlparse
 
-import requests
 from ..custom_outputs import (
     custom_output_payload_identity,
     custom_output_route_values,
     custom_output_section_identity,
 )
 from ..prompt.manager import PromptManager
+from ..services.http import BoundedRequestTimeout, bounded_get
 from ..services.logger import Logger
 from ..services.upload import Upload
 from ..utility import clean_json
@@ -976,23 +975,26 @@ class DocumentRequest(BaseModel):
 
             try:
                 self.print("WARN", f"[{attempt}] downloading [{page}]")
-                resp = requests.get(page)
+                resp = bounded_get(
+                    page,
+                    operation="page image",
+                    sleep_between_attempts=should_sleep,
+                )
                 resp.raise_for_status()
                 img = Image.open(BytesIO(resp.content))
                 if img:
                     self.page_image_dict[page] = len(self.page_images)
                     self.page_images.append(img)
                     pageImages.append(img)
+            except BoundedRequestTimeout as e:
+                self.print(
+                    "ERROR", f"[{attempt}] Failed to load image from {page}: {e}"
+                )
+                raise RuntimeError(str(e)) from e
             except Exception as e:
                 self.print(
                     "ERROR", f"[{attempt}] Failed to load image from {page}: {e}"
                 )
-                if attempt < 2:
-                    if should_sleep:
-                        time.sleep(2 * attempt + 1)
-                    return self.load_images(
-                        imgs, upload, attempt + 1, should_sleep=should_sleep
-                    )
 
         return pageImages
 

{groundx-3.8.2 → groundx-3.8.4}/src/groundx/extract/classes/groundx.py

@@ -1,11 +1,13 @@
-import json, requests, typing
+import json
+import typing
 from pathlib import Path
 
+import requests
+from ..services.http import bounded_get
+from ..services.upload import Upload
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 from typing_extensions import Annotated
 
-from ..services.upload import Upload
-
 
 class GroundXDocument(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
@@ -195,10 +197,12 @@ class XRayDocument(BaseModel):
 
         url = gx_doc.xray_url(base=base)
         try:
-            resp = requests.get(url)
+            resp = bounded_get(url, operation="X-ray JSON")
             resp.raise_for_status()
         except requests.RequestException as e:
             raise RuntimeError(f"Error fetching X-ray JSON from {url}: {e}")
+        except RuntimeError as e:
+            raise RuntimeError(f"Error fetching X-ray JSON from {url}: {e}") from e
 
         try:
             payload = resp.json()

{groundx-3.8.2 → groundx-3.8.4}/src/groundx/extract/custom_outputs.py

@@ -58,8 +58,26 @@ class _RouteContainer:
     page_numbers: typing.Tuple[int, ...] = ()
 
 
+@dataclasses.dataclass(frozen=True)
+class _ScalarCandidate:
+    value: typing.Any
+    quality: typing.Tuple[int, int, float]
+
+
 _REPEATED_STEP_KINDS = {"keys", "summary"}
 _EXTRACTED_FIELD_VALUE_KEYS = {"value", "confidence", "conflicts", "qa"}
+_DEFAULT_CANDIDATE_VALUES = {
+    "n/a",
+    "na",
+    "none",
+    "not applicable",
+    "not found",
+    "not provided",
+    "not specified",
+    "not stated",
+    "null",
+    "unknown",
+}
 _GET_ALIASES = {
     "chunkId": ("chunk_id",),
     "customChunkOutputs": ("custom_chunk_outputs",),
@@ -94,10 +112,12 @@ def reassemble_custom_outputs_from_xray(
         typing.Tuple[typing.Tuple[str, ...], typing.Tuple[typing.Any, ...]],
         typing.Dict[str, typing.Any],
     ] = {}
+    scalar_candidates: typing.Dict[str, _ScalarCandidate] = {}
     workflow_repeated_records: typing.Dict[
         typing.Tuple[str, typing.Tuple[typing.Any, ...]],
         typing.Dict[str, typing.Any],
     ] = {}
+    diagnostics: typing.List[CustomOutputDiagnostic] = []
 
     for route_index, route in enumerate(routes):
         route_hits[route_index] = False
@@ -143,14 +163,20 @@ def reassemble_custom_outputs_from_xray(
                     copy.deepcopy(route_value.value),
                     repeated_records=repeated_records,
                     record_key=record_key,
+                    route=route_map,
+                    page_numbers=route_container.page_numbers,
+                    scalar_candidates=scalar_candidates,
+                    diagnostics=diagnostics,
                 )
 
     relationships = workflow.get("output_relationships") if workflow else None
     relationship_output = None
-    diagnostics = _missing_required_route_diagnostics(
-        typing.cast(typing.Sequence[typing.Any], routes),
-        route_hits,
-        workflow_extract,
+    diagnostics.extend(
+        _missing_required_route_diagnostics(
+            typing.cast(typing.Sequence[typing.Any], routes),
+            route_hits,
+            workflow_extract,
+        )
     )
     if isinstance(relationships, list) and relationships:
         relationship_output, diagnostics = _apply_relationships(
@@ -159,6 +185,13 @@ def reassemble_custom_outputs_from_xray(
             diagnostics=diagnostics,
         )
         final_output = copy.deepcopy(relationship_output)
+    else:
+        diagnostics.extend(
+            _missing_relationship_metadata_diagnostics(
+                final_output,
+                workflow_extract,
+            )
+        )
 
     return CustomOutputReassemblyResult(
         final_output=final_output,
@@ -480,6 +513,56 @@ def _is_empty_output(value: typing.Any) -> bool:
     return False
 
 
+def _scalar_candidate(
+    value: typing.Any,
+    page_numbers: typing.Tuple[int, ...],
+) -> _ScalarCandidate:
+    return _ScalarCandidate(
+        value=value,
+        quality=_scalar_candidate_quality(value, page_numbers),
+    )
+
+
+def _scalar_candidate_quality(
+    value: typing.Any,
+    page_numbers: typing.Tuple[int, ...],
+) -> typing.Tuple[int, int, float]:
+    normalized = _normalized_candidate_value(value)
+    is_specific = 0 if _is_default_candidate_value(normalized) else 1
+    has_source_pages = 1 if page_numbers else 0
+    return (is_specific, has_source_pages, _candidate_confidence(value))
+
+
+def _normalized_candidate_value(value: typing.Any) -> typing.Any:
+    unwrapped = _unwrap_match_value(value)
+    if isinstance(unwrapped, str):
+        return " ".join(unwrapped.strip().casefold().split())
+    if isinstance(unwrapped, numbers.Real):
+        return float(unwrapped)
+    return unwrapped
+
+
+def _is_default_candidate_value(value: typing.Any) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, str):
+        return value in _DEFAULT_CANDIDATE_VALUES
+    return False
+
+
+def _candidate_confidence(value: typing.Any) -> float:
+    if not isinstance(value, typing.Mapping):
+        return 0.0
+    confidence = value.get("confidence")
+    if not isinstance(confidence, numbers.Real):
+        return 0.0
+    return max(0.0, min(1.0, float(confidence)))
+
+
+def _string_value(value: typing.Any) -> typing.Optional[str]:
+    return value if isinstance(value, str) else None
+
+
 def _effective_pointer(pointer: str, *, repeated: bool) -> str:
     if not repeated or "*" in pointer:
         return pointer
@@ -499,6 +582,10 @@ def _set_pointer(
         typing.Dict[str, typing.Any],
     ],
     record_key: typing.Tuple[typing.Any, ...],
+    route: typing.Mapping[str, typing.Any],
+    page_numbers: typing.Tuple[int, ...],
+    scalar_candidates: typing.Dict[str, _ScalarCandidate],
+    diagnostics: typing.List[CustomOutputDiagnostic],
 ) -> None:
     parts = _pointer_parts(pointer)
     if not parts:
@@ -538,7 +625,28 @@ def _set_pointer(
         if not isinstance(next_value, dict):
             return
         current = next_value
+    candidate = _scalar_candidate(value, page_numbers)
+    existing = scalar_candidates.get(pointer)
+    if existing is not None:
+        if candidate.quality < existing.quality:
+            return
+        if candidate.quality == existing.quality:
+            if _normalized_candidate_value(candidate.value) != _normalized_candidate_value(
+                existing.value
+            ):
+                diagnostics.append(
+                    CustomOutputDiagnostic(
+                        code="conflicting_output_candidates",
+                        message=f"multiple equal-quality candidates for [{pointer}]",
+                        severity="warning",
+                        workflow_group=_string_value(route.get("workflow_group")),
+                        workflow_field=_string_value(route.get("workflow_field")),
+                        final_path=pointer,
+                    )
+                )
+            return
     current[parts[-1]] = value
+    scalar_candidates[pointer] = candidate
 
 
 def _set_workflow_value(
@@ -670,6 +778,80 @@ def _missing_required_route_diagnostics(
     return diagnostics
 
 
+def _missing_relationship_metadata_diagnostics(
+    final_output: typing.Mapping[str, typing.Any],
+    workflow_extract: typing.Optional[typing.Mapping[str, typing.Any]],
+) -> typing.List[CustomOutputDiagnostic]:
+    if not isinstance(workflow_extract, typing.Mapping):
+        return []
+
+    diagnostics: typing.List[CustomOutputDiagnostic] = []
+    for group_name, group_spec in _relationship_intent_group_specs(workflow_extract):
+        if not isinstance(group_spec, typing.Mapping):
+            continue
+        if not _group_requires_relationship_metadata(group_spec):
+            continue
+        group_output = final_output.get(group_name)
+        if not isinstance(group_output, list) or len(group_output) == 0:
+            continue
+        diagnostics.append(
+            CustomOutputDiagnostic(
+                code="missing_output_relationships",
+                message=(
+                    f"workflow group [{group_name}] requires output relationship "
+                    "metadata but no output_relationships were provided"
+                ),
+                workflow_group=group_name,
+            )
+        )
+    return diagnostics
+
+
+def _relationship_intent_group_specs(
+    workflow_extract: typing.Mapping[str, typing.Any],
+) -> typing.Iterator[typing.Tuple[str, typing.Any]]:
+    seen: typing.Set[str] = set()
+    for container_key in (
+        "_groundx_persisted_extract",
+        "groups",
+        "prepared_final_groups",
+    ):
+        container = workflow_extract.get(container_key)
+        if not isinstance(container, typing.Mapping):
+            continue
+        for group_name, group_spec in container.items():
+            if isinstance(group_name, str) and group_name not in seen:
+                seen.add(group_name)
+                yield group_name, group_spec
+
+    for group_name, group_spec in workflow_extract.items():
+        if group_name in {
+            "_groundx_persisted_extract",
+            "groups",
+            "prepared_final_groups",
+            "workflow",
+        }:
+            continue
+        if isinstance(group_name, str) and group_name not in seen:
+            seen.add(group_name)
+            yield group_name, group_spec
+
+
+def _group_requires_relationship_metadata(
+    group_spec: typing.Mapping[str, typing.Any],
+) -> bool:
+    match_attrs = group_spec.get("match_attrs")
+    if isinstance(match_attrs, list) and len(match_attrs) > 0:
+        return True
+    passthrough = group_spec.get("passthrough")
+    if isinstance(passthrough, typing.Mapping) and isinstance(
+        passthrough.get("from"),
+        str,
+    ):
+        return True
+    return False
+
+
 def _route_is_required(
     route: typing.Mapping[str, typing.Any],
     workflow_extract: typing.Mapping[str, typing.Any],
@@ -834,7 +1016,11 @@ def _apply_relationships(
             )
             continue
 
-        parent_list = typing.cast(typing.List[typing.Dict[str, typing.Any]], parent_records)
+        parent_list = _dedupe_relationship_parents(
+            typing.cast(typing.List[typing.Dict[str, typing.Any]], parent_records),
+            typing.cast(typing.List[str], match_attrs),
+        )
+        result[parent_group] = parent_list
         child_list = typing.cast(typing.List[typing.Dict[str, typing.Any]], child_records)
         for parent in parent_list:
             parent.setdefault(parent_output_field, [])
@@ -883,6 +1069,50 @@ def _apply_relationships(
     return result, diagnostics
 
 
+def _dedupe_relationship_parents(
+    parent_list: typing.List[typing.Dict[str, typing.Any]],
+    match_attrs: typing.Sequence[str],
+) -> typing.List[typing.Dict[str, typing.Any]]:
+    deduped: typing.List[typing.Dict[str, typing.Any]] = []
+    by_key: typing.Dict[
+        typing.Tuple[typing.Tuple[str, typing.Any], ...],
+        typing.Dict[str, typing.Any],
+    ] = {}
+    for parent in parent_list:
+        parent_key = _match_key(parent, match_attrs)
+        if not parent_key:
+            deduped.append(parent)
+            continue
+        existing = by_key.get(parent_key)
+        if existing is None:
+            by_key[parent_key] = parent
+            deduped.append(parent)
+            continue
+        _merge_relationship_parent(existing, parent)
+    return deduped
+
+
+def _merge_relationship_parent(
+    target: typing.Dict[str, typing.Any],
+    source: typing.Mapping[str, typing.Any],
+) -> None:
+    for key, value in source.items():
+        if key not in target or _match_value_absent(target.get(key)):
+            target[key] = copy.deepcopy(value)
+            continue
+        target_value = target.get(key)
+        if isinstance(target_value, dict) and isinstance(value, typing.Mapping):
+            _merge_relationship_parent(
+                target_value,
+                typing.cast(typing.Mapping[str, typing.Any], value),
+            )
+            continue
+        if isinstance(target_value, list) and isinstance(value, list):
+            for item in value:
+                if item not in target_value:
+                    target_value.append(copy.deepcopy(item))
+
+
 def _relationship_application_order(
     relationships: typing.Sequence[typing.Any],
 ) -> typing.List[typing.Any]:

{groundx-3.8.2 → groundx-3.8.4}/src/groundx/extract/prompt/utility.py

@@ -1628,6 +1628,7 @@ def _collect_custom_workflow_routes(
     step_name: typing.Optional[str],
     steps_by_name: typing.Dict[str, typing.Dict[str, typing.Any]],
     prefix: typing.Tuple[str, ...] = (),
+    document_root: bool = False,
 ) -> typing.Tuple[
     typing.List[typing.Dict[str, typing.Any]],
     typing.List[typing.Dict[str, typing.Any]],
@@ -1655,6 +1656,7 @@ def _collect_custom_workflow_routes(
                         step_name,
                         steps_by_name,
                         (*prefix, field_name, "*"),
+                        document_root,
                     )
                     routes.extend(nested_routes)
                     leaves.extend(nested_leaves)
@@ -1683,7 +1685,11 @@ def _collect_custom_workflow_routes(
                 output_key_raw,
                 f"{field_path}.{_CUSTOM_WORKFLOW_FIELD_METADATA_KEY}",
             )
-            segments = (group_name, *prefix, field_name)
+            segments = (
+                (*prefix, field_name)
+                if document_root
+                else (group_name, *prefix, field_name)
+            )
             final_path = _encode_pointer(segments)
             level = typing.cast(str, step["level"])
             workflow_field = _custom_workflow_field_name(prefix, field_name)
@@ -1723,6 +1729,7 @@ def _collect_custom_workflow_routes(
                 step_name,
                 steps_by_name,
                 (*prefix, field_name),
+                document_root,
             )
             routes.extend(nested_routes)
             leaves.extend(nested_leaves)
@@ -1733,6 +1740,7 @@ def _collect_custom_workflow_routes(
                 step_name,
                 steps_by_name,
                 (*prefix, field_name),
+                document_root,
             )
             routes.extend(nested_routes)
             leaves.extend(nested_leaves)
@@ -1740,6 +1748,46 @@ def _collect_custom_workflow_routes(
     return routes, leaves
 
 
+def _agent_chain_group_roles(raw_chain: typing.Any) -> typing.Dict[str, str]:
+    roles: typing.Dict[str, str] = {}
+    if not isinstance(raw_chain, list):
+        return roles
+
+    for raw_stage in raw_chain:
+        if not isinstance(raw_stage, dict) or set(raw_stage.keys()) != {"parallel"}:
+            continue
+        raw_branches = raw_stage["parallel"]
+        if not isinstance(raw_branches, list):
+            continue
+        for raw_branch in raw_branches:
+            if not isinstance(raw_branch, dict):
+                continue
+            group = raw_branch.get("group")
+            chain = raw_branch.get("chain")
+            if not isinstance(group, str) or not isinstance(chain, list):
+                continue
+            suffixes = {
+                _agent_chain_task_suffix(task)
+                for task in chain
+                if isinstance(task, str)
+                and task in _CUSTOM_WORKFLOW_AGENT_CHAIN_SUPPORTED_TASKS
+            }
+            if len(suffixes) == 1:
+                roles[group] = next(iter(suffixes))
+    return roles
+
+
+def _is_document_root_statement_group(
+    group_metadata: typing.Mapping[str, typing.Any],
+    group_role: typing.Optional[str],
+) -> bool:
+    if group_role != "statement":
+        return False
+    return isinstance(group_metadata.get("final_value_aliases"), dict) or isinstance(
+        group_metadata.get("fill_rules"), list
+    )
+
+
 def _collect_pseudo_custom_workflow_routes(
     group_name: str,
     group: typing.Dict[str, typing.Any],
@@ -1830,6 +1878,9 @@ def _build_authored_custom_workflow_metadata(
 
     routes: typing.List[typing.Dict[str, typing.Any]] = []
     leaves: typing.List[typing.Dict[str, typing.Any]] = []
+    agent_chain_group_roles = _agent_chain_group_roles(
+        workflow.get(_CUSTOM_WORKFLOW_AGENT_CHAIN_KEY)
+    )
     for group_name, group in workflow_groups.items():
         group_step = workflow_group_metadata.get(group_name, {}).get(
             _CUSTOM_WORKFLOW_GROUP_METADATA_KEY
@@ -1846,11 +1897,16 @@ def _build_authored_custom_workflow_metadata(
                 workflow_field_paths.get(group_name, {}),
             )
         else:
+            document_root = _is_document_root_statement_group(
+                final_group_metadata.get(group_name, {}),
+                agent_chain_group_roles.get(group_name),
+            )
             group_routes, group_leaves = _collect_custom_workflow_routes(
                 fields,
                 group_name,
                 typing.cast(typing.Optional[str], group_step),
                 steps_by_name,
+                document_root=document_root,
             )
         routes.extend(group_routes)
         leaves.extend(group_leaves)