groundx 2.4.5__py3-none-any.whl → 2.4.9__py3-none-any.whl

groundx/extract/classes/field.py

@@ -0,0 +1,88 @@
+import dateparser, typing
+
+from pydantic import BaseModel
+
+
+class ExtractedField(BaseModel):
+    confidence: typing.Optional[str] = None
+    conflicts: typing.List[typing.Any] = []
+    key: str
+
+    value: typing.Union[str, float, typing.List[typing.Any]] = ""
+
+    def __init__(
+        self,
+        value: typing.Union[str, float, typing.List[typing.Any]],
+        **data: typing.Any,
+    ):
+        super().__init__(**data)
+
+        self.set_value(value)
+
+    def contains(self, other: "ExtractedField") -> bool:
+        self_val = self.get_value()
+        other_val = other.get_value()
+        if not (isinstance(self_val, (str, float, int))):
+            raise Exception(f"unexpected self field value type [{type(self_val)}]")
+
+        if self.equal_to_value(other_val):
+            return True
+
+        if other_val in self.conflicts:
+            return True
+
+        return False
+
+    def equal_to_field(self, other: "ExtractedField") -> bool:
+        self_val = self.get_value()
+        other_val = other.get_value()
+        if not (isinstance(self_val, (str, float, int))):
+            raise Exception(f"unexpected self field value type [{type(self_val)}]")
+
+        return self.equal_to_value(other_val)
+
+    def equal_to_value(self, other: typing.Any) -> bool:
+        if not (isinstance(other, (str, float, int))):
+            raise Exception(f"unexpected value type [{type(other)}]")
+
+        exist = self.get_value()
+        if isinstance(exist, int):
+            exist = float(exist)
+        if isinstance(other, int):
+            other = float(other)
+        if isinstance(exist, str):
+            exist = exist.lower()
+        if isinstance(other, str):
+            other = other.lower()
+
+        return type(other) == type(exist) and other == exist
+
+    def get_value(self) -> typing.Union[str, float, typing.List[typing.Any]]:
+        return self.value
+
+    def remove_conflict(self, value: typing.Any) -> None:
+        if value in self.conflicts:
+            self.conflicts.remove(value)
+        if not self.equal_to_value(value):
+            self.conflicts.append(self.get_value())
+
+    def set_value(
+        self, value: typing.Union[str, float, typing.List[typing.Any]]
+    ) -> None:
+        if isinstance(value, int):
+            self.value = float(value)
+        elif isinstance(value, str) and "date" in self.key.lower():
+            try:
+                dt = dateparser.parse(value)
+                if dt is None:
+                    self.value = value
+                else:
+                    self.value = dt.strftime("%Y-%m-%d")
+            except Exception as e:
+                print(f"date error [{value}]: [{e}]")
+                self.value = value
+        else:
+            self.value = value
+
+
+ExtractedField.model_rebuild()

groundx/extract/classes/groundx.py

@@ -0,0 +1,123 @@
+import json, requests, typing
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class GroundXDocument(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+    base_url: str
+    document_id: str = Field(alias="documentID")
+    task_id: str = Field(alias="taskID")
+
+    def xray_url(self, base: typing.Optional[str] = None) -> str:
+        if not base:
+            base = self.base_url
+        if base.endswith("/"):
+            base = base[:-1]
+        return f"{base}/layout/processed/{self.task_id}/{self.document_id}-xray.json"
+
+    def xray(
+        self,
+        clear_cache: bool = False,
+        is_test: bool = False,
+        base: typing.Optional[str] = None,
+    ) -> "XRayDocument":
+        return XRayDocument.download(
+            self, base=base, clear_cache=clear_cache, is_test=is_test
+        )
+
+
+class GroundXResponse(BaseModel):
+    code: int
+    document_id: str = Field(alias="documentID")
+    model_id: int = Field(alias="modelID")
+    processor_id: int = Field(alias="processorID")
+    result_url: str = Field(alias="resultURL")
+    task_id: str = Field(alias="taskID")
+
+
+class BoundingBox(BaseModel):
+    bottomRightX: float
+    bottomRightY: float
+    topLeftX: float
+    topLeftY: float
+    corrected: typing.Optional[bool]
+    pageNumber: typing.Optional[int]
+
+
+class Chunk(BaseModel):
+    boundingBoxes: typing.Optional[typing.List[BoundingBox]] = []
+    chunk: typing.Optional[str] = None
+    contentType: typing.Optional[typing.List[str]] = []
+    json_: typing.Optional[typing.List[typing.Any]] = Field(None, alias="json")
+    multimodalUrl: typing.Optional[str] = None
+    narrative: typing.Optional[typing.List[str]] = None
+    pageNumbers: typing.Optional[typing.List[int]] = []
+    sectionSummary: typing.Optional[str] = None
+    suggestedText: typing.Optional[str] = None
+    text: typing.Optional[str] = None
+
+
+class DocumentPage(BaseModel):
+    chunks: typing.List[Chunk]
+    height: float
+    pageNumber: int
+    pageUrl: str
+    width: float
+
+
+class XRayDocument(BaseModel):
+    chunks: typing.List[Chunk]
+    documentPages: typing.List[DocumentPage] = []
+    sourceUrl: str
+    fileKeywords: typing.Optional[str] = None
+    fileName: typing.Optional[str] = None
+    fileType: typing.Optional[str] = None
+    fileSummary: typing.Optional[str] = None
+    language: typing.Optional[str] = None
+
+    @classmethod
+    def download(
+        cls,
+        gx_doc: GroundXDocument,
+        clear_cache: bool = False,
+        is_test: bool = False,
+        base: typing.Optional[str] = None,
+    ) -> "XRayDocument":
+        cache_dir = Path(__file__).resolve().parent.parent / "cache"
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file = cache_dir / f"{gx_doc.document_id}-xray.json"
+
+        if not clear_cache and cache_file.exists():
+            try:
+                with cache_file.open("r", encoding="utf-8") as f:
+                    payload = json.load(f)
+
+            except Exception as e:
+                raise RuntimeError(
+                    f"Error loading cached X-ray JSON from {cache_file}: {e}"
+                )
+        else:
+            url = gx_doc.xray_url(base=base)
+            try:
+                resp = requests.get(url)
+                resp.raise_for_status()
+            except requests.RequestException as e:
+                raise RuntimeError(f"Error fetching X-ray JSON from {url}: {e}")
+
+            try:
+                payload = resp.json()
+            except ValueError as e:
+                raise RuntimeError(f"Invalid JSON returned from {url}: {e}")
+
+            if is_test is False:
+                try:
+                    with cache_file.open("w", encoding="utf-8") as f:
+                        json.dump(payload, f)
+                except Exception as e:
+                    print(
+                        f"Warning: failed to write X-ray JSON cache to {cache_file}: {e}"
+                    )
+
+        return cls(**payload)

groundx/extract/classes/post_process.py

@@ -0,0 +1,33 @@
+import typing
+
+
+def check_map(
+    fty: str,
+    sty: str,
+    val: str,
+    mp: typing.Dict[str, typing.Dict[str, str]],
+    should_warn: bool = True,
+) -> typing.Optional[str]:
+    if sty not in mp:
+        sty = ""
+    if sty not in mp:
+        return None
+
+    vl = val.lower().strip()
+
+    nmp = mp[sty]
+    if vl not in nmp:
+        if should_warn:
+            print(f"[arcadia-v1] {fty} not found [{sty}] [{vl}]")
+        return None
+
+    return nmp[vl]
+
+
+def check_valid(sty: str, val: str, valid: typing.Dict[str, typing.List[str]]) -> bool:
+    vl = val.lower().strip()
+
+    if sty not in valid:
+        sty = ""
+
+    return sty in valid and vl in valid[sty]

groundx/extract/classes/prompt.py

@@ -0,0 +1,36 @@
+import typing
+
+from pydantic import BaseModel
+
+from .utility import str_to_type_sequence
+
+
+class Prompt(BaseModel):
+    attr_name: str
+    prompt: str
+    type: typing.Union[str, typing.List[str]]
+
+    class Config:
+        validate_by_name = True
+
+    def valid_value(self, value: typing.Any) -> bool:
+        ty = self.type
+
+        types: typing.List[typing.Type[typing.Any]] = []
+        if isinstance(ty, list):
+            for t in ty:
+                if t == "int" or t == "float":
+                    types.extend([int, float])
+                elif t == "str":
+                    types.append(str)
+
+            return isinstance(value, tuple(types))
+
+        exp = str_to_type_sequence(ty)
+        for et in exp:
+            if et in (int, float):
+                types.extend([int, float])
+            else:
+                types.append(et)
+        types = list(dict.fromkeys(types))
+        return isinstance(value, tuple(types))

groundx/extract/classes/settings.py

@@ -0,0 +1,169 @@
+import json, typing, os
+
+from pydantic import BaseModel
+
+
+AWS_REGION: str = "AWS_REGION"
+AWS_DEFAULT_REGION: str = "AWS_DEFAULT_REGION"
+
+
+GX_AGENT_KEY: str = "GROUNDX_AGENT_API_KEY"
+CALLBACK_KEY: str = "GROUNDX_CALLBACK_API_KEY"
+GCP_CREDENTIALS: str = "GCP_CREDENTIALS"
+GX_API_KEY: str = "GROUNDX_API_KEY"
+GX_KEY: str = "GROUNDX_ACCESS_KEY_ID"
+GX_REGION: str = "GROUNDX_REGION"
+GX_DEFAULT_REGION: str = "GROUNDX_DEFAULT_REGION"
+GX_SECRET: str = "GROUNDX_SECRET_ACCESS_KEY"
+GX_TOKEN: str = "GROUNDX_SESSION_TOKEN"
+VALID_KEYS: str = "GROUNDX_VALID_API_KEYS"
+
+
+class AgentSettings(BaseModel):
+    api_base: typing.Optional[str] = None
+    api_key: typing.Optional[str] = None
+    imports: typing.List[str] = [
+        "csv",
+        "glob",
+        "io",
+        "json",
+        "markdown",
+        "numpy",
+        "os",
+        "pandas",
+        "posixpath",
+        "open",
+        "builtins.open",
+        "utils.safe_open",
+        "pydantic",
+        "typing",
+    ]
+    max_steps: int = 7
+    model_id: str = "gpt-5-mini"
+
+    def get_api_key(self) -> str:
+        if self.api_key:
+            return self.api_key
+
+        key = os.environ.get(GX_AGENT_KEY)
+        if key:
+            return key
+
+        raise Exception(f"you must set a valid agent api_key")
+
+
+class ContainerSettings(BaseModel):
+    broker: str
+    cache_to: int = 300
+    google_sheets_drive_id: typing.Optional[str] = None
+    google_sheets_template_id: typing.Optional[str] = None
+    log_level: str = "info"
+    metrics_broker: typing.Optional[str] = None
+    refresh_to: int = 60
+    service: str
+    task_to: int = 600
+    upload: "ContainerUploadSettings"
+    workers: int
+
+    callback_api_key: typing.Optional[str] = None
+    valid_api_keys: typing.Optional[typing.List[str]] = None
+
+    def get_callback_api_key(self) -> str:
+        if self.callback_api_key:
+            return self.callback_api_key
+
+        key = os.environ.get(CALLBACK_KEY)
+        if key:
+            return key
+
+        raise Exception(f"you must set a callback_api_key")
+
+    def get_valid_api_keys(self) -> typing.List[str]:
+        if self.valid_api_keys:
+            return self.valid_api_keys
+
+        keys: typing.Optional[str] = os.environ.get(VALID_KEYS)
+        if not keys:
+            raise Exception(f"you must set an array of valid_api_keys")
+
+        try:
+            data: typing.List[str] = json.loads(keys)
+        except Exception as e:
+            raise Exception(f"you must set an array of valid_api_keys: {e}")
+
+        return data
+
+    def loglevel(self) -> str:
+        return self.log_level.upper()
+
+    def status_broker(self) -> str:
+        if self.metrics_broker:
+            return self.metrics_broker
+
+        return self.broker
+
+
+class ContainerUploadSettings(BaseModel):
+    base_domain: str
+    base_path: str = "layout/processed/"
+    bucket: str
+    ssl: bool = False
+    type: str
+    url: str
+
+    key: typing.Optional[str] = None
+    region: typing.Optional[str] = None
+    secret: typing.Optional[str] = None
+    token: typing.Optional[str] = None
+
+    def get_key(self) -> typing.Optional[str]:
+        if self.key:
+            return self.key
+
+        return os.environ.get(GX_KEY)
+
+    def get_region(self) -> typing.Optional[str]:
+        if self.region:
+            return self.region
+
+        key = os.environ.get(GX_REGION)
+        if key:
+            return key
+
+        key = os.environ.get(AWS_REGION)
+        if key:
+            return key
+
+        key = os.environ.get(GX_DEFAULT_REGION)
+        if key:
+            return key
+
+        return os.environ.get(AWS_DEFAULT_REGION)
+
+    def get_secret(self) -> typing.Optional[str]:
+        if self.secret:
+            return self.secret
+
+        return os.environ.get(GX_SECRET)
+
+    def get_token(self) -> typing.Optional[str]:
+        if self.token:
+            return self.token
+
+        return os.environ.get(GX_TOKEN)
+
+
+class GroundXSettings(BaseModel):
+    api_key: typing.Optional[str] = None
+    base_url: typing.Optional[str] = None
+    upload_url: str = "https://upload.eyelevel.ai"
+
+    def get_api_key(self) -> str:
+        if self.api_key:
+            return self.api_key
+
+        key = os.environ.get(GX_API_KEY)
+        if key:
+            return key
+
+        raise Exception(f"you must set a valid GroundX api_key")

groundx/extract/classes/test_document.py

@@ -0,0 +1,126 @@
+import pytest, typing, unittest
+
+pytest.importorskip("PIL")
+
+from io import BytesIO
+from PIL import Image
+from unittest.mock import patch
+
+from .document import Document, DocumentRequest
+
+
+def DR(**data: typing.Any) -> DocumentRequest:
+    return DocumentRequest.model_validate(data)
+
+
+def test_request() -> DocumentRequest:
+    return DR(documentID="D", fileName="F", modelID=1, processorID=1, taskID="T")
+
+
+class DummyChunk:
+    def __init__(self, json_str: str):
+        self.sectionSummary = None
+        self.suggestedText = json_str
+
+
+class DummyDocumentPage:
+    def __init__(self, page_url: str):
+        self.pageUrl = page_url
+
+
+class DummyXRay:
+    def __init__(
+        self,
+        source_url: str,
+        chunks: typing.Optional[typing.List[DummyChunk]] = [],
+        document_pages: typing.Optional[typing.List[str]] = [],
+    ):
+        self.chunks = chunks
+        self.documentPages: typing.List[DummyDocumentPage] = []
+        if document_pages is not None:
+            for p in document_pages:
+                self.documentPages.append(DummyDocumentPage(p))
+        self.sourceUrl = source_url
+
+
+class TestDocument(unittest.TestCase):
+    def setUp(self) -> None:
+        patcher = patch(
+            "groundx.extract.classes.document.GroundXDocument.xray", autospec=True
+        )
+        self.mock_xray = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.mock_xray.return_value = DummyXRay("http://test.co", [])
+
+    def test_init_name(self) -> None:
+        st1: Document = Document.from_request(
+            base_url="",
+            req=test_request(),
+        )
+        self.assertEqual(st1.file_name, "F")
+        st2: Document = Document.from_request(
+            base_url="",
+            req=DR(
+                documentID="D", fileName="F.pdf", modelID=1, processorID=1, taskID="T"
+            ),
+        )
+        self.assertEqual(st2.file_name, "F.pdf")
+        st3: Document = Document.from_request(
+            base_url="",
+            req=DR(documentID="D", fileName="F.", modelID=1, processorID=1, taskID="T"),
+        )
+        self.assertEqual(st3.file_name, "F.")
+
+
+class TestDocumentRequest(unittest.TestCase):
+    def test_load_images_cached(self) -> None:
+        urls: typing.List[str] = [
+            "http://example.com/page1.png",
+            "http://example.com/page2.png",
+        ]
+
+        red_img = Image.new("RGB", (10, 10), color="red")
+        buf = BytesIO()
+        red_img.save(buf, format="PNG")
+
+        st = test_request()
+        st.page_images = [red_img, red_img]
+        st.page_image_dict = {
+            urls[0]: 0,
+            urls[1]: 1,
+        }
+        st.load_images(urls)
+        self.assertEqual(len(st.page_images), 2)
+        self.assertEqual(len(st.page_image_dict), 2)
+
+    def test_load_images_download(self) -> None:
+        urls = ["http://example.com/page1.png", "http://example.com/page2.png"]
+
+        red_img = Image.new("RGB", (10, 10), color="red")
+        buf = BytesIO()
+        red_img.save(buf, format="PNG")
+        img_bytes = buf.getvalue()
+
+        class DummyResp:
+            content = img_bytes
+
+            def raise_for_status(self) -> None:
+                pass
+
+        with patch("requests.get", return_value=DummyResp()):
+            st = test_request()
+            st.load_images(urls)
+
+            self.assertEqual(len(st.page_images), 2)
+            self.assertEqual(len(st.page_image_dict), 2)
+            for img in st.page_images:
+                self.assertIsInstance(img, Image.Image)
+                self.assertEqual(img.size, (10, 10))
+
+    def test_load_images_error(self) -> None:
+        urls = ["http://example.com/page1.png", "http://example.com/page2.png"]
+
+        st = test_request()
+        st.load_images(urls, should_sleep=False)
+        self.assertEqual(len(st.page_images), 0)
+        self.assertEqual(len(st.page_image_dict), 0)

groundx/extract/classes/test_field.py

@@ -0,0 +1,43 @@
+import pytest, typing, unittest
+
+pytest.importorskip("dateparser")
+
+from .field import ExtractedField
+
+
+def TestField(
+    name: str,
+    value: typing.Union[str, float, typing.List[typing.Any]],
+    conflicts: typing.List[typing.Any] = [],
+) -> ExtractedField:
+    return ExtractedField(
+        key=name.replace("_", " "),
+        value=value,
+        conflicts=conflicts,
+    )
+
+
+class TestExtractedField(unittest.TestCase):
+    def test_equalToValue_string(self):
+        ef = TestField("test", "hello")
+        self.assertTrue(ef.equal_to_value("hello"))
+        self.assertFalse(ef.equal_to_value("world"))
+
+    def test_equalToValue_int_float_equivalence(self):
+        ef = TestField("test", int(10))
+        self.assertTrue(ef.equal_to_value(10.0))
+        self.assertTrue(ef.equal_to_value(10))
+
+    def test_equalToValue_mismatch(self):
+        ef = TestField("test", 3.14)
+        self.assertFalse(ef.equal_to_value(2.71))
+
+    def test_set_value_dates(self):
+        ef1 = TestField("test date", "3/29/25")
+        self.assertEqual(ef1.get_value(), "2025-03-29")
+        ef2 = TestField("test date", "2025-03-29")
+        self.assertEqual(ef2.get_value(), "2025-03-29")
+
+
+if __name__ == "__main__":
+    unittest.main()