groundx 2.3.7__tar.gz → 2.4.0__tar.gz

{groundx-2.3.7 → groundx-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: groundx
-Version: 2.3.7
+Version: 2.4.0
 Summary: 
 License: MIT
 Requires-Python: >=3.8,<4.0

{groundx-2.3.7 → groundx-2.4.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "groundx"
 
 [tool.poetry]
 name = "groundx"
-version = "2.3.7"
+version = "2.4.0"
 description = ""
 readme = "README.md"
 authors = []

{groundx-2.3.7 → groundx-2.4.0}/src/groundx/core/client_wrapper.py

@@ -14,10 +14,10 @@ class BaseClientWrapper:
 
     def get_headers(self) -> typing.Dict[str, str]:
         headers: typing.Dict[str, str] = {
-            "User-Agent": "groundx/2.3.7",
+            "User-Agent": "groundx/2.4.0",
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "groundx",
-            "X-Fern-SDK-Version": "2.3.7",
+            "X-Fern-SDK-Version": "2.4.0",
         }
         headers["X-API-Key"] = self.api_key
         return headers

{groundx-2.3.7 → groundx-2.4.0}/src/groundx/documents/client.py

@@ -40,6 +40,8 @@ class DocumentsClient:
         self,
         *,
         documents: typing.Sequence[IngestRemoteDocument],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -51,6 +53,12 @@ class DocumentsClient:
         ----------
         documents : typing.Sequence[IngestRemoteDocument]
 
+        callback_url : typing.Optional[str]
+            An endpoint that will receive processing event updates as POST.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -77,7 +85,9 @@ class DocumentsClient:
             ],
         )
         """
-        _response = self._raw_client.ingest_remote(documents=documents, request_options=request_options)
+        _response = self._raw_client.ingest_remote(
+            documents=documents, callback_url=callback_url, callback_data=callback_data, request_options=request_options
+        )
         return _response.data
 
     def ingest_local(
@@ -124,7 +134,12 @@ class DocumentsClient:
         return _response.data
 
     def crawl_website(
-        self, *, websites: typing.Sequence[WebsiteSource], request_options: typing.Optional[RequestOptions] = None
+        self,
+        *,
+        websites: typing.Sequence[WebsiteSource],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
         Upload the content of a publicly accessible website for ingestion into a GroundX bucket. This is done by following links within a specified URL, recursively, up to a specified depth or number of pages.
@@ -138,6 +153,12 @@ class DocumentsClient:
         ----------
         websites : typing.Sequence[WebsiteSource]
 
+        callback_url : typing.Optional[str]
+            The URL that will receive processing event updates.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -165,7 +186,9 @@ class DocumentsClient:
             ],
         )
         """
-        _response = self._raw_client.crawl_website(websites=websites, request_options=request_options)
+        _response = self._raw_client.crawl_website(
+            websites=websites, callback_url=callback_url, callback_data=callback_data, request_options=request_options
+        )
         return _response.data
 
     def list(
@@ -490,6 +513,8 @@ class AsyncDocumentsClient:
         self,
         *,
         documents: typing.Sequence[IngestRemoteDocument],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -501,6 +526,12 @@ class AsyncDocumentsClient:
         ----------
         documents : typing.Sequence[IngestRemoteDocument]
 
+        callback_url : typing.Optional[str]
+            An endpoint that will receive processing event updates as POST.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -535,7 +566,9 @@ class AsyncDocumentsClient:
 
         asyncio.run(main())
         """
-        _response = await self._raw_client.ingest_remote(documents=documents, request_options=request_options)
+        _response = await self._raw_client.ingest_remote(
+            documents=documents, callback_url=callback_url, callback_data=callback_data, request_options=request_options
+        )
         return _response.data
 
     async def ingest_local(
@@ -594,7 +627,12 @@ class AsyncDocumentsClient:
         return _response.data
 
     async def crawl_website(
-        self, *, websites: typing.Sequence[WebsiteSource], request_options: typing.Optional[RequestOptions] = None
+        self,
+        *,
+        websites: typing.Sequence[WebsiteSource],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
         Upload the content of a publicly accessible website for ingestion into a GroundX bucket. This is done by following links within a specified URL, recursively, up to a specified depth or number of pages.
@@ -608,6 +646,12 @@ class AsyncDocumentsClient:
         ----------
         websites : typing.Sequence[WebsiteSource]
 
+        callback_url : typing.Optional[str]
+            The URL that will receive processing event updates.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -643,7 +687,9 @@ class AsyncDocumentsClient:
 
         asyncio.run(main())
         """
-        _response = await self._raw_client.crawl_website(websites=websites, request_options=request_options)
+        _response = await self._raw_client.crawl_website(
+            websites=websites, callback_url=callback_url, callback_data=callback_data, request_options=request_options
+        )
         return _response.data
 
     async def list(

{groundx-2.3.7 → groundx-2.4.0}/src/groundx/documents/raw_client.py

@@ -36,6 +36,8 @@ class RawDocumentsClient:
         self,
         *,
         documents: typing.Sequence[IngestRemoteDocument],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> HttpResponse[IngestResponse]:
         """
@@ -47,6 +49,12 @@ class RawDocumentsClient:
         ----------
         documents : typing.Sequence[IngestRemoteDocument]
 
+        callback_url : typing.Optional[str]
+            An endpoint that will receive processing event updates as POST.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -62,6 +70,8 @@ class RawDocumentsClient:
                 "documents": convert_and_respect_annotation_metadata(
                     object_=documents, annotation=typing.Sequence[IngestRemoteDocument], direction="write"
                 ),
+                "callbackUrl": callback_url,
+                "callbackData": callback_data,
             },
             headers={
                 "content-type": "application/json",
@@ -176,7 +186,12 @@ class RawDocumentsClient:
         raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
 
     def crawl_website(
-        self, *, websites: typing.Sequence[WebsiteSource], request_options: typing.Optional[RequestOptions] = None
+        self,
+        *,
+        websites: typing.Sequence[WebsiteSource],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> HttpResponse[IngestResponse]:
         """
         Upload the content of a publicly accessible website for ingestion into a GroundX bucket. This is done by following links within a specified URL, recursively, up to a specified depth or number of pages.
@@ -190,6 +205,12 @@ class RawDocumentsClient:
         ----------
         websites : typing.Sequence[WebsiteSource]
 
+        callback_url : typing.Optional[str]
+            The URL that will receive processing event updates.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -205,6 +226,8 @@ class RawDocumentsClient:
                 "websites": convert_and_respect_annotation_metadata(
                     object_=websites, annotation=typing.Sequence[WebsiteSource], direction="write"
                 ),
+                "callbackUrl": callback_url,
+                "callbackData": callback_data,
             },
             headers={
                 "content-type": "application/json",
@@ -724,6 +747,8 @@ class AsyncRawDocumentsClient:
         self,
         *,
         documents: typing.Sequence[IngestRemoteDocument],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> AsyncHttpResponse[IngestResponse]:
         """
@@ -735,6 +760,12 @@ class AsyncRawDocumentsClient:
         ----------
         documents : typing.Sequence[IngestRemoteDocument]
 
+        callback_url : typing.Optional[str]
+            An endpoint that will receive processing event updates as POST.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -750,6 +781,8 @@ class AsyncRawDocumentsClient:
                 "documents": convert_and_respect_annotation_metadata(
                     object_=documents, annotation=typing.Sequence[IngestRemoteDocument], direction="write"
                 ),
+                "callbackUrl": callback_url,
+                "callbackData": callback_data,
             },
             headers={
                 "content-type": "application/json",
@@ -864,7 +897,12 @@ class AsyncRawDocumentsClient:
         raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
 
     async def crawl_website(
-        self, *, websites: typing.Sequence[WebsiteSource], request_options: typing.Optional[RequestOptions] = None
+        self,
+        *,
+        websites: typing.Sequence[WebsiteSource],
+        callback_url: typing.Optional[str] = OMIT,
+        callback_data: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
     ) -> AsyncHttpResponse[IngestResponse]:
         """
         Upload the content of a publicly accessible website for ingestion into a GroundX bucket. This is done by following links within a specified URL, recursively, up to a specified depth or number of pages.
@@ -878,6 +916,12 @@ class AsyncRawDocumentsClient:
         ----------
         websites : typing.Sequence[WebsiteSource]
 
+        callback_url : typing.Optional[str]
+            The URL that will receive processing event updates.
+
+        callback_data : typing.Optional[str]
+            A string that is returned, along with processing event updates, to the callback URL.
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -893,6 +937,8 @@ class AsyncRawDocumentsClient:
                 "websites": convert_and_respect_annotation_metadata(
                     object_=websites, annotation=typing.Sequence[WebsiteSource], direction="write"
                 ),
+                "callbackUrl": callback_url,
+                "callbackData": callback_data,
             },
             headers={
                 "content-type": "application/json",

{groundx-2.3.7 → groundx-2.4.0}/src/groundx/ingest.py

@@ -125,7 +125,7 @@ def prep_documents(
     return remote_documents, local_documents
 
 
-def split_doc(file):
+def split_doc(file: Path) -> typing.List[Path]:
     if file.is_file() and (
         file.suffix.lower() in ALLOWED_SUFFIXES
         or file.suffix.lower() in SUFFIX_ALIASES
@@ -142,9 +142,11 @@ class GroundX(GroundXBase):
         self,
         *,
         documents: typing.Sequence[Document],
-        batch_size: typing.Optional[int] = 10,
-        wait_for_complete: typing.Optional[bool] = False,
-        upload_api: typing.Optional[str] = "https://api.eyelevel.ai/upload/file",
+        batch_size: int = 10,
+        wait_for_complete: bool = False,
+        upload_api: str = "https://api.eyelevel.ai/upload/file",
+        callback_url: typing.Optional[str] = None,
+        callback_data: typing.Optional[str] = None,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -165,6 +167,13 @@ class GroundX(GroundXBase):
         # and returns a presigned URL in a JSON dictionary with key 'URL'
         upload_api : typing.Optional[str]
 
+        # an endpoint that will receive processing event updates as POST
+        callback_url : typing.Optional[str]
+
+        # a string that is returned, along with processing event updates,
+        # to the callback URL.
+        callback_data : typing.Optional[str]
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -209,6 +218,8 @@ class GroundX(GroundXBase):
                     if len(remote_batch) >= n:
                         ingest = self.documents.ingest_remote(
                             documents=remote_batch,
+                            callback_url=callback_url,
+                            callback_data=callback_data,
                             request_options=request_options,
                         )
                         ingest, progress = self._monitor_batch(ingest, progress, pbar)
@@ -222,6 +233,8 @@ class GroundX(GroundXBase):
                 if remote_batch:
                     ingest = self.documents.ingest_remote(
                         documents=remote_batch,
+                        callback_data=callback_data,
+                        callback_url=callback_url,
                         request_options=request_options,
                     )
                     ingest, progress = self._monitor_batch(ingest, progress, pbar)
@@ -243,6 +256,8 @@ class GroundX(GroundXBase):
 
                         ingest = self.documents.ingest_remote(
                             documents=up_docs,
+                            callback_url=callback_url,
+                            callback_data=callback_data,
                             request_options=request_options,
                         )
                         ingest, progress = self._monitor_batch(ingest, progress, pbar)
@@ -258,6 +273,8 @@ class GroundX(GroundXBase):
 
                     ingest = self.documents.ingest_remote(
                         documents=up_docs,
+                        callback_data=callback_data,
+                        callback_url=callback_url,
                         request_options=request_options,
                     )
                     ingest, progress = self._monitor_batch(ingest, progress, pbar)
@@ -270,11 +287,13 @@ class GroundX(GroundXBase):
             raise ValueError("You have sent too many documents in this request")
 
 
-        up_docs, _ = self._process_local(local_documents, upload_api)
+        up_docs, _ = self._process_local(local_documents, upload_api, 0, None)
         remote_documents.extend(up_docs)
 
         return self.documents.ingest_remote(
             documents=remote_documents,
+            callback_url=callback_url,
+            callback_data=callback_data,
             request_options=request_options,
         )
 
@@ -283,8 +302,10 @@ class GroundX(GroundXBase):
         *,
         bucket_id: int,
         path: str,
-        batch_size: typing.Optional[int] = 10,
-        upload_api: typing.Optional[str] = "https://api.eyelevel.ai/upload/file",
+        batch_size: int = 10,
+        upload_api: str = "https://api.eyelevel.ai/upload/file",
+        callback_url: typing.Optional[str] = None,
+        callback_data: typing.Optional[str] = None,
         request_options: typing.Optional[RequestOptions] = None,
     ):
         """
@@ -300,6 +321,13 @@ class GroundX(GroundXBase):
         # and returns a presigned URL in a JSON dictionary with key 'URL'
         upload_api : typing.Optional[str]
 
+        # an endpoint that will receive processing event updates as POST
+        callback_url : typing.Optional[str]
+
+        # a string that is returned, along with processing event updates,
+        # to the callback URL.
+        callback_data : typing.Optional[str]
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -357,7 +385,7 @@ class GroundX(GroundXBase):
                 file_size = file.stat().st_size
 
                 if (current_batch_size + file_size > MAX_BATCH_SIZE_BYTES) or (len(current_batch) >= n):
-                    self._upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+                    self._upload_file_batch(bucket_id, current_batch, upload_api, callback_url, callback_data, request_options, pbar)
                     current_batch = []
                     current_batch_size = 0
 
@@ -365,13 +393,13 @@ class GroundX(GroundXBase):
                 current_batch_size += file_size
 
             if current_batch:
-                self._upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+                self._upload_file_batch(bucket_id, current_batch, upload_api, callback_url, callback_data, request_options, pbar)
 
     def _upload_file(
         self,
-        endpoint,
-        file_path,
-    ):
+        endpoint: str,
+        file_path: Path,
+    ) -> str:
         file_name = os.path.basename(file_path)
         file_extension = os.path.splitext(file_name)[1][1:].lower()
         if f".{file_extension}" in SUFFIX_ALIASES:
@@ -407,12 +435,12 @@ class GroundX(GroundXBase):
 
     def _process_local(
         self,
-        local_docs,
-        upload_api,
-        progress = None,
-        pbar = None,
-    ):
-        remote_docs = []
+        local_docs: typing.List[Document],
+        upload_api: str,
+        progress: float,
+        pbar: typing.Optional[typing.Any] = None,
+    ) -> typing.Tuple[typing.List[IngestRemoteDocument], float]:
+        remote_docs: typing.List[IngestRemoteDocument] = []
         for d in local_docs:
             splits = split_doc(Path(os.path.expanduser(d.file_path)))
 
@@ -439,23 +467,22 @@ class GroundX(GroundXBase):
                     )
                 )
 
-                if progress is not None and pbar is not None and pbar.update is not None:
+                progress -= 0.25
+                if pbar is not None and pbar.update is not None:
                     pbar.update(0.25)
-                    progress -= 0.25
 
         return remote_docs, progress
 
     def _monitor_batch(
         self,
-        ingest,
-        progress,
-        pbar,
-    ):
-        completed_files = set()
+        ingest: IngestResponse,
+        progress: float,
+        pbar: typing.Any,
+    ) -> typing.Tuple[IngestResponse, float]:
+        completed_files: typing.Set[str] = set()
 
         while (
-            ingest is not None
-            and ingest.ingest.status not in ["complete", "error", "cancelled"]
+            ingest.ingest.status not in ["complete", "error", "cancelled"]
         ):
             time.sleep(3)
             ingest = self.documents.get_processing_status_by_id(ingest.ingest.process_id)
@@ -494,13 +521,15 @@ class GroundX(GroundXBase):
 
     def _upload_file_batch(
         self,
-        bucket_id,
-        batch,
-        upload_api,
-        request_options,
-        pbar,
-    ):
-        docs = []
+        bucket_id: int,
+        batch: typing.List[Path],
+        upload_api: str,
+        callback_url: typing.Optional[str],
+        callback_data: typing.Optional[str],
+        request_options: typing.Optional[RequestOptions],
+        pbar: typing.Any,
+    ) -> None:
+        docs: typing.List[Document] = []
 
         progress =  float(len(batch))
         for file in batch:
@@ -526,7 +555,12 @@ class GroundX(GroundXBase):
             progress -= 0.25
 
         if docs:
-            ingest = self.ingest(documents=docs, request_options=request_options)
+            ingest = self.ingest(
+                documents=docs,
+                callback_data=callback_data,
+                callback_url=callback_url,
+                request_options=request_options,
+            )
             ingest, progress = self._monitor_batch(ingest, progress, pbar)
 
         if progress > 0:
@@ -540,6 +574,8 @@ class AsyncGroundX(AsyncGroundXBase):
         *,
         documents: typing.Sequence[Document],
         upload_api: str = "https://api.eyelevel.ai/upload/file",
+        callback_url: typing.Optional[str] = None,
+        callback_data: typing.Optional[str] = None,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -553,6 +589,13 @@ class AsyncGroundX(AsyncGroundXBase):
         # and returns a presigned URL in a JSON dictionary with key 'URL'
         upload_api : typing.Optional[str]
 
+        # an endpoint that will receive processing event updates as POST
+        callback_url : typing.Optional[str]
+
+        # a string that is returned, along with processing event updates,
+        # to the callback URL.
+        callback_data : typing.Optional[str]
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -621,14 +664,16 @@ class AsyncGroundX(AsyncGroundXBase):
 
         return await self.documents.ingest_remote(
             documents=remote_documents,
+            callback_url=callback_url,
+            callback_data=callback_data,
             request_options=request_options,
         )
 
     def _upload_file(
         self,
-        endpoint,
-        file_path,
-    ):
+        endpoint: str,
+        file_path: Path,
+    ) -> str:
         file_name = os.path.basename(file_path)
         file_extension = os.path.splitext(file_name)[1][1:].lower()
         if f".{file_extension}" in SUFFIX_ALIASES: