groundx 2.2.7__py3-none-any.whl → 2.2.8__py3-none-any.whl

groundx/core/client_wrapper.py

@@ -16,7 +16,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "groundx",
-            "X-Fern-SDK-Version": "2.2.7",
+            "X-Fern-SDK-Version": "2.2.8",
         }
         headers["X-API-Key"] = self.api_key
         return headers

groundx/ingest.py

@@ -1,17 +1,10 @@
-import aiohttp, io, json, mimetypes, requests, time, typing, os
-from asyncio import TimeoutError
+import requests, time, typing, os
 from pathlib import Path
 from tqdm import tqdm
 from urllib.parse import urlparse, urlunparse
 
-from json.decoder import JSONDecodeError
-
 from .client import GroundXBase, AsyncGroundXBase
-from .core.api_error import ApiError
-from .core.pydantic_utilities import parse_obj_as
 from .core.request_options import RequestOptions
-from .errors.bad_request_error import BadRequestError
-from .errors.unauthorized_error import UnauthorizedError
 from .types.document import Document
 from .types.document_type import DocumentType
 from .types.ingest_remote_document import IngestRemoteDocument
@@ -55,13 +48,30 @@ MAX_BATCH_SIZE = 50
 MIN_BATCH_SIZE = 1
 MAX_BATCH_SIZE_BYTES = 50 * 1024 * 1024
 
+def get_presigned_url(
+    endpoint: str,
+    file_name: str,
+    file_extension: str,
+) -> typing.Dict[str, typing.Any]:
+    params = {"name": file_name, "type": file_extension}
+    response = requests.get(endpoint, params=params)
+    response.raise_for_status()
+
+    return response.json()
+
+def strip_query_params(
+    url: str,
+) -> str:
+    parsed = urlparse(url)
+    clean_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
+
+    return clean_url
+
 def prep_documents(
     documents: typing.Sequence[Document],
 ) -> typing.Tuple[
     typing.List[IngestRemoteDocument],
-    typing.List[
-        typing.Tuple[str, typing.Tuple[typing.Union[str, None], typing.BinaryIO, str]]
-    ],
+    typing.List[Document],
 ]:
     """
     Process documents and separate them into remote and local documents.
@@ -80,9 +90,7 @@ def prep_documents(
         except ValueError:
             return False
 
-    local_documents: typing.List[
-        typing.Tuple[str, typing.Tuple[typing.Union[str, None], typing.BinaryIO, str]]
-    ] = []
+    local_documents: typing.List[Document] = []
     remote_documents: typing.List[IngestRemoteDocument] = []
 
     for document in documents:
@@ -100,53 +108,7 @@ def prep_documents(
             )
             remote_documents.append(remote_document)
         elif is_valid_local_path(document.file_path):
-            expanded_path = os.path.expanduser(document.file_path)
-            file_name = os.path.basename(expanded_path)
-            mime_type = mimetypes.guess_type(file_name)[0] or "application/octet-stream"
-            file_type = MIME_TO_DOCUMENT_TYPE.get(mime_type, None)
-            if document.file_type:
-                file_type = document.file_type
-                mime_type = DOCUMENT_TYPE_TO_MIME.get(
-                    document.file_type, "application/octet-stream"
-                )
-
-            if document.file_name:
-                file_name = document.file_name
-
-            try:
-                local_documents.append(
-                    (
-                        "blob",
-                        (
-                            file_name,
-                            open(expanded_path, "rb"),
-                            mime_type,
-                        ),
-                    )
-                )
-            except Exception as e:
-                raise ValueError(f"Error reading file {expanded_path}: {e}")
-
-            metadata = {
-                "bucketId": document.bucket_id,
-                "fileName": file_name,
-                "fileType": file_type,
-            }
-            if document.process_level:
-                metadata["processLevel"] = document.process_level
-            if document.search_data:
-                metadata["searchData"] = document.search_data
-
-            local_documents.append(
-                (
-                    "metadata",
-                    (
-                        f"data.json",
-                        io.BytesIO(json.dumps(metadata).encode("utf-8")),
-                        "application/json",
-                    ),
-                )
-            )
+            local_documents.append(document)
         else:
             raise ValueError(f"Invalid file path: {document.file_path}")
 
@@ -158,6 +120,7 @@ class GroundX(GroundXBase):
         self,
         *,
         documents: typing.Sequence[Document],
+        upload_api: typing.Optional[str] = "https://api.eyelevel.ai/upload/file",
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -167,6 +130,10 @@ class GroundX(GroundXBase):
         ----------
         documents : typing.Sequence[Document]
 
+        # an endpoint that accepts 'name' and 'type' query params
+        # and returns a presigned URL in a JSON dictionary with key 'URL'
+        upload_api : typing.Optional[str]
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -200,61 +167,39 @@ class GroundX(GroundXBase):
             raise ValueError("Documents must all be either local or remote, not a mix.")
 
         if len(remote_documents) > 0:
+            if len(remote_documents) > MAX_BATCH_SIZE:
+                raise ValueError("You have sent too many documents in this request")
+
             return self.documents.ingest_remote(
                 documents=remote_documents,
                 request_options=request_options,
             )
 
-        timeout = self._client_wrapper.get_timeout()
-        headers = self._client_wrapper.get_headers()
-        base_url = self._client_wrapper.get_base_url().rstrip("/")
-        follow_redirects = getattr(
-            self._client_wrapper.httpx_client, "follow_redirects", True
-        )
+        if len(local_documents) > MAX_BATCH_SIZE:
+            raise ValueError("You have sent too many documents in this request")
 
-        url = f"{base_url}/v1/ingest/documents/local"
-        _response = requests.post(
-            url,
-            files=local_documents,
-            headers=headers,
-            timeout=timeout,
-            allow_redirects=follow_redirects,
-        )
+        if len(local_documents) == 0:
+            raise ValueError("No valid documents were provided")
 
-        try:
-            if 200 <= _response.status_code < 300:
-                return typing.cast(
-                    IngestResponse,
-                    parse_obj_as(
-                        type_=IngestResponse,  # type: ignore
-                        object_=_response.json(),
-                    ),
-                )
-            if _response.status_code == 400:
-                raise BadRequestError(
-                    typing.cast(
-                        typing.Optional[typing.Any],
-                        parse_obj_as(
-                            type_=typing.Optional[typing.Any],  # type: ignore
-                            object_=_response.json(),
-                        ),
-                    )
-                )
-            if _response.status_code == 401:
-                raise UnauthorizedError(
-                    typing.cast(
-                        typing.Optional[typing.Any],
-                        parse_obj_as(
-                            type_=typing.Optional[typing.Any],  # type: ignore
-                            object_=_response.json(),
-                        ),
-                    )
+        docs: typing.List[IngestRemoteDocument] = []
+        for d in local_documents:
+            url = self._upload_file(upload_api, Path(os.path.expanduser(d.file_path)))
+
+            docs.append(
+                IngestRemoteDocument(
+                    bucket_id=d.bucket_id,
+                    file_name=d.file_name,
+                    file_type=d.file_type,
+                    process_level=d.process_level,
+                    search_data=d.search_data,
+                    source_url=url,
                 )
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
+            )
 
-        raise ApiError(status_code=_response.status_code, body=_response_json)
+        return self.documents.ingest_remote(
+            documents=docs,
+            request_options=request_options,
+        )
 
     def ingest_directory(
         self,
@@ -275,7 +220,7 @@ class GroundX(GroundXBase):
         batch_size : type.Optional[int]
 
         # an endpoint that accepts 'name' and 'type' query params
-        # and returns a presigned URL
+        # and returns a presigned URL in a JSON dictionary with key 'URL'
         upload_api : typing.Optional[str]
 
         request_options : typing.Optional[RequestOptions]
@@ -300,13 +245,6 @@ class GroundX(GroundXBase):
         )
         """
 
-        def get_presigned_url(endpoint, file_name, file_extension) -> typing.Dict[str, typing.Any]:
-            params = {"name": file_name, "type": file_extension}
-            response = requests.get(endpoint, params=params)
-            response.raise_for_status()
-
-            return response.json()
-
         def is_valid_local_directory(path: str) -> bool:
             expanded_path = os.path.expanduser(path)
             return os.path.isdir(expanded_path)
@@ -323,80 +261,7 @@ class GroundX(GroundXBase):
                 )
             ]
 
-            return matched_files
-
-        def strip_query_params(url: str) -> str:
-            parsed = urlparse(url)
-            clean_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
-            return clean_url
-
-        def _upload_file_batch(bucket_id, batch, upload_api, request_options, pbar):
-            docs = []
-
-            progress = len(batch)
-            for file in batch:
-                url = upload_file(upload_api, file)
-                docs.append(
-                    Document(
-                        bucket_id=bucket_id,
-                        file_path=url,
-                    ),
-                )
-                pbar.update(0.25)
-                progress -= 0.25
-
-            if docs:
-                ingest = self.ingest(documents=docs, request_options=request_options)
-
-                completed_files = set()
-
-                while (
-                    ingest is not None
-                    and ingest.ingest.status not in ["complete", "error", "cancelled"]
-                ):
-                    time.sleep(3)
-                    ingest = self.documents.get_processing_status_by_id(ingest.ingest.process_id)
-
-                    if ingest.ingest.progress and ingest.ingest.progress.processing:
-                        for doc in ingest.ingest.progress.processing.documents:
-                            if doc.status == "complete" and doc.document_id not in completed_files:
-                                pbar.update(0.75)
-                                progress -= 0.75
-
-                if ingest.ingest.status in ["error", "cancelled"]:
-                    raise ValueError(f"Ingest failed with status: {ingest.ingest.status}")
-
-                if progress > 0:
-                    pbar.update(progress)
-
-        def upload_file(endpoint, file_path) -> str:
-            file_name = os.path.basename(file_path)
-            file_extension = os.path.splitext(file_name)[1][1:].lower()
-
-            presigned_info = get_presigned_url(endpoint, file_name, file_extension)
-
-            upload_url = presigned_info["URL"]
-            headers = presigned_info.get("Header", {})
-            method = presigned_info.get("Method", "PUT").upper()
-
-            for key, value in headers.items():
-                if isinstance(value, list):
-                    headers[key] = value[0]
-
-            with open(file_path, "rb") as f:
-                file_data = f.read()
-
-            if method == "PUT":
-                upload_response = requests.put(upload_url, data=file_data, headers=headers)
-            else:
-                raise ValueError(f"Unsupported HTTP method: {method}")
-
-            if upload_response.status_code not in (200, 201):
-                raise Exception(
-                    f"Upload failed: {upload_response.status_code} - {upload_response.text}"
-                )
-
-            return strip_query_params(upload_url)
+            return matched_files      
 
         if bucket_id < 1:
             raise ValueError(f"Invalid bucket_id: {bucket_id}")
@@ -419,7 +284,7 @@ class GroundX(GroundXBase):
                 file_size = file.stat().st_size
 
                 if (current_batch_size + file_size > MAX_BATCH_SIZE_BYTES) or (len(current_batch) >= n):
-                    _upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+                    self._upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
                     current_batch = []
                     current_batch_size = 0
 
@@ -427,7 +292,89 @@ class GroundX(GroundXBase):
                 current_batch_size += file_size
 
             if current_batch:
-                _upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+                self._upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+
+    def _upload_file(
+        self,
+        endpoint,
+        file_path,
+    ):
+        file_name = os.path.basename(file_path)
+        file_extension = os.path.splitext(file_name)[1][1:].lower()
+
+        presigned_info = get_presigned_url(endpoint, file_name, file_extension)
+
+        upload_url = presigned_info["URL"]
+        headers = presigned_info.get("Header", {})
+        method = presigned_info.get("Method", "PUT").upper()
+
+        for key, value in headers.items():
+            if isinstance(value, list):
+                headers[key] = value[0]
+
+        try:
+            with open(file_path, "rb") as f:
+                file_data = f.read()
+        except Exception as e:
+            raise ValueError(f"Error reading file {file_path}: {e}")
+
+        if method == "PUT":
+            upload_response = requests.put(upload_url, data=file_data, headers=headers)
+        else:
+            raise ValueError(f"Unsupported HTTP method: {method}")
+
+        if upload_response.status_code not in (200, 201):
+            raise Exception(
+                f"Upload failed: {upload_response.status_code} - {upload_response.text}"
+            )
+
+        return strip_query_params(upload_url)
+
+    def _upload_file_batch(
+        self,
+        bucket_id,
+        batch,
+        upload_api,
+        request_options,
+        pbar,
+    ):
+        docs = []
+
+        progress = len(batch)
+        for file in batch:
+            url = self._upload_file(upload_api, file)
+            docs.append(
+                Document(
+                    bucket_id=bucket_id,
+                    file_path=url,
+                ),
+            )
+            pbar.update(0.25)
+            progress -= 0.25
+
+        if docs:
+            ingest = self.ingest(documents=docs, request_options=request_options)
+
+            completed_files = set()
+
+            while (
+                ingest is not None
+                and ingest.ingest.status not in ["complete", "error", "cancelled"]
+            ):
+                time.sleep(3)
+                ingest = self.documents.get_processing_status_by_id(ingest.ingest.process_id)
+
+                if ingest.ingest.progress and ingest.ingest.progress.processing:
+                    for doc in ingest.ingest.progress.processing.documents:
+                        if doc.status == "complete" and doc.document_id not in completed_files:
+                            pbar.update(0.75)
+                            progress -= 0.75
+
+            if ingest.ingest.status in ["error", "cancelled"]:
+                raise ValueError(f"Ingest failed with status: {ingest.ingest.status}")
+
+            if progress > 0:
+                pbar.update(progress)
 
 
 
@@ -436,6 +383,7 @@ class AsyncGroundX(AsyncGroundXBase):
         self,
         *,
         documents: typing.Sequence[Document],
+        upload_api: str = "https://api.eyelevel.ai/upload/file",
         request_options: typing.Optional[RequestOptions] = None,
     ) -> IngestResponse:
         """
@@ -445,6 +393,10 @@ class AsyncGroundX(AsyncGroundXBase):
         ----------
         documents : typing.Sequence[Document]
 
+        # an endpoint that accepts 'name' and 'type' query params
+        # and returns a presigned URL in a JSON dictionary with key 'URL'
+        upload_api : typing.Optional[str]
+
         request_options : typing.Optional[RequestOptions]
             Request-specific configuration.
 
@@ -483,49 +435,72 @@ class AsyncGroundX(AsyncGroundXBase):
             raise ValueError("Documents must all be either local or remote, not a mix.")
 
         if len(remote_documents) > 0:
+            if len(remote_documents) > MAX_BATCH_SIZE:
+                raise ValueError("You have sent too many documents in this request")
+
             return await self.documents.ingest_remote(
                 documents=remote_documents,
                 request_options=request_options,
             )
 
-        timeout = self._client_wrapper.get_timeout()
-        headers = self._client_wrapper.get_headers()
-        base_url = self._client_wrapper.get_base_url().rstrip("/")
+        if len(local_documents) > MAX_BATCH_SIZE:
+            raise ValueError("You have sent too many documents in this request")
+
+        if len(local_documents) == 0:
+            raise ValueError("No valid documents were provided")
 
-        url = f"{base_url}/v1/ingest/documents/local"
+        docs: typing.List[IngestRemoteDocument] = []
+        for d in local_documents:
+            url = self._upload_file(upload_api, Path(os.path.expanduser(d.file_path)))
+
+            docs.append(
+                IngestRemoteDocument(
+                    bucket_id=d.bucket_id,
+                    file_name=d.file_name,
+                    file_type=d.file_type,
+                    process_level=d.process_level,
+                    search_data=d.search_data,
+                    source_url=url,
+                )
+            )
+
+        return await self.documents.ingest_remote(
+            documents=docs,
+            request_options=request_options,
+        )
+
+    def _upload_file(
+        self,
+        endpoint,
+        file_path,
+    ):
+        file_name = os.path.basename(file_path)
+        file_extension = os.path.splitext(file_name)[1][1:].lower()
+
+        presigned_info = get_presigned_url(endpoint, file_name, file_extension)
+
+        upload_url = presigned_info["URL"]
+        headers = presigned_info.get("Header", {})
+        method = presigned_info.get("Method", "PUT").upper()
+
+        for key, value in headers.items():
+            if isinstance(value, list):
+                headers[key] = value[0]
 
         try:
-            async with aiohttp.ClientSession() as session:
-                data = aiohttp.FormData()
-                for field_name, (file_name, file_obj, content_type) in local_documents:
-                    data.add_field(
-                        name=field_name,
-                        value=file_obj,
-                        filename=file_name,
-                        content_type=content_type,
-                    )
+            with open(file_path, "rb") as f:
+                file_data = f.read()
+        except Exception as e:
+            raise ValueError(f"Error reading file {file_path}: {e}")
 
-                async with session.post(
-                    url, data=data, headers=headers, timeout=timeout
-                ) as response:
-                    if 200 <= response.status < 300:
-                        response_data = await response.json()
-                        return typing.cast(
-                            IngestResponse,
-                            parse_obj_as(
-                                type_=IngestResponse,  # type: ignore
-                                object_=response_data,
-                            ),
-                        )
-                    if response.status == 400:
-                        raise BadRequestError(await response.json())
-                    if response.status == 401:
-                        raise UnauthorizedError(await response.json())
-
-                    raise ApiError(
-                        status_code=response.status, body=await response.text()
-                    )
-        except TimeoutError:
-            raise ApiError(status_code=408, body="Request timed out")
-        except aiohttp.ClientError as e:
-            raise ApiError(status_code=500, body=str(e))
+        if method == "PUT":
+            upload_response = requests.put(upload_url, data=file_data, headers=headers)
+        else:
+            raise ValueError(f"Unsupported HTTP method: {method}")
+
+        if upload_response.status_code not in (200, 201):
+            raise Exception(
+                f"Upload failed: {upload_response.status_code} - {upload_response.text}"
+            )
+
+        return strip_query_params(upload_url)

{groundx-2.2.7.dist-info → groundx-2.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: groundx
-Version: 2.2.7
+Version: 2.2.8
 Summary: 
 License: MIT
 Requires-Python: >=3.8,<4.0

{groundx-2.2.7.dist-info → groundx-2.2.8.dist-info}/RECORD

@@ -4,7 +4,7 @@ groundx/buckets/client.py,sha256=4jlc9vfIult1mMJ4FZW4_KFJybZPStZt1FUplIgrxbU,239
 groundx/client.py,sha256=dIW9OyrMyfC1N7HSxRrHh0w_8rJ8osNUOPdYD6ueQ6g,6515
 groundx/core/__init__.py,sha256=SQ85PF84B9MuKnBwHNHWemSGuy-g_515gFYNFhvEE0I,1438
 groundx/core/api_error.py,sha256=RE8LELok2QCjABadECTvtDp7qejA1VmINCh6TbqPwSE,426
-groundx/core/client_wrapper.py,sha256=Bhc6L2UfeJoET17u-IIW6OWHD5GwdYaita2HNWDJjr4,1802
+groundx/core/client_wrapper.py,sha256=DckEncGF_W3v8kguKAXWcn2um2B9k-nkWvKmH0HoyGQ,1802
 groundx/core/datetime_utils.py,sha256=nBys2IsYrhPdszxGKCNRPSOCwa-5DWOHG95FB8G9PKo,1047
 groundx/core/file.py,sha256=d4NNbX8XvXP32z8KpK2Xovv33nFfruIrpz0QWxlgpZk,2663
 groundx/core/http_client.py,sha256=Z77OIxIbL4OAB2IDqjRq_sYa5yNYAWfmdhdCSSvh6Y4,19552
@@ -26,7 +26,7 @@ groundx/groups/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
 groundx/groups/client.py,sha256=bytQRh9m7e4vIuYHb7dD1kCTQZvyBxedCqGnmmLqrsI,35237
 groundx/health/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
 groundx/health/client.py,sha256=fcTa21RWPyBuT77PQ0EncC6rBaW_DrYlRvudy9-0H58,7545
-groundx/ingest.py,sha256=RTgmeg_4cEaZynSEyf-3ArKGBcnhbcZhJl7BAeUeAMU,18187
+groundx/ingest.py,sha256=mQB__GQmIDP6W5jGhmjORXh2o6HHTseHiahZz2Es-tM,16119
 groundx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 groundx/search/__init__.py,sha256=RagVzjShP33mDg9o4N3kGzV0egL1RYNjCpXPE8VzMYE,145
 groundx/search/client.py,sha256=zrrqFy0HowDUYPsMU4nfvDV2RgmkEQ4E8WYNktu3xcs,18684
@@ -81,7 +81,7 @@ groundx/types/subscription_detail.py,sha256=WNfUw2EMVECIvNYcV2s51zZ6T3Utc4zYXw63
 groundx/types/subscription_detail_meters.py,sha256=lBa8-1QlMVHjr5RLGqhiTKnD1KMM0AAHTWvz9TVtG8w,830
 groundx/types/website_source.py,sha256=3WeRCiilNKKBTfhwgjo3jbcVI3vLTeM-KxI6dVzpg9o,1578
 groundx/version.py,sha256=1yVogKaq260fQfckM2RYN2144SEw0QROsZW8ICtkG4U,74
-groundx-2.2.7.dist-info/LICENSE,sha256=dFE6nY1bHnSn6NqmdlghlU1gQqLqYNphrceGVehSa7o,1065
-groundx-2.2.7.dist-info/METADATA,sha256=Gx6smhve9G7ECJbxQBrw3pJ_LosNgtehM3VETuW0c9I,5173
-groundx-2.2.7.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-groundx-2.2.7.dist-info/RECORD,,
+groundx-2.2.8.dist-info/LICENSE,sha256=dFE6nY1bHnSn6NqmdlghlU1gQqLqYNphrceGVehSa7o,1065
+groundx-2.2.8.dist-info/METADATA,sha256=Q67rLMlsO72-NH2qvK8Bw4ciSTDS5C4LNoPiBk8UWyI,5173
+groundx-2.2.8.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+groundx-2.2.8.dist-info/RECORD,,