groundx 2.2.4__py3-none-any.whl → 2.2.7__py3-none-any.whl

groundx/core/client_wrapper.py

@@ -16,7 +16,7 @@ class BaseClientWrapper:
         headers: typing.Dict[str, str] = {
             "X-Fern-Language": "Python",
             "X-Fern-SDK-Name": "groundx",
-            "X-Fern-SDK-Version": "2.2.4",
+            "X-Fern-SDK-Version": "2.2.7",
         }
         headers["X-API-Key"] = self.api_key
         return headers

groundx/ingest.py

@@ -1,6 +1,8 @@
-import aiohttp, io, json, mimetypes, requests, typing, os
+import aiohttp, io, json, mimetypes, requests, time, typing, os
 from asyncio import TimeoutError
-from urllib.parse import urlparse
+from pathlib import Path
+from tqdm import tqdm
+from urllib.parse import urlparse, urlunparse
 
 from json.decoder import JSONDecodeError
 
@@ -11,6 +13,7 @@ from .core.request_options import RequestOptions
 from .errors.bad_request_error import BadRequestError
 from .errors.unauthorized_error import UnauthorizedError
 from .types.document import Document
+from .types.document_type import DocumentType
 from .types.ingest_remote_document import IngestRemoteDocument
 from .types.ingest_response import IngestResponse
 
@@ -19,6 +22,14 @@ OMIT = typing.cast(typing.Any, ...)
 
 
 DOCUMENT_TYPE_TO_MIME = {
+    "bmp": "image/bmp",
+    "gif": "image/gif",
+    "heif": "image/heif",
+    "hwp": "application/x-hwp",
+    "ico": "image/vnd.microsoft.icon",
+    "svg": "image/svg",
+    "tiff": "image/tiff",
+    "webp": "image/webp",
     "txt": "text/plain",
     "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
     "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
@@ -32,6 +43,17 @@ DOCUMENT_TYPE_TO_MIME = {
 }
 MIME_TO_DOCUMENT_TYPE = {v: k for k, v in DOCUMENT_TYPE_TO_MIME.items()}
 
+ALLOWED_SUFFIXES = {f".{k}": v for k, v in DOCUMENT_TYPE_TO_MIME.items()}
+
+SUFFIX_ALIASES = {
+    ".jpeg": ".jpg",
+    ".heic": ".heif",
+    ".tif": ".tiff",
+}
+
+MAX_BATCH_SIZE = 50
+MIN_BATCH_SIZE = 1
+MAX_BATCH_SIZE_BYTES = 50 * 1024 * 1024
 
 def prep_documents(
     documents: typing.Sequence[Document],
@@ -234,6 +256,180 @@ class GroundX(GroundXBase):
 
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    def ingest_directory(
+        self,
+        *,
+        bucket_id: int,
+        path: str,
+        batch_size: typing.Optional[int] = 10,
+        upload_api: typing.Optional[str] = "https://api.eyelevel.ai/upload/file",
+        request_options: typing.Optional[RequestOptions] = None,
+    ):
+        """
+        Ingest documents from a local directory into a GroundX bucket.
+
+        Parameters
+        ----------
+        bucket_id : int
+        path : str
+        batch_size : type.Optional[int]
+
+        # an endpoint that accepts 'name' and 'type' query params
+        # and returns a presigned URL
+        upload_api : typing.Optional[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        IngestResponse
+            Documents successfully uploaded
+
+        Examples
+        --------
+        from groundx import Document, GroundX
+
+        client = GroundX(
+            api_key="YOUR_API_KEY",
+        )
+
+        client.ingest_directory(
+            bucket_id=0,
+            path="/path/to/directory"
+        )
+        """
+
+        def get_presigned_url(endpoint, file_name, file_extension) -> typing.Dict[str, typing.Any]:
+            params = {"name": file_name, "type": file_extension}
+            response = requests.get(endpoint, params=params)
+            response.raise_for_status()
+
+            return response.json()
+
+        def is_valid_local_directory(path: str) -> bool:
+            expanded_path = os.path.expanduser(path)
+            return os.path.isdir(expanded_path)
+
+        def load_directory_files(directory: str) -> typing.List[Path]:
+            dir_path = Path(directory)
+
+            matched_files = [
+                file
+                for file in dir_path.rglob("*")
+                if file.is_file() and (
+                    file.suffix.lower() in ALLOWED_SUFFIXES
+                    or file.suffix.lower() in SUFFIX_ALIASES
+                )
+            ]
+
+            return matched_files
+
+        def strip_query_params(url: str) -> str:
+            parsed = urlparse(url)
+            clean_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
+            return clean_url
+
+        def _upload_file_batch(bucket_id, batch, upload_api, request_options, pbar):
+            docs = []
+
+            progress = len(batch)
+            for file in batch:
+                url = upload_file(upload_api, file)
+                docs.append(
+                    Document(
+                        bucket_id=bucket_id,
+                        file_path=url,
+                    ),
+                )
+                pbar.update(0.25)
+                progress -= 0.25
+
+            if docs:
+                ingest = self.ingest(documents=docs, request_options=request_options)
+
+                completed_files = set()
+
+                while (
+                    ingest is not None
+                    and ingest.ingest.status not in ["complete", "error", "cancelled"]
+                ):
+                    time.sleep(3)
+                    ingest = self.documents.get_processing_status_by_id(ingest.ingest.process_id)
+
+                    if ingest.ingest.progress and ingest.ingest.progress.processing:
+                        for doc in ingest.ingest.progress.processing.documents:
+                            if doc.status == "complete" and doc.document_id not in completed_files:
+                                pbar.update(0.75)
+                                progress -= 0.75
+
+                if ingest.ingest.status in ["error", "cancelled"]:
+                    raise ValueError(f"Ingest failed with status: {ingest.ingest.status}")
+
+                if progress > 0:
+                    pbar.update(progress)
+
+        def upload_file(endpoint, file_path) -> str:
+            file_name = os.path.basename(file_path)
+            file_extension = os.path.splitext(file_name)[1][1:].lower()
+
+            presigned_info = get_presigned_url(endpoint, file_name, file_extension)
+
+            upload_url = presigned_info["URL"]
+            headers = presigned_info.get("Header", {})
+            method = presigned_info.get("Method", "PUT").upper()
+
+            for key, value in headers.items():
+                if isinstance(value, list):
+                    headers[key] = value[0]
+
+            with open(file_path, "rb") as f:
+                file_data = f.read()
+
+            if method == "PUT":
+                upload_response = requests.put(upload_url, data=file_data, headers=headers)
+            else:
+                raise ValueError(f"Unsupported HTTP method: {method}")
+
+            if upload_response.status_code not in (200, 201):
+                raise Exception(
+                    f"Upload failed: {upload_response.status_code} - {upload_response.text}"
+                )
+
+            return strip_query_params(upload_url)
+
+        if bucket_id < 1:
+            raise ValueError(f"Invalid bucket_id: {bucket_id}")
+
+        if is_valid_local_directory(path) is not True:
+            raise ValueError(f"Invalid directory path: {path}")
+
+        files = load_directory_files(path)
+
+        if len(files) < 1:
+            raise ValueError(f"No supported files found in directory: {path}")
+
+        current_batch: typing.List[Path] = []
+        current_batch_size: int = 0
+
+        n = max(MIN_BATCH_SIZE, min(batch_size or MIN_BATCH_SIZE, MAX_BATCH_SIZE))
+
+        with tqdm(total=len(files), desc="Ingesting Files", unit="file") as pbar:
+            for file in files:
+                file_size = file.stat().st_size
+
+                if (current_batch_size + file_size > MAX_BATCH_SIZE_BYTES) or (len(current_batch) >= n):
+                    _upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+                    current_batch = []
+                    current_batch_size = 0
+
+                current_batch.append(file)
+                current_batch_size += file_size
+
+            if current_batch:
+                _upload_file_batch(bucket_id, current_batch, upload_api, request_options, pbar)
+
+
 
 class AsyncGroundX(AsyncGroundXBase):
     async def ingest(

{groundx-2.2.4.dist-info → groundx-2.2.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: groundx
-Version: 2.2.4
+Version: 2.2.7
 Summary: 
 License: MIT
 Requires-Python: >=3.8,<4.0
@@ -26,6 +26,7 @@ Requires-Dist: pydantic (>=1.9.2)
 Requires-Dist: pydantic-core (>=2.18.2,<3.0.0)
 Requires-Dist: requests (>=2.4.0)
 Requires-Dist: tqdm (>=4.60.0)
+Requires-Dist: types-tqdm (>=4.60.0)
 Requires-Dist: typing_extensions (>=4.0.0)
 Description-Content-Type: text/markdown
 

{groundx-2.2.4.dist-info → groundx-2.2.7.dist-info}/RECORD

@@ -4,7 +4,7 @@ groundx/buckets/client.py,sha256=4jlc9vfIult1mMJ4FZW4_KFJybZPStZt1FUplIgrxbU,239
 groundx/client.py,sha256=dIW9OyrMyfC1N7HSxRrHh0w_8rJ8osNUOPdYD6ueQ6g,6515
 groundx/core/__init__.py,sha256=SQ85PF84B9MuKnBwHNHWemSGuy-g_515gFYNFhvEE0I,1438
 groundx/core/api_error.py,sha256=RE8LELok2QCjABadECTvtDp7qejA1VmINCh6TbqPwSE,426
-groundx/core/client_wrapper.py,sha256=rpvFZnPH9F7U4KGeUUqy3rGA57uer9q3SdfSnzptNJ0,1802
+groundx/core/client_wrapper.py,sha256=Bhc6L2UfeJoET17u-IIW6OWHD5GwdYaita2HNWDJjr4,1802
 groundx/core/datetime_utils.py,sha256=nBys2IsYrhPdszxGKCNRPSOCwa-5DWOHG95FB8G9PKo,1047
 groundx/core/file.py,sha256=d4NNbX8XvXP32z8KpK2Xovv33nFfruIrpz0QWxlgpZk,2663
 groundx/core/http_client.py,sha256=Z77OIxIbL4OAB2IDqjRq_sYa5yNYAWfmdhdCSSvh6Y4,19552
@@ -26,7 +26,7 @@ groundx/groups/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
 groundx/groups/client.py,sha256=bytQRh9m7e4vIuYHb7dD1kCTQZvyBxedCqGnmmLqrsI,35237
 groundx/health/__init__.py,sha256=FTtvy8EDg9nNNg9WCatVgKTRYV8-_v1roeGPAKoa_pw,65
 groundx/health/client.py,sha256=fcTa21RWPyBuT77PQ0EncC6rBaW_DrYlRvudy9-0H58,7545
-groundx/ingest.py,sha256=snQ586PRmV_s3VQNBqYfKM0Lo_AaRvft5mX4sT4k-l0,11536
+groundx/ingest.py,sha256=RTgmeg_4cEaZynSEyf-3ArKGBcnhbcZhJl7BAeUeAMU,18187
 groundx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 groundx/search/__init__.py,sha256=RagVzjShP33mDg9o4N3kGzV0egL1RYNjCpXPE8VzMYE,145
 groundx/search/client.py,sha256=zrrqFy0HowDUYPsMU4nfvDV2RgmkEQ4E8WYNktu3xcs,18684
@@ -81,7 +81,7 @@ groundx/types/subscription_detail.py,sha256=WNfUw2EMVECIvNYcV2s51zZ6T3Utc4zYXw63
 groundx/types/subscription_detail_meters.py,sha256=lBa8-1QlMVHjr5RLGqhiTKnD1KMM0AAHTWvz9TVtG8w,830
 groundx/types/website_source.py,sha256=3WeRCiilNKKBTfhwgjo3jbcVI3vLTeM-KxI6dVzpg9o,1578
 groundx/version.py,sha256=1yVogKaq260fQfckM2RYN2144SEw0QROsZW8ICtkG4U,74
-groundx-2.2.4.dist-info/LICENSE,sha256=dFE6nY1bHnSn6NqmdlghlU1gQqLqYNphrceGVehSa7o,1065
-groundx-2.2.4.dist-info/METADATA,sha256=hIIy9ZCSg8stpVhmHV-LFw44ky_Gtd7lSvOY6FLJzCc,5136
-groundx-2.2.4.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-groundx-2.2.4.dist-info/RECORD,,
+groundx-2.2.7.dist-info/LICENSE,sha256=dFE6nY1bHnSn6NqmdlghlU1gQqLqYNphrceGVehSa7o,1065
+groundx-2.2.7.dist-info/METADATA,sha256=Gx6smhve9G7ECJbxQBrw3pJ_LosNgtehM3VETuW0c9I,5173
+groundx-2.2.7.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+groundx-2.2.7.dist-info/RECORD,,