unstructured-ingest 1.0.8__py3-none-any.whl → 1.0.11__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.8"  # pragma: no cover
+__version__ = "1.0.11"  # pragma: no cover

unstructured_ingest/processes/connectors/google_drive.py

@@ -153,7 +153,13 @@ class GoogleDriveIndexer(Indexer):
         """
         try:
             # A very minimal call: list 1 file from the drive.
-            client.list(spaces="drive", pageSize=1, fields="files(id)").execute()
+            client.list(
+                supportsAllDrives=True,
+                includeItemsFromAllDrives=True,
+                spaces="drive",
+                pageSize=1,
+                fields="files(id)",
+            ).execute()
         except HttpError as e:
             error_content = e.content.decode() if hasattr(e, "content") else ""
             lower_error = error_content.lower()
@@ -183,6 +189,8 @@ class GoogleDriveIndexer(Indexer):
             page_token = None
             while True:
                 response = files_client.list(
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True,
                     spaces="drive",
                     q=query,
                     fields="nextPageToken, files(id, mimeType, fileExtension)",
@@ -251,6 +259,8 @@ class GoogleDriveIndexer(Indexer):
                 else:
                     # Non-recursive: check for at least one immediate non-folder child.
                     response = client.list(
+                        supportsAllDrives=True,
+                        includeItemsFromAllDrives=True,
                         spaces="drive",
                         fields="files(id)",
                         pageSize=1,
@@ -348,6 +358,8 @@ class GoogleDriveIndexer(Indexer):
         files_response = []
         while not done:
             response: dict = files_client.list(
+                supportsAllDrives=True,
+                includeItemsFromAllDrives=True,
                 spaces="drive",
                 fields=fields_input,
                 corpora="user",
@@ -381,7 +393,9 @@ class GoogleDriveIndexer(Indexer):
         return files_response
 
     def get_root_info(self, files_client, object_id: str) -> dict:
-        return files_client.get(fileId=object_id, fields=",".join(self.fields)).execute()
+        return files_client.get(
+            supportsAllDrives=True, fileId=object_id, fields=",".join(self.fields)
+        ).execute()
 
     def get_files(
         self,

unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -172,7 +172,7 @@ class IbmWatsonxUploaderConfig(UploaderConfig):
     namespace: str = Field(description="Namespace name")
     table: str = Field(description="Table name")
     max_retries: int = Field(
-        default=5, description="Maximum number of retries to upload data", ge=2, le=10
+        default=5, description="Maximum number of retries to upload data", ge=2, le=500
     )
     record_id_key: str = Field(
         default=RECORD_ID_LABEL,

unstructured_ingest/processes/connectors/pinecone.py

@@ -1,6 +1,7 @@
 import json
 import re
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, Optional
 
 from pydantic import Field, Secret
@@ -18,11 +19,14 @@ from unstructured_ingest.interfaces import (
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.constants import RECORD_ID_LABEL
 from unstructured_ingest.utils.data_prep import (
     flatten_dict,
     generator_batching_wbytes,
     get_enhanced_element_id,
+    get_json_data,
+    write_data,
 )
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
@@ -162,6 +166,28 @@ class PineconeUploadStager(UploadStager):
             "metadata": metadata,
         }
 
+    def stream_update(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
+        with input_file.open() as in_f:
+            reader = ndjson.reader(in_f)
+            with output_file.open("w") as out_f:
+                writer = ndjson.writer(out_f)
+                for element in reader:
+                    if "embeddings" not in element:
+                        continue
+                    conformed_element = self.conform_dict(element_dict=element, file_data=file_data)
+                    writer.write(row=conformed_element)
+                    writer.f.flush()
+
+    def process_whole(self, input_file: Path, output_file: Path, file_data: FileData) -> None:
+        elements_contents = get_json_data(path=input_file)
+
+        conformed_elements = [
+            self.conform_dict(element_dict=element, file_data=file_data)
+            for element in elements_contents
+            if "embeddings" in element
+        ]
+        write_data(path=output_file, data=conformed_elements)
+
 
 @dataclass
 class PineconeUploader(VectorDBUploader):

{unstructured_ingest-1.0.8.dist-info → unstructured_ingest-1.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.8
+Version: 1.0.11
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.8.dist-info → unstructured_ingest-1.0.11.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=Ca4MzLfEjKrGXx21Kyt3Ve65pu59qVvEIU_io-qxQ9o,42
+unstructured_ingest/__version__.py,sha256=T2mg8xL1j5lQc6Bl6xC62EjS3ZMzCbWBXW81Kgto5vs,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -72,7 +72,7 @@ unstructured_ingest/processes/connectors/delta_table.py,sha256=2DFox_Vzoopt_D3Jy
 unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMkcd8lcLJC0uqbo4izjdZ3rU,5294
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=6h1CdqznJmzeWxGfXrFLdNdT23PExGnUMMX7usK_4Kk,10013
-unstructured_ingest/processes/connectors/google_drive.py,sha256=YMuobb9AZq-0pp6rbYyXNizs3jA6z3nQjO9cK_nncXQ,21936
+unstructured_ingest/processes/connectors/google_drive.py,sha256=-bpOd2kIffghRBCmhEXQ0TPRZAKoE6hFtzA7hfw6QN0,22421
 unstructured_ingest/processes/connectors/jira.py,sha256=eG8yTn8ZVEz7rBJ-ha8i_d9hEh6VALN6QJT_vbYvbL0,17142
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9VhzKZuuQUpDS-cVNAQ2g,7426
@@ -81,7 +81,7 @@ unstructured_ingest/processes/connectors/mongodb.py,sha256=1g_5bfbS6lah3nsOXqLAa
 unstructured_ingest/processes/connectors/neo4j.py,sha256=ztxvI9KY8RF5kYUuMGSzzN5mz7Fu_4Ai9P7dqCpJLc0,20267
 unstructured_ingest/processes/connectors/onedrive.py,sha256=VBkKlbJgR7uKlKTnjNybAw6ZawLKflDPpy2uVvgWYWw,19296
 unstructured_ingest/processes/connectors/outlook.py,sha256=zHM5frO7CqQG0-KcTyX49aZeSlsvVrl8kh_lR_ESgQw,9275
-unstructured_ingest/processes/connectors/pinecone.py,sha256=BdO1PS_Y6FOeL-7uPl-Eh6ij1wHOwMkopOzKQGQ9Ac0,13979
+unstructured_ingest/processes/connectors/pinecone.py,sha256=pSREUNsQqel6q1EFZsFWelg-uZgGubQY5m_6nVnBFKs,15090
 unstructured_ingest/processes/connectors/redisdb.py,sha256=YzvSlfHs83XWsWMaIC3bV5enKfxejMQ9BQ8CtXfnJ5o,6923
 unstructured_ingest/processes/connectors/salesforce.py,sha256=OaKEWCqZrirHqFJ650K5jSPwYlWefPOapas8Y-4D9oc,11661
 unstructured_ingest/processes/connectors/sharepoint.py,sha256=PowaqMzWr-VCW1rnwcAeRhHyE55kJ9J9FCVlrmtzN0E,4827
@@ -115,7 +115,7 @@ unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=2ZV6b2E2pIsf_ab1Lty
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
 unstructured_ingest/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/processes/connectors/ibm_watsonx/__init__.py,sha256=kf0UpgdAY2KK1R1FbAB6GEBBAIOeYQ8cZIr3bp660qM,374
-unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=SpZIqjreXpLTpZfezhG6xkZ_h7w-QWmdjXDBG6mlddQ,11729
+unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py,sha256=cQvtLVKCTiqFeImz2OwvgdFL7YnMwLLT1_B_ueHy8Qo,11730
 unstructured_ingest/processes/connectors/kafka/__init__.py,sha256=pFN2cWwAStiGTAsQ616GIWKi_hDv0s74ZvNqhJEp1Pc,751
 unstructured_ingest/processes/connectors/kafka/cloud.py,sha256=Ki6iOLoZ86tYWdnLnMWYvb2hUCneKqo4mTJcfXh7YoQ,3432
 unstructured_ingest/processes/connectors/kafka/kafka.py,sha256=7NMvWijfoliyAgnmz8TM8oJt5x7RDzC-ABPdYAm7J3w,10306
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.8.dist-info/METADATA,sha256=IzN7b_dpadQBKpp59jO7VfWzgQfJrF8ykGLo7epNMeY,8719
-unstructured_ingest-1.0.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.8.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.8.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.8.dist-info/RECORD,,
+unstructured_ingest-1.0.11.dist-info/METADATA,sha256=qD4WgfLZBApr6D_Zwg_9_1QBXjmKcQgxt-lpIja1liE,8720
+unstructured_ingest-1.0.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.11.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.11.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.11.dist-info/RECORD,,