unstructured-ingest 1.2.14__py3-none-any.whl → 1.2.18__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.14"  # pragma: no cover
+__version__ = "1.2.18"  # pragma: no cover

unstructured_ingest/embed/bedrock.py

@@ -27,12 +27,24 @@ if TYPE_CHECKING:
     from botocore.client import BaseClient
 
     class BedrockRuntimeClient(BaseClient):
-        def invoke_model(self, body: str, modelId: str, accept: str, contentType: str) -> dict:
+        def invoke_model(
+            self,
+            body: str,
+            modelId: str,
+            accept: str,
+            contentType: str,
+            inferenceProfileId: str = None,
+        ) -> dict:
             pass
 
     class AsyncBedrockRuntimeClient(BaseClient):
         async def invoke_model(
-            self, body: str, modelId: str, accept: str, contentType: str
+            self,
+            body: str,
+            modelId: str,
+            accept: str,
+            contentType: str,
+            inferenceProfileId: str = None,
         ) -> dict:
             pass
 
@@ -63,10 +75,10 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         description="aws secret access key", default=None
     )
     region_name: str = Field(
-        description="aws region name", 
+        description="aws region name",
         default_factory=lambda: (
-            os.getenv("BEDROCK_REGION_NAME") or 
-            os.getenv("AWS_DEFAULT_REGION") or 
+            os.getenv("BEDROCK_REGION_NAME") or
+            os.getenv("AWS_DEFAULT_REGION") or
             "us-west-2"
         )
     )
@@ -79,6 +91,10 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         alias="model_name",
         description="AWS Bedrock model name",
     )
+    inference_profile_id: str | None = Field(
+        description="AWS Bedrock inference profile ID",
+        default_factory=lambda: os.getenv("BEDROCK_INFERENCE_PROFILE_ID"),
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -218,12 +234,18 @@ class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
         bedrock_client = self.config.get_client()
         # invoke bedrock API
         try:
-            response = bedrock_client.invoke_model(
-                body=json.dumps(body),
-                modelId=self.config.embedder_model_name,
-                accept="application/json",
-                contentType="application/json",
-            )
+            invoke_params = {
+                "body": json.dumps(body),
+                "modelId": self.config.embedder_model_name,
+                "accept": "application/json",
+                "contentType": "application/json",
+            }
+
+            # Add inference profile if configured
+            if self.config.inference_profile_id:
+                invoke_params["inferenceProfileId"] = self.config.inference_profile_id
+
+            response = bedrock_client.invoke_model(**invoke_params)
         except Exception as e:
             raise self.wrap_error(e=e)
 
@@ -264,12 +286,18 @@ class AsyncBedrockEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
             async with self.config.get_async_client() as bedrock_client:
                 # invoke bedrock API
                 try:
-                    response = await bedrock_client.invoke_model(
-                        body=json.dumps(body),
-                        modelId=self.config.embedder_model_name,
-                        accept="application/json",
-                        contentType="application/json",
-                    )
+                    invoke_params = {
+                        "body": json.dumps(body),
+                        "modelId": self.config.embedder_model_name,
+                        "accept": "application/json",
+                        "contentType": "application/json",
+                    }
+
+                    # Add inference profile if configured
+                    if self.config.inference_profile_id:
+                        invoke_params["inferenceProfileId"] = self.config.inference_profile_id
+
+                    response = await bedrock_client.invoke_model(**invoke_params)
                 except Exception as e:
                     raise self.wrap_error(e=e)
                 async with response.get("body") as client_response:

unstructured_ingest/processes/connectors/databricks/volumes.py

@@ -76,7 +76,9 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         if isinstance(e, ValueError):
             error_message = e.args[0]
             message_split = error_message.split(":")
-            if message_split[0].endswith("auth"):
+            if (message_split[0].endswith("auth")) or (
+                "Client authentication failed" in error_message
+            ):
                 return UserAuthError(e)
         if isinstance(e, DatabricksError):
             reverse_mapping = {v: k for k, v in STATUS_CODE_MAPPING.items()}

unstructured_ingest/processes/connectors/google_drive.py

@@ -189,6 +189,9 @@ class GoogleDriveIndexer(Indexer):
         """
         count = 0
         stack = [folder_id]
+        # Pre-compute lower-case extension set for O(1) lookup
+        valid_exts = set(e.lower() for e in extensions) if extensions else None
+
         while stack:
             current_folder = stack.pop()
             # Always list all items under the current folder.
@@ -212,7 +215,6 @@ class GoogleDriveIndexer(Indexer):
                         if extensions:
                             # Use a case-insensitive comparison for the file extension.
                             file_ext = (item.get("fileExtension") or "").lower()
-                            valid_exts = [e.lower() for e in extensions]
                             if file_ext in valid_exts:
                                 count += 1
                         else:

unstructured_ingest/processes/connectors/milvus.py

@@ -1,6 +1,7 @@
 import json
 from contextlib import contextmanager
 from dataclasses import dataclass, field
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from dateutil import parser
@@ -97,6 +98,13 @@ class MilvusUploadStager(UploadStager):
             return timestamp
         except ValueError:
             pass
+
+        try:
+            dt = datetime.fromisoformat(date_string.replace("Z", "+00:00"))
+            return dt.timestamp()
+        except ValueError:
+            pass
+
         return parser.parse(date_string).timestamp()
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:

unstructured_ingest/processes/connectors/notion/types/page.py

@@ -1,5 +1,5 @@
 # https://developers.notion.com/reference/page
-from dataclasses import dataclass
+from dataclasses import dataclass, fields
 from typing import Optional
 
 from unstructured_ingest.processes.connectors.notion.interfaces import FromJSONMixin
@@ -28,18 +28,25 @@ class Page(FromJSONMixin):
 
     @classmethod
     def from_dict(cls, data: dict):
+        data = data.copy()  # Don't modify the original
         created_by = data.pop("created_by")
         last_edited_by = data.pop("last_edited_by")
         icon = data.pop("icon")
         cover = data.pop("cover")
         parent = data.pop("parent")
+
+        # Filter data to only include fields that exist in the dataclass
+        filtered_data = {
+            k: v for k, v in data.items() if k in {field.name for field in fields(cls)}
+        }
+
         page = cls(
             created_by=PartialUser.from_dict(created_by),
             last_edited_by=PartialUser.from_dict(last_edited_by),
             icon=FileObject.from_dict(icon) if icon else None,
             cover=FileObject.from_dict(cover) if cover else None,
             parent=Parent.from_dict(parent),
-            **data,
+            **filtered_data,
         )
 
         return page

unstructured_ingest/processes/connectors/sql/snowflake.py

@@ -233,18 +233,19 @@ class SnowflakeUploader(SQLUploader):
             output.append(tuple(parsed))
         return output
 
-    def _parse_values(self, columns: list[str]) -> str:
+    def _parse_select(self, columns: list[str]) -> str:
         embeddings_dimension = self.embeddings_dimension
         parsed_values = []
-        for col in columns:
+        for i, col in enumerate(columns):
+            argument_selector = f"${i + 1}"
             if col in _VECTOR_COLUMNS and embeddings_dimension:
                 parsed_values.append(
-                    f"PARSE_JSON({self.values_delimiter})::VECTOR(FLOAT,{embeddings_dimension})"
+                    f"PARSE_JSON({argument_selector})::VECTOR(FLOAT,{embeddings_dimension})"
                 )
             elif col in _ARRAY_COLUMNS or col in _VECTOR_COLUMNS:
-                parsed_values.append(f"PARSE_JSON({self.values_delimiter})")
+                parsed_values.append(f"PARSE_JSON({argument_selector})")
             else:
-                parsed_values.append(self.values_delimiter)
+                parsed_values.append(argument_selector)
         return ",".join(parsed_values)
 
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
@@ -262,10 +263,11 @@ class SnowflakeUploader(SQLUploader):
         df.replace({np.nan: None}, inplace=True)
 
         columns = list(df.columns)
-        stmt = "INSERT INTO {table_name} ({columns}) SELECT {values}".format(
+        stmt = "INSERT INTO {table_name} ({columns}) SELECT {select} FROM VALUES ({values})".format(
             table_name=self.upload_config.table_name,
             columns=",".join(columns),
-            values=self._parse_values(columns),
+            select=self._parse_select(columns),
+            values=",".join([self.values_delimiter for _ in columns]),
         )
         logger.info(
             f"writing a total of {len(df)} elements via"
@@ -276,10 +278,7 @@ class SnowflakeUploader(SQLUploader):
         for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
             with self.connection_config.get_cursor() as cursor:
                 values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
-                # TODO: executemany break on 'Binding data in type (list) is not supported'
-                for val in values:
-                    logger.debug(f"running query: {stmt}\nwith values: {val}")
-                    cursor.execute(stmt, val)
+                cursor.executemany(stmt, values)
 
 
 snowflake_source_entry = SourceRegistryEntry(

unstructured_ingest/processes/connectors/sql/sql.py

@@ -64,6 +64,12 @@ def parse_date_string(date_value: Union[str, int]) -> datetime:
         return datetime.fromtimestamp(timestamp)
     except Exception as e:
         logger.debug(f"date {date_value} string not a timestamp: {e}")
+
+    if isinstance(date_value, str):
+        try:
+            return datetime.fromisoformat(date_value)
+        except Exception:
+            pass
     return parser.parse(date_value)
 
 

{unstructured_ingest-1.2.14.dist-info → unstructured_ingest-1.2.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.14
+Version: 1.2.18
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.2.14.dist-info → unstructured_ingest-1.2.18.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=3NalubdNM3MTB5UJ7yWDPdOo9kbE1-bL_USqp5DEYnk,43
+unstructured_ingest/__version__.py,sha256=2oXTzaWwDAqSzco9vnQ5X3trGI9q_sP5tVMO58snQ_4,43
 unstructured_ingest/error.py,sha256=chM7zQSTKjaKaQt_2_QkoZDUwY5XPNeACML7JqOWRLY,4036
 unstructured_ingest/errors_v2.py,sha256=chM7zQSTKjaKaQt_2_QkoZDUwY5XPNeACML7JqOWRLY,4036
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -23,7 +23,7 @@ unstructured_ingest/data_types/entities.py,sha256=ECc6EkZ5_ZUvK7uaALYOynfFmofIrH
 unstructured_ingest/data_types/file_data.py,sha256=J0RQa7YXhhxiLVzhPbF5Hl2nzSpxLFK9vrP6RTBWlSg,3833
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/embed/azure_openai.py,sha256=Q_buBkAcx9FBuTsAqKbRU8vd9vDh8JoDOEth4fFxHbg,2160
-unstructured_ingest/embed/bedrock.py,sha256=BnlKOuL1e4FfR4nV_Ro3A26Fqj3Pq-ZBaSNAgIzUQH0,10955
+unstructured_ingest/embed/bedrock.py,sha256=pUt0VctaxYMkCsyoYoZ-OAiVAM9LmyQN0mp5D5dU-ao,11867
 unstructured_ingest/embed/huggingface.py,sha256=6Gx9L3xa3cv9fX4AMuLsePJQF4T_jwkKjovfqF5X1NM,2435
 unstructured_ingest/embed/interfaces.py,sha256=VCrCSJiEfIxKB4NL4AHgKb-0vB_SEekb47zMUW6gWf0,5211
 unstructured_ingest/embed/mixedbreadai.py,sha256=uKTqzoi4M_WeYZu-qc_TSxwJONOESzxVbBLUbD1Wbns,3922
@@ -72,11 +72,11 @@ unstructured_ingest/processes/connectors/delta_table.py,sha256=SWf-ekUzwzU3TuLHp
 unstructured_ingest/processes/connectors/discord.py,sha256=Ou6b4WKhHqIb1NZBlvgN_4m5zv6OEMHyFE3j_YSqQaY,5333
 unstructured_ingest/processes/connectors/github.py,sha256=-zb9CZbMvbQdXcy7PYWbG-l_-MObxhk1OARGeWkl7FI,7867
 unstructured_ingest/processes/connectors/gitlab.py,sha256=3bFyn8chtAWkrDIXwlVhcZUKuwZLPHBtE7Dq5WTuTt4,10126
-unstructured_ingest/processes/connectors/google_drive.py,sha256=O6RtOQH9dHBUtwKsa9BiVQjBDvcSjwvJsp8cmaJXYSs,35325
+unstructured_ingest/processes/connectors/google_drive.py,sha256=0Ya7TrYOxZsOzAY_xHpwPUNjzskSahnaMGYfEn33VAA,35396
 unstructured_ingest/processes/connectors/jira.py,sha256=5BZeYHoWKzcwZKUbJDFsdCUcSHbLDZTErOdNbteDwI0,20290
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=Tsp9d9YSx2zPh4yl2U--P6cMIQSKMzsFAGyNXXtdS-4,7529
-unstructured_ingest/processes/connectors/milvus.py,sha256=Bkt4u1zzrKqpO0CZbmuFfbtd824ws5XouiTAnc4I4BM,12102
+unstructured_ingest/processes/connectors/milvus.py,sha256=YZaMHGD79tDX5-UpyZR1TFrL460D-JctMdNh-zpSU2o,12301
 unstructured_ingest/processes/connectors/mongodb.py,sha256=zhGWnEJYZnKzjuElyYAEJUT3M7J5m0e48TpVPdiKsBA,15412
 unstructured_ingest/processes/connectors/neo4j.py,sha256=jmnxQmi8EjS22mFKfcdXajZrxoKEkrzHRtrP6QeTuFI,20353
 unstructured_ingest/processes/connectors/onedrive.py,sha256=qhIeFWotFuIxt1Ehg-6IEWXaDu4p-Zhy0u14CfDcnZo,20142
@@ -92,7 +92,7 @@ unstructured_ingest/processes/connectors/assets/__init__.py,sha256=47DEQpj8HBSa-
 unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=8a9HTcRWA6IuswSD632b_uZSO6Dax_0rUYnflqktcek,226
 unstructured_ingest/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/processes/connectors/databricks/__init__.py,sha256=RtKAPyNtXh6fzEsOQ08pA0-vC1uMr3KqYG6cqiBoo70,2133
-unstructured_ingest/processes/connectors/databricks/volumes.py,sha256=sX_6TNrcp5T0lGiMk-z58adyMCPIkoHOe1LyHAKZy6I,8369
+unstructured_ingest/processes/connectors/databricks/volumes.py,sha256=agcDtOweLkqSJwg4fdcZwFq8NUvGynTRVr_buOFUpGk,8454
 unstructured_ingest/processes/connectors/databricks/volumes_aws.py,sha256=WhGTp6aRTLSdc4GChCL4mz2b-IanderW8j1IqezX6YA,2958
 unstructured_ingest/processes/connectors/databricks/volumes_azure.py,sha256=pF2d6uAIbwJJUeOIG5xknUMCGc5d9Aztmc2776wp-a0,3740
 unstructured_ingest/processes/connectors/databricks/volumes_gcp.py,sha256=y9AvVl6PtnIxlTlrPj_wyHBDBRJNq3uoTOuZwTryNg8,2994
@@ -141,7 +141,7 @@ unstructured_ingest/processes/connectors/notion/types/block.py,sha256=pEhy3fFCXS
 unstructured_ingest/processes/connectors/notion/types/database.py,sha256=4VNhpX06dGKmA-COLLzh-sfdSoxQzdUxl1Sk05_B258,2636
 unstructured_ingest/processes/connectors/notion/types/date.py,sha256=VNLs5nTAIGWKxkFUwE13Yoeo5kVQiI-bxUVhjW9SWhE,753
 unstructured_ingest/processes/connectors/notion/types/file.py,sha256=MpEWi7OE0mpA3efq11HJQJTlaVpMMM8cXVE_Pk0m0kg,1315
-unstructured_ingest/processes/connectors/notion/types/page.py,sha256=0fExZsJHXBzaRLwJAKpZwtnfQf_gZ7KnTIbyIyDYC4Q,1471
+unstructured_ingest/processes/connectors/notion/types/page.py,sha256=lfw5fu8UpkmHHv-d4pSTqPbwJt5rnw0eveHf6TFzxmU,1743
 unstructured_ingest/processes/connectors/notion/types/parent.py,sha256=l-EJBKU0HNpDg7p87cATqw0WlUSATD9btyVF7B2A2nI,1706
 unstructured_ingest/processes/connectors/notion/types/rich_text.py,sha256=LPeyFconK_-8Kl3DSLFiCmxwXH3LWthBiYSzj4FAJKY,5483
 unstructured_ingest/processes/connectors/notion/types/user.py,sha256=Bs9hqsMPsfXtMJq1pf-tSgoexVjx__jKdJdfcCyMggM,1964
@@ -206,8 +206,8 @@ unstructured_ingest/processes/connectors/sql/__init__.py,sha256=WNO7jSL1ABw7K5Ix
 unstructured_ingest/processes/connectors/sql/databricks_delta_tables.py,sha256=JFzU84OUIqnKNweH60GbAif6N22KwHWAWOslSGTC62g,9369
 unstructured_ingest/processes/connectors/sql/postgres.py,sha256=kDIL8Cj45EDpKqit1_araRpP4v3cb__QbYqoINg9f2k,5403
 unstructured_ingest/processes/connectors/sql/singlestore.py,sha256=B46lpvyAj1AArpACi9MXbXD1-52zF6Dsj3RJtD1g4r0,5955
-unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=dkGIFz_VIVhew_FjbuO8r3cVluw7VIUdvV6VjkAItP8,11369
-unstructured_ingest/processes/connectors/sql/sql.py,sha256=jIwAck_vFlsMczH7BOyI-iZC_lrLAV-1eqmGtKkPNQc,16170
+unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=zQ6XjLTORkh3bhE9i3u5byB5Myo9UEacUEFHnh3ymIQ,11306
+unstructured_ingest/processes/connectors/sql/sql.py,sha256=GQOlh2onsnJ3_tk-k6t5LWHlgDRCFp1mwH5VO-YDvU4,16317
 unstructured_ingest/processes/connectors/sql/sqlite.py,sha256=2SbwuTlVUztJeuVSEw_--cVP3THJRr2gFySLXF2xkMU,5598
 unstructured_ingest/processes/connectors/sql/vastdb.py,sha256=trhvUBumDmj2rLjmxFBKw9L9wF6ZpssF0wfmRaG97H0,9803
 unstructured_ingest/processes/connectors/weaviate/__init__.py,sha256=1Vnz8hm_Cf3NkQUTz5ZD4QkbLSVql4UvRoY2j2FnC9k,853
@@ -235,8 +235,8 @@ unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3r
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
 unstructured_ingest/utils/tls.py,sha256=Ra8Mii1F4VqErRreg76PBI0eAqPBC009l0sSHa8FdnA,448
-unstructured_ingest-1.2.14.dist-info/METADATA,sha256=GXAxy85oatbdkNDhFN2Yt0g84XuQfdsuAFXeh_U17tI,8827
-unstructured_ingest-1.2.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.2.14.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.2.14.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.2.14.dist-info/RECORD,,
+unstructured_ingest-1.2.18.dist-info/METADATA,sha256=EifLrammakugSnn80GasBgwUar3hrk4l-tj6Sd1b-DM,8827
+unstructured_ingest-1.2.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.2.18.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.2.18.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.2.18.dist-info/RECORD,,