unstructured-ingest 1.2.14__tar.gz → 1.2.18__tar.gz

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.14
+Version: 1.2.18
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.2.18/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.18"  # pragma: no cover

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/embed/bedrock.py

@@ -27,12 +27,24 @@ if TYPE_CHECKING:
     from botocore.client import BaseClient
 
     class BedrockRuntimeClient(BaseClient):
-        def invoke_model(self, body: str, modelId: str, accept: str, contentType: str) -> dict:
+        def invoke_model(
+            self,
+            body: str,
+            modelId: str,
+            accept: str,
+            contentType: str,
+            inferenceProfileId: str = None,
+        ) -> dict:
             pass
 
     class AsyncBedrockRuntimeClient(BaseClient):
         async def invoke_model(
-            self, body: str, modelId: str, accept: str, contentType: str
+            self,
+            body: str,
+            modelId: str,
+            accept: str,
+            contentType: str,
+            inferenceProfileId: str = None,
         ) -> dict:
             pass
 
@@ -63,10 +75,10 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         description="aws secret access key", default=None
     )
     region_name: str = Field(
-        description="aws region name", 
+        description="aws region name",
         default_factory=lambda: (
-            os.getenv("BEDROCK_REGION_NAME") or 
-            os.getenv("AWS_DEFAULT_REGION") or 
+            os.getenv("BEDROCK_REGION_NAME") or
+            os.getenv("AWS_DEFAULT_REGION") or
             "us-west-2"
         )
     )
@@ -79,6 +91,10 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         alias="model_name",
         description="AWS Bedrock model name",
     )
+    inference_profile_id: str | None = Field(
+        description="AWS Bedrock inference profile ID",
+        default_factory=lambda: os.getenv("BEDROCK_INFERENCE_PROFILE_ID"),
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -218,12 +234,18 @@ class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
         bedrock_client = self.config.get_client()
         # invoke bedrock API
         try:
-            response = bedrock_client.invoke_model(
-                body=json.dumps(body),
-                modelId=self.config.embedder_model_name,
-                accept="application/json",
-                contentType="application/json",
-            )
+            invoke_params = {
+                "body": json.dumps(body),
+                "modelId": self.config.embedder_model_name,
+                "accept": "application/json",
+                "contentType": "application/json",
+            }
+
+            # Add inference profile if configured
+            if self.config.inference_profile_id:
+                invoke_params["inferenceProfileId"] = self.config.inference_profile_id
+
+            response = bedrock_client.invoke_model(**invoke_params)
         except Exception as e:
             raise self.wrap_error(e=e)
 
@@ -264,12 +286,18 @@ class AsyncBedrockEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
             async with self.config.get_async_client() as bedrock_client:
                 # invoke bedrock API
                 try:
-                    response = await bedrock_client.invoke_model(
-                        body=json.dumps(body),
-                        modelId=self.config.embedder_model_name,
-                        accept="application/json",
-                        contentType="application/json",
-                    )
+                    invoke_params = {
+                        "body": json.dumps(body),
+                        "modelId": self.config.embedder_model_name,
+                        "accept": "application/json",
+                        "contentType": "application/json",
+                    }
+
+                    # Add inference profile if configured
+                    if self.config.inference_profile_id:
+                        invoke_params["inferenceProfileId"] = self.config.inference_profile_id
+
+                    response = await bedrock_client.invoke_model(**invoke_params)
                 except Exception as e:
                     raise self.wrap_error(e=e)
                 async with response.get("body") as client_response:

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/databricks/volumes.py

@@ -76,7 +76,9 @@ class DatabricksVolumesConnectionConfig(ConnectionConfig, ABC):
         if isinstance(e, ValueError):
             error_message = e.args[0]
             message_split = error_message.split(":")
-            if message_split[0].endswith("auth"):
+            if (message_split[0].endswith("auth")) or (
+                "Client authentication failed" in error_message
+            ):
                 return UserAuthError(e)
         if isinstance(e, DatabricksError):
             reverse_mapping = {v: k for k, v in STATUS_CODE_MAPPING.items()}

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/google_drive.py

@@ -189,6 +189,9 @@ class GoogleDriveIndexer(Indexer):
         """
         count = 0
         stack = [folder_id]
+        # Pre-compute lower-case extension set for O(1) lookup
+        valid_exts = set(e.lower() for e in extensions) if extensions else None
+
         while stack:
             current_folder = stack.pop()
             # Always list all items under the current folder.
@@ -212,7 +215,6 @@ class GoogleDriveIndexer(Indexer):
                         if extensions:
                             # Use a case-insensitive comparison for the file extension.
                             file_ext = (item.get("fileExtension") or "").lower()
-                            valid_exts = [e.lower() for e in extensions]
                             if file_ext in valid_exts:
                                 count += 1
                         else:

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/milvus.py

@@ -1,6 +1,7 @@
 import json
 from contextlib import contextmanager
 from dataclasses import dataclass, field
+from datetime import datetime
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from dateutil import parser
@@ -97,6 +98,13 @@ class MilvusUploadStager(UploadStager):
             return timestamp
         except ValueError:
             pass
+
+        try:
+            dt = datetime.fromisoformat(date_string.replace("Z", "+00:00"))
+            return dt.timestamp()
+        except ValueError:
+            pass
+
         return parser.parse(date_string).timestamp()
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/notion/types/page.py

@@ -1,5 +1,5 @@
 # https://developers.notion.com/reference/page
-from dataclasses import dataclass
+from dataclasses import dataclass, fields
 from typing import Optional
 
 from unstructured_ingest.processes.connectors.notion.interfaces import FromJSONMixin
@@ -28,18 +28,25 @@ class Page(FromJSONMixin):
 
     @classmethod
     def from_dict(cls, data: dict):
+        data = data.copy()  # Don't modify the original
         created_by = data.pop("created_by")
         last_edited_by = data.pop("last_edited_by")
         icon = data.pop("icon")
         cover = data.pop("cover")
         parent = data.pop("parent")
+
+        # Filter data to only include fields that exist in the dataclass
+        filtered_data = {
+            k: v for k, v in data.items() if k in {field.name for field in fields(cls)}
+        }
+
         page = cls(
             created_by=PartialUser.from_dict(created_by),
             last_edited_by=PartialUser.from_dict(last_edited_by),
             icon=FileObject.from_dict(icon) if icon else None,
             cover=FileObject.from_dict(cover) if cover else None,
             parent=Parent.from_dict(parent),
-            **data,
+            **filtered_data,
         )
 
         return page

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/sql/snowflake.py

@@ -233,18 +233,19 @@ class SnowflakeUploader(SQLUploader):
             output.append(tuple(parsed))
         return output
 
-    def _parse_values(self, columns: list[str]) -> str:
+    def _parse_select(self, columns: list[str]) -> str:
         embeddings_dimension = self.embeddings_dimension
         parsed_values = []
-        for col in columns:
+        for i, col in enumerate(columns):
+            argument_selector = f"${i + 1}"
             if col in _VECTOR_COLUMNS and embeddings_dimension:
                 parsed_values.append(
-                    f"PARSE_JSON({self.values_delimiter})::VECTOR(FLOAT,{embeddings_dimension})"
+                    f"PARSE_JSON({argument_selector})::VECTOR(FLOAT,{embeddings_dimension})"
                 )
             elif col in _ARRAY_COLUMNS or col in _VECTOR_COLUMNS:
-                parsed_values.append(f"PARSE_JSON({self.values_delimiter})")
+                parsed_values.append(f"PARSE_JSON({argument_selector})")
             else:
-                parsed_values.append(self.values_delimiter)
+                parsed_values.append(argument_selector)
         return ",".join(parsed_values)
 
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
@@ -262,10 +263,11 @@ class SnowflakeUploader(SQLUploader):
         df.replace({np.nan: None}, inplace=True)
 
         columns = list(df.columns)
-        stmt = "INSERT INTO {table_name} ({columns}) SELECT {values}".format(
+        stmt = "INSERT INTO {table_name} ({columns}) SELECT {select} FROM VALUES ({values})".format(
             table_name=self.upload_config.table_name,
             columns=",".join(columns),
-            values=self._parse_values(columns),
+            select=self._parse_select(columns),
+            values=",".join([self.values_delimiter for _ in columns]),
         )
         logger.info(
             f"writing a total of {len(df)} elements via"
@@ -276,10 +278,7 @@ class SnowflakeUploader(SQLUploader):
         for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
             with self.connection_config.get_cursor() as cursor:
                 values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
-                # TODO: executemany break on 'Binding data in type (list) is not supported'
-                for val in values:
-                    logger.debug(f"running query: {stmt}\nwith values: {val}")
-                    cursor.execute(stmt, val)
+                cursor.executemany(stmt, values)
 
 
 snowflake_source_entry = SourceRegistryEntry(

{unstructured_ingest-1.2.14 → unstructured_ingest-1.2.18}/unstructured_ingest/processes/connectors/sql/sql.py

@@ -64,6 +64,12 @@ def parse_date_string(date_value: Union[str, int]) -> datetime:
         return datetime.fromtimestamp(timestamp)
     except Exception as e:
         logger.debug(f"date {date_value} string not a timestamp: {e}")
+
+    if isinstance(date_value, str):
+        try:
+            return datetime.fromisoformat(date_value)
+        except Exception:
+            pass
     return parser.parse(date_value)
 
 

unstructured_ingest-1.2.14/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.14"  # pragma: no cover