unstructured-ingest 1.0.24__tar.gz → 1.0.28__tar.gz

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.24
+Version: 1.0.28
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.28/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.28"  # pragma: no cover

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/embed/vertexai.py

@@ -34,7 +34,7 @@ ApiKeyType = Secret[Annotated[dict, BeforeValidator(conform_string_to_dict)]]
 class VertexAIEmbeddingConfig(EmbeddingConfig):
     api_key: ApiKeyType = Field(description="API key for Vertex AI")
     embedder_model_name: Optional[str] = Field(
-        default="textembedding-gecko@001", alias="model_name", description="Vertex AI model name"
+        default="text-embedding-005", alias="model_name", description="Vertex AI model name"
     )
 
     def wrap_error(self, e: Exception) -> Exception:

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/processes/connectors/notion/types/blocks/callout.py

@@ -51,14 +51,49 @@ class ExternalIcon(FromJSONMixin, GetHTMLMixin):
             return None
 
 
+@dataclass
+class FileIconContent(FromJSONMixin):
+    url: str
+    expiry_time: Optional[str] = None  # Add expiry_time if needed
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        # Only include expiry_time if it exists in the dictionary
+        # Notion API might not always include it
+        init_data = {"url": data.get("url")}
+        if "expiry_time" in data:
+            init_data["expiry_time"] = data.get("expiry_time")
+        return cls(**init_data)
+
+
+@dataclass
+class FileIcon(FromJSONMixin, GetHTMLMixin):
+    file: FileIconContent
+    type: str = "file"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(file=FileIconContent.from_dict(data=data.pop("file")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        # Render the file URL, similar to how ExternalIcon is handled
+        if self.file:
+            # Could potentially render an <img> tag, but sticking to URL for consistency
+            return A([Href(self.file.url)], [f"[File Icon: {self.file.url}]"])
+        else:
+            return None
+
+
 class Icon(FromJSONMixin):
     @classmethod
-    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon]:
+    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon, FileIcon]:
         t = data.get("type")
         if t == "emoji":
             return EmojiIcon.from_dict(data)
         elif t == "external":
             return ExternalIcon.from_dict(data)
+        elif t == "file":
+            return FileIcon.from_dict(data)
         else:
             raise ValueError(f"Unexpected icon type: {t} ({data})")
 
@@ -66,7 +101,7 @@ class Icon(FromJSONMixin):
 @dataclass
 class Callout(BlockBase):
     color: str
-    icon: Optional[Union[EmojiIcon, ExternalIcon]] = None
+    icon: Optional[Union[EmojiIcon, ExternalIcon, FileIcon]] = None
     rich_text: List[RichText] = field(default_factory=list)
 
     @staticmethod
@@ -76,9 +111,11 @@ class Callout(BlockBase):
     @classmethod
     def from_dict(cls, data: dict):
         rich_text = data.pop("rich_text", [])
+        icon_data = data.pop("icon", None)
+        icon = Icon.from_dict(icon_data) if icon_data else None
         return cls(
             color=data["color"],
-            icon=Icon.from_dict(data.pop("icon")),
+            icon=icon,
             rich_text=[RichText.from_dict(rt) for rt in rich_text],
         )
 

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/processes/connectors/notion/types/user.py

@@ -36,14 +36,16 @@ class User(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")
 
 
 @dataclass
@@ -69,11 +71,13 @@ class Bots(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/processes/connectors/sharepoint.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, AsyncIterator
+from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
 
 from pydantic import Field
 
@@ -39,6 +39,10 @@ class SharepointAccessConfig(OnedriveAccessConfig):
 
 
 class SharepointConnectionConfig(OnedriveConnectionConfig):
+    user_pname: Optional[str] = Field(
+        default=None,
+        description="User principal name or service account, usually your Azure AD email.",
+    )
     site: str = Field(
         description="Sharepoint site url. Process either base url e.g \
                     https://[tenant].sharepoint.com  or relative sites \

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/processes/connectors/sql/snowflake.py

@@ -37,8 +37,8 @@ if TYPE_CHECKING:
 
 CONNECTOR_TYPE = "snowflake"
 
+EMBEDDINGS_COLUMN = "embeddings"
 _ARRAY_COLUMNS = (
-    "embeddings",
     "languages",
     "link_urls",
     "link_texts",
@@ -47,6 +47,7 @@ _ARRAY_COLUMNS = (
     "emphasized_text_contents",
     "emphasized_text_tags",
 )
+_VECTOR_COLUMNS = (EMBEDDINGS_COLUMN,)
 
 
 class SnowflakeAccessConfig(SQLAccessConfig):
@@ -174,6 +175,33 @@ class SnowflakeUploader(SQLUploader):
     connector_type: str = CONNECTOR_TYPE
     values_delimiter: str = "?"
 
+    _embeddings_dimension: Optional[int] = None
+
+    @property
+    def embeddings_dimension(self) -> Optional[int]:
+        """
+        Get the dimension of the embeddings column in the Snowflake table.
+        If the column is not present or is not of type VECTOR, returns None.
+        """
+        if self._embeddings_dimension is None:
+            with self.connection_config.get_cursor() as cursor:
+                embeddings_column = cursor.execute(
+                    f"SHOW COLUMNS LIKE '{EMBEDDINGS_COLUMN}' IN {self.upload_config.table_name}"
+                ).fetchone()
+            if embeddings_column:
+                data_type = {}
+                if isinstance(embeddings_column, dict):
+                    data_type = json.loads(embeddings_column.get("data_type", "{}"))
+                elif isinstance(embeddings_column, tuple):
+                    data_type = json.loads(embeddings_column[3] or "{}")
+                if isinstance(data_type, dict) and data_type.get("type") == "VECTOR":
+                    self._embeddings_dimension = data_type.get("dimension")
+        # If the _embeddings_dimension is still None, it means the column
+        # is not present or not a VECTOR type
+        if self._embeddings_dimension is None:
+            self._embeddings_dimension = 0
+        return self._embeddings_dimension
+
     @requires_dependencies(["pandas"], extras="snowflake")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         super().run(path=path, file_data=file_data, **kwargs)
@@ -193,7 +221,7 @@ class SnowflakeUploader(SQLUploader):
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))
-                elif column_name in _ARRAY_COLUMNS:
+                elif column_name in _ARRAY_COLUMNS or column_name in _VECTOR_COLUMNS:
                     if not isinstance(value, list) and (
                         value is None or pd.isna(value)
                     ):  # pandas is nan
@@ -206,16 +234,18 @@ class SnowflakeUploader(SQLUploader):
         return output
 
     def _parse_values(self, columns: list[str]) -> str:
-        return ",".join(
-            [
-                (
-                    f"PARSE_JSON({self.values_delimiter})"
-                    if col in _ARRAY_COLUMNS
-                    else self.values_delimiter
+        embeddings_dimension = self.embeddings_dimension
+        parsed_values = []
+        for col in columns:
+            if col in _VECTOR_COLUMNS and embeddings_dimension:
+                parsed_values.append(
+                    f"PARSE_JSON({self.values_delimiter})::VECTOR(FLOAT,{embeddings_dimension})"
                 )
-                for col in columns
-            ]
-        )
+            elif col in _ARRAY_COLUMNS or col in _VECTOR_COLUMNS:
+                parsed_values.append(f"PARSE_JSON({self.values_delimiter})")
+            else:
+                parsed_values.append(self.values_delimiter)
+        return ",".join(parsed_values)
 
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         import numpy as np
@@ -228,8 +258,8 @@ class SnowflakeUploader(SQLUploader):
                 f"record id column "
                 f"{self.upload_config.record_id_key}, skipping delete"
             )
+        df = self._fit_to_schema(df=df, add_missing_columns=True, case_sensitive=False)
         df.replace({np.nan: None}, inplace=True)
-        self._fit_to_schema(df=df)
 
         columns = list(df.columns)
         stmt = "INSERT INTO {table_name} ({columns}) SELECT {values}".format(

{unstructured_ingest-1.0.24 → unstructured_ingest-1.0.28}/unstructured_ingest/processes/connectors/sql/sql.py

@@ -339,12 +339,16 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
-    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+    def _fit_to_schema(
+        self, df: "DataFrame", add_missing_columns: bool = True, case_sensitive: bool = True
+    ) -> "DataFrame":
         import pandas as pd
 
         table_columns = self.get_table_columns()
-        columns = set(df.columns)
-        schema_fields = set(table_columns)
+        columns = set(df.columns if case_sensitive else df.columns.str.lower())
+        schema_fields = set(
+            table_columns if case_sensitive else {col.lower() for col in table_columns}
+        )
         columns_to_drop = columns - schema_fields
         missing_columns = schema_fields - columns
 

unstructured_ingest-1.0.24/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.24"  # pragma: no cover