unstructured-ingest 1.0.24__py3-none-any.whl → 1.0.28__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.24"  # pragma: no cover
+__version__ = "1.0.28"  # pragma: no cover

unstructured_ingest/embed/vertexai.py

@@ -34,7 +34,7 @@ ApiKeyType = Secret[Annotated[dict, BeforeValidator(conform_string_to_dict)]]
 class VertexAIEmbeddingConfig(EmbeddingConfig):
     api_key: ApiKeyType = Field(description="API key for Vertex AI")
     embedder_model_name: Optional[str] = Field(
-        default="textembedding-gecko@001", alias="model_name", description="Vertex AI model name"
+        default="text-embedding-005", alias="model_name", description="Vertex AI model name"
     )
 
     def wrap_error(self, e: Exception) -> Exception:

unstructured_ingest/processes/connectors/notion/types/blocks/callout.py

@@ -51,14 +51,49 @@ class ExternalIcon(FromJSONMixin, GetHTMLMixin):
             return None
 
 
+@dataclass
+class FileIconContent(FromJSONMixin):
+    url: str
+    expiry_time: Optional[str] = None  # Add expiry_time if needed
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        # Only include expiry_time if it exists in the dictionary
+        # Notion API might not always include it
+        init_data = {"url": data.get("url")}
+        if "expiry_time" in data:
+            init_data["expiry_time"] = data.get("expiry_time")
+        return cls(**init_data)
+
+
+@dataclass
+class FileIcon(FromJSONMixin, GetHTMLMixin):
+    file: FileIconContent
+    type: str = "file"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(file=FileIconContent.from_dict(data=data.pop("file")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        # Render the file URL, similar to how ExternalIcon is handled
+        if self.file:
+            # Could potentially render an <img> tag, but sticking to URL for consistency
+            return A([Href(self.file.url)], [f"[File Icon: {self.file.url}]"])
+        else:
+            return None
+
+
 class Icon(FromJSONMixin):
     @classmethod
-    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon]:
+    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon, FileIcon]:
         t = data.get("type")
         if t == "emoji":
             return EmojiIcon.from_dict(data)
         elif t == "external":
             return ExternalIcon.from_dict(data)
+        elif t == "file":
+            return FileIcon.from_dict(data)
         else:
             raise ValueError(f"Unexpected icon type: {t} ({data})")
 
@@ -66,7 +101,7 @@ class Icon(FromJSONMixin):
 @dataclass
 class Callout(BlockBase):
     color: str
-    icon: Optional[Union[EmojiIcon, ExternalIcon]] = None
+    icon: Optional[Union[EmojiIcon, ExternalIcon, FileIcon]] = None
     rich_text: List[RichText] = field(default_factory=list)
 
     @staticmethod
@@ -76,9 +111,11 @@ class Callout(BlockBase):
     @classmethod
     def from_dict(cls, data: dict):
         rich_text = data.pop("rich_text", [])
+        icon_data = data.pop("icon", None)
+        icon = Icon.from_dict(icon_data) if icon_data else None
         return cls(
             color=data["color"],
-            icon=Icon.from_dict(data.pop("icon")),
+            icon=icon,
             rich_text=[RichText.from_dict(rt) for rt in rich_text],
         )
 

unstructured_ingest/processes/connectors/notion/types/user.py

@@ -36,14 +36,16 @@ class User(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")
 
 
 @dataclass
@@ -69,11 +71,13 @@ class Bots(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")

unstructured_ingest/processes/connectors/sharepoint.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, AsyncIterator
+from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
 
 from pydantic import Field
 
@@ -39,6 +39,10 @@ class SharepointAccessConfig(OnedriveAccessConfig):
 
 
 class SharepointConnectionConfig(OnedriveConnectionConfig):
+    user_pname: Optional[str] = Field(
+        default=None,
+        description="User principal name or service account, usually your Azure AD email.",
+    )
     site: str = Field(
         description="Sharepoint site url. Process either base url e.g \
                     https://[tenant].sharepoint.com  or relative sites \

unstructured_ingest/processes/connectors/sql/snowflake.py

@@ -37,8 +37,8 @@ if TYPE_CHECKING:
 
 CONNECTOR_TYPE = "snowflake"
 
+EMBEDDINGS_COLUMN = "embeddings"
 _ARRAY_COLUMNS = (
-    "embeddings",
     "languages",
     "link_urls",
     "link_texts",
@@ -47,6 +47,7 @@ _ARRAY_COLUMNS = (
     "emphasized_text_contents",
     "emphasized_text_tags",
 )
+_VECTOR_COLUMNS = (EMBEDDINGS_COLUMN,)
 
 
 class SnowflakeAccessConfig(SQLAccessConfig):
@@ -174,6 +175,33 @@ class SnowflakeUploader(SQLUploader):
     connector_type: str = CONNECTOR_TYPE
     values_delimiter: str = "?"
 
+    _embeddings_dimension: Optional[int] = None
+
+    @property
+    def embeddings_dimension(self) -> Optional[int]:
+        """
+        Get the dimension of the embeddings column in the Snowflake table.
+        If the column is not present or is not of type VECTOR, returns None.
+        """
+        if self._embeddings_dimension is None:
+            with self.connection_config.get_cursor() as cursor:
+                embeddings_column = cursor.execute(
+                    f"SHOW COLUMNS LIKE '{EMBEDDINGS_COLUMN}' IN {self.upload_config.table_name}"
+                ).fetchone()
+            if embeddings_column:
+                data_type = {}
+                if isinstance(embeddings_column, dict):
+                    data_type = json.loads(embeddings_column.get("data_type", "{}"))
+                elif isinstance(embeddings_column, tuple):
+                    data_type = json.loads(embeddings_column[3] or "{}")
+                if isinstance(data_type, dict) and data_type.get("type") == "VECTOR":
+                    self._embeddings_dimension = data_type.get("dimension")
+        # If the _embeddings_dimension is still None, it means the column
+        # is not present or not a VECTOR type
+        if self._embeddings_dimension is None:
+            self._embeddings_dimension = 0
+        return self._embeddings_dimension
+
     @requires_dependencies(["pandas"], extras="snowflake")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         super().run(path=path, file_data=file_data, **kwargs)
@@ -193,7 +221,7 @@ class SnowflakeUploader(SQLUploader):
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))
-                elif column_name in _ARRAY_COLUMNS:
+                elif column_name in _ARRAY_COLUMNS or column_name in _VECTOR_COLUMNS:
                     if not isinstance(value, list) and (
                         value is None or pd.isna(value)
                     ):  # pandas is nan
@@ -206,16 +234,18 @@ class SnowflakeUploader(SQLUploader):
         return output
 
     def _parse_values(self, columns: list[str]) -> str:
-        return ",".join(
-            [
-                (
-                    f"PARSE_JSON({self.values_delimiter})"
-                    if col in _ARRAY_COLUMNS
-                    else self.values_delimiter
+        embeddings_dimension = self.embeddings_dimension
+        parsed_values = []
+        for col in columns:
+            if col in _VECTOR_COLUMNS and embeddings_dimension:
+                parsed_values.append(
+                    f"PARSE_JSON({self.values_delimiter})::VECTOR(FLOAT,{embeddings_dimension})"
                 )
-                for col in columns
-            ]
-        )
+            elif col in _ARRAY_COLUMNS or col in _VECTOR_COLUMNS:
+                parsed_values.append(f"PARSE_JSON({self.values_delimiter})")
+            else:
+                parsed_values.append(self.values_delimiter)
+        return ",".join(parsed_values)
 
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         import numpy as np
@@ -228,8 +258,8 @@ class SnowflakeUploader(SQLUploader):
                 f"record id column "
                 f"{self.upload_config.record_id_key}, skipping delete"
             )
+        df = self._fit_to_schema(df=df, add_missing_columns=True, case_sensitive=False)
         df.replace({np.nan: None}, inplace=True)
-        self._fit_to_schema(df=df)
 
         columns = list(df.columns)
         stmt = "INSERT INTO {table_name} ({columns}) SELECT {values}".format(

unstructured_ingest/processes/connectors/sql/sql.py

@@ -339,12 +339,16 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
-    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+    def _fit_to_schema(
+        self, df: "DataFrame", add_missing_columns: bool = True, case_sensitive: bool = True
+    ) -> "DataFrame":
         import pandas as pd
 
         table_columns = self.get_table_columns()
-        columns = set(df.columns)
-        schema_fields = set(table_columns)
+        columns = set(df.columns if case_sensitive else df.columns.str.lower())
+        schema_fields = set(
+            table_columns if case_sensitive else {col.lower() for col in table_columns}
+        )
         columns_to_drop = columns - schema_fields
         missing_columns = schema_fields - columns
 

{unstructured_ingest-1.0.24.dist-info → unstructured_ingest-1.0.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.24
+Version: 1.0.28
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.24.dist-info → unstructured_ingest-1.0.28.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=p1Nz9H4WBA_aI3GL1htUsWwzMmx5t9ktPqeOxmax3ms,43
+unstructured_ingest/__version__.py,sha256=9ilMs9aEgY_oAgkTk4JfWz5bhMHPPXDWre-49z0HXgo,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -30,7 +30,7 @@ unstructured_ingest/embed/mixedbreadai.py,sha256=uKTqzoi4M_WeYZu-qc_TSxwJONOESzx
 unstructured_ingest/embed/octoai.py,sha256=yZuD7R4mEKS4Jjyae_IrNWogMPOFFS8gW5oUllj3ROU,4540
 unstructured_ingest/embed/openai.py,sha256=TMEOPVfm_OSs4tb3Ymd6q5J49R_-YKvO4TOqCHb3bwk,4647
 unstructured_ingest/embed/togetherai.py,sha256=EehrzTRx4sd_P6AG9JkHAGwTG-o93GMaV5ufmJaxKWs,3629
-unstructured_ingest/embed/vertexai.py,sha256=jA3Y-AysVVaYwqkVd_OgRKF0JdHLAgZlRgfgddcZV2o,3763
+unstructured_ingest/embed/vertexai.py,sha256=DphvPhiYdXTMrQxJCd-64vMs4iVdLY_BphHqz3n5HfM,3758
 unstructured_ingest/embed/voyageai.py,sha256=EOrYzaoXOZ6C4fNkMlCgb8KA8rdfgVXN3USMFpnn0Bs,4698
 unstructured_ingest/interfaces/__init__.py,sha256=QIkWqjsq9INTa89gPuXlMlQL4s3y5TqLmPkuVuTyXcs,795
 unstructured_ingest/interfaces/connector.py,sha256=qUFFJ3qgDMenTCZMtVRjq1DIwsVak6pxNjQOH2eVkMw,1623
@@ -84,7 +84,7 @@ unstructured_ingest/processes/connectors/outlook.py,sha256=zHM5frO7CqQG0-KcTyX49
 unstructured_ingest/processes/connectors/pinecone.py,sha256=pSREUNsQqel6q1EFZsFWelg-uZgGubQY5m_6nVnBFKs,15090
 unstructured_ingest/processes/connectors/redisdb.py,sha256=YzvSlfHs83XWsWMaIC3bV5enKfxejMQ9BQ8CtXfnJ5o,6923
 unstructured_ingest/processes/connectors/salesforce.py,sha256=OaKEWCqZrirHqFJ650K5jSPwYlWefPOapas8Y-4D9oc,11661
-unstructured_ingest/processes/connectors/sharepoint.py,sha256=PowaqMzWr-VCW1rnwcAeRhHyE55kJ9J9FCVlrmtzN0E,4827
+unstructured_ingest/processes/connectors/sharepoint.py,sha256=jI-erp4YUfHxPeUTcfHSPEG3w0wjSBYfAnMg1WT6lfw,4996
 unstructured_ingest/processes/connectors/slack.py,sha256=EkFj9PcAu5_gF2xLogikKDADLbJYq-_jvchzYrTdLO4,9224
 unstructured_ingest/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/processes/connectors/vectara.py,sha256=xrC6jkgW8BII4UjdzUelDu122xT484cpfMTK2wl-sko,12292
@@ -144,12 +144,12 @@ unstructured_ingest/processes/connectors/notion/types/file.py,sha256=MpEWi7OE0mp
 unstructured_ingest/processes/connectors/notion/types/page.py,sha256=0fExZsJHXBzaRLwJAKpZwtnfQf_gZ7KnTIbyIyDYC4Q,1471
 unstructured_ingest/processes/connectors/notion/types/parent.py,sha256=l-EJBKU0HNpDg7p87cATqw0WlUSATD9btyVF7B2A2nI,1706
 unstructured_ingest/processes/connectors/notion/types/rich_text.py,sha256=LPeyFconK_-8Kl3DSLFiCmxwXH3LWthBiYSzj4FAJKY,5483
-unstructured_ingest/processes/connectors/notion/types/user.py,sha256=AKV2ZUcOPe8a98VmyOEaQf-ow6lR5kzudvmTH9BJTKU,1824
+unstructured_ingest/processes/connectors/notion/types/user.py,sha256=Bs9hqsMPsfXtMJq1pf-tSgoexVjx__jKdJdfcCyMggM,1964
 unstructured_ingest/processes/connectors/notion/types/blocks/__init__.py,sha256=mp-jlTLXntT94jdG3koguXTwQ4q_a-ZRR9M_yYew3Jc,1505
 unstructured_ingest/processes/connectors/notion/types/blocks/bookmark.py,sha256=-SDNODMDsp92-YGtKR0ZDAdcWJ7v-YMyrXf2iat-9oU,1191
 unstructured_ingest/processes/connectors/notion/types/blocks/breadcrumb.py,sha256=uR2EwXyAMCBSj-nG4Vp_biZg4CMNXQt4HVwCUS2K08Q,493
 unstructured_ingest/processes/connectors/notion/types/blocks/bulleted_list_item.py,sha256=VyP0l6mNk4-u9vWQOC0Y_lkKhv81b7z74UMcCSiycbo,993
-unstructured_ingest/processes/connectors/notion/types/blocks/callout.py,sha256=MUi8TSn2fL-mfd-yiwOEG5nvjN550YzFAiwSC88UBGA,2619
+unstructured_ingest/processes/connectors/notion/types/blocks/callout.py,sha256=F3blVavN8K6DFcqqP-rQBStksKs1RUJmIabiUeJajx4,3849
 unstructured_ingest/processes/connectors/notion/types/blocks/child_database.py,sha256=0HkAZHFQtRa8rN48JW7x2pJzvJScjCl5yDhmym8UPHc,544
 unstructured_ingest/processes/connectors/notion/types/blocks/child_page.py,sha256=r4fp_O0NYP4UxUfncJdhOGl7C_spprqzq3etK9tILIE,564
 unstructured_ingest/processes/connectors/notion/types/blocks/code.py,sha256=DkhXFYkNzSswwuGh0it4p-RucTAuQPPxPuyEUj09OkI,1404
@@ -206,8 +206,8 @@ unstructured_ingest/processes/connectors/sql/__init__.py,sha256=WNO7jSL1ABw7K5Ix
 unstructured_ingest/processes/connectors/sql/databricks_delta_tables.py,sha256=_IZFFGQUsHV9ScOOISrm6c5FSd4PnX91ePj_COat-gk,9320
 unstructured_ingest/processes/connectors/sql/postgres.py,sha256=kDIL8Cj45EDpKqit1_araRpP4v3cb__QbYqoINg9f2k,5403
 unstructured_ingest/processes/connectors/sql/singlestore.py,sha256=B46lpvyAj1AArpACi9MXbXD1-52zF6Dsj3RJtD1g4r0,5955
-unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=GSEoNrIoJM7p-Q-PrFiONamoxWzjQG8wZJG3mw5Uwdk,9589
-unstructured_ingest/processes/connectors/sql/sql.py,sha256=yUGnv4MF_vT3VHdg7hhGiTD0be94ll-HyhHmRKQp_vQ,15712
+unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=dkGIFz_VIVhew_FjbuO8r3cVluw7VIUdvV6VjkAItP8,11369
+unstructured_ingest/processes/connectors/sql/sql.py,sha256=e2GKJXBKAPpp-H14PMLMUXSa6pfKctEAVOlH9JqfHF4,15885
 unstructured_ingest/processes/connectors/sql/sqlite.py,sha256=V3OfRrXGGhTa_R2FPA-ysn95HHCv9x_VEBKVDsSGsbs,5549
 unstructured_ingest/processes/connectors/sql/vastdb.py,sha256=trhvUBumDmj2rLjmxFBKw9L9wF6ZpssF0wfmRaG97H0,9803
 unstructured_ingest/processes/connectors/weaviate/__init__.py,sha256=1Vnz8hm_Cf3NkQUTz5ZD4QkbLSVql4UvRoY2j2FnC9k,853
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.24.dist-info/METADATA,sha256=Ssmaf7onq6HIFmhR7f2mMPoS2gqGy6dmvxo605W_dWU,8691
-unstructured_ingest-1.0.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.24.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.24.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.24.dist-info/RECORD,,
+unstructured_ingest-1.0.28.dist-info/METADATA,sha256=_e-2mJSqWwsdod99-didta-wEjNVIaIua_EkdBU5ZHY,8691
+unstructured_ingest-1.0.28.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.28.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.28.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.28.dist-info/RECORD,,