unstructured-ingest 1.0.23__py3-none-any.whl → 1.0.27__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.23"  # pragma: no cover
+__version__ = "1.0.27"  # pragma: no cover

unstructured_ingest/processes/connectors/jira.py

@@ -3,7 +3,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any, Callable, Generator, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, List, Optional, Union
 
 from pydantic import BaseModel, Field, Secret
 
@@ -139,28 +139,8 @@ class JiraConnectionConfig(ConnectionConfig):
     def get_client(self) -> Generator["Jira", None, None]:
         from atlassian import Jira
 
-        class CustomJira(Jira):
-            """
-            Custom Jira class to fix the issue with the get_project_issues_count method.
-            This class inherits from the original Jira class and overrides the method to
-            handle the response correctly.
-            Once the issue is fixed in the original library, this class can be removed.
-            """
-
-            def __init__(self, *args, **kwargs):
-                super().__init__(*args, **kwargs)
-
-            def get_project_issues_count(self, project: str) -> int:
-                jql = f'project = "{project}" '
-                response = self.jql(jql, fields="*none")
-                response = cast("dict", response)
-                if "total" in response:
-                    return response["total"]
-                else:
-                    return len(response["issues"])
-
         access_configs = self.access_config.get_secret_value()
-        with CustomJira(
+        with Jira(
             url=self.url,
             username=self.username,
             password=access_configs.password,
@@ -206,13 +186,21 @@ class JiraIndexer(Indexer):
             )
         logger.info("Connection to Jira successful.")
 
-    def _get_issues_within_projects(self) -> Generator[JiraIssueMetadata, None, None]:
+    def run_jql(self, jql: str, **kwargs) -> Generator[JiraIssueMetadata, None, None]:
         with self.connection_config.get_client() as client:
-            fields = ["key", "id"]
-            jql = "project in ({})".format(", ".join(self.index_config.projects))
-            jql = self._update_jql(jql)
-            for issue in api_token_based_generator(client.enhanced_jql, jql=jql, fields=fields):
-                yield JiraIssueMetadata.model_validate(issue)
+            if client.cloud:
+                for issue in api_token_based_generator(client.enhanced_jql, jql=jql, **kwargs):
+                    yield JiraIssueMetadata.model_validate(issue)
+            else:
+                for issue in api_page_based_generator(client.jql, jql=jql, **kwargs):
+                    yield JiraIssueMetadata.model_validate(issue)
+
+    def _get_issues_within_projects(self) -> Generator[JiraIssueMetadata, None, None]:
+        fields = ["key", "id", "status"]
+        jql = "project in ({})".format(", ".join(self.index_config.projects))
+        jql = self._update_jql(jql)
+        logger.debug(f"running jql: {jql}")
+        return self.run_jql(jql=jql, fields=fields)
 
     def _get_issues_within_single_board(self, board_id: str) -> List[JiraIssueMetadata]:
         with self.connection_config.get_client() as client:
@@ -223,6 +211,7 @@ class JiraIndexer(Indexer):
                 )
             else:
                 jql = "ORDER BY id"
+            logger.debug(f"running jql for board {board_id}: {jql}")
             for issue in api_page_based_generator(
                 fn=client.get_issues_for_board, board_id=board_id, fields=fields, jql=jql
             ):
@@ -244,12 +233,11 @@ class JiraIndexer(Indexer):
         return jql
 
     def _get_issues_by_keys(self) -> Generator[JiraIssueMetadata, None, None]:
-        with self.connection_config.get_client() as client:
-            fields = ["key", "id"]
-            jql = "key in ({})".format(", ".join(self.index_config.issues))
-            jql = self._update_jql(jql)
-            for issue in api_token_based_generator(client.enhanced_jql, jql=jql, fields=fields):
-                yield JiraIssueMetadata.model_validate(issue)
+        fields = ["key", "id"]
+        jql = "key in ({})".format(", ".join(self.index_config.issues))
+        jql = self._update_jql(jql)
+        logger.debug(f"running jql: {jql}")
+        return self.run_jql(jql=jql, fields=fields)
 
     def _create_file_data_from_issue(self, issue: JiraIssueMetadata) -> FileData:
         # Build metadata

unstructured_ingest/processes/connectors/notion/types/blocks/callout.py

@@ -51,14 +51,49 @@ class ExternalIcon(FromJSONMixin, GetHTMLMixin):
             return None
 
 
+@dataclass
+class FileIconContent(FromJSONMixin):
+    url: str
+    expiry_time: Optional[str] = None  # Add expiry_time if needed
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        # Only include expiry_time if it exists in the dictionary
+        # Notion API might not always include it
+        init_data = {"url": data.get("url")}
+        if "expiry_time" in data:
+            init_data["expiry_time"] = data.get("expiry_time")
+        return cls(**init_data)
+
+
+@dataclass
+class FileIcon(FromJSONMixin, GetHTMLMixin):
+    file: FileIconContent
+    type: str = "file"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(file=FileIconContent.from_dict(data=data.pop("file")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        # Render the file URL, similar to how ExternalIcon is handled
+        if self.file:
+            # Could potentially render an <img> tag, but sticking to URL for consistency
+            return A([Href(self.file.url)], [f"[File Icon: {self.file.url}]"])
+        else:
+            return None
+
+
 class Icon(FromJSONMixin):
     @classmethod
-    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon]:
+    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon, FileIcon]:
         t = data.get("type")
         if t == "emoji":
             return EmojiIcon.from_dict(data)
         elif t == "external":
             return ExternalIcon.from_dict(data)
+        elif t == "file":
+            return FileIcon.from_dict(data)
         else:
             raise ValueError(f"Unexpected icon type: {t} ({data})")
 
@@ -66,7 +101,7 @@ class Icon(FromJSONMixin):
 @dataclass
 class Callout(BlockBase):
     color: str
-    icon: Optional[Union[EmojiIcon, ExternalIcon]] = None
+    icon: Optional[Union[EmojiIcon, ExternalIcon, FileIcon]] = None
     rich_text: List[RichText] = field(default_factory=list)
 
     @staticmethod
@@ -76,9 +111,11 @@ class Callout(BlockBase):
     @classmethod
     def from_dict(cls, data: dict):
         rich_text = data.pop("rich_text", [])
+        icon_data = data.pop("icon", None)
+        icon = Icon.from_dict(icon_data) if icon_data else None
         return cls(
             color=data["color"],
-            icon=Icon.from_dict(data.pop("icon")),
+            icon=icon,
             rich_text=[RichText.from_dict(rt) for rt in rich_text],
         )
 

unstructured_ingest/processes/connectors/notion/types/user.py

@@ -36,14 +36,16 @@ class User(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")
 
 
 @dataclass
@@ -69,11 +71,13 @@ class Bots(FromJSONMixin, GetHTMLMixin):
     def get_text(self) -> Optional[str]:
         text = self.name
         if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
+            text = f"[{text}]({self.avatar_url})"
         return text
 
     def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
+        if self.avatar_url and self.name:
             return A([Href(self.avatar_url)], self.name)
-        else:
+        elif self.name:
             return Div([], self.name)
+        else:
+            return Div([], "")

unstructured_ingest/processes/connectors/sql/snowflake.py

@@ -37,8 +37,8 @@ if TYPE_CHECKING:
 
 CONNECTOR_TYPE = "snowflake"
 
+EMBEDDINGS_COLUMN = "embeddings"
 _ARRAY_COLUMNS = (
-    "embeddings",
     "languages",
     "link_urls",
     "link_texts",
@@ -47,6 +47,7 @@ _ARRAY_COLUMNS = (
     "emphasized_text_contents",
     "emphasized_text_tags",
 )
+_VECTOR_COLUMNS = (EMBEDDINGS_COLUMN,)
 
 
 class SnowflakeAccessConfig(SQLAccessConfig):
@@ -174,6 +175,33 @@ class SnowflakeUploader(SQLUploader):
     connector_type: str = CONNECTOR_TYPE
     values_delimiter: str = "?"
 
+    _embeddings_dimension: Optional[int] = None
+
+    @property
+    def embeddings_dimension(self) -> Optional[int]:
+        """
+        Get the dimension of the embeddings column in the Snowflake table.
+        If the column is not present or is not of type VECTOR, returns None.
+        """
+        if self._embeddings_dimension is None:
+            with self.connection_config.get_cursor() as cursor:
+                embeddings_column = cursor.execute(
+                    f"SHOW COLUMNS LIKE '{EMBEDDINGS_COLUMN}' IN {self.upload_config.table_name}"
+                ).fetchone()
+            if embeddings_column:
+                data_type = {}
+                if isinstance(embeddings_column, dict):
+                    data_type = json.loads(embeddings_column.get("data_type", "{}"))
+                elif isinstance(embeddings_column, tuple):
+                    data_type = json.loads(embeddings_column[3] or "{}")
+                if isinstance(data_type, dict) and data_type.get("type") == "VECTOR":
+                    self._embeddings_dimension = data_type.get("dimension")
+        # If the _embeddings_dimension is still None, it means the column
+        # is not present or not a VECTOR type
+        if self._embeddings_dimension is None:
+            self._embeddings_dimension = 0
+        return self._embeddings_dimension
+
     @requires_dependencies(["pandas"], extras="snowflake")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         super().run(path=path, file_data=file_data, **kwargs)
@@ -193,7 +221,7 @@ class SnowflakeUploader(SQLUploader):
                         parsed.append(None)
                     else:
                         parsed.append(parse_date_string(value))
-                elif column_name in _ARRAY_COLUMNS:
+                elif column_name in _ARRAY_COLUMNS or column_name in _VECTOR_COLUMNS:
                     if not isinstance(value, list) and (
                         value is None or pd.isna(value)
                     ):  # pandas is nan
@@ -206,16 +234,18 @@ class SnowflakeUploader(SQLUploader):
         return output
 
     def _parse_values(self, columns: list[str]) -> str:
-        return ",".join(
-            [
-                (
-                    f"PARSE_JSON({self.values_delimiter})"
-                    if col in _ARRAY_COLUMNS
-                    else self.values_delimiter
+        embeddings_dimension = self.embeddings_dimension
+        parsed_values = []
+        for col in columns:
+            if col in _VECTOR_COLUMNS and embeddings_dimension:
+                parsed_values.append(
+                    f"PARSE_JSON({self.values_delimiter})::VECTOR(FLOAT,{embeddings_dimension})"
                 )
-                for col in columns
-            ]
-        )
+            elif col in _ARRAY_COLUMNS or col in _VECTOR_COLUMNS:
+                parsed_values.append(f"PARSE_JSON({self.values_delimiter})")
+            else:
+                parsed_values.append(self.values_delimiter)
+        return ",".join(parsed_values)
 
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         import numpy as np
@@ -228,8 +258,8 @@ class SnowflakeUploader(SQLUploader):
                 f"record id column "
                 f"{self.upload_config.record_id_key}, skipping delete"
             )
+        df = self._fit_to_schema(df=df, add_missing_columns=True, case_sensitive=False)
         df.replace({np.nan: None}, inplace=True)
-        self._fit_to_schema(df=df)
 
         columns = list(df.columns)
         stmt = "INSERT INTO {table_name} ({columns}) SELECT {values}".format(

unstructured_ingest/processes/connectors/sql/sql.py

@@ -339,12 +339,16 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
-    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+    def _fit_to_schema(
+        self, df: "DataFrame", add_missing_columns: bool = True, case_sensitive: bool = True
+    ) -> "DataFrame":
         import pandas as pd
 
         table_columns = self.get_table_columns()
-        columns = set(df.columns)
-        schema_fields = set(table_columns)
+        columns = set(df.columns if case_sensitive else df.columns.str.lower())
+        schema_fields = set(
+            table_columns if case_sensitive else {col.lower() for col in table_columns}
+        )
         columns_to_drop = columns - schema_fields
         missing_columns = schema_fields - columns
 

{unstructured_ingest-1.0.23.dist-info → unstructured_ingest-1.0.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.23
+Version: 1.0.27
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.23.dist-info → unstructured_ingest-1.0.27.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=xbdPxvOGZJUW_s_LZYTaPijNvLNKSjZuHlwNDGHpDjE,43
+unstructured_ingest/__version__.py,sha256=E5vx6gXBg117MrnhZuy3hX22Pf4UAQZstAs7OEF1Lio,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -73,7 +73,7 @@ unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMk
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=6h1CdqznJmzeWxGfXrFLdNdT23PExGnUMMX7usK_4Kk,10013
 unstructured_ingest/processes/connectors/google_drive.py,sha256=BIFBZGp26JlBBOcXy5Gq0UoNzWv6pwRKhEAHMVMI2_M,25050
-unstructured_ingest/processes/connectors/jira.py,sha256=Hw07c2HT2vA2l2wpoYWXPNtLbnWreXCIRimAxm0Gfpw,19055
+unstructured_ingest/processes/connectors/jira.py,sha256=a7OuVi4RFfr22Tqgk60lwmtWTRBw2fI1m8KPqfA8Ffo,18504
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9VhzKZuuQUpDS-cVNAQ2g,7426
 unstructured_ingest/processes/connectors/milvus.py,sha256=Jr9cul7By03tGAPFnFBoqncnNWwbhKd-qbmkuqnin8U,8908
@@ -144,12 +144,12 @@ unstructured_ingest/processes/connectors/notion/types/file.py,sha256=MpEWi7OE0mp
 unstructured_ingest/processes/connectors/notion/types/page.py,sha256=0fExZsJHXBzaRLwJAKpZwtnfQf_gZ7KnTIbyIyDYC4Q,1471
 unstructured_ingest/processes/connectors/notion/types/parent.py,sha256=l-EJBKU0HNpDg7p87cATqw0WlUSATD9btyVF7B2A2nI,1706
 unstructured_ingest/processes/connectors/notion/types/rich_text.py,sha256=LPeyFconK_-8Kl3DSLFiCmxwXH3LWthBiYSzj4FAJKY,5483
-unstructured_ingest/processes/connectors/notion/types/user.py,sha256=AKV2ZUcOPe8a98VmyOEaQf-ow6lR5kzudvmTH9BJTKU,1824
+unstructured_ingest/processes/connectors/notion/types/user.py,sha256=Bs9hqsMPsfXtMJq1pf-tSgoexVjx__jKdJdfcCyMggM,1964
 unstructured_ingest/processes/connectors/notion/types/blocks/__init__.py,sha256=mp-jlTLXntT94jdG3koguXTwQ4q_a-ZRR9M_yYew3Jc,1505
 unstructured_ingest/processes/connectors/notion/types/blocks/bookmark.py,sha256=-SDNODMDsp92-YGtKR0ZDAdcWJ7v-YMyrXf2iat-9oU,1191
 unstructured_ingest/processes/connectors/notion/types/blocks/breadcrumb.py,sha256=uR2EwXyAMCBSj-nG4Vp_biZg4CMNXQt4HVwCUS2K08Q,493
 unstructured_ingest/processes/connectors/notion/types/blocks/bulleted_list_item.py,sha256=VyP0l6mNk4-u9vWQOC0Y_lkKhv81b7z74UMcCSiycbo,993
-unstructured_ingest/processes/connectors/notion/types/blocks/callout.py,sha256=MUi8TSn2fL-mfd-yiwOEG5nvjN550YzFAiwSC88UBGA,2619
+unstructured_ingest/processes/connectors/notion/types/blocks/callout.py,sha256=F3blVavN8K6DFcqqP-rQBStksKs1RUJmIabiUeJajx4,3849
 unstructured_ingest/processes/connectors/notion/types/blocks/child_database.py,sha256=0HkAZHFQtRa8rN48JW7x2pJzvJScjCl5yDhmym8UPHc,544
 unstructured_ingest/processes/connectors/notion/types/blocks/child_page.py,sha256=r4fp_O0NYP4UxUfncJdhOGl7C_spprqzq3etK9tILIE,564
 unstructured_ingest/processes/connectors/notion/types/blocks/code.py,sha256=DkhXFYkNzSswwuGh0it4p-RucTAuQPPxPuyEUj09OkI,1404
@@ -206,8 +206,8 @@ unstructured_ingest/processes/connectors/sql/__init__.py,sha256=WNO7jSL1ABw7K5Ix
 unstructured_ingest/processes/connectors/sql/databricks_delta_tables.py,sha256=_IZFFGQUsHV9ScOOISrm6c5FSd4PnX91ePj_COat-gk,9320
 unstructured_ingest/processes/connectors/sql/postgres.py,sha256=kDIL8Cj45EDpKqit1_araRpP4v3cb__QbYqoINg9f2k,5403
 unstructured_ingest/processes/connectors/sql/singlestore.py,sha256=B46lpvyAj1AArpACi9MXbXD1-52zF6Dsj3RJtD1g4r0,5955
-unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=GSEoNrIoJM7p-Q-PrFiONamoxWzjQG8wZJG3mw5Uwdk,9589
-unstructured_ingest/processes/connectors/sql/sql.py,sha256=yUGnv4MF_vT3VHdg7hhGiTD0be94ll-HyhHmRKQp_vQ,15712
+unstructured_ingest/processes/connectors/sql/snowflake.py,sha256=dkGIFz_VIVhew_FjbuO8r3cVluw7VIUdvV6VjkAItP8,11369
+unstructured_ingest/processes/connectors/sql/sql.py,sha256=e2GKJXBKAPpp-H14PMLMUXSa6pfKctEAVOlH9JqfHF4,15885
 unstructured_ingest/processes/connectors/sql/sqlite.py,sha256=V3OfRrXGGhTa_R2FPA-ysn95HHCv9x_VEBKVDsSGsbs,5549
 unstructured_ingest/processes/connectors/sql/vastdb.py,sha256=trhvUBumDmj2rLjmxFBKw9L9wF6ZpssF0wfmRaG97H0,9803
 unstructured_ingest/processes/connectors/weaviate/__init__.py,sha256=1Vnz8hm_Cf3NkQUTz5ZD4QkbLSVql4UvRoY2j2FnC9k,853
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.23.dist-info/METADATA,sha256=b0LZ3XzhlhUgDZd4mEUPxxhOT-lqKAOnDfiQeJhCgoA,8691
-unstructured_ingest-1.0.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.23.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.23.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.23.dist-info/RECORD,,
+unstructured_ingest-1.0.27.dist-info/METADATA,sha256=lridvITtHe951XlNndhSuKB2kRNGqqaimYW9SvuJ0Kg,8691
+unstructured_ingest-1.0.27.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.27.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.27.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.27.dist-info/RECORD,,