unstructured-ingest 1.0.18__py3-none-any.whl → 1.0.21__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.18"  # pragma: no cover
+__version__ = "1.0.21"  # pragma: no cover

unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql

@@ -1,4 +1,4 @@
-CREATE TABLE elements (
+CREATE TABLE IF NOT EXISTS `elements` (
     id STRING NOT NULL PRIMARY KEY,
     record_id STRING NOT NULL,
     element_id STRING NOT NULL,
@@ -7,4 +7,3 @@ CREATE TABLE elements (
     type STRING,
     metadata VARIANT
 );
-

unstructured_ingest/processes/connectors/databricks/volumes_table.py

@@ -136,7 +136,7 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     def get_table_columns(self) -> dict[str, str]:
         if self._columns is None:
             with self.get_cursor() as cursor:
-                cursor.execute(f"SELECT * from {self.upload_config.table_name} LIMIT 1")
+                cursor.execute(f"SELECT * from `{self.upload_config.table_name}` LIMIT 1")
                 self._columns = {desc[0]: desc[1] for desc in cursor.description}
         return self._columns
 
@@ -152,7 +152,7 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
         )
         with self.get_cursor() as cursor:
             cursor.execute(
-                f"DELETE FROM {self.upload_config.table_name} WHERE {RECORD_ID_LABEL} = '{file_data.identifier}'"  # noqa: E501
+                f"DELETE FROM `{self.upload_config.table_name}` WHERE {RECORD_ID_LABEL} = '{file_data.identifier}'"  # noqa: E501
             )
             results = cursor.fetchall()
             deleted_rows = results[0][0]

unstructured_ingest/processes/connectors/jira.py

@@ -3,7 +3,7 @@ from collections import abc
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union, cast
 
 from pydantic import Field, Secret
 
@@ -169,8 +169,28 @@ class JiraConnectionConfig(ConnectionConfig):
     def get_client(self) -> Generator["Jira", None, None]:
         from atlassian import Jira
 
+        class CustomJira(Jira):
+            """
+            Custom Jira class to fix the issue with the get_project_issues_count method.
+            This class inherits from the original Jira class and overrides the method to
+            handle the response correctly.
+            Once the issue is fixed in the original library, this class can be removed.
+            """
+
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+
+            def get_project_issues_count(self, project: str) -> int:
+                jql = f'project = "{project}" '
+                response = self.jql(jql, fields="*none")
+                response = cast("dict", response)
+                if "total" in response:
+                    return response["total"]
+                else:
+                    return len(response["issues"])
+
         access_configs = self.access_config.get_secret_value()
-        with Jira(
+        with CustomJira(
             url=self.url,
             username=self.username,
             password=access_configs.password,

unstructured_ingest/processes/connectors/weaviate/weaviate.py

@@ -234,15 +234,32 @@ class WeaviateUploader(VectorDBUploader, ABC):
         self.create_destination(**kwargs)
 
     def format_destination_name(self, destination_name: str) -> str:
-        # Weaviate naming requirements:
-        # must be alphanumeric and underscores only
+        """
+        Weaviate Collection naming conventions:
+            1. must begin with an uppercase letter
+            2. must be alphanumeric and underscores only
+        """
+
+        # Check if the first character is an uppercase letter
+        if not re.match(r"^[a-zA-Z]", destination_name):
+            raise ValueError("Collection name must start with an uppercase letter")
+        # Replace all non-alphanumeric characters with underscores
         formatted = re.sub(r"[^a-zA-Z0-9]", "_", destination_name)
-        # must begin with capital letter
-        return formatted.capitalize()
+        # Make the first character uppercase and leave the rest as is
+        if len(formatted) == 1:
+            formatted = formatted.capitalize()
+        else:
+            formatted = formatted[0].capitalize() + formatted[1:]
+        if formatted != destination_name:
+            logger.warning(
+                f"Given Collection name '{destination_name}' doesn't follow naming conventions. "
+                f"Renaming to '{formatted}'"
+            )
+        return formatted
 
     def create_destination(
         self,
-        destination_name: str = "unstructuredautocreated",
+        destination_name: str = "Unstructuredautocreated",
         vector_length: Optional[int] = None,
         **kwargs: Any,
     ) -> bool:
@@ -250,18 +267,18 @@ class WeaviateUploader(VectorDBUploader, ABC):
         collection_name = self.format_destination_name(collection_name)
         self.upload_config.collection = collection_name
 
-        connectors_dir = Path(__file__).parents[1]
-        collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
-        with collection_config_file.open() as f:
-            collection_config = json.load(f)
-        collection_config["class"] = collection_name
-
         if not self._collection_exists():
-            logger.info(f"creating weaviate collection '{collection_name}' with default configs")
+            connectors_dir = Path(__file__).parents[1]
+            collection_config_file = connectors_dir / "assets" / "weaviate_collection_config.json"
+            with collection_config_file.open() as f:
+                collection_config = json.load(f)
+            collection_config["class"] = collection_name
+
+            logger.info(f"Creating weaviate collection '{collection_name}' with default configs")
             with self.connection_config.get_client() as weaviate_client:
                 weaviate_client.collections.create_from_dict(config=collection_config)
                 return True
-        logger.debug(f"collection with name '{collection_name}' already exists, skipping creation")
+        logger.debug(f"Collection with name '{collection_name}' already exists, skipping creation")
         return False
 
     def check_for_errors(self, client: "WeaviateClient") -> None:

{unstructured_ingest-1.0.18.dist-info → unstructured_ingest-1.0.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.18
+Version: 1.0.21
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.18.dist-info → unstructured_ingest-1.0.21.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=_WdLW6DLLv5QXi1-R0iwdBbseUYvOyWeY5pyAXABOCY,43
+unstructured_ingest/__version__.py,sha256=_fAo4tbdJV7k_s1lgXUPPmLFVpxbTy7HhoN9KbPxQ4Y,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -73,7 +73,7 @@ unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMk
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=6h1CdqznJmzeWxGfXrFLdNdT23PExGnUMMX7usK_4Kk,10013
 unstructured_ingest/processes/connectors/google_drive.py,sha256=BIFBZGp26JlBBOcXy5Gq0UoNzWv6pwRKhEAHMVMI2_M,25050
-unstructured_ingest/processes/connectors/jira.py,sha256=eG8yTn8ZVEz7rBJ-ha8i_d9hEh6VALN6QJT_vbYvbL0,17142
+unstructured_ingest/processes/connectors/jira.py,sha256=alnwUYyID-mUIlGq1xh5QGEw2iZ2RwbOIyptev3dI6Q,18011
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9VhzKZuuQUpDS-cVNAQ2g,7426
 unstructured_ingest/processes/connectors/milvus.py,sha256=Jr9cul7By03tGAPFnFBoqncnNWwbhKd-qbmkuqnin8U,8908
@@ -89,7 +89,7 @@ unstructured_ingest/processes/connectors/slack.py,sha256=EkFj9PcAu5_gF2xLogikKDA
 unstructured_ingest/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/processes/connectors/vectara.py,sha256=xrC6jkgW8BII4UjdzUelDu122xT484cpfMTK2wl-sko,12292
 unstructured_ingest/processes/connectors/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=dUZZDNkyvQXKqoAThRz3ek7zaUE2l_LAQimlG5WZhH4,211
+unstructured_ingest/processes/connectors/assets/databricks_delta_table_schema.sql,sha256=8a9HTcRWA6IuswSD632b_uZSO6Dax_0rUYnflqktcek,226
 unstructured_ingest/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
 unstructured_ingest/processes/connectors/databricks/__init__.py,sha256=RtKAPyNtXh6fzEsOQ08pA0-vC1uMr3KqYG6cqiBoo70,2133
 unstructured_ingest/processes/connectors/databricks/volumes.py,sha256=OWQrne9-5hPzc-kxGa2P53M3DoksDzMDyjLhQyihdCo,8020
@@ -97,7 +97,7 @@ unstructured_ingest/processes/connectors/databricks/volumes_aws.py,sha256=WhGTp6
 unstructured_ingest/processes/connectors/databricks/volumes_azure.py,sha256=pF2d6uAIbwJJUeOIG5xknUMCGc5d9Aztmc2776wp-a0,3740
 unstructured_ingest/processes/connectors/databricks/volumes_gcp.py,sha256=y9AvVl6PtnIxlTlrPj_wyHBDBRJNq3uoTOuZwTryNg8,2994
 unstructured_ingest/processes/connectors/databricks/volumes_native.py,sha256=pivySGMmFSsyuB42ARAWAPXFQ7qTQxO3dfEoE23pBNM,3104
-unstructured_ingest/processes/connectors/databricks/volumes_table.py,sha256=tqi6PpYpIBMTZcYZXl5Lw0YuawyDvjHI08TKPFFTTr0,8194
+unstructured_ingest/processes/connectors/databricks/volumes_table.py,sha256=K-EBsV99I9ubD3A0cqAJTC4vpSwrnBeACFGWbgGCSsY,8198
 unstructured_ingest/processes/connectors/duckdb/__init__.py,sha256=Dr6BRJJGefJnnp_vn5W5gBd7vrCCXTMLweuDIqTP-fM,558
 unstructured_ingest/processes/connectors/duckdb/base.py,sha256=bTLhilg6mgERNCpeeNNl7wxy3xkOt23O9XpCyD0WVY4,2945
 unstructured_ingest/processes/connectors/duckdb/duckdb.py,sha256=jsmibTd_yvYzkCT05HhCJvplyobtjfNILC3zyTuCcVY,4464
@@ -214,7 +214,7 @@ unstructured_ingest/processes/connectors/weaviate/__init__.py,sha256=1Vnz8hm_Cf3
 unstructured_ingest/processes/connectors/weaviate/cloud.py,sha256=tDQ4Vfph1RwADzS0Lk4TSoeT6TZ2gX9DNi78yXkgDw0,6245
 unstructured_ingest/processes/connectors/weaviate/embedded.py,sha256=buizqBd6PSbd9VgRrOj43GZEorBpDFkUIkE6sN9emhw,3008
 unstructured_ingest/processes/connectors/weaviate/local.py,sha256=4fgZsL9dgnWuaSNqVlKROm-S3Ql3naLmKvigLBgUQdw,2195
-unstructured_ingest/processes/connectors/weaviate/weaviate.py,sha256=SqtGcQgejGH0N1R49tGrUtGcTB8mt7sywXmWFTIcpB8,12866
+unstructured_ingest/processes/connectors/weaviate/weaviate.py,sha256=yB67gxvo3X0UaP_mNeB0HbSWXst7ur0E2QKwLA0gIS4,13647
 unstructured_ingest/processes/connectors/zendesk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/processes/connectors/zendesk/client.py,sha256=GvPIpx4aYdD58-edHgvCFjFao94uR0O5Yf4dT9NCmSk,11952
 unstructured_ingest/processes/connectors/zendesk/zendesk.py,sha256=j5zS_7vJmYDEQtysz_UfwIUH65gc4r-Zjc1LocJr9FM,9033
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.18.dist-info/METADATA,sha256=Ab6dhItl8CiP5OYQContbtpnfBpz77OsIecAyjgb_DA,8694
-unstructured_ingest-1.0.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.18.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.18.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.18.dist-info/RECORD,,
+unstructured_ingest-1.0.21.dist-info/METADATA,sha256=lYMmxWJ0ySauI_NWrAQo4YZQ7pXAK4bZ0dX0XIsgacE,8694
+unstructured_ingest-1.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.21.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.21.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.21.dist-info/RECORD,,