unstructured-ingest 0.5.12__py3-none-any.whl → 0.5.14__py3-none-any.whl

test/integration/connectors/test_zendesk.py

@@ -11,7 +11,7 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError
-from unstructured_ingest.v2.processes.connectors.zendesk import (
+from unstructured_ingest.v2.processes.connectors.zendesk.zendesk import (
     CONNECTOR_TYPE,
     ZendeskAccessConfig,
     ZendeskConnectionConfig,

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.12"  # pragma: no cover
+__version__ = "0.5.14"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -64,6 +64,8 @@ from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
 from .slack import slack_source_entry
 from .vectara import CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE
 from .vectara import vectara_destination_entry
+from .zendesk.zendesk import CONNECTOR_TYPE as ZENDESK_CONNECTOR_TYPE
+from .zendesk.zendesk import zendesk_source_entry
 
 add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
@@ -119,3 +121,4 @@ add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
 add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
 
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
+add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)

unstructured_ingest/v2/processes/connectors/neo4j.py

@@ -154,7 +154,7 @@ class Neo4jUploadStager(UploadStager):
             self._add_entities(element, graph, element_node)
 
             if self._is_chunk(element):
-                for origin_element in format_and_truncate_orig_elements(element):
+                for origin_element in format_and_truncate_orig_elements(element, include_text=True):
                     origin_element_node = self._create_element_node(origin_element)
 
                     graph.add_edge(
@@ -327,13 +327,16 @@ class Neo4jUploader(Uploader):
     async def _create_vector_index(
         self, client: AsyncDriver, dimensions: int, similarity_function: SimilarityFunction
     ) -> None:
+        import neo4j.exceptions
+
         label = Label.CHUNK
         logger.info(
             f"Creating index on nodes labeled '{label.value}' if it does not already exist."
         )
         index_name = f"{label.value.lower()}_vector"
-        await client.execute_query(
-            f"""
+        try:
+            await client.execute_query(
+                f"""
             CREATE VECTOR INDEX {index_name} IF NOT EXISTS
             FOR (n:{label.value}) ON n.embedding
             OPTIONS {{indexConfig: {{
@@ -341,7 +344,12 @@ class Neo4jUploader(Uploader):
                 `vector.dimensions`: {dimensions}}}
             }}
             """
-        )
+            )
+        except neo4j.exceptions.ClientError as e:
+            if e.code == "Neo.ClientError.Schema.EquivalentSchemaRuleAlreadyExists":
+                logger.info(f"Index on nodes labeled '{label.value}' already exists.")
+            else:
+                raise
 
     async def _delete_old_data_if_exists(self, file_data: FileData, client: AsyncDriver) -> None:
         logger.info(f"Deleting old data for the record '{file_data.identifier}' (if present).")

unstructured_ingest/v2/processes/connectors/utils.py

@@ -31,7 +31,9 @@ def conform_string_to_dict(value: Any) -> dict:
     raise ValidationError(f"Input could not be mapped to a valid dict: {value}")
 
 
-def format_and_truncate_orig_elements(element: dict) -> list[dict[str, Any]]:
+def format_and_truncate_orig_elements(
+    element: dict, include_text: bool = False
+) -> list[dict[str, Any]]:
     """
     This function is used to format and truncate the orig_elements field in the metadata.
     This is used to remove the text field and other larger fields from the orig_elements
@@ -42,7 +44,8 @@ def format_and_truncate_orig_elements(element: dict) -> list[dict[str, Any]]:
     orig_elements = []
     if raw_orig_elements is not None:
         for element in elements_from_base64_gzipped_json(raw_orig_elements):
-            element.pop("text", None)
+            if not include_text:
+                element.pop("text", None)
             for prop in (
                 "image_base64",
                 "text_as_html",

unstructured_ingest/v2/processes/connectors/zendesk/__init__.py

@@ -1,31 +0,0 @@
-from unstructured_ingest.v2.processes.connector_registry import (
-    add_source_entry,
-)
-
-from .zendesk import (
-    CONNECTOR_TYPE,
-    ZendeskAccessConfig,
-    ZendeskClient,
-    ZendeskConnectionConfig,
-    ZendeskDownloader,
-    ZendeskDownloaderConfig,
-    ZendeskIndexer,
-    ZendeskIndexerConfig,
-    ZendeskTicket,
-    zendesk_source_entry,
-)
-
-__all__ = [
-    "add_source_entry",
-    "zendesk_source_entry",
-    "ZendeskAccessConfig",
-    "ZendeskClient",
-    "ZendeskConnectionConfig",
-    "ZendeskDownloader",
-    "ZendeskDownloaderConfig",
-    "ZendeskIndexer",
-    "ZendeskIndexerConfig",
-    "ZendeskTicket",
-]
-
-add_source_entry(source_type=CONNECTOR_TYPE, entry=zendesk_source_entry)

{unstructured_ingest-0.5.12.dist-info → unstructured_ingest-0.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.12
+Version: 0.5.14
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: dataclasses_json
+Requires-Dist: pandas
+Requires-Dist: opentelemetry-sdk
 Requires-Dist: python-dateutil
-Requires-Dist: click
 Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: pandas
-Requires-Dist: opentelemetry-sdk
+Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
 Provides-Extra: csv
@@ -66,28 +66,28 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
+Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -98,19 +98,19 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -128,10 +128,10 @@ Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
@@ -152,8 +152,8 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -183,12 +183,12 @@ Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
-Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
+Requires-Dist: vastdb; extra == "vastdb"
 Provides-Extra: zendesk
 Requires-Dist: httpx; extra == "zendesk"
-Requires-Dist: bs4; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
+Requires-Dist: bs4; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
@@ -204,8 +204,8 @@ Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.12.dist-info → unstructured_ingest-0.5.14.dist-info}/RECORD

@@ -25,7 +25,7 @@ test/integration/connectors/test_redis.py,sha256=1aKwOb-K4zCxZwHmgW_WzGJwqLntbWT
 test/integration/connectors/test_s3.py,sha256=E1dypeag_E3OIfpQWIz3jb7ctRHRD63UtyTrzyvJzpc,7473
 test/integration/connectors/test_sharepoint.py,sha256=weGby5YD6se7R7KLEq96hxUZYPzwoqZqXXTPhtQWZsQ,7646
 test/integration/connectors/test_vectara.py,sha256=4kKOOTGUjeZw2jKRcgVDI7ifbRPRZfjjVO4d_7H5C6I,8710
-test/integration/connectors/test_zendesk.py,sha256=6Xsxxav9b1NBp_zd66S_sE4Nn5iO6Et4a5zgGR2-Y04,4159
+test/integration/connectors/test_zendesk.py,sha256=Jc1GcMBnCrpzm6_6tJi-FdYxSs15xnp94a7kVwrObMc,4167
 test/integration/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/databricks/test_volumes_native.py,sha256=KqiapQAV0s_Zv0CO8BwYoiCk30dwrSZzuigUWNRIem0,9559
 test/integration/connectors/discord/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -111,7 +111,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=RrghV4wKBupxOt4yEkkqnsRGy9iuYIS7TnVXDOGMwI8,43
+unstructured_ingest/__version__.py,sha256=-WADymCsmQMFkVZ96tUEJfzNLFjyNrB26q5Or_LtNJs,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=7DOnDpGvUNlCoFR7UPRGmOarqH5sFtuUOO5vf8X3oTM,31489
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -426,7 +426,7 @@ unstructured_ingest/v2/processes/embedder.py,sha256=4x-Rt5UCvwdgihDAr24hvTGDEd1C
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
 unstructured_ingest/v2/processes/partitioner.py,sha256=HxopDSbovLh_1epeGeVtuWEX7v5KG35BowwKIJ_y4e8,9910
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=rkEQVVgcHoY3jwgW_5PH_NzdXIEwtBLs9Dk4VzmTZMA,6387
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=ebLvZes84qRx4eS20SkvlVH6WIIM76hifyUgkUJ-dfg,6588
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=eeZJe-bBNxt5Sa-XEFCdcGeJCguJU5WN2Mv9kLp5dVQ,8917
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=E6fB4anCd_gtSzVUsZ5pDrfdxs5AWERQM_NEfeenfEs,18202
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=ngPDpU0oZ6m5sxIlB6u5ebQpqCS_SJ-_amCC1KQ03EQ,11529
@@ -442,7 +442,7 @@ unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=VRDAiou_7oWOIAgQTdOG
 unstructured_ingest/v2/processes/connectors/local.py,sha256=ZvWTj6ZYkwnvQMNFsZWoaQyp9zp0WVqAywMaHJ2kcAc,7153
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=wmcu9NVy3gYlQGT25inN5w_QrhFoL8-hRq0pJFSNw8g,8866
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNNeAywXVpaIiND4b5JTAFlYjLjw,14273
-unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=QO8D0fptJ6C_K4m34KcYvAcrBgdve_Kcb0UVvY4edXs,18405
+unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=vxf6Xuh-OMS09Y-mIF0PIwrFauqRtoI7vjeLBXsFwTk,18744
 unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=EM9fq67RsiudZvZbi6nDXkS-i6W0xLvbkNvD0G-Ni5E,17779
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=O9lC4mZ9V_exg9apiCJSWHsgkuYDSEOlI6CaUS5ZB7c,13961
@@ -450,7 +450,7 @@ unstructured_ingest/v2/processes/connectors/redisdb.py,sha256=p0AY4ukBNpwAemV4bW
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=2T9Bm1H_ALwHhG_YP7vsuUUW-mUg61zcaae3aa9BnN4,4827
 unstructured_ingest/v2/processes/connectors/slack.py,sha256=Z73VmQ3oUY09KoLEi5OBdQeDt4ONEY_02SglWQc6HXE,9252
-unstructured_ingest/v2/processes/connectors/utils.py,sha256=ru_4e5lo5t1jJhR8sGYa5nNhX3gKTgC5B7Oze9qQJjo,2000
+unstructured_ingest/v2/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/v2/processes/connectors/vectara.py,sha256=BlI_4nkpNR99aYxDd9eusm5LQsVB9EI0r-5Kc1D7pgQ,12255
 unstructured_ingest/v2/processes/connectors/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/processes/connectors/assets/weaviate_collection_config.json,sha256=SJlIO0kXxy866tWQ8bEzvwLwflsoUMIS-OKlxMvHIuE,504
@@ -572,12 +572,12 @@ unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=UZ_s8dnVNx9BWFG2fPah4VbQbgEDF4nP78bQeU3jg08,12821
-unstructured_ingest/v2/processes/connectors/zendesk/__init__.py,sha256=XMNocKJ3FHDfy36p_KHhH7ALi0-ji6NhGuQNCV2E4vY,699
+unstructured_ingest/v2/processes/connectors/zendesk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/processes/connectors/zendesk/client.py,sha256=MNyI6SUuxZHf_6zONoC6jR2f9BvyTYoMyGKDOhl4kgs,7897
 unstructured_ingest/v2/processes/connectors/zendesk/zendesk.py,sha256=vQHZa5YYiDPXXPRAPMnPXhh0QzXeiBVx_YIWskZBQIc,15465
-unstructured_ingest-0.5.12.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.5.12.dist-info/METADATA,sha256=-IT5G8E8WZLSABmgyUZtxzQElpUVzKuVPqK54zFC-V0,8465
-unstructured_ingest-0.5.12.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-unstructured_ingest-0.5.12.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.5.12.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.5.12.dist-info/RECORD,,
+unstructured_ingest-0.5.14.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.5.14.dist-info/METADATA,sha256=MWLSj6JZslpZjjlZhvn34zn_AJdlVHZibG2RPEbaMjE,8465
+unstructured_ingest-0.5.14.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+unstructured_ingest-0.5.14.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.5.14.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.5.14.dist-info/RECORD,,