unstructured-ingest 0.3.11__py3-none-any.whl → 0.3.12__py3-none-any.whl

{unstructured_ingest-0.3.11.dist-info → unstructured_ingest-0.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.11
+Version: 0.3.12
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,13 +23,13 @@ Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: ndjson
+Requires-Dist: python-dateutil
+Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
 Requires-Dist: dataclasses-json
-Requires-Dist: pydantic>=2.7
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
-Requires-Dist: click
 Requires-Dist: tqdm
+Requires-Dist: click
+Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
@@ -45,8 +45,8 @@ Provides-Extra: biomed
 Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -61,8 +61,8 @@ Requires-Dist: unstructured[tsv]; extra == "csv"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: doc
@@ -70,8 +70,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -90,12 +90,12 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
@@ -121,19 +121,19 @@ Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: neo4j
 Requires-Dist: neo4j; extra == "neo4j"
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
@@ -142,8 +142,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
 Provides-Extra: pinecone
@@ -158,6 +158,8 @@ Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
+Provides-Extra: redis
+Requires-Dist: redis; extra == "redis"
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: rst
@@ -165,16 +167,16 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
@@ -187,6 +189,8 @@ Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
 Provides-Extra: vectara
+Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"

{unstructured_ingest-0.3.11.dist-info → unstructured_ingest-0.3.12.dist-info}/RECORD

@@ -10,15 +10,16 @@ test/integration/connectors/test_azure_ai_search.py,sha256=EGV-G_Lq3h6pHhhmmQGWj
 test/integration/connectors/test_chroma.py,sha256=KQCzBJsOHAOtg0Ehp0tNtuYchFtiSmhHDKyOju33kJg,3686
 test/integration/connectors/test_confluence.py,sha256=adJxIggjuO-jgMimBZdv_AqWeBFlQoodELucIYwWC98,3546
 test/integration/connectors/test_delta_table.py,sha256=xsnJmwlWVQrccYeAtpt2lm0DYm2jGxiKXeERQXqCDCM,6884
-test/integration/connectors/test_kafka.py,sha256=FtHLptvS9V3Br7wCm2Xyh_ulz8_wWvCOKKEd0xD9LyM,10758
 test/integration/connectors/test_lancedb.py,sha256=U2HfIrf6iJ7lYMn-vz0j-LesVyDY-jc9QrQhlJVhG9Q,9183
-test/integration/connectors/test_milvus.py,sha256=abYQOjF8grEFj3FB1_wQgFSbWPFWfZ2pEsgKarfKJE4,6574
+test/integration/connectors/test_milvus.py,sha256=aRT5SpJHY4NA8pG_LcVTJwYwvLw2W_OOE-NIfDq03SE,7015
 test/integration/connectors/test_mongodb.py,sha256=UZ4eo61MisCw4s0p7HWaediN7M-lSddMDs71RFgdmJs,12347
 test/integration/connectors/test_neo4j.py,sha256=Esiq_Z9k1JLrWNXPmLBsX3LLwyEozwKoxX7iwMEJjRM,8252
-test/integration/connectors/test_onedrive.py,sha256=KIkBwKh1hnv203VCL2UABnDkS_bP4NxOFm1AL8EPGLA,3554
+test/integration/connectors/test_onedrive.py,sha256=Bp9Ayv59JnfsjSwqbQ-zYvg-XAPGgZfKJ45Asc0y1bM,3808
 test/integration/connectors/test_pinecone.py,sha256=suPFi40d6rHXurQQLIpCzW5XRTdgzlP-f-KLPhGCUHo,10208
 test/integration/connectors/test_qdrant.py,sha256=hyuqSJDaylkQVxWh7byD8jo8bwPuBxSa8MWRD3sBu-Y,7906
+test/integration/connectors/test_redis.py,sha256=Q_KAZPNE9NIoRN2UsbXtc1fe_aJg66RbSQtS3OKNpc0,4327
 test/integration/connectors/test_s3.py,sha256=PJaAwFRF2lXMQlkbv9JHpngPc6706ML7zowOlXT3TcY,7033
+test/integration/connectors/test_vectara.py,sha256=_FQHbhxL3f1rLV9MrHOvcljm_4qTVf5xl-Q7MplE_xs,8688
 test/integration/connectors/databricks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/databricks/test_volumes_native.py,sha256=ig60-nCdLF0GsgJowG9eRaG28iuoYHtuf12HdK6OE1I,7764
 test/integration/connectors/duckdb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,17 +51,17 @@ test/integration/connectors/weaviate/test_local.py,sha256=SK6iEwQUKiCd0X99BEk8Gl
 test/integration/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/embedders/conftest.py,sha256=B2W771RbijR7G_GybsCzRyIvOzXqzbKZdRIlNDd5AGY,334
 test/integration/embedders/test_azure_openai.py,sha256=6tFpKFBFRXD49imhhRzsvy3MPtuZ4L1PtnKyMVBRAqc,1808
-test/integration/embedders/test_bedrock.py,sha256=0oBRNS_DtFDGQ22Z1T3t6VOJ31PrItgvnJpqcLe9Fg4,1903
+test/integration/embedders/test_bedrock.py,sha256=ZyS17PLaHOeh-ykrd71Jkgg_ext7aOadxvxVJ_4IvFE,2852
 test/integration/embedders/test_huggingface.py,sha256=0mMTOO-Nh7KB70AGs_7LLQIxMYrnSPqyihriUeqACbM,1007
 test/integration/embedders/test_mixedbread.py,sha256=RrLv8SByMNXsgrlh94RbaT-VyxZ4-DILO-OPpmOwvSI,1441
-test/integration/embedders/test_octoai.py,sha256=LnR0BLttamW5PGid6jFxATDAi0x7hq5iWMXurbHP6TI,1328
-test/integration/embedders/test_openai.py,sha256=0jlFqEeeCneIWX9tGyC3TXeUNqsMXR7u5n7uEIaAQKo,1328
-test/integration/embedders/test_togetherai.py,sha256=0W1ScD5yb1D9hPC2ewUsuCHLUOpCuM083YMBhqAI9fw,1395
+test/integration/embedders/test_octoai.py,sha256=oQYpYh2XaKhiqtnOSpH0rP9TQrzykZ1-3C3jZRurPu8,1734
+test/integration/embedders/test_openai.py,sha256=s4_XGQfVpsTb4hKh2QZkXdOG_MnF5OQgL98kzNjTFCg,1664
+test/integration/embedders/test_togetherai.py,sha256=3otyr6i9smJMyXbhKCcaC2gx813rqGaZTKi2sEM7GIQ,1707
 test/integration/embedders/test_vertexai.py,sha256=OtoFzmrWWhGIO5Bbl5zt_4sp6qRHZxtaDQKpGcfzNLM,1345
 test/integration/embedders/test_voyageai.py,sha256=Zqf7nn1AxfBDBr5A9Jr-5pxes4QNvfKiyeGexCCm4nY,1346
 test/integration/embedders/utils.py,sha256=3AMKMBpgBep_0jFqrqMHH8BJo6w60kpouSZ5JPJTwIA,1850
 test/integration/partitioners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/partitioners/test_partitioner.py,sha256=KEpnhsz2YNAoQ2UZGOTsi1_uk1h4Vg-gGTsy5Fe9OCw,2846
+test/integration/partitioners/test_partitioner.py,sha256=MEQJbRoc01uPLT6O8CkXeQF_DXK21nz3KVJkzkBtsgM,2835
 test/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/test_chunking_utils.py,sha256=0iPwfnMPpyTm-yOE0BXMnEQQP4iguS6NhOqgMQU5nhk,1390
 test/unit/test_error.py,sha256=RflmngCdFNKOLXVfLnUdNfY3Mfg3k7DTEzfIl0B-syU,840
@@ -69,7 +70,7 @@ test/unit/test_logger.py,sha256=0SKndXE_VRd8XmUHkrj7zuBQHZscXx3ZQllMEOvtF9Y,2380
 test/unit/test_utils.py,sha256=Q6mp9YZPah8z3-2lreyRbmAc7m2Y_w26_N9vocSInoA,5421
 test/unit/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/embed/test_mixedbreadai.py,sha256=XFNJDP5pIgF3eQYwBiuEWmH3zZWx72Wpwyv-Q4m0DJg,1332
-test/unit/embed/test_octoai.py,sha256=Ha9EgAW64Q45hFj51tToe8RyKXWXwqAkdDqSFDMu37Q,831
+test/unit/embed/test_octoai.py,sha256=pouR4J6B_mrlu4TsA5yr2Ln_LCYL2pGBojXY5KEqvKI,1053
 test/unit/embed/test_openai.py,sha256=0O1yshDcE0BMKv1yJqrNuiNLSdPhLpKqJ-D_wmnidsM,831
 test/unit/embed/test_vertexai.py,sha256=Pl7COc9E3tf_yGidkTEmTizNGyZF1F5zuL2TgPTMnfI,1048
 test/unit/embed/test_voyageai.py,sha256=DviCOJFhe5H4e26-kNyX3JNe8h3qB5Yl0KOe8rQEMrc,981
@@ -94,7 +95,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=CJalz6YpEm8DAhzCP5dryU5ddzKQaSQOzTObKxfOVHs,43
+unstructured_ingest/__version__.py,sha256=R522TM0FvpKddIRo55tqz-j1ENS8k4uXjk60bKhQ50M,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -263,15 +264,15 @@ unstructured_ingest/connector/notion/types/database_properties/url.py,sha256=iXQ
 unstructured_ingest/connector/notion/types/database_properties/verification.py,sha256=J_DLjY-v2T6xDGMQ7FkI0YMKMA6SG6Y3yYW7qUD1hKA,2334
 unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/embed/azure_openai.py,sha256=4YBOIxv66wVZ5EqNNC4uCDPNJ3VrsLPe5wwagT6zqe0,1001
-unstructured_ingest/embed/bedrock.py,sha256=-PRdZsF44vwi6G4G75gdO31AJKfZWClOXkJQAk7rEO8,3096
+unstructured_ingest/embed/bedrock.py,sha256=qb1eo1Uooz2JxhHcqvJDTYGbuwaqyRTD0ZepQzlL9_o,4455
 unstructured_ingest/embed/huggingface.py,sha256=2cBiQhOhfWHX3hS-eKjocysOkUaRlyRfUj9Kxjrp6cE,1934
-unstructured_ingest/embed/interfaces.py,sha256=au4Xp8ciDvo4bidlUbazFW2aC7NZW5-UDLKXBFVzAX4,2025
+unstructured_ingest/embed/interfaces.py,sha256=XsPtb53367KCkH-ItwWQ_EQ-sYWHaekhxkF4PwHCNXc,2210
 unstructured_ingest/embed/mixedbreadai.py,sha256=OwFWWukvkQaXhjgs6b6N6D4w7sYrtcHNhsHAj-Bocj4,4268
-unstructured_ingest/embed/octoai.py,sha256=jHytDfQgup0v1PBcmlMv1nIh9Obg8WGO5qtLmN-Ot5g,1473
-unstructured_ingest/embed/openai.py,sha256=JXo4boivNoo2lBzHuS4Z0FZ1zlgUGAPVt0X3HY540ZU,1282
-unstructured_ingest/embed/togetherai.py,sha256=BL7NzExSE-laQqrp4ybUgoZ9JG_eop4hk-s2yCO_d5c,1451
+unstructured_ingest/embed/octoai.py,sha256=0LVZlbOMUuxwZV0QHhGWUlneWDX3fCklPRTuc4huze0,3007
+unstructured_ingest/embed/openai.py,sha256=5M2idJ7Ynx_3-FXwm9mTjGnNiww0DSuZmbuvi2YAUqk,2543
+unstructured_ingest/embed/togetherai.py,sha256=2jXYFB9QTDUlSKc_j32bMrwKu7YQA0oF893rGSmlXr8,2374
 unstructured_ingest/embed/vertexai.py,sha256=X5bGJdXyR5nAFH_ocAVgEowmd60nOBykyfclYo3VfBM,2808
-unstructured_ingest/embed/voyageai.py,sha256=bjom9QqWmH1Mv08ewg8ZG7gO3rQPMVS0_ztm2KBAOjI,1821
+unstructured_ingest/embed/voyageai.py,sha256=BfYa-oedkq-56j5_0rDjOLy18b9zC0zagaoPHJry5xA,2958
 unstructured_ingest/enhanced_dataclass/__init__.py,sha256=gDZOUsv5eo-8jm4Yu7DdDwi101aGbfG7JctTdOYnTOM,151
 unstructured_ingest/enhanced_dataclass/core.py,sha256=d6aUkDynuKX87cHx9_N5UDUWrvISR4jYRFRTvd_avlI,3038
 unstructured_ingest/enhanced_dataclass/dataclasses.py,sha256=aZMsoCzAGRb8Rmh3BTSBFtNr6FmFTY93KYGLk3gYJKQ,1949
@@ -360,10 +361,11 @@ unstructured_ingest/utils/string_and_date_utils.py,sha256=kijtPlGAbH376vVjFSo5H_
 unstructured_ingest/utils/table.py,sha256=aWjcowDVSClNpEAdR6PY3H7khKu4T6T3QqQE6GjmQ_M,3469
 unstructured_ingest/v2/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
 unstructured_ingest/v2/constants.py,sha256=pDspTYz-nEojHBqrZNfssGEiujmVa02pIWL63PQP9sU,103
+unstructured_ingest/v2/errors.py,sha256=y1tGvobuhQdcR9vw5APuFigiQSfsQKrAYGDr4biGDdw,207
 unstructured_ingest/v2/logger.py,sha256=wcln4s5Nyp2fjjJux9iM3d6t9aQFNJ2H1IAZXmIknjI,4323
 unstructured_ingest/v2/main.py,sha256=WFdLEqEXRy6E9_G-dF20MK2AtgX51Aan1sp_N67U2B8,172
 unstructured_ingest/v2/otel.py,sha256=2fGj1c7cVcC3J8NwL6MNYhyPEAXiB33DsilvRDkrdLo,4130
-unstructured_ingest/v2/unstructured_api.py,sha256=f_6NK0QOVwjAFJvlyvzu0IaXb6QQgRNJleYxB1KvzKE,3856
+unstructured_ingest/v2/unstructured_api.py,sha256=g6AO2Vy0lpy6-ooOvdgfJvIRhearPKArp3ggIdApG8I,4514
 unstructured_ingest/v2/utils.py,sha256=HHli5rHDBm6flUeQ_ovVDvtOdnzzL4FvNyw6jsHIJfw,2041
 unstructured_ingest/v2/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/cli/cli.py,sha256=qHXIs-PcvMgDZhP1AR9iDMxh8FXBMJCEDksPBfiMULE,648
@@ -406,11 +408,11 @@ unstructured_ingest/v2/processes/embedder.py,sha256=xCBpaL07WnVUOUW8SHktaf1vwBGZ
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
 unstructured_ingest/v2/processes/partitioner.py,sha256=agpHwB9FR8OZVQqE7zFEb0IcDPCOPA_BZjLzLF71nOY,8194
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=CTWLEmaKLTjbqeUQGI0fxJobsqDOc1d2ZKJoXh98Lww,5432
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=bmogp1sPbRS-RndN0R8V8gY4uaTkpmNJv-035-Y5SGU,5835
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=eeZJe-bBNxt5Sa-XEFCdcGeJCguJU5WN2Mv9kLp5dVQ,8917
 unstructured_ingest/v2/processes/connectors/astradb.py,sha256=xhUMoUdnrfAY1isZGqsV4lZUsnZNpbvgLyQWQbR4hVo,14814
 unstructured_ingest/v2/processes/connectors/azure_ai_search.py,sha256=ngPDpU0oZ6m5sxIlB6u5ebQpqCS_SJ-_amCC1KQ03EQ,11529
-unstructured_ingest/v2/processes/connectors/chroma.py,sha256=G1DQHhhFQCS2RLF0cVvoUH9QO8KkVjIyNZ9nKh__aHw,7220
+unstructured_ingest/v2/processes/connectors/chroma.py,sha256=VHCnM56qNXuHzovJihrNfJnZbWLJShOe8j12PJFrbL0,7219
 unstructured_ingest/v2/processes/connectors/confluence.py,sha256=-Y1OU_ZXhZQNj5NH3EN01CP8QKKZJaJ9xkXoAlSgnIk,7604
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=i7vuNKsUkN93JRVmg4--MO0ZgbjvhIqt46oYqk9zFSQ,12250
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=SotSXZQ85_6TO906YvFi3yTml8jE9A_zV6nBJ4oTx8A,7075
@@ -418,16 +420,18 @@ unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=ufE65Z8q_tC4oppGg5B
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=5k7pdAzJGXSdyPCzW9vu2OaAjGVTo2JevDyGaXM1Hvk,13370
 unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=VRDAiou_7oWOIAgQTdOGQWxudzQEDopXM8XkfkQ2j6g,5004
 unstructured_ingest/v2/processes/connectors/local.py,sha256=ZvWTj6ZYkwnvQMNFsZWoaQyp9zp0WVqAywMaHJ2kcAc,7153
-unstructured_ingest/v2/processes/connectors/milvus.py,sha256=I57hyH5nz_p7utmUOkvt_6vCPxNIVQMoukplUgIyYi8,8503
+unstructured_ingest/v2/processes/connectors/milvus.py,sha256=wmcu9NVy3gYlQGT25inN5w_QrhFoL8-hRq0pJFSNw8g,8866
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNNeAywXVpaIiND4b5JTAFlYjLjw,14273
-unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=lRtWe6xWYogT-y_r_o7HWvlFMf_OIPGQq_Z-5v7IOq0,14163
-unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=H8qk53YJXAPrPyISze0dybZdDFv5B7dVO3fIr10dVU8,15982
+unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=QTw_Kq1_kHMdqsaBST6yW8vl-SYXVQFlIofDP1W_IuI,14250
+unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=d6gC40YmfqBNXxizAt4MO4OOu5BoCZ7SAe1AbNwTP0E,18322
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=cohF7gBj0opSGKXlENSdGfTtyIKMHd1pwu4ydeb7JAY,10605
+unstructured_ingest/v2/processes/connectors/redisdb.py,sha256=p0AY4ukBNpwAemV4bWzpScvVbLTVlI3DzsCNUKiBI5M,6757
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=Ndn2Wm7RupfjAtlLxxQwJueeE0V8aGMbNVPuFq9nqdQ,19730
 unstructured_ingest/v2/processes/connectors/slack.py,sha256=Z73VmQ3oUY09KoLEi5OBdQeDt4ONEY_02SglWQc6HXE,9252
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=8kd0g7lo9NqnpaIkjeO-Ut6erhwUNH_gS9koevpe3WE,878
+unstructured_ingest/v2/processes/connectors/vectara.py,sha256=BlI_4nkpNR99aYxDd9eusm5LQsVB9EI0r-5Kc1D7pgQ,12255
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=jO71UTC7bLA_N12CrLWJzh_yZML5gfT7VohxzCpUGWg,1848
 unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=kI_ThB5e-DS8-GiQP5TQ8cP3fiGRm-V2AuNlGoSjH6I,6613
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=TA2e_1SIr4VaEI62873eyReCNfgmQ51_2Pko2I04pPM,2747
@@ -451,8 +455,8 @@ unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=l3TRKPEb0AJ7e0VS
 unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=JsxXKXNI20mdgwR_A6Rnf4u8fsFwLe3AkJmIe_3NEKY,6150
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/v2/processes/connectors/kafka/__init__.py,sha256=mQJ9Ex-QCfhz-BB5YWTfbPf7xGLd1i7FpjRr0ukbhNw,754
-unstructured_ingest/v2/processes/connectors/kafka/cloud.py,sha256=1SqNdY8Q8JwwB57wk9efxKv_BCeUkxZJ2HJ526wuCMw,3294
-unstructured_ingest/v2/processes/connectors/kafka/kafka.py,sha256=a-LWqYeJAK-g32UPgvvDt6W7dJp85N66aR_EKSR66RU,9685
+unstructured_ingest/v2/processes/connectors/kafka/cloud.py,sha256=GdAeQ8Uz-6v1C5byBHtjfevVfbzW3obScBFFLRTb0ps,3441
+unstructured_ingest/v2/processes/connectors/kafka/kafka.py,sha256=UfS41jzV9VxekS6AwWHhURJmJ7RUAw5iiIrj75BWrXQ,10255
 unstructured_ingest/v2/processes/connectors/kafka/local.py,sha256=lUkmfbTxyQW87CXxbJaijIT6foV09Gi-IG9o08OgiEs,2581
 unstructured_ingest/v2/processes/connectors/lancedb/__init__.py,sha256=LW37xZrn48JeHluRNulLTreUPdaF-ZU81F7MCUHcCv8,1253
 unstructured_ingest/v2/processes/connectors/lancedb/aws.py,sha256=eeXWsh8UeVm1Ur53C4MEnpLplfO8U91KYgk--0kk5pE,1413
@@ -464,7 +468,7 @@ unstructured_ingest/v2/processes/connectors/lancedb/local.py,sha256=_7-6iO6B60gA
 unstructured_ingest/v2/processes/connectors/qdrant/__init__.py,sha256=xM19uYzAuGizVoZIM_hnVZ5AcBN69aOBGpqZcpWPtuE,760
 unstructured_ingest/v2/processes/connectors/qdrant/cloud.py,sha256=accJ4sNWBVWV-KiVBDBDBYYx5A9CUoikP5NCErRmfik,1624
 unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ4DU8yhJWMpL82QYwBVdPTxxNuV127U,1588
-unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=ITRYXKYEFhlagSe-AKKGRvC8jzyWmhQLfHbFb0ax8o8,5438
+unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=BHI7HYSdbS05j2vrjyDvLzVG1WfsM8osKeq-lttlybQ,5437
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
 unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=E16CXRBw8fZKTuXIECns5wif_I07oncBHskVxHC4p7w,1448
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
@@ -477,9 +481,9 @@ unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=X1yv1H_orDQ-J965EMXhR2XaURqe8vovSi9n1fk85B4,10499
-unstructured_ingest-0.3.11.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.3.11.dist-info/METADATA,sha256=4-py_Sf-ahdzXF0l1evY4aI_s5Vz4oc6Gtenhegc6Vo,7623
-unstructured_ingest-0.3.11.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.3.11.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.3.11.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.3.11.dist-info/RECORD,,
+unstructured_ingest-0.3.12.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.3.12.dist-info/METADATA,sha256=nNPregI5d4D8fHqXxTPkKmn7bqmfUX5RB-AcMsgj0J4,7769
+unstructured_ingest-0.3.12.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.3.12.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.3.12.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.3.12.dist-info/RECORD,,

test/integration/connectors/test_kafka.py

@@ -1,304 +0,0 @@
-import json
-import os
-import tempfile
-import time
-from pathlib import Path
-
-import pytest
-from confluent_kafka import Consumer, KafkaError, KafkaException, Producer
-from confluent_kafka.admin import AdminClient, NewTopic
-
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-    SOURCE_TAG,
-    env_setup_path,
-)
-from test.integration.connectors.utils.docker_compose import docker_compose_context
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
-from unstructured_ingest.v2.processes.connectors.kafka.cloud import (
-    CloudKafkaAccessConfig,
-    CloudKafkaConnectionConfig,
-    CloudKafkaDownloader,
-    CloudKafkaDownloaderConfig,
-    CloudKafkaIndexer,
-    CloudKafkaIndexerConfig,
-)
-from unstructured_ingest.v2.processes.connectors.kafka.local import (
-    CONNECTOR_TYPE,
-    LocalKafkaConnectionConfig,
-    LocalKafkaDownloader,
-    LocalKafkaDownloaderConfig,
-    LocalKafkaIndexer,
-    LocalKafkaIndexerConfig,
-    LocalKafkaUploader,
-    LocalKafkaUploaderConfig,
-)
-
-SEED_MESSAGES = 10
-TOPIC = "fake-topic"
-
-
-def get_admin_client() -> AdminClient:
-    conf = {
-        "bootstrap.servers": "localhost:29092",
-    }
-    return AdminClient(conf)
-
-
-@pytest.fixture
-def docker_compose_ctx():
-    with docker_compose_context(docker_compose_path=env_setup_path / "kafka") as ctx:
-        yield ctx
-
-
-def wait_for_topic(
-    topic: str,
-    retries: int = 10,
-    interval: int = 1,
-    exists: bool = True,
-    admin_client=None,
-):
-    if admin_client is None:
-        admin_client = get_admin_client()
-    current_topics = admin_client.list_topics().topics
-    attempts = 0
-    while (topic not in current_topics) == exists and attempts < retries:
-        attempts += 1
-        print(
-            "Attempt {}: Waiting for topic {} to {} exist. Current topics: [{}]".format(
-                attempts, topic, "" if exists else "not", ", ".join(current_topics)
-            )
-        )
-        time.sleep(interval)
-        current_topics = admin_client.list_topics().topics
-    if (topic not in current_topics) == exists:
-        raise TimeoutError(f"Timeout out waiting for topic {topic} to exist")
-
-
-@pytest.fixture
-def kafka_seed_topic(docker_compose_ctx) -> str:
-    conf = {
-        "bootstrap.servers": "localhost:29092",
-    }
-    producer = Producer(conf)
-    for i in range(SEED_MESSAGES):
-        message = f"This is some text for message {i}"
-        producer.produce(topic=TOPIC, value=message)
-    producer.flush(timeout=10)
-    print(f"kafka topic {TOPIC} seeded with {SEED_MESSAGES} messages")
-    wait_for_topic(topic=TOPIC)
-    return TOPIC
-
-
-@pytest.fixture
-def kafka_upload_topic(docker_compose_ctx) -> str:
-    admin_client = get_admin_client()
-    admin_client.create_topics([NewTopic(TOPIC, 1, 1)])
-    return TOPIC
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
-async def test_kafka_source_local(kafka_seed_topic: str):
-    connection_config = LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092)
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        download_config = LocalKafkaDownloaderConfig(download_dir=tempdir_path)
-        indexer = LocalKafkaIndexer(
-            connection_config=connection_config,
-            index_config=LocalKafkaIndexerConfig(topic=kafka_seed_topic, num_messages_to_consume=5),
-        )
-        downloader = LocalKafkaDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        indexer.precheck()
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="kafka-local", expected_num_files=5, validate_downloaded_files=True
-            ),
-        )
-
-
-@pytest.fixture
-def kafka_seed_topic_cloud(expected_messages: int = 5) -> int:
-    conf = {
-        "bootstrap.servers": os.environ["KAFKA_BOOTSTRAP_SERVER"],
-        "sasl.username": os.environ["KAFKA_API_KEY"],
-        "sasl.password": os.environ["KAFKA_SECRET"],
-        "sasl.mechanism": "PLAIN",
-        "security.protocol": "SASL_SSL",
-    }
-    admin_client = AdminClient(conf)
-    try:
-        res = admin_client.delete_topics([TOPIC], operation_timeout=10)
-        for topic, f in res.items():
-            f.result()
-            print(f"Topic {topic} removed")
-            wait_for_topic(TOPIC, 5, 1, False, admin_client)
-    except Exception:
-        pass
-
-    cluster_meta = admin_client.list_topics()
-    current_topics = [topic for topic in cluster_meta.topics if topic != "__consumer_offsets"]
-
-    assert TOPIC not in current_topics, f"Topic {TOPIC} shouldn't exist"
-
-    # Kafka Cloud allows to use replication_factor=1 only for Dedicated clusters.
-    topic_obj = NewTopic(TOPIC, num_partitions=1, replication_factor=3)
-
-    res = admin_client.create_topics([topic_obj], operation_timeout=10, validate_only=False)
-    for topic, f in res.items():
-        f.result()
-
-    producer = Producer(conf)
-    for i in range(expected_messages):
-        message = f"This is some text for message {i}"
-        producer.produce(topic=TOPIC, value=message)
-    producer.flush(timeout=10)
-    return expected_messages
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
-@requires_env("KAFKA_API_KEY", "KAFKA_SECRET", "KAFKA_BOOTSTRAP_SERVER")
-async def test_kafka_source_cloud(kafka_seed_topic_cloud: int):
-    """
-    In order to have this test succeed, you need to create cluster on Confluent Cloud,
-    and create the API key with admin privileges. By default, user account keys have it.
-    """
-
-    expected_messages = kafka_seed_topic_cloud
-
-    connection_config = CloudKafkaConnectionConfig(
-        bootstrap_server=os.environ["KAFKA_BOOTSTRAP_SERVER"],
-        port=9092,
-        access_config=CloudKafkaAccessConfig(
-            kafka_api_key=os.environ["KAFKA_API_KEY"],
-            secret=os.environ["KAFKA_SECRET"],
-        ),
-    )
-
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        download_config = CloudKafkaDownloaderConfig(download_dir=tempdir_path)
-        indexer = CloudKafkaIndexer(
-            connection_config=connection_config,
-            index_config=CloudKafkaIndexerConfig(
-                topic=TOPIC,
-                num_messages_to_consume=expected_messages,
-            ),
-        )
-        downloader = CloudKafkaDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        indexer.precheck()
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="kafka-cloud",
-                exclude_fields_extend=["connector_type"],
-                expected_num_files=expected_messages,
-                validate_downloaded_files=True,
-                validate_file_data=True,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
-def test_kafka_source_local_precheck_fail_no_cluster():
-    connection_config = LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092)
-    indexer = LocalKafkaIndexer(
-        connection_config=connection_config,
-        index_config=LocalKafkaIndexerConfig(topic=TOPIC, num_messages_to_consume=5),
-    )
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
-def test_kafka_source_local_precheck_fail_no_topic(kafka_seed_topic: str):
-    connection_config = LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092)
-    indexer = LocalKafkaIndexer(
-        connection_config=connection_config,
-        index_config=LocalKafkaIndexerConfig(topic="topic", num_messages_to_consume=5),
-    )
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-def get_all_messages(topic: str, max_empty_messages: int = 5) -> list[dict]:
-    conf = {
-        "bootstrap.servers": "localhost:29092",
-        "group.id": "default_group_id",
-        "enable.auto.commit": "false",
-        "auto.offset.reset": "earliest",
-    }
-    consumer = Consumer(conf)
-    consumer.subscribe([topic])
-    messages = []
-    try:
-        empty_count = 0
-        while empty_count < max_empty_messages:
-            msg = consumer.poll(timeout=1)
-            if msg is None:
-                empty_count += 1
-                continue
-            if msg.error():
-                if msg.error().code() == KafkaError._PARTITION_EOF:
-                    break
-                else:
-                    raise KafkaException(msg.error())
-            try:
-                message = json.loads(msg.value().decode("utf8"))
-                messages.append(message)
-            finally:
-                consumer.commit(asynchronous=False)
-    finally:
-        print("closing consumer")
-        consumer.close()
-    return messages
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
-async def test_kafka_destination_local(upload_file: Path, kafka_upload_topic: str):
-    uploader = LocalKafkaUploader(
-        connection_config=LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092),
-        upload_config=LocalKafkaUploaderConfig(topic=TOPIC, batch_size=10),
-    )
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    uploader.precheck()
-    if uploader.is_async():
-        await uploader.run_async(path=upload_file, file_data=file_data)
-    else:
-        uploader.run(path=upload_file, file_data=file_data)
-    all_messages = get_all_messages(topic=kafka_upload_topic)
-    with upload_file.open("r") as upload_fs:
-        content_to_upload = json.load(upload_fs)
-    assert len(all_messages) == len(content_to_upload), (
-        f"expected number of messages ({len(content_to_upload)}) doesn't "
-        f"match how many messages read off of kakfa topic {kafka_upload_topic}: {len(all_messages)}"
-    )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
-def test_kafka_destination_local_precheck_fail_no_cluster():
-    uploader = LocalKafkaUploader(
-        connection_config=LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092),
-        upload_config=LocalKafkaUploaderConfig(topic=TOPIC, batch_size=10),
-    )
-    with pytest.raises(DestinationConnectionError):
-        uploader.precheck()