unstructured-ingest 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

{unstructured_ingest-0.1.1.dist-info → unstructured_ingest-0.2.1.dist-info}/RECORD

@@ -4,17 +4,21 @@ test/integration/utils.py,sha256=CWqzEGw6TA_ZoP9hRUkW64TWYssooBbufcTRmbJvod8,401
 test/integration/chunkers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/chunkers/test_chunkers.py,sha256=pqn1Rqh36jZTJL4qpU0iuOMFAEQ-LrKAPOgWtQMAt_I,1482
 test/integration/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/conftest.py,sha256=Q8ScDzrzO2o-8D_kYFt8LL7QAhoFTRRtKJKMc2hLMcI,345
-test/integration/connectors/test_s3.py,sha256=fK0soCTkNxp-4hm4O2LPrhlZXvYmaeTmeEgeNh1b0k8,5839
+test/integration/connectors/conftest.py,sha256=6dVNMBrL6WIO4KXA-0nf2tNrPYk_tsor8uomi6fbi3Q,727
+test/integration/connectors/test_delta_table.py,sha256=4_KPyQJpd6DmyIjjtXWPMw6NNf7xULRkxmqfbvmZ80g,5018
+test/integration/connectors/test_s3.py,sha256=1ErPRpNmbg-88ig80SfIyxujF7xnAWtI42WSue4sgKU,5850
 test/integration/connectors/databricks_tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/databricks_tests/test_volumes_native.py,sha256=kS45mnNu9_U4qV3cxByEFXCYLEBWRy-fxxhzR3r93cs,5685
+test/integration/connectors/databricks_tests/test_volumes_native.py,sha256=k4lALbwNtlyuI3wd3OHoBULI21E3Ck2Fo8EJXaVfwgw,5812
 test/integration/connectors/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/sql/test_postgres.py,sha256=A9vWj5pBdoEyL2m6d3e2Ep8ZZcnLhdXkaHPPlkTStbg,6581
-test/integration/connectors/sql/test_sqlite.py,sha256=F6Ljb6npmFZlq_5pvJj-0Hkk2mC3T-pMAGyhDm1UtM4,5702
+test/integration/connectors/sql/test_postgres.py,sha256=gDBuNyvWmpVPmDrSSYC99z3t17B_a196P1MwIAOp5Dk,6584
+test/integration/connectors/sql/test_singlestore.py,sha256=wGI3-lc6qh0qN4-WD9VtiXBB9MlekeqK402_9EXQyX0,5876
+test/integration/connectors/sql/test_snowflake.py,sha256=XXU2-2z_k8jHWP684v2IuaGOlV3cmPpg3RxkwMp08v8,6998
+test/integration/connectors/sql/test_sqlite.py,sha256=51QrFufAq-XxNjHAkmPWxdJUkGdIRRIGKeRT09A5pkA,5704
 test/integration/connectors/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/utils/constants.py,sha256=0zSPnsZVqJuNhXduXvdXFQLZTRIQa5Fo_1qjBYVCfb8,209
-test/integration/connectors/utils/docker_compose.py,sha256=6XeYOKQFZCBRLEmcgH2mmBAaVs6R6jCWAhJLjq6p-aM,1771
-test/integration/connectors/utils/validation.py,sha256=Sf0ELATWG5K3E3d5S_ArtZeFFYdzoI5jN86U4DiqNyw,8422
+test/integration/connectors/utils/docker.py,sha256=-wknXRVlzr3BVPdEhCyJgsdNjO9aSb2xjb-mQ306j7Q,2256
+test/integration/connectors/utils/docker_compose.py,sha256=GVTB6Cel05c0VQ2n4AwkQQx_cBfz13ZTs1HpbaYipNU,2223
+test/integration/connectors/utils/validation.py,sha256=gnflehoYbFkSBJdXQV-7HwcrlL_Cuqni2ri1YmArjT0,12019
 test/integration/embedders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/embedders/conftest.py,sha256=B2W771RbijR7G_GybsCzRyIvOzXqzbKZdRIlNDd5AGY,334
 test/integration/embedders/test_bedrock.py,sha256=0oBRNS_DtFDGQ22Z1T3t6VOJ31PrItgvnJpqcLe9Fg4,1903
@@ -43,7 +47,7 @@ test/unit/embed/test_openai.py,sha256=0O1yshDcE0BMKv1yJqrNuiNLSdPhLpKqJ-D_wmnids
 test/unit/embed/test_vertexai.py,sha256=Pl7COc9E3tf_yGidkTEmTizNGyZF1F5zuL2TgPTMnfI,1048
 test/unit/embed/test_voyageai.py,sha256=DviCOJFhe5H4e26-kNyX3JNe8h3qB5Yl0KOe8rQEMrc,981
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=ch9Ch304-rlC6iFyomBT7OHb9bvtQNzaejmd5QwbzKE,42
+unstructured_ingest/__version__.py,sha256=Hmm5OuicK0ynl_R5DSnpRYWJpEXwe7guJdsAMHH7K60,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=m03BgenxSA34HbW157L7V9TGxK_dTG7N2AnAhF31W-U,31364
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -310,7 +314,7 @@ unstructured_ingest/v2/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LG
 unstructured_ingest/v2/logger.py,sha256=wcln4s5Nyp2fjjJux9iM3d6t9aQFNJ2H1IAZXmIknjI,4323
 unstructured_ingest/v2/main.py,sha256=WFdLEqEXRy6E9_G-dF20MK2AtgX51Aan1sp_N67U2B8,172
 unstructured_ingest/v2/otel.py,sha256=2fGj1c7cVcC3J8NwL6MNYhyPEAXiB33DsilvRDkrdLo,4130
-unstructured_ingest/v2/unstructured_api.py,sha256=1EQVuL-TojmFxWoWFzXr1oCFPEC3IkCQqhjXM8spdTY,3373
+unstructured_ingest/v2/unstructured_api.py,sha256=HqOaQ80YTdAnFj_2Ce108g7Pp3-F9Qg329Uw2OXtRmA,3375
 unstructured_ingest/v2/utils.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/v2/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/cli/cli.py,sha256=qHXIs-PcvMgDZhP1AR9iDMxh8FXBMJCEDksPBfiMULE,648
@@ -321,13 +325,13 @@ unstructured_ingest/v2/cli/base/dest.py,sha256=zDjqek7anr0JQ2ptEl8KIAsUXuCuHRnBQ
 unstructured_ingest/v2/cli/base/importer.py,sha256=nRt0QQ3qpi264-n_mR0l55C2ddM8nowTNzT1jsWaam8,1128
 unstructured_ingest/v2/cli/base/src.py,sha256=cpQ43qQju4e5s_YSaPxUtA70BaisRkTBdjtlPhqn5Mg,2872
 unstructured_ingest/v2/cli/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/v2/cli/utils/click.py,sha256=HCEcdHf8Lck0zcx3kidKjLbHDHXIBxPRL2MGgtKtDlg,6967
-unstructured_ingest/v2/cli/utils/model_conversion.py,sha256=uJQKpbTC5ysOdVaRq2SWEjG8btBimVZYzX9NVL7xnzs,7500
+unstructured_ingest/v2/cli/utils/click.py,sha256=1_eJgrwS2DFBl1jZPLsj1vgVgR7agFBIEBe4A_n7mH4,7827
+unstructured_ingest/v2/cli/utils/model_conversion.py,sha256=7eEIkk1KU51-ZNiIfI1KRxlwITNW1xl1YxMAG8BcTk0,7604
 unstructured_ingest/v2/interfaces/__init__.py,sha256=Rfa8crx6De7WNOK-EjsWWwFVpsUfCc6gY8B8tQ3ae9I,899
 unstructured_ingest/v2/interfaces/connector.py,sha256=qUFFJ3qgDMenTCZMtVRjq1DIwsVak6pxNjQOH2eVkMw,1623
 unstructured_ingest/v2/interfaces/downloader.py,sha256=Lj3nTY1hPA71GfNeedFVCdHdZsHLle8qrx5RtXAy9GY,2940
-unstructured_ingest/v2/interfaces/file_data.py,sha256=ieJK-hqHCEOmoYNGoFbCHziSaZyMtRS9VpSoYbwoKCE,1944
-unstructured_ingest/v2/interfaces/indexer.py,sha256=Bd1S-gTLsxhJBLEh1lYm_gXqwQLaEZMoqPq9yGxtN_E,713
+unstructured_ingest/v2/interfaces/file_data.py,sha256=D71bXImJ7Pyjtl3I3pa2O2B2iBqIaY-mC-hdoEF3RmI,1983
+unstructured_ingest/v2/interfaces/indexer.py,sha256=gsa1MLhFa82BzD2h4Yb7ons0VxRwKINZOrzvHAahwVU,846
 unstructured_ingest/v2/interfaces/process.py,sha256=BgglTu5K93FnDDopZKKr_rkK2LTZOguR6kcQjKHjF40,392
 unstructured_ingest/v2/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
 unstructured_ingest/v2/interfaces/upload_stager.py,sha256=ZFkDxcwKn-6EPrTbdBEgOkz1kGAq4gUtze98KP48KG4,1146
@@ -335,13 +339,13 @@ unstructured_ingest/v2/interfaces/uploader.py,sha256=JmZDl1blJa5rS61YHCae3Hfet84
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/interfaces.py,sha256=-Y6gPnl-SbNxIx5-dQCmiYSPKUMjivrRlBLIKIUWVeM,8658
 unstructured_ingest/v2/pipeline/otel.py,sha256=K3pQvWVgWzyOWMKCBUofsH7wTZPJ0Ysw5sLjMBLW41I,1088
-unstructured_ingest/v2/pipeline/pipeline.py,sha256=x6hanD7Cj7Wd5MBUvb33UwXQMZxubzwlAiYyBCMukuc,15693
+unstructured_ingest/v2/pipeline/pipeline.py,sha256=7Yg8_xwlSX6lA-oPGlTcn6KXZ9kc51zsoJxME5TiUlw,15956
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=rYVcHSXeQSzWszg6VmtYlNc66Gsx-22Ti0BxPyQaJak,3135
 unstructured_ingest/v2/pipeline/steps/download.py,sha256=lzvOl5SoUK6OCCVVeG4CzdPIGj6eKKCGdciNo_0RMNk,8173
 unstructured_ingest/v2/pipeline/steps/embed.py,sha256=-YFvmchdsonWiSXxaD7PJfuUUtMLklaQM_8kZCQxCdM,3113
 unstructured_ingest/v2/pipeline/steps/filter.py,sha256=q7bNieaFMprqoF8Mx7w-ZN6jyA5peiGeTGyPtvcV-uw,1199
-unstructured_ingest/v2/pipeline/steps/index.py,sha256=nfDo-wt5sooKtMHKG7sI42m1L44uw-pxErDlDB1engw,2678
+unstructured_ingest/v2/pipeline/steps/index.py,sha256=YUUf1sYZRZSrRgapca3Sfzk1sNPJ05yyTQ5wKlyDjEo,3543
 unstructured_ingest/v2/pipeline/steps/partition.py,sha256=9MQViptxK3ALKco8uE4gK9PpEoGq5JjzyU14C_18blU,3193
 unstructured_ingest/v2/pipeline/steps/stage.py,sha256=cphKgHScLz2rNLZRI5Olsb6dAH-MKGu3p6MYS1BEzkA,2246
 unstructured_ingest/v2/pipeline/steps/uncompress.py,sha256=CFSy4tGp6BAvF0oIwWFN8v4zFzh5pRDeESjEn5iP9hE,1756
@@ -353,13 +357,13 @@ unstructured_ingest/v2/processes/embedder.py,sha256=PQn0IO8xbGRQHpcT2VVl-J8gTJ5H
 unstructured_ingest/v2/processes/filter.py,sha256=kjUmMw2SDq2bme0JCAOxs6cJriIG6Ty09KOznS-xz08,2145
 unstructured_ingest/v2/processes/partitioner.py,sha256=2Lhztd730soVC2TOqrn_ba7CGZna8AHHpqJY2ZUYVxE,7776
 unstructured_ingest/v2/processes/uncompress.py,sha256=Z_XfsITGdyaRwhtNUc7bMj5Y2jLuBge8KoK4nxhqKag,2425
-unstructured_ingest/v2/processes/connectors/__init__.py,sha256=glyowqb93_NNreQXoRLbF0PvzMc6Ptv0ARfl3xfSH4E,4967
+unstructured_ingest/v2/processes/connectors/__init__.py,sha256=zMO50wOGWOJrCTdh19Najj-i5tfMUyf977TKz4yN04A,5249
 unstructured_ingest/v2/processes/connectors/airtable.py,sha256=Yi7PEv_FejZ9_y3BPY3gu5YGVfeLh-9YX-qLyQHjJsY,8921
-unstructured_ingest/v2/processes/connectors/astradb.py,sha256=ZctZRfXcOAMBGPkKgHvhTmV_-2F0YN5vqwfY9UCHIlU,5791
+unstructured_ingest/v2/processes/connectors/astradb.py,sha256=k6zaxm05-ESpRV6w1jgrtfE10-I2Z50kafURxxJVzdk,14043
 unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=S55v7TXu30rEdgythMBB_2VcuomyMPmcPtLYykbhw_E,8466
 unstructured_ingest/v2/processes/connectors/chroma.py,sha256=skrxRPHZ8y3JxNa0dt5SVitHiDQ5WVxLvY_kh2-QUrQ,8029
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=SONLywyEfoAlLc-HPabXeGzoiwKnekMHIbRMXd4CGXs,12146
-unstructured_ingest/v2/processes/connectors/databricks_volumes.py,sha256=BQHHpCDwE51inD3pZF4tL4zLr7lv6iBcwnA1NazrHqY,9423
+unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=ZZfdNTw1W0ISQGWCtM1JuIME26FYzuPBOqRKql0wlLg,7013
 unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=ojxMUHkLa6ZG50aTGn2YWhDHZ1n38uFRn5p8_ghAIvM,16762
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=7xOQthcqBd9auJxB0nxZlhh1vdjXpMX_CtQZa6YfZz0,13088
 unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=8bGHbZctJ_Tl1AUSMnI7CCZ7CgEtTRVcRuvlB1HPlqQ,5907
@@ -372,11 +376,11 @@ unstructured_ingest/v2/processes/connectors/outlook.py,sha256=NK67Pd8Nk5oUIXTK-s
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=k_GH55S_OQ6-wCLC6gkhRrNpXIFECYZ_2Gjz_XRtY6Y,7561
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=hOaV5gBcHFc6N5Rbu3MgM-5Aol1ht-QkNIN4PqjvfxE,19665
-unstructured_ingest/v2/processes/connectors/singlestore.py,sha256=4rVvWKK2iQr03Ff6cB5zjfE1MpN0JyIGpCxxFCDI6hc,5563
+unstructured_ingest/v2/processes/connectors/slack.py,sha256=b9IanzUApUexiJzuNg7PR3tujOoeG8dhM0L0v4MDuPw,9256
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=8kd0g7lo9NqnpaIkjeO-Ut6erhwUNH_gS9koevpe3WE,878
 unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=Ss0YyD5T6k-00eJ6dr5lSo2H0LcOjVTMmozehyTvnAo,8866
 unstructured_ingest/v2/processes/connectors/databricks/__init__.py,sha256=jO71UTC7bLA_N12CrLWJzh_yZML5gfT7VohxzCpUGWg,1848
-unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=db4PxE1LiKWUq0b9THABFRChArAfHps89pZBglqEg3c,6521
+unstructured_ingest/v2/processes/connectors/databricks/volumes.py,sha256=8FasrRcoqa9zrhmnbfYN_rBBTH6xBXM50TzGsUMEm98,6581
 unstructured_ingest/v2/processes/connectors/databricks/volumes_aws.py,sha256=I1MJwe5LOxoPLjwo00H0XbXO6u_SJHWYgsj4s6ePoyI,2754
 unstructured_ingest/v2/processes/connectors/databricks/volumes_azure.py,sha256=P4rfcE3td7WyuuguRgUnGQytCMDpfeYrrpshBZuVynY,3539
 unstructured_ingest/v2/processes/connectors/databricks/volumes_gcp.py,sha256=UUotY_-HpgSEJkvdQfZTlbxY7CRLZ4ctL8TlryeFvxk,2790
@@ -385,18 +389,20 @@ unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Yp
 unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=Y01BuVRql0Kvzc_cdaZE9dDGYjJzrwJu-etfUrEGcUU,7061
 unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=Cjk0LUxqOCDbme0GmnD_5_b1hfStjI23cKw6BquKNrg,5488
 unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=NNAxIRdOQxUncfwhu7J7SnQRM6BSStNOyQZi-4E51iY,5816
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=eFcrpSAB8wbLHuCiDb-2QpEUtgEEUA_iSqcT81H2-3Q,11472
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=5uZ_nGBXNQgwvfjNcor6mwzbYOHeja4-EV3nNCXvxaQ,11512
 unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=-_pYHbsBG9FyRyNIaf_xyFbPiiR7pnWEEg_8mp0rIZ8,7053
 unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=je1BDqFWlyMfPa4oAMMNFQLLQtCY9quuqx3xjTwF8OQ,6251
 unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=dwpyqDq0qceCBWX3zM1hiUlgXB4hzX6ObOr-sh-5CJs,6926
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
-unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=tr3SZH0tz04XSxqGRkUu__tL_0zn0bSms2jILE-3Rug,543
-unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=oMwfYCycX-jTSKW-c6o6K09aU74Wn1B_G3Ib20oYi1A,6050
-unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=MbSvYSjhgGj8HHI7P-gH5bQ0Lqxtf8BEFsKNmCUfzug,9807
-unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=LxC2Q_rPHytbTDflmWzj4H5Jx-41phKnfp6FCpDe-UY,5701
-unstructured_ingest-0.1.1.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.1.1.dist-info/METADATA,sha256=LQ_M1kX7q7rGBvslwml9KbrJGJHAaA_SLWM64BBaZrg,7188
-unstructured_ingest-0.1.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.1.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.1.1.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.1.1.dist-info/RECORD,,
+unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=D43wrV2ADvQsToIYwbEWnZ7mhzlsYcZMFCqf6jIC7dQ,1333
+unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=__Wf5lkCQGhbtEH_2DxfNmQyWP-UKC9o_KEawG81jY0,4905
+unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=YrmhAL1RQ1c5-2fnR3UAyj_4KfvjYTQ2cWzpvsdJOnU,5535
+unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=0s0oBfMttPg5JL6jn8SsoCeTSRoXXdVy2bJAZv_hiSk,5576
+unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=rWDkefUnYkzJT0mhIcHxieECdaIWLTvbDcOcZgLA4FQ,11636
+unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=9605K36nQ5-gBxzt1daYKYotON1SE85RETusqCJrbdk,5230
+unstructured_ingest-0.2.1.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.2.1.dist-info/METADATA,sha256=NBV3OAonxt8Y0Tra7LWqQBoLSROwA106sf8vDCsXu2k,7271
+unstructured_ingest-0.2.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.2.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.2.1.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.2.1.dist-info/RECORD,,

unstructured_ingest/v2/processes/connectors/databricks_volumes.py

@@ -1,250 +0,0 @@
-import os
-from dataclasses import dataclass
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generator, Optional
-
-from pydantic import Field, Secret
-
-from unstructured_ingest.error import (
-    DestinationConnectionError,
-    SourceConnectionError,
-    SourceConnectionNetworkError,
-)
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import (
-    AccessConfig,
-    ConnectionConfig,
-    Downloader,
-    DownloaderConfig,
-    DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
-    Indexer,
-    IndexerConfig,
-    SourceIdentifiers,
-    Uploader,
-    UploaderConfig,
-)
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
-    DestinationRegistryEntry,
-    SourceRegistryEntry,
-)
-
-if TYPE_CHECKING:
-    from databricks.sdk import WorkspaceClient
-
-CONNECTOR_TYPE = "databricks_volumes"
-
-
-class DatabricksVolumesAccessConfig(AccessConfig):
-    account_id: Optional[str] = Field(
-        default=None,
-        description="The Databricks account ID for the Databricks "
-        "accounts endpoint. Only has effect when Host is "
-        "either https://accounts.cloud.databricks.com/ (AWS), "
-        "https://accounts.azuredatabricks.net/ (Azure), "
-        "or https://accounts.gcp.databricks.com/ (GCP).",
-    )
-    client_id: Optional[str] = Field(default=None, description="Client ID of the OAuth app.")
-    client_secret: Optional[str] = Field(
-        default=None, description="Client Secret of the OAuth app."
-    )
-    token: Optional[str] = Field(
-        default=None,
-        description="The Databricks personal access token (PAT) (AWS, Azure, and GCP) or "
-        "Azure Active Directory (Azure AD) token (Azure).",
-    )
-    profile: Optional[str] = None
-    azure_workspace_resource_id: Optional[str] = Field(
-        default=None,
-        description="The Azure Resource Manager ID for the Azure Databricks workspace, "
-        "which is exchanged for a Databricks host URL.",
-    )
-    azure_client_secret: Optional[str] = Field(
-        default=None, description="The Azure AD service principal’s client secret."
-    )
-    azure_client_id: Optional[str] = Field(
-        default=None, description="The Azure AD service principal’s application ID."
-    )
-    azure_tenant_id: Optional[str] = Field(
-        default=None, description="The Azure AD service principal’s tenant ID."
-    )
-    azure_environment: Optional[str] = Field(
-        default=None,
-        description="The Azure environment type for a " "specific set of API endpoints",
-        examples=["Public", "UsGov", "China", "Germany"],
-    )
-    auth_type: Optional[str] = Field(
-        default=None,
-        description="When multiple auth attributes are available in the "
-        "environment, use the auth type specified by this "
-        "argument. This argument also holds the currently "
-        "selected auth.",
-    )
-    google_credentials: Optional[str] = None
-    google_service_account: Optional[str] = None
-
-
-class DatabricksVolumesConnectionConfig(ConnectionConfig):
-    access_config: Secret[DatabricksVolumesAccessConfig] = Field(
-        default=DatabricksVolumesAccessConfig(), validate_default=True
-    )
-    host: Optional[str] = Field(
-        default=None,
-        description="The Databricks host URL for either the "
-        "Databricks workspace endpoint or the "
-        "Databricks accounts endpoint.",
-    )
-    volume: str = Field(description="Name of volume in the Unity Catalog")
-    catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
-    volume_path: Optional[str] = Field(
-        default=None, description="Optional path within the volume to write to"
-    )
-    databricks_schema: str = Field(
-        default="default",
-        alias="schema",
-        description="Schema associated with the volume to write to in the Unity Catalog service",
-    )
-
-    @property
-    def path(self) -> str:
-        path = f"/Volumes/{self.catalog}/{self.databricks_schema}/{self.volume}"
-        if self.volume_path:
-            path = f"{path}/{self.volume_path}"
-        return path
-
-    @requires_dependencies(dependencies=["databricks.sdk"], extras="databricks-volumes")
-    def get_client(self) -> "WorkspaceClient":
-        from databricks.sdk import WorkspaceClient
-
-        return WorkspaceClient(
-            host=self.host,
-            **self.access_config.get_secret_value().model_dump(),
-        )
-
-
-@dataclass
-class DatabricksVolumesIndexerConfig(IndexerConfig):
-    recursive: bool = False
-
-
-@dataclass
-class DatabricksVolumesIndexer(Indexer):
-    index_config: DatabricksVolumesIndexerConfig
-    connection_config: DatabricksVolumesConnectionConfig
-    connector_type: str = CONNECTOR_TYPE
-
-    def precheck(self) -> None:
-        try:
-            self.connection_config.get_client()
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        for file_info in self.connection_config.get_client().dbfs.list(
-            path=self.connection_config.path, recursive=self.index_config.recursive
-        ):
-            if file_info.is_dir:
-                continue
-            rel_path = file_info.path.replace(self.connection_config.path, "")
-            if rel_path.startswith("/"):
-                rel_path = rel_path[1:]
-            filename = Path(file_info.path).name
-            yield FileData(
-                identifier=file_info.path,
-                connector_type=CONNECTOR_TYPE,
-                source_identifiers=SourceIdentifiers(
-                    filename=filename,
-                    rel_path=rel_path,
-                    fullpath=file_info.path,
-                ),
-                additional_metadata={
-                    "catalog": self.connection_config.catalog,
-                },
-                metadata=FileDataSourceMetadata(
-                    url=file_info.path, date_modified=str(file_info.modification_time)
-                ),
-            )
-
-
-@dataclass
-class DatabricksVolumesDownloaderConfig(DownloaderConfig):
-    pass
-
-
-@dataclass
-class DatabricksVolumesDownloader(Downloader):
-    download_config: DatabricksVolumesDownloaderConfig
-    connection_config: DatabricksVolumesConnectionConfig
-    connector_type: str = CONNECTOR_TYPE
-
-    def precheck(self) -> None:
-        try:
-            self.connection_config.get_client()
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    def get_download_path(self, file_data: FileData) -> Path:
-        return self.download_config.download_dir / Path(file_data.source_identifiers.relative_path)
-
-    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
-        download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.info(f"Writing {file_data.identifier} to {download_path}")
-        try:
-            with self.connection_config.get_client().dbfs.download(path=file_data.identifier) as c:
-                read_content = c._read_handle.read()
-            with open(download_path, "wb") as f:
-                f.write(read_content)
-        except Exception as e:
-            logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
-            raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
-
-        return self.generate_download_response(file_data=file_data, download_path=download_path)
-
-
-class DatabricksVolumesUploaderConfig(UploaderConfig):
-    overwrite: bool = Field(
-        default=False, description="If true, an existing file will be overwritten."
-    )
-
-
-@dataclass
-class DatabricksVolumesUploader(Uploader):
-    upload_config: DatabricksVolumesUploaderConfig
-    connection_config: DatabricksVolumesConnectionConfig
-    connector_type: str = CONNECTOR_TYPE
-
-    def precheck(self) -> None:
-        try:
-            assert self.connection_config.get_client().current_user.me().active
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise DestinationConnectionError(f"failed to validate connection: {e}")
-
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        output_path = os.path.join(self.connection_config.path, path.name)
-        with open(path, "rb") as elements_file:
-            self.connection_config.get_client().files.upload(
-                file_path=output_path,
-                contents=elements_file,
-                overwrite=self.upload_config.overwrite,
-            )
-
-
-databricks_volumes_destination_entry = DestinationRegistryEntry(
-    connection_config=DatabricksVolumesConnectionConfig,
-    uploader=DatabricksVolumesUploader,
-    uploader_config=DatabricksVolumesUploaderConfig,
-)
-
-databricks_volumes_source_entry = SourceRegistryEntry(
-    connection_config=DatabricksVolumesConnectionConfig,
-    indexer=DatabricksVolumesIndexer,
-    indexer_config=DatabricksVolumesIndexerConfig,
-    downloader=DatabricksVolumesDownloader,
-    downloader_config=DatabricksVolumesDownloaderConfig,
-)

unstructured_ingest/v2/processes/connectors/singlestore.py

@@ -1,156 +0,0 @@
-import json
-from dataclasses import dataclass
-from datetime import date, datetime
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
-
-import numpy as np
-import pandas as pd
-from dateutil import parser
-from pydantic import Field, Secret
-
-from unstructured_ingest.utils.data_prep import batch_generator
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.table import convert_to_pandas_dataframe
-from unstructured_ingest.v2.interfaces import (
-    AccessConfig,
-    ConnectionConfig,
-    FileData,
-    Uploader,
-    UploaderConfig,
-    UploadStager,
-    UploadStagerConfig,
-)
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
-    DestinationRegistryEntry,
-)
-
-if TYPE_CHECKING:
-    from singlestoredb.connection import Connection
-
-CONNECTOR_TYPE = "singlestore"
-
-
-class SingleStoreAccessConfig(AccessConfig):
-    password: Optional[str] = Field(default=None, description="SingleStore password")
-
-
-class SingleStoreConnectionConfig(ConnectionConfig):
-    host: Optional[str] = Field(default=None, description="SingleStore host")
-    port: Optional[int] = Field(default=None, description="SingleStore port")
-    user: Optional[str] = Field(default=None, description="SingleStore user")
-    database: Optional[str] = Field(default=None, description="SingleStore database")
-    access_config: Secret[SingleStoreAccessConfig]
-
-    @requires_dependencies(["singlestoredb"], extras="singlestore")
-    def get_connection(self) -> "Connection":
-        import singlestoredb as s2
-
-        conn = s2.connect(
-            host=self.host,
-            port=self.port,
-            database=self.database,
-            user=self.user,
-            password=self.access_config.get_secret_value().password,
-        )
-        return conn
-
-
-class SingleStoreUploadStagerConfig(UploadStagerConfig):
-    drop_empty_cols: bool = Field(default=False, description="Drop any columns that have no data")
-
-
-@dataclass
-class SingleStoreUploadStager(UploadStager):
-    upload_stager_config: SingleStoreUploadStagerConfig
-
-    @staticmethod
-    def parse_date_string(date_string: str) -> date:
-        try:
-            timestamp = float(date_string)
-            return datetime.fromtimestamp(timestamp)
-        except Exception as e:
-            logger.debug(f"date {date_string} string not a timestamp: {e}")
-        return parser.parse(date_string)
-
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
-        output_path = Path(output_dir) / Path(f"{output_filename}.csv")
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-
-        df = convert_to_pandas_dataframe(
-            elements_dict=elements_contents,
-            drop_empty_cols=self.upload_stager_config.drop_empty_cols,
-        )
-        datetime_columns = [
-            "data_source_date_created",
-            "data_source_date_modified",
-            "data_source_date_processed",
-        ]
-        for column in filter(lambda x: x in df.columns, datetime_columns):
-            df[column] = df[column].apply(self.parse_date_string)
-        if "data_source_record_locator" in df.columns:
-            df["data_source_record_locator"] = df["data_source_record_locator"].apply(
-                lambda x: json.dumps(x) if x else None
-            )
-
-        with output_path.open("w") as output_file:
-            df.to_csv(output_file, index=False)
-        return output_path
-
-
-class SingleStoreUploaderConfig(UploaderConfig):
-    table_name: str = Field(description="SingleStore table to write contents to")
-    batch_size: int = Field(default=100, description="Batch size when writing to SingleStore")
-
-
-@dataclass
-class SingleStoreUploader(Uploader):
-    connection_config: SingleStoreConnectionConfig
-    upload_config: SingleStoreUploaderConfig
-    connector_type: str = CONNECTOR_TYPE
-
-    def upload_csv(self, csv_path: Path) -> None:
-        df = pd.read_csv(csv_path)
-        logger.debug(
-            f"uploading {len(df)} entries to {self.connection_config.database} "
-            f"db in table {self.upload_config.table_name}"
-        )
-        stmt = "INSERT INTO {} ({}) VALUES ({})".format(
-            self.upload_config.table_name,
-            ", ".join(df.columns),
-            ", ".join(["%s"] * len(df.columns)),
-        )
-        logger.debug(f"sql statement: {stmt}")
-        df.replace({np.nan: None}, inplace=True)
-        data_as_tuples = list(df.itertuples(index=False, name=None))
-        with self.connection_config.get_connection() as conn:
-            with conn.cursor() as cur:
-                for chunk in batch_generator(
-                    data_as_tuples, batch_size=self.upload_config.batch_size
-                ):
-                    cur.executemany(stmt, chunk)
-                    conn.commit()
-
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        if path.suffix != ".csv":
-            raise ValueError(f"Only .csv files are supported: {path}")
-        self.upload_csv(csv_path=path)
-
-
-singlestore_destination_entry = DestinationRegistryEntry(
-    connection_config=SingleStoreConnectionConfig,
-    uploader=SingleStoreUploader,
-    uploader_config=SingleStoreUploaderConfig,
-    upload_stager=SingleStoreUploadStager,
-    upload_stager_config=SingleStoreUploadStagerConfig,
-)