unstructured-ingest 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

{unstructured_ingest-0.0.6.dist-info → unstructured_ingest-0.0.8.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=WuSmHtFa_1AeNyEP--6JEbVnwCjy8XsRpGHgW9T644I,42
+unstructured_ingest/__version__.py,sha256=mhrX1cgjjM8K7dLNNIYu3tS3ABCdK59YLgH5zvk67BI,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/evaluate.py,sha256=R-mKLFXbVX1xQ1tjGsLHjdP-TbSSV-925IHzggW_bIg,9793
 unstructured_ingest/interfaces.py,sha256=AeEywcSKCMA5AiEdENLpu_yPcXp_c6wpvESePfC00yo,31214
@@ -254,9 +254,9 @@ unstructured_ingest/utils/google_filetype.py,sha256=YVspEkiiBrRUSGVeVbsavvLvTmiz
 unstructured_ingest/utils/string_and_date_utils.py,sha256=hnGglD8Z626vLhH_UV4QybF_P62vwWRcA8CLk2x-s40,1377
 unstructured_ingest/utils/table.py,sha256=aWjcowDVSClNpEAdR6PY3H7khKu4T6T3QqQE6GjmQ_M,3469
 unstructured_ingest/v2/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/v2/example.py,sha256=qkwmpMxUlaJXdDNKQ4LlUt3XGxgTUU3CXGGO57eW5Gs,1644
 unstructured_ingest/v2/logger.py,sha256=akcghdHwpKM3CfoeFzir0zmc7R9Hk7zjquU-X-gwUIw,4324
 unstructured_ingest/v2/main.py,sha256=WFdLEqEXRy6E9_G-dF20MK2AtgX51Aan1sp_N67U2B8,172
+unstructured_ingest/v2/otel.py,sha256=jD-zuezaU5BHQEZfPSEusXNmesEvtrcfNjVPlQp-cmE,4130
 unstructured_ingest/v2/utils.py,sha256=ykmyvmRMHGahkpKbkFbJfEHwNjZccKqbYsixUtUtrFw,1478
 unstructured_ingest/v2/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/cli/cli.py,sha256=qHXIs-PcvMgDZhP1AR9iDMxh8FXBMJCEDksPBfiMULE,648
@@ -275,62 +275,63 @@ unstructured_ingest/v2/interfaces/downloader.py,sha256=PKT1kr79Mz1urW_8xCyq9sBuK
 unstructured_ingest/v2/interfaces/file_data.py,sha256=w6sBMCDH1va6XbvVaZyb7EJendXRCa_mo1Qv8uTWzCU,1898
 unstructured_ingest/v2/interfaces/indexer.py,sha256=Bd1S-gTLsxhJBLEh1lYm_gXqwQLaEZMoqPq9yGxtN_E,713
 unstructured_ingest/v2/interfaces/process.py,sha256=BgglTu5K93FnDDopZKKr_rkK2LTZOguR6kcQjKHjF40,392
-unstructured_ingest/v2/interfaces/processor.py,sha256=t1LIrkubfbqt7RMZ9bABrxd0Z9TJxG6zqozBC5Pi4Yc,1615
+unstructured_ingest/v2/interfaces/processor.py,sha256=1taXZVAHKnWH420N1v-JNXfRGq5roTaYvxqcO1EzpnQ,1772
 unstructured_ingest/v2/interfaces/upload_stager.py,sha256=ZFkDxcwKn-6EPrTbdBEgOkz1kGAq4gUtze98KP48KG4,1146
-unstructured_ingest/v2/interfaces/uploader.py,sha256=ymEC-0JFTvjuAFsz9QLRF_6rFNFlDURkAgZ1tBUb2ec,1009
+unstructured_ingest/v2/interfaces/uploader.py,sha256=JmZDl1blJa5rS61YHCae3Hfet84ixSSJ_NYRjflYsbY,1168
 unstructured_ingest/v2/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-unstructured_ingest/v2/pipeline/interfaces.py,sha256=Z50-6XFZNajfmJbLKunLxw3RuYMzCYiUp6F0jhQwERE,6441
-unstructured_ingest/v2/pipeline/pipeline.py,sha256=rOiTGLW6e1kifSA_0bCzFktDVKN7eXG_BzEKtLBTjCA,14736
+unstructured_ingest/v2/pipeline/interfaces.py,sha256=V9zlEWI5OqTXnV4epfc_Y2TDZyPZCEB4qPam7NlUMEI,8354
+unstructured_ingest/v2/pipeline/otel.py,sha256=K3pQvWVgWzyOWMKCBUofsH7wTZPJ0Ysw5sLjMBLW41I,1088
+unstructured_ingest/v2/pipeline/pipeline.py,sha256=D8AlVCflOjytyqhNwhpzyVJs-pHJ-FdPt9o1OJXAe-A,15010
 unstructured_ingest/v2/pipeline/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/v2/pipeline/steps/chunk.py,sha256=I5JQn9rVJu9zEnsAZsJzDnxuASp2hdkF8ZRW4dOtgb0,3124
 unstructured_ingest/v2/pipeline/steps/download.py,sha256=IwysS1_YZRuraIifBW94LWhPNDYU-oaeFkEbgPNpBag,7690
 unstructured_ingest/v2/pipeline/steps/embed.py,sha256=5wONbMvT_hZRZtHPgquok1ryC66dajCU5iifVfIaP9Y,3102
 unstructured_ingest/v2/pipeline/steps/filter.py,sha256=1HM6aBZ5YI0wHQjMXx4KISsiueRlLXVn0mYyiXLMgy4,1188
-unstructured_ingest/v2/pipeline/steps/index.py,sha256=oyYFtDgWg-CJoBM75YTln1t6DjGoLooz2stEeiaB08Q,2537
+unstructured_ingest/v2/pipeline/steps/index.py,sha256=0h5sc5mlnMuyxPKmbm4sY6LytqZiAWcP_FJvsYQF4WA,2632
 unstructured_ingest/v2/pipeline/steps/partition.py,sha256=2NuXpDQ9brf7D4vPhbalCGpjw80XRGYZAAO-Ist1yKs,3182
 unstructured_ingest/v2/pipeline/steps/stage.py,sha256=6gAPzp46DrsOtL914hqgATRDCMvBRI7VtvlsFuMWc4I,2211
 unstructured_ingest/v2/pipeline/steps/uncompress.py,sha256=fEjHPdXnMKpmN5vhqbrNOrVsQGKcoKxRSKE5PrViE9I,2389
-unstructured_ingest/v2/pipeline/steps/upload.py,sha256=5Y6oZV5IyfOvXr8RPYjEfy8_yp4XGatoY0tsznlH_wA,2278
+unstructured_ingest/v2/pipeline/steps/upload.py,sha256=G9z8QQe9b_WokI5qyr4UOOqaepEVgwFqMn9pWcta9gI,1917
 unstructured_ingest/v2/processes/__init__.py,sha256=FaHWSCGyc7GWVnAsNEUUj7L8hT8gCVY3_hUE2VzWtUg,462
 unstructured_ingest/v2/processes/chunker.py,sha256=W2qPIddT-QEhHTKpA1krUhBrR0UFOq4nuko2eBjlG_I,6675
 unstructured_ingest/v2/processes/connector_registry.py,sha256=vkEe6jpgdYtZCxMj59s5atWGgmPuxAEXRUoTt-MJ7wc,2198
 unstructured_ingest/v2/processes/embedder.py,sha256=1AYF0o41tYtQv-ArGCc1PKGnlmNFDiFmhhpgEuG2d4I,5939
 unstructured_ingest/v2/processes/filter.py,sha256=eiAxdYiX8wd4vmD4J40x5t5wwJNmoGa5z33Z9Q-knK8,2145
-unstructured_ingest/v2/processes/partitioner.py,sha256=wI0BCFNGdnDFKuEGVeR1Lt65qn5ZSVYMSfMRJ_N3ryI,9782
+unstructured_ingest/v2/processes/partitioner.py,sha256=s7R7KVR-w7EtmqyieC-z-ZFv8H5bPn7IvXgqZddyoF8,10040
 unstructured_ingest/v2/processes/uncompress.py,sha256=LL68WLq4EfZGORvv5vaQx03EXfiA7k62sbzElPEN8AM,1557
 unstructured_ingest/v2/processes/connectors/__init__.py,sha256=4zKMtzTqSzBKiHG92oE5jZUWw7Dc_RZ8c1VvwePrpjo,4801
-unstructured_ingest/v2/processes/connectors/astradb.py,sha256=dexaljYJ08V4URm971hhmZE9hKRCFHO-JNstXNeh4GI,5924
-unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=Tx2ux-w7CgEZ4bn0r0dtAUzAy4HB9PvpQRoh52oU0ac,8603
-unstructured_ingest/v2/processes/connectors/chroma.py,sha256=a5Ebd5Rj7OF0bTpj88aIRh6HsKJJYFDhVe-7RQEuzvs,7916
-unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=5-yKJDnT_iKTDsmMC-HZvUXiSpuNxmvhWQ-MiaZlLY8,12240
-unstructured_ingest/v2/processes/connectors/databricks_volumes.py,sha256=kn5xBdcB2UfnuYxNj0URvBD4WUNIUn0Roi_9mpYOMII,6120
-unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=9ZNNiX69-5beVIQrrYB5uU8x9F4KAogZsPiRshhEuso,16898
+unstructured_ingest/v2/processes/connectors/astradb.py,sha256=bjlzJVNANnpTxRm8Ba8ZS0KetJ_yxmEyEoPJDwUkcOw,5774
+unstructured_ingest/v2/processes/connectors/azure_cognitive_search.py,sha256=S55v7TXu30rEdgythMBB_2VcuomyMPmcPtLYykbhw_E,8466
+unstructured_ingest/v2/processes/connectors/chroma.py,sha256=-ZZLwNS10hyW1-tb_Wl0YBswMc-_xPAp0nRCcmDqLas,7766
+unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=SONLywyEfoAlLc-HPabXeGzoiwKnekMHIbRMXd4CGXs,12146
+unstructured_ingest/v2/processes/connectors/databricks_volumes.py,sha256=K0Sjt57vsVxL2eImqHzu7LnAONPUVTcDw2-hdLcWjV0,5984
+unstructured_ingest/v2/processes/connectors/elasticsearch.py,sha256=0O7l5LTIKw5bEiA0Nnm1umRjMaUhjJr7XihYSzn1a9g,16750
 unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=zRhhBCDFF4QzBpT2Ij1uXd5jdKTc_JR9WwfSLV9ynQc,12890
-unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=oTLbRsapX9nDvUM-rjSGW0lrCuGyJERi13ycPS00lgU,5698
-unstructured_ingest/v2/processes/connectors/local.py,sha256=5sY8su9nI3PoV-Y7o0jaM2lNtTL1CYnwaeBI1puddBI,7074
-unstructured_ingest/v2/processes/connectors/milvus.py,sha256=J3Zjg7dqhqQt35AYWiUt812bzfwHa9hVapWiWxswhHQ,7265
-unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=3FavmgKjtQADSyuH3EMIkfUgmRjIQfc0wVDlvLpd7Hs,5098
+unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=vF-Owg2ZDI4nC7sk-Ied-5o_qkfwJzDr3uztOeS8kC0,5653
+unstructured_ingest/v2/processes/connectors/local.py,sha256=sXO-t_HZbq3rE3RzPUiWPnlrCHDixcSxz6epg4XgyYo,6786
+unstructured_ingest/v2/processes/connectors/milvus.py,sha256=hNMtjdNF6Nv8E_0n5uDpki1kAFdtPplq_5N0W92rrVs,7761
+unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=q_GRuG2RQ5-8ajefifKuhFO52wCVhtU9j4ZIEf5hNas,4948
 unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=-B8ELr0rnspzrTy6HBvgbvuiF1eEKRQyCT1ocwmET5Q,9145
 unstructured_ingest/v2/processes/connectors/opensearch.py,sha256=dEj4yYCgdhCD6376kuhKH7NVO5v-vsrN8dbULjEXfss,6811
-unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=cfegFH1t7cSjjfebPnbnaOXgOMFlIET8I73YGfR4uco,6115
+unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=_RwrV7qVbMOsvD8LuGYKQO3UrSFJvgjTGzx304Jj4l0,5459
 unstructured_ingest/v2/processes/connectors/salesforce.py,sha256=2CiO2ZZiZ1Y1-nB7wcDlDVcpW2B7ut9wCj66rkkqho0,11616
 unstructured_ingest/v2/processes/connectors/sharepoint.py,sha256=_ZTjtFNcKcJ0z4cvEZml18TdOMm-Kbwlz8nxTTjp9nc,19500
-unstructured_ingest/v2/processes/connectors/singlestore.py,sha256=Pkax_54XGhbp-BeC_jsLxRHEJlJeYMYsH5r0H1tVi8w,5526
-unstructured_ingest/v2/processes/connectors/sql.py,sha256=UU4EoDMy--QyHnY-Bw96QXcGFmnQAyHO_QhkdjYZi30,9386
+unstructured_ingest/v2/processes/connectors/singlestore.py,sha256=4rVvWKK2iQr03Ff6cB5zjfE1MpN0JyIGpCxxFCDI6hc,5563
+unstructured_ingest/v2/processes/connectors/sql.py,sha256=tDWL3YqL8MQuLsjW8A-KUkpSLh1iOn934OWfzPkqils,9298
 unstructured_ingest/v2/processes/connectors/utils.py,sha256=nmpZZCeX0O7rGrwHSWM_heBgpZK9tKT6EV1Moer-z40,576
-unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=5tQbo0jfqhl61Uc6DWgLo4sb5tFL_4BO00H0tq0WoqU,9207
+unstructured_ingest/v2/processes/connectors/weaviate.py,sha256=4hBQ9GWbBv6ti9futVJCShNugDC6Vh7Hy9ZhEC4XDpM,8958
 unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Ypl_n6sl7I1JqX6bGSG0t_FqvCqE3Cy24og,1846
-unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=ovlU5s_s5vfjRqECXOlOTNONlaZ-hWqcjWJ2eZv8iIY,6214
-unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=MerISu_HxJ3n4-4VVnz9tBAbkCbhQmt0k_iHSSZvZYI,4435
-unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=_I_GXnjrzrfJO0zoXgaLQX3foIvXYl_-Cm3yfHOCkUs,4775
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=Nn9rOkQKH9aq575R-h-l9xZCFsLfrYiry8L5xbHP_DQ,12556
-unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=HoExrmUHigyWRBgEMneKigjcROc-GH_Jp2aUyWkHBH0,6006
-unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=tKxH6h80i7BDzEoMsWZZBLwJ1wpJ-WPxUwT2ezWRVLI,6394
-unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=dpLFv5vGtw8aiLDgrf6E7elg8Pm2dKF0MQnnl84rIqM,6056
+unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=GrC44PnA8PLHUJQ4aH3gETxL8v8UvknbKptxiXweqdc,6214
+unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=dVKVCdbKdNSkkZBYJL14-u7aXOr50mzKCmhCVAneuqI,4435
+unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=ZnoSGAZ4wtOhyg8G3PLYFMpbMVsBffvW-qp5jWwEDuA,4775
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=AAP7NNxO5smWYwzsAVbfs91FzpAteZeFI8vZj34ytgg,12441
+unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=O1j0hIYWI4lPpTQ5hsEKV8usDCrUm-t1qVcSNKsJQd0,6006
+unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=cOMvMh0C9rtyEPJ0X59Fn-qb11LFUMRfeUgsi3QRWUk,6390
+unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=NkcU4U2DQWWuM8UHkez67C3SEOZpVyRtmtNS-z-F0Fw,6056
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
-unstructured_ingest-0.0.6.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.0.6.dist-info/METADATA,sha256=0U9Y5j8pliTPcb2IDkfh40eUwcu-Fn70OTXyXjnKCo8,24733
-unstructured_ingest-0.0.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-unstructured_ingest-0.0.6.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.0.6.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
-unstructured_ingest-0.0.6.dist-info/RECORD,,
+unstructured_ingest-0.0.8.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.0.8.dist-info/METADATA,sha256=ooohUbW_ADaqx0TqLjxrj-T8wj6Q9r4xdMu380kvQ5k,28109
+unstructured_ingest-0.0.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+unstructured_ingest-0.0.8.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.0.8.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
+unstructured_ingest-0.0.8.dist-info/RECORD,,

unstructured_ingest/v2/example.py

@@ -1,37 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.v2.interfaces import ProcessorConfig
-from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.pipeline.pipeline import Pipeline
-from unstructured_ingest.v2.processes.chunker import ChunkerConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
-    S3ConnectionConfig,
-    S3DownloaderConfig,
-    S3IndexerConfig,
-)
-from unstructured_ingest.v2.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.v2.processes.embedder import EmbedderConfig
-from unstructured_ingest.v2.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest"
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"Writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(
-            work_dir=str(work_dir.resolve()), tqdm=True, reprocess=True, verbose=True
-        ),
-        indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
-        downloader_config=S3DownloaderConfig(download_dir=download_path),
-        source_connection_config=S3ConnectionConfig(anonymous=True),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()