unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/cli/configs/embed.py

@@ -1,74 +0,0 @@
-from dataclasses import dataclass
-from typing import Any
-
-import click
-from dataclasses_json.core import Json
-from unstructured.embed import EMBEDDING_PROVIDER_TO_CLASS_MAP
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-
-
-@dataclass
-class EmbedderCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--embedding-provider"],
-                help="Type of the embedding class to be used. Can be one of: "
-                f"{list(EMBEDDING_PROVIDER_TO_CLASS_MAP)}",
-                type=click.Choice(list(EMBEDDING_PROVIDER_TO_CLASS_MAP)),
-            ),
-            click.Option(
-                ["--embedding-api-key"],
-                help="API key for the embedding model, for the case an API key is needed.",
-                type=str,
-                default=None,
-            ),
-            click.Option(
-                ["--embedding-model-name"],
-                help="Embedding model name, if needed. "
-                "Chooses a particular LLM between different options, to embed with it.",
-                type=str,
-                default=None,
-            ),
-            click.Option(
-                ["--embedding-aws-access-key-id"],
-                help="AWS access key used for AWS-based embedders, such as bedrock",
-                type=str,
-                default=None,
-            ),
-            click.Option(
-                ["--embedding-aws-secret-access-key"],
-                help="AWS secret key used for AWS-based embedders, such as bedrock",
-                type=str,
-                default=None,
-            ),
-            click.Option(
-                ["--embedding-aws-region"],
-                help="AWS region used for AWS-based embedders, such as bedrock",
-                type=str,
-                default="us-west-2",
-            ),
-        ]
-        return options
-
-    @classmethod
-    def from_dict(cls, kvs: Json, **kwargs: Any):
-        """
-        Extension of the dataclass from_dict() to avoid a naming conflict with other CLI params.
-        This allows CLI arguments to be prepended with embedding_ during CLI invocation but
-        doesn't require that as part of the field names in this class
-        """
-        if isinstance(kvs, dict):
-            new_kvs = {
-                k[len("embedding_") :]: v  # noqa: E203
-                for k, v in kvs.items()
-                if k.startswith("embedding_")
-            }
-            if len(new_kvs.keys()) == 0:
-                return None
-            if not new_kvs.get("provider"):
-                return None
-            return super().from_dict(new_kvs, **kwargs)
-        return super().from_dict(kvs, **kwargs)

unstructured_ingest/v2/cli/configs/filter.py

@@ -1,28 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-
-
-@dataclass
-class FilterCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which types of "
-                "local files are accepted, e.g. '*.html,*.txt'",
-            ),
-            click.Option(
-                ["--max-file-size"],
-                default=None,
-                type=click.IntRange(min=1),
-                help="Max file size to process in bytes",
-            ),
-        ]
-        return options

unstructured_ingest/v2/cli/configs/partition.py

@@ -1,99 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString, Dict
-
-
-@dataclass
-class PartitionerCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--strategy"],
-                default="auto",
-                help="The method that will be used to process the documents. "
-                "Default: auto. Other strategies include `fast` and `hi_res`.",
-            ),
-            click.Option(
-                ["--ocr-languages"],
-                default=None,
-                type=DelimitedString(delimiter="+"),
-                help="A list of language packs to specify which languages to use for OCR, "
-                "separated by '+' e.g. 'eng+deu' to use the English and German language packs. "
-                "The appropriate Tesseract "
-                "language pack needs to be installed.",
-            ),
-            click.Option(
-                ["--encoding"],
-                default=None,
-                help="Text encoding to use when reading documents. By default the encoding is "
-                "detected automatically.",
-            ),
-            click.Option(
-                ["--skip-infer-table-types"],
-                type=DelimitedString(),
-                default=None,
-                help="Optional list of document types to skip table extraction on",
-            ),
-            click.Option(
-                ["--additional-partition-args"],
-                type=Dict(),
-                help="A json string representation of values to pass through to partition()",
-            ),
-            click.Option(
-                ["--fields-include"],
-                type=DelimitedString(),
-                default=["element_id", "text", "type", "metadata", "embeddings"],
-                help="Comma-delimited list. If set, include the specified top-level "
-                "fields in an element.",
-            ),
-            click.Option(
-                ["--flatten-metadata"],
-                is_flag=True,
-                default=False,
-                help="Results in flattened json elements. "
-                "Specifically, the metadata key values are brought to "
-                "the top-level of the element, and the `metadata` key itself is removed.",
-            ),
-            click.Option(
-                ["--metadata-include"],
-                default=[],
-                type=DelimitedString(),
-                help="Comma-delimited list. If set, include the specified metadata "
-                "fields if they exist and drop all other fields. ",
-            ),
-            click.Option(
-                ["--metadata-exclude"],
-                default=[],
-                type=DelimitedString(),
-                help="Comma-delimited list. If set, drop the specified metadata "
-                "fields if they exist.",
-            ),
-            click.Option(
-                ["--partition-by-api"],
-                is_flag=True,
-                default=False,
-                help="Use a remote API to partition the files."
-                " Otherwise, use the function from partition.auto",
-            ),
-            click.Option(
-                ["--partition-endpoint"],
-                default="https://api.unstructured.io/general/v0/general",
-                help="If partitioning via api, use the following host. "
-                "Default: https://api.unstructured.io/general/v0/general",
-            ),
-            click.Option(
-                ["--api-key"],
-                default=None,
-                help="API Key for partition endpoint.",
-            ),
-            click.Option(
-                ["--hi-res-model-name"],
-                default=None,
-                help="Model name for hi-res strategy.",
-            ),
-        ]
-        return options

unstructured_ingest/v2/cli/configs/processor.py

@@ -1,88 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.interfaces.processor import DEFAULT_WORK_DIR
-
-
-@dataclass
-class ProcessorCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--reprocess"],
-                is_flag=True,
-                default=False,
-                help="Reprocess a downloaded file even if the relevant structured "
-                "output .json file in output directory already exists.",
-            ),
-            click.Option(
-                ["--work-dir"],
-                type=str,
-                default=DEFAULT_WORK_DIR,
-                show_default=True,
-                help="Where to place working files when processing each step",
-            ),
-            click.Option(
-                ["--num-processes"],
-                default=2,
-                show_default=True,
-                type=click.IntRange(min=1),
-                help="Number of parallel processes with which to process docs",
-            ),
-            click.Option(
-                ["--max-connections"],
-                default=None,
-                show_default=True,
-                type=click.IntRange(min=1),
-                help="Max number of connections allowed when running an async step",
-            ),
-            click.Option(
-                ["--raise-on-error"],
-                is_flag=True,
-                default=False,
-                help="Is set, will raise error if any doc in the pipeline fail. Otherwise will "
-                "log error and continue with other docs",
-            ),
-            click.Option(
-                ["--re-download"],
-                is_flag=True,
-                default=False,
-                help="Re-download files even if they are already present in download dir.",
-            ),
-            click.Option(
-                ["--preserve-downloads"],
-                is_flag=True,
-                default=False,
-                help="Preserve downloaded files. Otherwise each file is removed "
-                "after being processed successfully.",
-            ),
-            click.Option(
-                ["--download-only"],
-                is_flag=True,
-                default=False,
-                help="Download any files that are not already present in either --download-dir or "
-                "the default download ~/.cache/... location in case --download-dir "
-                "is not specified and "
-                "skip processing them through unstructured.",
-            ),
-            click.Option(
-                ["--max-docs"],
-                default=None,
-                type=int,
-                help="If specified, process at most the specified number of documents.",
-            ),
-            click.Option(
-                ["--uncompress"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="Uncompress any archived files. Currently supporting zip and tar "
-                "files based on file extension.",
-            ),
-            click.Option(["--verbose"], is_flag=True, default=False),
-            click.Option(["--tqdm"], is_flag=True, default=False, help="Show progress bar"),
-        ]
-        return options

unstructured_ingest/v2/cli/interfaces.py

@@ -1,27 +0,0 @@
-from abc import ABC, abstractmethod
-
-import click
-
-
-class CliConfig(ABC):
-    @staticmethod
-    @abstractmethod
-    def get_cli_options() -> list[click.Option]:
-        pass
-
-    @classmethod
-    def add_cli_options(cls, cmd: click.Command) -> None:
-        options_to_add = cls.get_cli_options()
-        CliConfig.add_params(cmd, params=options_to_add)
-
-    @staticmethod
-    def add_params(cmd: click.Command, params: list[click.Parameter]):
-        existing_opts = []
-        for param in cmd.params:
-            existing_opts.extend(param.opts)
-        for param in params:
-            for opt in param.opts:
-                if opt in existing_opts:
-                    raise ValueError(f"{opt} is already defined on the command {cmd.name}")
-                existing_opts.append(opt)
-                cmd.params.append(param)

unstructured_ingest/v2/pipeline/utils.py

@@ -1,15 +0,0 @@
-import json
-from datetime import datetime
-from pathlib import Path
-
-
-def sterilize_dict(data: dict) -> dict:
-    def json_serial(obj):
-        if isinstance(obj, Path):
-            return obj.as_posix()
-        if isinstance(obj, datetime):
-            return obj.isoformat()
-        raise TypeError("Type %s not serializable" % type(obj))
-
-    data_s = json.dumps(data, default=json_serial)
-    return json.loads(data_s)

unstructured_ingest-0.0.3.dist-info/METADATA

@@ -1,175 +0,0 @@
-Metadata-Version: 2.1
-Name: unstructured-ingest
-Version: 0.0.3
-Summary: A library that prepares raw documents for downstream ML tasks.
-Home-page: https://github.com/Unstructured-IO/unstructured-ingest
-Author: Unstructured Technologies
-Author-email: devops@unstructuredai.io
-License: Apache-2.0
-Keywords: NLP PDF HTML CV XML parsing preprocessing
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: Education
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: >=3.9.0,<3.13
-Description-Content-Type: text/markdown
-Requires-Dist: unstructured
-Requires-Dist: python-dateutil
-Requires-Dist: pandas
-Provides-Extra: airtable
-Requires-Dist: pyairtable ; extra == 'airtable'
-Provides-Extra: astra
-Requires-Dist: astrapy ; extra == 'astra'
-Provides-Extra: azure
-Requires-Dist: fsspec ; extra == 'azure'
-Requires-Dist: adlfs ; extra == 'azure'
-Provides-Extra: azure-cognitive-search
-Requires-Dist: azure-search-documents ; extra == 'azure-cognitive-search'
-Provides-Extra: bedrock
-Requires-Dist: boto3 ; extra == 'bedrock'
-Requires-Dist: langchain-community ; extra == 'bedrock'
-Provides-Extra: biomed
-Requires-Dist: bs4 ; extra == 'biomed'
-Provides-Extra: box
-Requires-Dist: fsspec ; extra == 'box'
-Requires-Dist: boxfs ; extra == 'box'
-Provides-Extra: chroma
-Requires-Dist: typer <=0.9.0 ; extra == 'chroma'
-Requires-Dist: importlib-metadata >=7.1.0 ; extra == 'chroma'
-Requires-Dist: chromadb ; extra == 'chroma'
-Provides-Extra: clarifai
-Requires-Dist: clarifai ; extra == 'clarifai'
-Provides-Extra: confluence
-Requires-Dist: atlassian-python-api ; extra == 'confluence'
-Provides-Extra: csv
-Requires-Dist: unstructured[tsv] ; extra == 'csv'
-Provides-Extra: databricks-volumes
-Requires-Dist: databricks-sdk ; extra == 'databricks-volumes'
-Provides-Extra: delta-table
-Requires-Dist: fsspec ; extra == 'delta-table'
-Requires-Dist: deltalake ; extra == 'delta-table'
-Provides-Extra: discord
-Requires-Dist: discord-py ; extra == 'discord'
-Provides-Extra: doc
-Requires-Dist: unstructured[docx] ; extra == 'doc'
-Provides-Extra: docx
-Requires-Dist: unstructured[docx] ; extra == 'docx'
-Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs ; extra == 'dropbox'
-Requires-Dist: fsspec ; extra == 'dropbox'
-Provides-Extra: elasticsearch
-Requires-Dist: elasticsearch[async] ; extra == 'elasticsearch'
-Provides-Extra: embed-huggingface
-Requires-Dist: sentence-transformers ; extra == 'embed-huggingface'
-Requires-Dist: langchain-community ; extra == 'embed-huggingface'
-Requires-Dist: huggingface ; extra == 'embed-huggingface'
-Provides-Extra: embed-octoai
-Requires-Dist: tiktoken ; extra == 'embed-octoai'
-Requires-Dist: openai ; extra == 'embed-octoai'
-Provides-Extra: embed-vertexai
-Requires-Dist: langchain ; extra == 'embed-vertexai'
-Requires-Dist: langchain-community ; extra == 'embed-vertexai'
-Requires-Dist: langchain-google-vertexai ; extra == 'embed-vertexai'
-Provides-Extra: embed-voyageai
-Requires-Dist: langchain ; extra == 'embed-voyageai'
-Requires-Dist: langchain-voyageai ; extra == 'embed-voyageai'
-Provides-Extra: epub
-Requires-Dist: unstructured[epub] ; extra == 'epub'
-Provides-Extra: gcs
-Requires-Dist: fsspec ; extra == 'gcs'
-Requires-Dist: bs4 ; extra == 'gcs'
-Requires-Dist: gcsfs ; extra == 'gcs'
-Provides-Extra: github
-Requires-Dist: pygithub >1.58.0 ; extra == 'github'
-Provides-Extra: gitlab
-Requires-Dist: python-gitlab ; extra == 'gitlab'
-Provides-Extra: google-drive
-Requires-Dist: google-api-python-client ; extra == 'google-drive'
-Provides-Extra: hubspot
-Requires-Dist: urllib3 ; extra == 'hubspot'
-Requires-Dist: hubspot-api-client ; extra == 'hubspot'
-Provides-Extra: jira
-Requires-Dist: atlassian-python-api ; extra == 'jira'
-Provides-Extra: kafka
-Requires-Dist: confluent-kafka ; extra == 'kafka'
-Provides-Extra: md
-Requires-Dist: unstructured[md] ; extra == 'md'
-Provides-Extra: milvus
-Requires-Dist: pymilvus ; extra == 'milvus'
-Provides-Extra: mongodb
-Requires-Dist: pymongo ; extra == 'mongodb'
-Provides-Extra: msg
-Requires-Dist: unstructured[msg] ; extra == 'msg'
-Provides-Extra: notion
-Requires-Dist: notion-client ; extra == 'notion'
-Requires-Dist: htmlBuilder ; extra == 'notion'
-Provides-Extra: odt
-Requires-Dist: unstructured[odt] ; extra == 'odt'
-Provides-Extra: onedrive
-Requires-Dist: bs4 ; extra == 'onedrive'
-Requires-Dist: msal ; extra == 'onedrive'
-Requires-Dist: Office365-REST-Python-Client ; extra == 'onedrive'
-Provides-Extra: openai
-Requires-Dist: tiktoken ; extra == 'openai'
-Requires-Dist: langchain-community ; extra == 'openai'
-Requires-Dist: openai ; extra == 'openai'
-Provides-Extra: opensearch
-Requires-Dist: opensearch-py ; extra == 'opensearch'
-Provides-Extra: org
-Requires-Dist: unstructured[org] ; extra == 'org'
-Provides-Extra: outlook
-Requires-Dist: msal ; extra == 'outlook'
-Requires-Dist: Office365-REST-Python-Client ; extra == 'outlook'
-Provides-Extra: pdf
-Requires-Dist: unstructured[pdf] ; extra == 'pdf'
-Provides-Extra: pinecone
-Requires-Dist: pinecone-client >=3.7.1 ; extra == 'pinecone'
-Provides-Extra: postgres
-Requires-Dist: psycopg2-binary ; extra == 'postgres'
-Provides-Extra: ppt
-Requires-Dist: unstructured[pptx] ; extra == 'ppt'
-Provides-Extra: pptx
-Requires-Dist: unstructured[pptx] ; extra == 'pptx'
-Provides-Extra: qdrant
-Requires-Dist: qdrant-client ; extra == 'qdrant'
-Provides-Extra: reddit
-Requires-Dist: praw ; extra == 'reddit'
-Provides-Extra: rst
-Requires-Dist: unstructured[rst] ; extra == 'rst'
-Provides-Extra: rtf
-Requires-Dist: unstructured[rtf] ; extra == 'rtf'
-Provides-Extra: s3
-Requires-Dist: fsspec ; extra == 's3'
-Requires-Dist: s3fs ; extra == 's3'
-Provides-Extra: salesforce
-Requires-Dist: simple-salesforce ; extra == 'salesforce'
-Provides-Extra: sftp
-Requires-Dist: fsspec ; extra == 'sftp'
-Requires-Dist: paramiko ; extra == 'sftp'
-Provides-Extra: sharepoint
-Requires-Dist: msal ; extra == 'sharepoint'
-Requires-Dist: Office365-REST-Python-Client ; extra == 'sharepoint'
-Provides-Extra: singlestore
-Requires-Dist: singlestoredb ; extra == 'singlestore'
-Provides-Extra: slack
-Requires-Dist: slack-sdk ; extra == 'slack'
-Provides-Extra: tsv
-Requires-Dist: unstructured[tsv] ; extra == 'tsv'
-Provides-Extra: weaviate
-Requires-Dist: weaviate-client ; extra == 'weaviate'
-Provides-Extra: wikipedia
-Requires-Dist: wikipedia ; extra == 'wikipedia'
-Provides-Extra: xlsx
-Requires-Dist: unstructured[xlsx] ; extra == 'xlsx'
-
-# Unstructured Ingest  
-
-For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.