unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/connector/fsspec/gcs.py

@@ -1,82 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Type
-
-from unstructured_ingest.connector.fsspec.fsspec import (
-    FsspecDestinationConnector,
-    FsspecIngestDoc,
-    FsspecSourceConnector,
-    FsspecWriteConfig,
-    SimpleFsspecConfig,
-)
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.interfaces import AccessConfig
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.string_and_date_utils import json_to_dict
-
-
-@dataclass
-class GcsAccessConfig(AccessConfig):
-    token: t.Optional[str] = enhanced_field(
-        default=None, sensitive=True, overload_name="service_account_key"
-    )
-
-    def __post_init__(self):
-        ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
-
-        # Case: null value
-        if not self.token:
-            return
-        # Case: one of auth constants
-        if self.token in ALLOWED_AUTH_VALUES:
-            return
-        # Case: token as json
-        if isinstance(json_to_dict(self.token), dict):
-            self.token = json_to_dict(self.token)
-            return
-        # Case: path to token
-        if Path(self.token).is_file():
-            return
-
-        raise ValueError("Invalid auth token value")
-
-
-@dataclass
-class GcsWriteConfig(FsspecWriteConfig):
-    pass
-
-
-@dataclass
-class SimpleGcsConfig(SimpleFsspecConfig):
-    access_config: GcsAccessConfig = None
-
-
-@dataclass
-class GcsIngestDoc(FsspecIngestDoc):
-    connector_config: SimpleGcsConfig
-    registry_name: str = "gcs"
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
-    def get_file(self):
-        super().get_file()
-
-
-@dataclass
-class GcsSourceConnector(FsspecSourceConnector):
-    connector_config: SimpleGcsConfig
-
-    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
-    def initialize(self):
-        super().initialize()
-
-    def __post_init__(self):
-        self.ingest_doc_cls: Type[GcsIngestDoc] = GcsIngestDoc
-
-
-@dataclass
-class GcsDestinationConnector(FsspecDestinationConnector):
-    connector_config: SimpleGcsConfig
-    write_config: GcsWriteConfig

unstructured_ingest/connector/fsspec/s3.py

@@ -1,62 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-from typing import Type
-
-from unstructured_ingest.connector.fsspec.fsspec import (
-    FsspecDestinationConnector,
-    FsspecIngestDoc,
-    FsspecSourceConnector,
-    FsspecWriteConfig,
-    SimpleFsspecConfig,
-)
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.interfaces import AccessConfig
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-
-@dataclass
-class S3AccessConfig(AccessConfig):
-    anon: bool = enhanced_field(default=False, overload_name="anonymous")
-    endpoint_url: t.Optional[str] = None
-    key: t.Optional[str] = enhanced_field(default=None, sensitive=True)
-    secret: t.Optional[str] = enhanced_field(default=None, sensitive=True)
-    token: t.Optional[str] = enhanced_field(default=None, sensitive=True)
-
-
-@dataclass
-class S3WriteConfig(FsspecWriteConfig):
-    pass
-
-
-@dataclass
-class SimpleS3Config(SimpleFsspecConfig):
-    access_config: S3AccessConfig = enhanced_field(default=None)
-
-
-@dataclass
-class S3IngestDoc(FsspecIngestDoc):
-    connector_config: SimpleS3Config
-    remote_file_path: str
-    registry_name: str = "s3"
-
-    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
-    def get_file(self):
-        super().get_file()
-
-
-@dataclass
-class S3SourceConnector(FsspecSourceConnector):
-    connector_config: SimpleS3Config
-
-    def __post_init__(self):
-        self.ingest_doc_cls: Type[S3IngestDoc] = S3IngestDoc
-
-
-@dataclass
-class S3DestinationConnector(FsspecDestinationConnector):
-    connector_config: SimpleS3Config
-    write_config: S3WriteConfig
-
-    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
-    def initialize(self):
-        super().initialize()

unstructured_ingest/connector/fsspec/sftp.py

@@ -1,81 +0,0 @@
-import os
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Type
-from urllib.parse import urlparse
-
-from unstructured_ingest.connector.fsspec.fsspec import (
-    FsspecIngestDoc,
-    FsspecSourceConnector,
-    SimpleFsspecConfig,
-)
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.interfaces import AccessConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-
-@dataclass
-class SftpAccessConfig(AccessConfig):
-    username: str
-    password: str = enhanced_field(sensitive=True)
-    host: str = ""
-    port: int = 22
-    look_for_keys: bool = False
-    allow_agent: bool = False
-
-
-@dataclass
-class SimpleSftpConfig(SimpleFsspecConfig):
-    access_config: SftpAccessConfig = None
-
-    def __post_init__(self):
-        super().__post_init__()
-
-        _, ext = os.path.splitext(self.remote_url)
-        parsed_url = urlparse(self.remote_url)
-        if ext:
-            # We only want the file_path if it has an extension
-            self.file_path = Path(self.remote_url).name
-            self.dir_path = Path(parsed_url.path).parent.as_posix().lstrip("/")
-            self.path_without_protocol = self.dir_path
-        else:
-            self.file_path = ""
-            self.dir_path = parsed_url.path.lstrip("/")
-            self.path_without_protocol = self.dir_path
-        self.access_config.host = parsed_url.hostname or self.access_config.host
-        self.access_config.port = parsed_url.port or self.access_config.port
-
-
-@dataclass
-class SftpIngestDoc(FsspecIngestDoc):
-    connector_config: SimpleSftpConfig
-    registry_name: str = "sftp"
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
-    def get_file(self):
-        super().get_file()
-
-
-@dataclass
-class SftpSourceConnector(FsspecSourceConnector):
-    connector_config: SimpleSftpConfig
-
-    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
-    def initialize(self):
-        super().initialize()
-
-    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
-    def check_connection(self):
-        from fsspec.implementations.sftp import SFTPFileSystem
-
-        try:
-            SFTPFileSystem(**self.connector_config.get_access_config())
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    def __post_init__(self):
-        self.ingest_doc_cls: Type[SftpIngestDoc] = SftpIngestDoc

unstructured_ingest/connector/git.py

@@ -1,124 +0,0 @@
-import fnmatch
-import typing as t
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    IngestDocCleanupMixin,
-    SourceConnectorCleanupMixin,
-)
-from unstructured_ingest.logger import logger
-
-
-@dataclass
-class GitAccessConfig(AccessConfig):
-    access_token: t.Optional[str] = enhanced_field(
-        default=None, sensitive=True, overload_name="git_access_token"
-    )
-
-
-@dataclass
-class SimpleGitConfig(BaseConnectorConfig):
-    url: str
-    access_config: GitAccessConfig
-    branch: t.Optional[str] = enhanced_field(default=None, overload_name="git_branch")
-    file_glob: t.Optional[t.List[str]] = enhanced_field(default=None, overload_name="git_file_glob")
-    repo_path: str = field(init=False, repr=False)
-
-
-@dataclass
-class GitIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
-    connector_config: SimpleGitConfig = field(repr=False)
-    path: str
-
-    @property
-    def filename(self):
-        return (Path(self.read_config.download_dir) / self.path).resolve()
-
-    @property
-    def _output_filename(self):
-        return Path(self.processor_config.output_dir) / f"{self.path}.json"
-
-    @property
-    def record_locator(self) -> t.Dict[str, t.Any]:
-        record_locator = {
-            "repo_path": self.connector_config.repo_path,
-            "file_path": self.path,
-        }
-        if self.connector_config.branch is not None:
-            record_locator["branch"] = self.connector_config.branch
-        return record_locator
-
-    def _create_full_tmp_dir_path(self):
-        """includes directories in in the gitlab repository"""
-        self.filename.parent.mkdir(parents=True, exist_ok=True)
-
-    def update_source_metadata(self, **kwargs):
-        raise NotImplementedError()
-
-    @SourceConnectionError.wrap
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        """Fetches the "remote" doc and stores it locally on the filesystem."""
-        self._create_full_tmp_dir_path()
-        self._fetch_and_write()
-
-    def _fetch_content(self) -> None:
-        raise NotImplementedError()
-
-    def _fetch_and_write(self) -> None:
-        raise NotImplementedError()
-
-
-@dataclass
-class GitSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    connector_config: SimpleGitConfig
-
-    def initialize(self):
-        pass
-
-    def check_connection(self):
-        pass
-
-    @staticmethod
-    def is_file_type_supported(path: str) -> bool:
-        # Workaround to ensure that auto.partition isn't fed with .yaml, .py, etc. files
-        # TODO: What to do with no filenames? e.g. LICENSE, Makefile, etc.
-        supported = path.endswith(
-            (
-                ".md",
-                ".txt",
-                ".pdf",
-                ".doc",
-                ".docx",
-                ".eml",
-                ".heic",
-                ".html",
-                ".png",
-                ".jpg",
-                ".ppt",
-                ".pptx",
-                ".xml",
-            ),
-        )
-        if not supported:
-            logger.debug(
-                f"The file {path!r} is discarded as it does not contain a supported filetype.",
-            )
-        return supported
-
-    def does_path_match_glob(self, path: str) -> bool:
-        if not self.connector_config.file_glob:
-            return True
-        patterns = self.connector_config.file_glob
-        for pattern in patterns:
-            if fnmatch.filter([path], pattern):
-                return True
-        logger.debug(f"the file {path!r} is discarded as it does not match any given glob.")
-        return False

unstructured_ingest/connector/github.py

@@ -1,174 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-from datetime import datetime
-from urllib.parse import urlparse
-
-from unstructured_ingest.connector.git import (
-    GitIngestDoc,
-    GitSourceConnector,
-    SimpleGitConfig,
-)
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import SourceMetadata
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-if t.TYPE_CHECKING:
-    from github.Repository import Repository
-
-
-@dataclass
-class SimpleGitHubConfig(SimpleGitConfig):
-    def __post_init__(self):
-        parsed_gh_url = urlparse(self.url)
-        path_fragments = [fragment for fragment in parsed_gh_url.path.split("/") if fragment]
-
-        # If a scheme and netloc are provided, ensure they are correct
-        # Additionally, ensure that the path contains two fragments
-        if (
-            (parsed_gh_url.scheme and parsed_gh_url.scheme != "https")
-            or (parsed_gh_url.netloc and parsed_gh_url.netloc != "github.com")
-            or len(path_fragments) != 2
-        ):
-            raise ValueError(
-                'Please provide a valid URL, e.g. "https://github.com/Unstructured-IO/unstructured"'
-                ' or a repository owner/name pair, e.g. "Unstructured-IO/unstructured".',
-            )
-
-        # If there's no issues, store the core repository info
-        self.repo_path = parsed_gh_url.path
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["github"], extras="github")
-    def get_repo(self) -> "Repository":
-        from github import Github
-
-        github = Github(self.access_config.access_token)
-        return github.get_repo(self.repo_path)
-
-
-@dataclass
-class GitHubIngestDoc(GitIngestDoc):
-    connector_config: SimpleGitHubConfig
-    registry_name: str = "github"
-
-    @property
-    def date_created(self) -> t.Optional[str]:
-        return None
-
-    @requires_dependencies(["github"], extras="github")
-    def _fetch_file(self):
-        from github.GithubException import UnknownObjectException
-
-        try:
-            content_file = self.connector_config.get_repo().get_contents(self.path)
-        except UnknownObjectException:
-            logger.error(f"File doesn't exists {self.connector_config.url}/{self.path}")
-            return None
-
-        return content_file
-
-    @SourceConnectionNetworkError.wrap
-    @requires_dependencies(["requests"], extras="github")
-    def _fetch_content(self, content_file):
-        import requests
-
-        contents = b""
-        if (
-            not content_file.content  # type: ignore
-            and content_file.encoding == "none"  # type: ignore
-            and content_file.size  # type: ignore
-        ):
-            logger.info("File too large for the GitHub API, using direct download link instead.")
-            # NOTE: Maybe add a raise_for_status to catch connection timeout or HTTP Errors?
-            response = requests.get(content_file.download_url)  # type: ignore
-            if response.status_code != 200:
-                logger.info("Direct download link has failed... Skipping this file.")
-                return None
-            else:
-                contents = response.content
-        else:
-            contents = content_file.decoded_content  # type: ignore
-        return contents
-
-    def update_source_metadata(self, **kwargs):
-        content_file = kwargs.get("content_file", self._fetch_file())
-        if content_file is None:
-            self.source_metadata = SourceMetadata(
-                exists=False,
-            )
-            return
-
-        date_modified = datetime.strptime(
-            content_file.last_modified,
-            "%a, %d %b %Y %H:%M:%S %Z",
-        ).isoformat()
-        self.source_metadata = SourceMetadata(
-            date_modified=date_modified,
-            version=content_file.etag,
-            source_url=content_file.download_url,
-            exists=True,
-        )
-
-    def _fetch_and_write(self) -> None:
-        content_file = self._fetch_file()
-        self.update_source_metadata(content_file=content_file)
-        contents = self._fetch_content(content_file)
-        if contents is None:
-            raise ValueError(
-                f"Failed to retrieve file from repo "
-                f"{self.connector_config.url}/{self.path}. Check logs",
-            )
-        with open(self.filename, "wb") as f:
-            f.write(contents)
-
-
-@dataclass
-class GitHubSourceConnector(GitSourceConnector):
-    connector_config: SimpleGitHubConfig
-
-    @requires_dependencies(["github"], extras="github")
-    def check_connection(self):
-        from github import Consts
-        from github.GithubRetry import GithubRetry
-        from github.Requester import Requester
-
-        try:
-            requester = Requester(
-                auth=self.connector_config.access_config.access_token,
-                base_url=Consts.DEFAULT_BASE_URL,
-                timeout=Consts.DEFAULT_TIMEOUT,
-                user_agent=Consts.DEFAULT_USER_AGENT,
-                per_page=Consts.DEFAULT_PER_PAGE,
-                verify=True,
-                retry=GithubRetry(),
-                pool_size=None,
-            )
-            url_base = (
-                "/repositories/" if isinstance(self.connector_config.repo_path, int) else "/repos/"
-            )
-            url = f"{url_base}{self.connector_config.repo_path}"
-            headers, _ = requester.requestJsonAndCheck("HEAD", url)
-            logger.debug(f"headers from HEAD request: {headers}")
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    def get_ingest_docs(self):
-        repo = self.connector_config.get_repo()
-        # Load the Git tree with all files, and then create Ingest docs
-        # for all blobs, i.e. all files, ignoring directories
-        sha = self.connector_config.branch or repo.default_branch
-        git_tree = repo.get_git_tree(sha, recursive=True)
-        return [
-            GitHubIngestDoc(
-                connector_config=self.connector_config,
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                path=element.path,
-            )
-            for element in git_tree.tree
-            if element.type == "blob"
-            and self.is_file_type_supported(element.path)
-            and (not self.connector_config.file_glob or self.does_path_match_glob(element.path))
-        ]

unstructured_ingest/connector/gitlab.py

@@ -1,142 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-from urllib.parse import urlparse
-
-from unstructured_ingest.connector.git import (
-    GitIngestDoc,
-    GitSourceConnector,
-    SimpleGitConfig,
-)
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import SourceMetadata
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-if t.TYPE_CHECKING:
-    from gitlab.v4.objects.projects import Project
-
-
-@dataclass
-class SimpleGitlabConfig(SimpleGitConfig):
-    base_url: str = "https://gitlab.com"
-
-    def __post_init__(self):
-        parsed_gh_url = urlparse(self.url)
-        # If a scheme or netloc are provided, use the parsed base url
-        if parsed_gh_url.scheme or parsed_gh_url.netloc:
-            self.base_url = f"{parsed_gh_url.scheme}://{parsed_gh_url.netloc}"
-        self.repo_path = parsed_gh_url.path
-        while self.repo_path.startswith("/"):
-            self.repo_path = self.repo_path[1:]
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["gitlab"], extras="gitlab")
-    def get_project(self) -> "Project":
-        from gitlab import Gitlab
-
-        gitlab = Gitlab(self.base_url, private_token=self.access_config.access_token)
-        return gitlab.projects.get(self.repo_path)
-
-
-@dataclass
-class GitLabIngestDoc(GitIngestDoc):
-    connector_config: SimpleGitlabConfig
-    registry_name: str = "gitlab"
-
-    @property
-    def date_created(self) -> t.Optional[str]:
-        return None
-
-    @property
-    def date_modified(self) -> t.Optional[str]:
-        return None
-
-    @property
-    def source_url(self) -> t.Optional[str]:
-        return None
-
-    @SourceConnectionNetworkError.wrap
-    @requires_dependencies(["gitlab"], extras="gitlab")
-    def _fetch_content(self):
-        from gitlab.exceptions import GitlabHttpError
-
-        try:
-            project = self.connector_config.get_project()
-            content_file = project.files.get(
-                self.path,
-                ref=self.connector_config.branch or project.default_branch,
-            )
-        except GitlabHttpError as e:
-            if e.response_code == 404:
-                logger.error(f"File doesn't exists {self.connector_config.url}/{self.path}")
-                return None
-            raise
-        return content_file
-
-    def update_source_metadata(self, **kwargs):
-        content_file = kwargs.get("content_file", self._fetch_content())
-        if content_file is None:
-            self.source_metadata = SourceMetadata(
-                exists=None,
-            )
-            return
-        self.source_metadata = SourceMetadata(
-            version=content_file.attributes.get("last_commit_id", ""),
-            exists=True,
-        )
-
-    def _fetch_and_write(self) -> None:
-        content_file = self._fetch_content()
-        self.update_source_metadata(content_file=content_file)
-        if content_file is None:
-            raise ValueError(
-                f"Failed to retrieve file from repo "
-                f"{self.connector_config.url}/{self.path}. Check logs.",
-            )
-        contents = content_file.decode()
-        with open(self.filename, "wb") as f:
-            f.write(contents)
-
-
-@dataclass
-class GitLabSourceConnector(GitSourceConnector):
-    connector_config: SimpleGitlabConfig
-
-    @requires_dependencies(["gitlab"], extras="gitlab")
-    def check_connection(self):
-        from gitlab import Gitlab
-        from gitlab.exceptions import GitlabError
-
-        try:
-            gitlab = Gitlab(
-                self.connector_config.base_url,
-                private_token=self.connector_config.access_config.access_token,
-            )
-            gitlab.auth()
-        except GitlabError as gitlab_error:
-            logger.error(f"failed to validate connection: {gitlab_error}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {gitlab_error}")
-
-    def get_ingest_docs(self):
-        # Load the Git tree with all files, and then create Ingest docs
-        # for all blobs, i.e. all files, ignoring directories
-        project = self.connector_config.get_project()
-        ref = self.connector_config.branch or project.default_branch
-        git_tree = project.repository_tree(
-            ref=ref,
-            recursive=True,
-            iterator=True,
-            all=True,
-        )
-        return [
-            GitLabIngestDoc(
-                connector_config=self.connector_config,
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                path=element["path"],
-            )
-            for element in git_tree
-            if element["type"] == "blob"
-            and self.is_file_type_supported(element["path"])
-            and (not self.connector_config.file_glob or self.does_path_match_glob(element["path"]))
-        ]