unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/connector/fsspec/sftp.py

@@ -0,0 +1,81 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Type
+from urllib.parse import urlparse
+
+from unstructured_ingest.connector.fsspec.fsspec import (
+    FsspecIngestDoc,
+    FsspecSourceConnector,
+    SimpleFsspecConfig,
+)
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.interfaces import AccessConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+
+@dataclass
+class SftpAccessConfig(AccessConfig):
+    username: str
+    password: str = enhanced_field(sensitive=True)
+    host: str = ""
+    port: int = 22
+    look_for_keys: bool = False
+    allow_agent: bool = False
+
+
+@dataclass
+class SimpleSftpConfig(SimpleFsspecConfig):
+    access_config: SftpAccessConfig = None
+
+    def __post_init__(self):
+        super().__post_init__()
+
+        _, ext = os.path.splitext(self.remote_url)
+        parsed_url = urlparse(self.remote_url)
+        if ext:
+            # We only want the file_path if it has an extension
+            self.file_path = Path(self.remote_url).name
+            self.dir_path = Path(parsed_url.path).parent.as_posix().lstrip("/")
+            self.path_without_protocol = self.dir_path
+        else:
+            self.file_path = ""
+            self.dir_path = parsed_url.path.lstrip("/")
+            self.path_without_protocol = self.dir_path
+        self.access_config.host = parsed_url.hostname or self.access_config.host
+        self.access_config.port = parsed_url.port or self.access_config.port
+
+
+@dataclass
+class SftpIngestDoc(FsspecIngestDoc):
+    connector_config: SimpleSftpConfig
+    registry_name: str = "sftp"
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def get_file(self):
+        super().get_file()
+
+
+@dataclass
+class SftpSourceConnector(FsspecSourceConnector):
+    connector_config: SimpleSftpConfig
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def initialize(self):
+        super().initialize()
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def check_connection(self):
+        from fsspec.implementations.sftp import SFTPFileSystem
+
+        try:
+            SFTPFileSystem(**self.connector_config.get_access_config())
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def __post_init__(self):
+        self.ingest_doc_cls: Type[SftpIngestDoc] = SftpIngestDoc

unstructured_ingest/connector/git.py

@@ -0,0 +1,124 @@
+import fnmatch
+import typing as t
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    BaseConnectorConfig,
+    BaseSingleIngestDoc,
+    BaseSourceConnector,
+    IngestDocCleanupMixin,
+    SourceConnectorCleanupMixin,
+)
+from unstructured_ingest.logger import logger
+
+
+@dataclass
+class GitAccessConfig(AccessConfig):
+    access_token: t.Optional[str] = enhanced_field(
+        default=None, sensitive=True, overload_name="git_access_token"
+    )
+
+
+@dataclass
+class SimpleGitConfig(BaseConnectorConfig):
+    url: str
+    access_config: GitAccessConfig
+    branch: t.Optional[str] = enhanced_field(default=None, overload_name="git_branch")
+    file_glob: t.Optional[t.List[str]] = enhanced_field(default=None, overload_name="git_file_glob")
+    repo_path: str = field(init=False, repr=False)
+
+
+@dataclass
+class GitIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
+    connector_config: SimpleGitConfig = field(repr=False)
+    path: str
+
+    @property
+    def filename(self):
+        return (Path(self.read_config.download_dir) / self.path).resolve()
+
+    @property
+    def _output_filename(self):
+        return Path(self.processor_config.output_dir) / f"{self.path}.json"
+
+    @property
+    def record_locator(self) -> t.Dict[str, t.Any]:
+        record_locator = {
+            "repo_path": self.connector_config.repo_path,
+            "file_path": self.path,
+        }
+        if self.connector_config.branch is not None:
+            record_locator["branch"] = self.connector_config.branch
+        return record_locator
+
+    def _create_full_tmp_dir_path(self):
+        """includes directories in in the gitlab repository"""
+        self.filename.parent.mkdir(parents=True, exist_ok=True)
+
+    def update_source_metadata(self, **kwargs):
+        raise NotImplementedError()
+
+    @SourceConnectionError.wrap
+    @BaseSingleIngestDoc.skip_if_file_exists
+    def get_file(self):
+        """Fetches the "remote" doc and stores it locally on the filesystem."""
+        self._create_full_tmp_dir_path()
+        self._fetch_and_write()
+
+    def _fetch_content(self) -> None:
+        raise NotImplementedError()
+
+    def _fetch_and_write(self) -> None:
+        raise NotImplementedError()
+
+
+@dataclass
+class GitSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
+    connector_config: SimpleGitConfig
+
+    def initialize(self):
+        pass
+
+    def check_connection(self):
+        pass
+
+    @staticmethod
+    def is_file_type_supported(path: str) -> bool:
+        # Workaround to ensure that auto.partition isn't fed with .yaml, .py, etc. files
+        # TODO: What to do with no filenames? e.g. LICENSE, Makefile, etc.
+        supported = path.endswith(
+            (
+                ".md",
+                ".txt",
+                ".pdf",
+                ".doc",
+                ".docx",
+                ".eml",
+                ".heic",
+                ".html",
+                ".png",
+                ".jpg",
+                ".ppt",
+                ".pptx",
+                ".xml",
+            ),
+        )
+        if not supported:
+            logger.debug(
+                f"The file {path!r} is discarded as it does not contain a supported filetype.",
+            )
+        return supported
+
+    def does_path_match_glob(self, path: str) -> bool:
+        if not self.connector_config.file_glob:
+            return True
+        patterns = self.connector_config.file_glob
+        for pattern in patterns:
+            if fnmatch.filter([path], pattern):
+                return True
+        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
+        return False

unstructured_ingest/connector/github.py

@@ -0,0 +1,173 @@
+import typing as t
+from dataclasses import dataclass
+from datetime import datetime
+from urllib.parse import urlparse
+
+import requests
+
+from unstructured_ingest.connector.git import (
+    GitIngestDoc,
+    GitSourceConnector,
+    SimpleGitConfig,
+)
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
+from unstructured_ingest.interfaces import SourceMetadata
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if t.TYPE_CHECKING:
+    from github.Repository import Repository
+
+
+@dataclass
+class SimpleGitHubConfig(SimpleGitConfig):
+    def __post_init__(self):
+        parsed_gh_url = urlparse(self.url)
+        path_fragments = [fragment for fragment in parsed_gh_url.path.split("/") if fragment]
+
+        # If a scheme and netloc are provided, ensure they are correct
+        # Additionally, ensure that the path contains two fragments
+        if (
+            (parsed_gh_url.scheme and parsed_gh_url.scheme != "https")
+            or (parsed_gh_url.netloc and parsed_gh_url.netloc != "github.com")
+            or len(path_fragments) != 2
+        ):
+            raise ValueError(
+                'Please provide a valid URL, e.g. "https://github.com/Unstructured-IO/unstructured"'
+                ' or a repository owner/name pair, e.g. "Unstructured-IO/unstructured".',
+            )
+
+        # If there's no issues, store the core repository info
+        self.repo_path = parsed_gh_url.path
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["github"], extras="github")
+    def get_repo(self) -> "Repository":
+        from github import Github
+
+        github = Github(self.access_config.access_token)
+        return github.get_repo(self.repo_path)
+
+
+@dataclass
+class GitHubIngestDoc(GitIngestDoc):
+    connector_config: SimpleGitHubConfig
+    registry_name: str = "github"
+
+    @property
+    def date_created(self) -> t.Optional[str]:
+        return None
+
+    @requires_dependencies(["github"], extras="github")
+    def _fetch_file(self):
+        from github.GithubException import UnknownObjectException
+
+        try:
+            content_file = self.connector_config.get_repo().get_contents(self.path)
+        except UnknownObjectException:
+            logger.error(f"File doesn't exists {self.connector_config.url}/{self.path}")
+            return None
+
+        return content_file
+
+    @SourceConnectionNetworkError.wrap
+    def _fetch_content(self, content_file):
+        contents = b""
+        if (
+            not content_file.content  # type: ignore
+            and content_file.encoding == "none"  # type: ignore
+            and content_file.size  # type: ignore
+        ):
+            logger.info("File too large for the GitHub API, using direct download link instead.")
+            # NOTE: Maybe add a raise_for_status to catch connection timeout or HTTP Errors?
+            response = requests.get(content_file.download_url)  # type: ignore
+            if response.status_code != 200:
+                logger.info("Direct download link has failed... Skipping this file.")
+                return None
+            else:
+                contents = response.content
+        else:
+            contents = content_file.decoded_content  # type: ignore
+        return contents
+
+    def update_source_metadata(self, **kwargs):
+        content_file = kwargs.get("content_file", self._fetch_file())
+        if content_file is None:
+            self.source_metadata = SourceMetadata(
+                exists=False,
+            )
+            return
+
+        date_modified = datetime.strptime(
+            content_file.last_modified,
+            "%a, %d %b %Y %H:%M:%S %Z",
+        ).isoformat()
+        self.source_metadata = SourceMetadata(
+            date_modified=date_modified,
+            version=content_file.etag,
+            source_url=content_file.download_url,
+            exists=True,
+        )
+
+    def _fetch_and_write(self) -> None:
+        content_file = self._fetch_file()
+        self.update_source_metadata(content_file=content_file)
+        contents = self._fetch_content(content_file)
+        if contents is None:
+            raise ValueError(
+                f"Failed to retrieve file from repo "
+                f"{self.connector_config.url}/{self.path}. Check logs",
+            )
+        with open(self.filename, "wb") as f:
+            f.write(contents)
+
+
+@dataclass
+class GitHubSourceConnector(GitSourceConnector):
+    connector_config: SimpleGitHubConfig
+
+    @requires_dependencies(["github"], extras="github")
+    def check_connection(self):
+        from github import Consts
+        from github.GithubRetry import GithubRetry
+        from github.Requester import Requester
+
+        try:
+            requester = Requester(
+                auth=self.connector_config.access_config.access_token,
+                base_url=Consts.DEFAULT_BASE_URL,
+                timeout=Consts.DEFAULT_TIMEOUT,
+                user_agent=Consts.DEFAULT_USER_AGENT,
+                per_page=Consts.DEFAULT_PER_PAGE,
+                verify=True,
+                retry=GithubRetry(),
+                pool_size=None,
+            )
+            url_base = (
+                "/repositories/" if isinstance(self.connector_config.repo_path, int) else "/repos/"
+            )
+            url = f"{url_base}{self.connector_config.repo_path}"
+            headers, _ = requester.requestJsonAndCheck("HEAD", url)
+            logger.debug(f"headers from HEAD request: {headers}")
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def get_ingest_docs(self):
+        repo = self.connector_config.get_repo()
+        # Load the Git tree with all files, and then create Ingest docs
+        # for all blobs, i.e. all files, ignoring directories
+        sha = self.connector_config.branch or repo.default_branch
+        git_tree = repo.get_git_tree(sha, recursive=True)
+        return [
+            GitHubIngestDoc(
+                connector_config=self.connector_config,
+                processor_config=self.processor_config,
+                read_config=self.read_config,
+                path=element.path,
+            )
+            for element in git_tree.tree
+            if element.type == "blob"
+            and self.is_file_type_supported(element.path)
+            and (not self.connector_config.file_glob or self.does_path_match_glob(element.path))
+        ]

unstructured_ingest/connector/gitlab.py

@@ -0,0 +1,142 @@
+import typing as t
+from dataclasses import dataclass
+from urllib.parse import urlparse
+
+from unstructured_ingest.connector.git import (
+    GitIngestDoc,
+    GitSourceConnector,
+    SimpleGitConfig,
+)
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
+from unstructured_ingest.interfaces import SourceMetadata
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if t.TYPE_CHECKING:
+    from gitlab.v4.objects.projects import Project
+
+
+@dataclass
+class SimpleGitlabConfig(SimpleGitConfig):
+    base_url: str = "https://gitlab.com"
+
+    def __post_init__(self):
+        parsed_gh_url = urlparse(self.url)
+        # If a scheme or netloc are provided, use the parsed base url
+        if parsed_gh_url.scheme or parsed_gh_url.netloc:
+            self.base_url = f"{parsed_gh_url.scheme}://{parsed_gh_url.netloc}"
+        self.repo_path = parsed_gh_url.path
+        while self.repo_path.startswith("/"):
+            self.repo_path = self.repo_path[1:]
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["gitlab"], extras="gitlab")
+    def get_project(self) -> "Project":
+        from gitlab import Gitlab
+
+        gitlab = Gitlab(self.base_url, private_token=self.access_config.access_token)
+        return gitlab.projects.get(self.repo_path)
+
+
+@dataclass
+class GitLabIngestDoc(GitIngestDoc):
+    connector_config: SimpleGitlabConfig
+    registry_name: str = "gitlab"
+
+    @property
+    def date_created(self) -> t.Optional[str]:
+        return None
+
+    @property
+    def date_modified(self) -> t.Optional[str]:
+        return None
+
+    @property
+    def source_url(self) -> t.Optional[str]:
+        return None
+
+    @SourceConnectionNetworkError.wrap
+    @requires_dependencies(["gitlab"], extras="gitlab")
+    def _fetch_content(self):
+        from gitlab.exceptions import GitlabHttpError
+
+        try:
+            project = self.connector_config.get_project()
+            content_file = project.files.get(
+                self.path,
+                ref=self.connector_config.branch or project.default_branch,
+            )
+        except GitlabHttpError as e:
+            if e.response_code == 404:
+                logger.error(f"File doesn't exists {self.connector_config.url}/{self.path}")
+                return None
+            raise
+        return content_file
+
+    def update_source_metadata(self, **kwargs):
+        content_file = kwargs.get("content_file", self._fetch_content())
+        if content_file is None:
+            self.source_metadata = SourceMetadata(
+                exists=None,
+            )
+            return
+        self.source_metadata = SourceMetadata(
+            version=content_file.attributes.get("last_commit_id", ""),
+            exists=True,
+        )
+
+    def _fetch_and_write(self) -> None:
+        content_file = self._fetch_content()
+        self.update_source_metadata(content_file=content_file)
+        if content_file is None:
+            raise ValueError(
+                f"Failed to retrieve file from repo "
+                f"{self.connector_config.url}/{self.path}. Check logs.",
+            )
+        contents = content_file.decode()
+        with open(self.filename, "wb") as f:
+            f.write(contents)
+
+
+@dataclass
+class GitLabSourceConnector(GitSourceConnector):
+    connector_config: SimpleGitlabConfig
+
+    @requires_dependencies(["gitlab"], extras="gitlab")
+    def check_connection(self):
+        from gitlab import Gitlab
+        from gitlab.exceptions import GitlabError
+
+        try:
+            gitlab = Gitlab(
+                self.connector_config.base_url,
+                private_token=self.connector_config.access_config.access_token,
+            )
+            gitlab.auth()
+        except GitlabError as gitlab_error:
+            logger.error(f"failed to validate connection: {gitlab_error}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {gitlab_error}")
+
+    def get_ingest_docs(self):
+        # Load the Git tree with all files, and then create Ingest docs
+        # for all blobs, i.e. all files, ignoring directories
+        project = self.connector_config.get_project()
+        ref = self.connector_config.branch or project.default_branch
+        git_tree = project.repository_tree(
+            ref=ref,
+            recursive=True,
+            iterator=True,
+            all=True,
+        )
+        return [
+            GitLabIngestDoc(
+                connector_config=self.connector_config,
+                processor_config=self.processor_config,
+                read_config=self.read_config,
+                path=element["path"],
+            )
+            for element in git_tree
+            if element["type"] == "blob"
+            and self.is_file_type_supported(element["path"])
+            and (not self.connector_config.file_glob or self.does_path_match_glob(element["path"]))
+        ]