unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/connector/reddit.py

@@ -1,166 +0,0 @@
-import typing as t
-from dataclasses import dataclass, field
-from datetime import datetime
-from pathlib import Path
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    IngestDocCleanupMixin,
-    SourceConnectorCleanupMixin,
-    SourceMetadata,
-)
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-if t.TYPE_CHECKING:
-    from praw import Reddit
-
-
-@dataclass
-class RedditAccessConfig(AccessConfig):
-    client_secret: t.Optional[str] = enhanced_field(default=None, sensitive=True)
-
-
-@dataclass
-class SimpleRedditConfig(BaseConnectorConfig):
-    access_config: RedditAccessConfig
-    subreddit_name: str
-    num_posts: int
-    user_agent: str
-    client_id: str
-    search_query: t.Optional[str] = None
-
-    def __post_init__(self):
-        if self.num_posts <= 0:
-            raise ValueError("The number of Reddit posts to fetch must be positive.")
-
-
-@dataclass
-class RedditIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
-    connector_config: SimpleRedditConfig = field(repr=False)
-    post_id: str
-    registry_name: str = "reddit"
-
-    def _create_full_tmp_dir_path(self):
-        self.filename.parent.mkdir(parents=True, exist_ok=True)
-
-    @SourceConnectionNetworkError.wrap
-    @requires_dependencies(["praw"])
-    def get_post(self):
-        from praw import Reddit
-        from praw.models import Submission
-
-        reddit = Reddit(
-            client_id=self.connector_config.client_id,
-            client_secret=self.connector_config.access_config.client_secret,
-            user_agent=self.connector_config.user_agent,
-        )
-        post = Submission(reddit, self.post_id)
-        return post
-
-    def update_source_metadata(self, **kwargs):
-        post = kwargs.get("post", self.get_post())
-        if post is None:
-            self.source_metadata = SourceMetadata(
-                exists=False,
-            )
-            return
-
-        file_exists = (post.author != "[deleted]" or post.auth is not None) and (
-            post.selftext != "[deleted]" or post.selftext != "[removed]"
-        )
-
-        self.source_metadata = SourceMetadata(
-            date_created=datetime.utcfromtimestamp(post.created_utc).isoformat(),
-            source_url=post.permalink,
-            exists=file_exists,
-        )
-
-    @SourceConnectionError.wrap
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        """Fetches the "remote" doc and stores it locally on the filesystem."""
-        self._create_full_tmp_dir_path()
-        # Write the title plus the body, if any
-        post = self.get_post()
-        self.update_source_metadata(post=post)
-        if post is None:
-            raise ValueError(
-                f"Failed to retrieve post {self.post_id}",
-            )
-
-        text_to_write = f"# {post.title}\n{post.selftext}"
-        with open(self.filename, "w", encoding="utf8") as f:
-            f.write(text_to_write)
-
-    @property
-    def filename(self) -> Path:
-        return (Path(self.read_config.download_dir) / f"{self.post_id}.md").resolve()
-
-    @property
-    def _output_filename(self):
-        return Path(self.processor_config.output_dir) / f"{self.post_id}.json"
-
-    @property
-    def date_modified(self) -> t.Optional[str]:
-        return None
-
-    @property
-    def version(self) -> t.Optional[str]:
-        return None
-
-
-@dataclass
-class RedditSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    connector_config: SimpleRedditConfig
-    _reddit: t.Optional["Reddit"] = field(init=False, default=None)
-
-    @property
-    def reddit(self) -> "Reddit":
-        from praw import Reddit
-
-        if self._reddit is None:
-            self._reddit = Reddit(
-                client_id=self.connector_config.client_id,
-                client_secret=self.connector_config.access_config.client_secret,
-                user_agent=self.connector_config.user_agent,
-            )
-        return self._reddit
-
-    @requires_dependencies(["praw"], extras="reddit")
-    def initialize(self):
-        _ = self.reddit
-
-    def check_connection(self):
-        from praw.endpoints import API_PATH
-        from prawcore import ResponseException
-
-        try:
-            self.reddit._objectify_request(method="HEAD", params=None, path=API_PATH["me"])
-        except ResponseException as response_error:
-            logger.error(f"failed to validate connection: {response_error}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {response_error}")
-
-    def get_ingest_docs(self):
-        subreddit = self.reddit.subreddit(self.connector_config.subreddit_name)
-        if self.connector_config.search_query:
-            posts = subreddit.search(
-                self.connector_config.search_query,
-                limit=self.connector_config.num_posts,
-            )
-        else:
-            posts = subreddit.hot(limit=self.connector_config.num_posts)
-        return [
-            RedditIngestDoc(
-                connector_config=self.connector_config,
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                post_id=post.id,
-            )
-            for post in posts
-        ]

unstructured_ingest/connector/registry.py

@@ -1,109 +0,0 @@
-import json
-from typing import Dict, Type, cast
-
-from unstructured_ingest.connector.airtable import AirtableIngestDoc
-from unstructured_ingest.connector.astradb import AstraDBIngestDoc
-from unstructured_ingest.connector.biomed import BiomedIngestDoc
-from unstructured_ingest.connector.confluence import ConfluenceIngestDoc
-from unstructured_ingest.connector.delta_table import DeltaTableIngestDoc
-from unstructured_ingest.connector.discord import DiscordIngestDoc
-from unstructured_ingest.connector.elasticsearch import (
-    ElasticsearchIngestDoc,
-    ElasticsearchIngestDocBatch,
-)
-from unstructured_ingest.connector.fsspec.azure import AzureBlobStorageIngestDoc
-from unstructured_ingest.connector.fsspec.box import BoxIngestDoc
-from unstructured_ingest.connector.fsspec.dropbox import DropboxIngestDoc
-from unstructured_ingest.connector.fsspec.gcs import GcsIngestDoc
-from unstructured_ingest.connector.fsspec.s3 import S3IngestDoc
-from unstructured_ingest.connector.fsspec.sftp import SftpIngestDoc
-from unstructured_ingest.connector.github import GitHubIngestDoc
-from unstructured_ingest.connector.gitlab import GitLabIngestDoc
-from unstructured_ingest.connector.google_drive import GoogleDriveIngestDoc
-from unstructured_ingest.connector.hubspot import HubSpotIngestDoc
-from unstructured_ingest.connector.jira import JiraIngestDoc
-from unstructured_ingest.connector.kafka import KafkaIngestDoc
-from unstructured_ingest.connector.local import LocalIngestDoc
-from unstructured_ingest.connector.mongodb import MongoDBIngestDoc, MongoDBIngestDocBatch
-from unstructured_ingest.connector.notion.connector import (
-    NotionDatabaseIngestDoc,
-    NotionPageIngestDoc,
-)
-from unstructured_ingest.connector.onedrive import OneDriveIngestDoc
-from unstructured_ingest.connector.opensearch import OpenSearchIngestDoc, OpenSearchIngestDocBatch
-from unstructured_ingest.connector.outlook import OutlookIngestDoc
-from unstructured_ingest.connector.reddit import RedditIngestDoc
-from unstructured_ingest.connector.salesforce import SalesforceIngestDoc
-from unstructured_ingest.connector.sharepoint import SharepointIngestDoc
-from unstructured_ingest.connector.slack import SlackIngestDoc
-from unstructured_ingest.connector.wikipedia import (
-    WikipediaIngestHTMLDoc,
-    WikipediaIngestSummaryDoc,
-    WikipediaIngestTextDoc,
-)
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.interfaces import BaseIngestDoc
-
-INGEST_DOC_NAME_TO_CLASS: Dict[str, Type[EnhancedDataClassJsonMixin]] = {
-    "airtable": AirtableIngestDoc,
-    "astradb": AstraDBIngestDoc,
-    "azure": AzureBlobStorageIngestDoc,
-    "biomed": BiomedIngestDoc,
-    "box": BoxIngestDoc,
-    "confluence": ConfluenceIngestDoc,
-    "delta-table": DeltaTableIngestDoc,
-    "discord": DiscordIngestDoc,
-    "dropbox": DropboxIngestDoc,
-    "elasticsearch": ElasticsearchIngestDoc,
-    "elasticsearch_batch": ElasticsearchIngestDocBatch,
-    "gcs": GcsIngestDoc,
-    "github": GitHubIngestDoc,
-    "gitlab": GitLabIngestDoc,
-    "google_drive": GoogleDriveIngestDoc,
-    "hubspot": HubSpotIngestDoc,
-    "jira": JiraIngestDoc,
-    "kafka": KafkaIngestDoc,
-    "local": LocalIngestDoc,
-    "mongodb": MongoDBIngestDoc,
-    "mongodb_batch": MongoDBIngestDocBatch,
-    "notion_database": NotionDatabaseIngestDoc,
-    "notion_page": NotionPageIngestDoc,
-    "onedrive": OneDriveIngestDoc,
-    "opensearch": OpenSearchIngestDoc,
-    "opensearch_batch": OpenSearchIngestDocBatch,
-    "outlook": OutlookIngestDoc,
-    "reddit": RedditIngestDoc,
-    "s3": S3IngestDoc,
-    "salesforce": SalesforceIngestDoc,
-    "sftp": SftpIngestDoc,
-    "sharepoint": SharepointIngestDoc,
-    "slack": SlackIngestDoc,
-    "wikipedia_html": WikipediaIngestHTMLDoc,
-    "wikipedia_text": WikipediaIngestTextDoc,
-    "wikipedia_summary": WikipediaIngestSummaryDoc,
-}
-
-
-def create_ingest_doc_from_json(ingest_doc_json: str) -> BaseIngestDoc:
-    try:
-        ingest_doc_dict: dict = json.loads(ingest_doc_json)
-    except TypeError as te:
-        raise TypeError(
-            f"failed to load json string when deserializing IngestDoc: {ingest_doc_json}",
-        ) from te
-    return create_ingest_doc_from_dict(ingest_doc_dict)
-
-
-def create_ingest_doc_from_dict(ingest_doc_dict: dict) -> BaseIngestDoc:
-    ingest_doc_dict = ingest_doc_dict.copy()
-    if "registry_name" not in ingest_doc_dict:
-        raise ValueError(f"registry_name not present in ingest doc: {ingest_doc_dict}")
-    registry_name = ingest_doc_dict.pop("registry_name")
-    try:
-        ingest_doc_cls = INGEST_DOC_NAME_TO_CLASS[registry_name]
-        return cast(BaseIngestDoc, ingest_doc_cls.from_dict(ingest_doc_dict))
-    except KeyError:
-        raise ValueError(
-            f"Error: Received unknown IngestDoc name: {registry_name} while deserializing",
-            "IngestDoc.",
-        )

unstructured_ingest/connector/salesforce.py

@@ -1,301 +0,0 @@
-"""
-Salesforce Connector
-Able to download Account, Case, Campaign, EmailMessage, Lead
-Salesforce returns everything as a list of json.
-This saves each entry as a separate file to be partitioned.
-Using JWT authorization
-https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_auth_key_and_cert.htm
-https://developer.salesforce.com/docs/atlas.en-us.sfdx_dev.meta/sfdx_dev/sfdx_dev_auth_connected_app.htm
-"""
-
-import json
-import typing as t
-from collections import OrderedDict
-from dataclasses import dataclass, field
-from datetime import datetime
-from email.utils import formatdate
-from pathlib import Path
-from string import Template
-from textwrap import dedent
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    IngestDocCleanupMixin,
-    SourceConnectorCleanupMixin,
-    SourceMetadata,
-)
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-
-class MissingCategoryError(Exception):
-    """There are no categories with that name."""
-
-
-SALESFORCE_API_VERSION = "57.0"
-
-ACCEPTED_CATEGORIES = ["Account", "Case", "Campaign", "EmailMessage", "Lead"]
-
-EMAIL_TEMPLATE = Template(
-    """MIME-Version: 1.0
-Date: $date
-Message-ID: $message_identifier
-Subject: $subject
-From: $from_email
-To: $to_email
-Content-Type: multipart/alternative; boundary="00000000000095c9b205eff92630"
---00000000000095c9b205eff92630
-Content-Type: text/plain; charset="UTF-8"
-$textbody
---00000000000095c9b205eff92630
-Content-Type: text/html; charset="UTF-8"
-$htmlbody
---00000000000095c9b205eff92630--
-""",
-)
-
-
-@dataclass
-class SalesforceAccessConfig(AccessConfig):
-    consumer_key: str = enhanced_field(sensitive=True)
-    private_key: str = enhanced_field(sensitive=True)
-
-    @requires_dependencies(["cryptography"])
-    def get_private_key_value_and_type(self) -> t.Tuple[str, t.Type]:
-        from cryptography.hazmat.primitives import serialization
-
-        try:
-            serialization.load_pem_private_key(data=self.private_key.encode("utf-8"), password=None)
-        except ValueError:
-            pass
-        else:
-            return self.private_key, str
-
-        if Path(self.private_key).is_file():
-            return self.private_key, Path
-
-        raise ValueError("private_key does not contain PEM private key or path")
-
-
-@dataclass
-class SimpleSalesforceConfig(BaseConnectorConfig):
-    """Connector specific attributes"""
-
-    access_config: SalesforceAccessConfig
-    categories: t.List[str]
-    username: str
-    recursive: bool = False
-
-    @requires_dependencies(["simple_salesforce"], extras="salesforce")
-    def get_client(self):
-        from simple_salesforce import Salesforce
-
-        pkey_value, pkey_type = self.access_config.get_private_key_value_and_type()
-
-        return Salesforce(
-            username=self.username,
-            consumer_key=self.access_config.consumer_key,
-            privatekey_file=pkey_value if pkey_type is Path else None,
-            privatekey=pkey_value if pkey_type is str else None,
-            version=SALESFORCE_API_VERSION,
-        )
-
-
-@dataclass
-class SalesforceIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
-    connector_config: SimpleSalesforceConfig
-    record_type: str
-    record_id: str
-    registry_name: str = "salesforce"
-    _record: OrderedDict = field(default_factory=lambda: OrderedDict())
-
-    @property
-    def record(self):
-        if not self._record:
-            self._record = self.get_record()
-        return self._record
-
-    def get_file_extension(self) -> str:
-        if self.record_type == "EmailMessage":
-            extension = ".eml"
-        elif self.record_type in ["Account", "Lead", "Case", "Campaign"]:
-            extension = ".xml"
-        else:
-            raise MissingCategoryError(
-                f"There are no categories with the name: {self.record_type}",
-            )
-        return extension
-
-    def _tmp_download_file(self) -> Path:
-        record_file = self.record_id + self.get_file_extension()
-        return Path(self.read_config.download_dir) / self.record_type / record_file
-
-    @property
-    def _output_filename(self) -> Path:
-        record_file = self.record_id + self.get_file_extension() + ".json"
-        return Path(self.processor_config.output_dir) / self.record_type / record_file
-
-    def _create_full_tmp_dir_path(self):
-        self._tmp_download_file().parent.mkdir(parents=True, exist_ok=True)
-
-    def _xml_for_record(self, record: OrderedDict) -> str:
-        """Creates partitionable xml file from a record"""
-        import xml.etree.ElementTree as ET
-
-        def flatten_dict(data, parent, prefix=""):
-            for key, value in data.items():
-                if isinstance(value, OrderedDict):
-                    flatten_dict(value, parent, prefix=f"{prefix}{key}.")
-                else:
-                    item = ET.Element("item")
-                    item.text = f"{prefix}{key}: {value}"
-                    parent.append(item)
-
-        root = ET.Element("root")
-        flatten_dict(record, root)
-        xml_string = ET.tostring(root, encoding="utf-8", xml_declaration=True).decode()
-        return xml_string
-
-    def _eml_for_record(self, email_json: t.Dict[str, t.Any]) -> str:
-        from dateutil import parser  # type: ignore
-
-        """Recreates standard expected .eml format using template."""
-        eml = EMAIL_TEMPLATE.substitute(
-            date=formatdate(parser.parse(email_json.get("MessageDate")).timestamp()),
-            message_identifier=email_json.get("MessageIdentifier"),
-            subject=email_json.get("Subject"),
-            from_email=email_json.get("FromAddress"),
-            to_email=email_json.get("ToAddress"),
-            textbody=email_json.get("TextBody"),
-            # TODO: This is a hack to get emails to process correctly.
-            # The HTML partitioner seems to have issues with <br> and text without tags like <p>
-            htmlbody=email_json.get("HtmlBody", "")  # "" because you can't .replace None
-            .replace("<br />", "<p>")
-            .replace("<body", "<body><p"),
-        )
-        return dedent(eml)
-
-    @SourceConnectionNetworkError.wrap
-    def _get_response(self):
-        client = self.connector_config.get_client()
-        return client.query_all(
-            f"select FIELDS(STANDARD) from {self.record_type} where Id='{self.record_id}'",
-        )
-
-    def get_record(self) -> OrderedDict:
-        # Get record from Salesforce based on id
-        response = self._get_response()
-        logger.debug(f"response was returned for salesforce record id: {self.record_id}")
-        records = response["records"]
-        if not records:
-            raise ValueError(
-                f"No record found with record id {self.record_id}: {json.dumps(response)}"
-            )
-        record_json = records[0]
-        return record_json
-
-    def update_source_metadata(self) -> None:  # type: ignore
-        record_json = self.record
-
-        date_format = "%Y-%m-%dT%H:%M:%S.000+0000"
-        self.source_metadata = SourceMetadata(
-            date_created=datetime.strptime(record_json["CreatedDate"], date_format).isoformat(),
-            date_modified=datetime.strptime(
-                record_json["LastModifiedDate"],
-                date_format,
-            ).isoformat(),
-            # SystemModstamp is Timestamp if record has been modified by person or automated system
-            version=record_json.get("SystemModstamp"),
-            source_url=record_json["attributes"].get("url"),
-            exists=True,
-        )
-
-    @SourceConnectionError.wrap
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        """Saves individual json records locally."""
-        self._create_full_tmp_dir_path()
-        record = self.record
-
-        self.update_source_metadata()
-
-        try:
-            if self.record_type == "EmailMessage":
-                document = self._eml_for_record(record)
-            else:
-                document = self._xml_for_record(record)
-
-            with open(self._tmp_download_file(), "w") as page_file:
-                page_file.write(document)
-
-        except Exception as e:
-            logger.error(
-                f"Error while downloading and saving file: {self.record_id}.",
-            )
-            logger.error(e)
-
-    @property
-    def filename(self):
-        """The filename of the file created from a Salesforce record"""
-        return self._tmp_download_file()
-
-
-@dataclass
-class SalesforceSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    connector_config: SimpleSalesforceConfig
-
-    def __post_init__(self):
-        self.ingest_doc_cls: t.Type[SalesforceIngestDoc] = SalesforceIngestDoc
-
-    def initialize(self):
-        pass
-
-    @requires_dependencies(["simple_salesforce"], extras="salesforce")
-    def check_connection(self):
-        from simple_salesforce.exceptions import SalesforceError
-
-        try:
-            self.connector_config.get_client()
-        except SalesforceError as salesforce_error:
-            logger.error(f"failed to validate connection: {salesforce_error}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {salesforce_error}")
-
-    @requires_dependencies(["simple_salesforce"], extras="salesforce")
-    def get_ingest_docs(self) -> t.List[SalesforceIngestDoc]:
-        """Get Salesforce Ids for the records.
-        Send them to next phase where each doc gets downloaded into the
-        appropriate format for partitioning.
-        """
-        from simple_salesforce.exceptions import SalesforceMalformedRequest
-
-        client = self.connector_config.get_client()
-
-        ingest_docs = []
-        for record_type in self.connector_config.categories:
-            if record_type not in ACCEPTED_CATEGORIES:
-                raise ValueError(f"{record_type} not currently an accepted Salesforce category")
-
-            try:
-                # Get ids from Salesforce
-                records = client.query_all(
-                    f"select Id from {record_type}",
-                )
-                for record in records["records"]:
-                    ingest_docs.append(
-                        SalesforceIngestDoc(
-                            connector_config=self.connector_config,
-                            processor_config=self.processor_config,
-                            read_config=self.read_config,
-                            record_type=record_type,
-                            record_id=record["Id"],
-                        ),
-                    )
-            except SalesforceMalformedRequest as e:
-                raise SalesforceMalformedRequest(f"Problem with Salesforce query: {e}")
-
-        return ingest_docs