unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/connector/notion/types/date.py

@@ -1,26 +0,0 @@
-# https://developers.notion.com/reference/property-value-object#date-property-values
-from dataclasses import dataclass
-from typing import Optional
-
-from htmlBuilder.tags import Div, HtmlTag
-
-from unstructured_ingest.connector.notion.interfaces import FromJSONMixin, GetHTMLMixin
-
-
-@dataclass
-class Date(FromJSONMixin, GetHTMLMixin):
-    start: str
-    end: Optional[str] = None
-    time_zone: Optional[str] = None
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-    def get_html(self) -> Optional[HtmlTag]:
-        text = f"{self.start}"
-        if end := self.end:
-            text += f" - {end}"
-        if self.time_zone:
-            text += f" {self.time_zone}"
-        return Div([], text)

unstructured_ingest/connector/notion/types/file.py

@@ -1,51 +0,0 @@
-# https://developers.notion.com/reference/file-object
-from dataclasses import dataclass
-from typing import Optional
-
-from htmlBuilder.attributes import Href
-from htmlBuilder.tags import A, HtmlTag
-
-from unstructured_ingest.connector.notion.interfaces import FromJSONMixin, GetHTMLMixin
-
-
-@dataclass
-class External(FromJSONMixin):
-    url: str
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-
-@dataclass
-class File(FromJSONMixin):
-    url: str
-    expiry_time: str
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-
-@dataclass
-class FileObject(FromJSONMixin, GetHTMLMixin):
-    type: str
-    external: Optional[External] = None
-    file: Optional[File] = None
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        t = data["type"]
-        file_object = cls(type=t)
-        if t == "external":
-            file_object.external = External.from_dict(data["external"])
-        elif t == "file":
-            file_object.file = File.from_dict(data["file"])
-        return file_object
-
-    def get_html(self) -> Optional[HtmlTag]:
-        if self.file:
-            return A([Href(self.file.url)], self.file.url)
-        if self.external:
-            return A([Href(self.external.url)], self.external.url)
-        return None

unstructured_ingest/connector/notion/types/user.py

@@ -1,76 +0,0 @@
-# https://developers.notion.com/reference/user
-from dataclasses import dataclass, field
-from typing import Optional
-
-from htmlBuilder.attributes import Href
-from htmlBuilder.tags import A, Div, HtmlTag
-
-from unstructured_ingest.connector.notion.interfaces import FromJSONMixin, GetHTMLMixin
-
-
-@dataclass
-class PartialUser(FromJSONMixin):
-    id: str
-    object: str = "user"
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(id=data["id"])
-
-
-@dataclass
-class User(FromJSONMixin, GetHTMLMixin):
-    object: dict
-    id: str
-    type: Optional[str] = None
-    name: Optional[str] = None
-    avatar_url: Optional[str] = None
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-    def get_text(self) -> Optional[str]:
-        text = self.name
-        if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
-        return text
-
-    def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
-            return A([Href(self.avatar_url)], self.name)
-        else:
-            return Div([], self.name)
-
-
-@dataclass
-class People(User):
-    person: dict = field(default_factory=dict)
-
-
-@dataclass
-class Bots(FromJSONMixin, GetHTMLMixin):
-    object: dict
-    id: str
-    bot: dict
-    owner: dict
-    type: str
-    workspace_name: str
-    name: Optional[str] = None
-    avatar_url: Optional[str] = None
-
-    @classmethod
-    def from_dict(cls, data: dict):
-        return cls(**data)
-
-    def get_text(self) -> Optional[str]:
-        text = self.name
-        if self.avatar_url:
-            text = f"[{text}]({self.avatar_url}"
-        return text
-
-    def get_html(self) -> Optional[HtmlTag]:
-        if self.avatar_url:
-            return A([Href(self.avatar_url)], self.name)
-        else:
-            return Div([], self.name)

unstructured_ingest/connector/onedrive.py

@@ -1,232 +0,0 @@
-import typing as t
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    IngestDocCleanupMixin,
-    SourceConnectorCleanupMixin,
-    SourceMetadata,
-)
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.string_and_date_utils import ensure_isoformat_datetime
-
-if t.TYPE_CHECKING:
-    from office365.graph_client import GraphClient
-    from office365.onedrive.driveitems.driveItem import DriveItem
-MAX_MB_SIZE = 512_000_000
-
-
-@dataclass
-class OneDriveAccessConfig(AccessConfig):
-    client_credential: str = enhanced_field(repr=False, sensitive=True, overload_name="client_cred")
-
-
-@dataclass
-class SimpleOneDriveConfig(BaseConnectorConfig):
-    access_config: OneDriveAccessConfig
-    client_id: str
-    user_pname: str
-    tenant: str = field(repr=False)
-    authority_url: t.Optional[str] = field(repr=False, default="https://login.microsoftonline.com")
-    path: t.Optional[str] = field(default="")
-    recursive: bool = False
-
-    def __post_init__(self):
-        if not (self.client_id and self.access_config.client_credential and self.user_pname):
-            raise ValueError(
-                "Please provide all the following mandatory values:"
-                "\n-ms-client_id\n-ms-client_cred\n-ms-user-pname",
-            )
-        self.token_factory = self._acquire_token
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["msal"])
-    def _acquire_token(self):
-        from msal import ConfidentialClientApplication
-
-        try:
-            app = ConfidentialClientApplication(
-                authority=f"{self.authority_url}/{self.tenant}",
-                client_id=self.client_id,
-                client_credential=self.access_config.client_credential,
-            )
-            token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
-        except ValueError as exc:
-            logger.error("Couldn't set up credentials for OneDrive")
-            raise exc
-        return token
-
-
-@dataclass
-class OneDriveIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
-    connector_config: SimpleOneDriveConfig
-    file_name: str
-    file_path: str
-    registry_name: str = "onedrive"
-
-    def __post_init__(self):
-        self.ext = Path(self.file_name).suffix
-        if not self.ext:
-            raise ValueError("Unsupported file without extension.")
-
-        self.server_relative_path = self.file_path + "/" + self.file_name
-        self._set_download_paths()
-
-    def _set_download_paths(self) -> None:
-        """Parses the folder structure from the source and creates the download and output paths"""
-        download_path = Path(f"{self.read_config.download_dir}")
-        output_path = Path(f"{self.processor_config.output_dir}")
-
-        if parent_path := self.file_path:
-            download_path = (
-                download_path if parent_path == "" else (download_path / parent_path).resolve()
-            )
-            output_path = (
-                output_path if parent_path == "" else (output_path / parent_path).resolve()
-            )
-
-        self.download_dir = download_path
-        self.download_filepath = (download_path / self.file_name).resolve()
-        output_filename = output_filename = self.file_name + ".json"
-        self.output_dir = output_path
-        self.output_filepath = (output_path / output_filename).resolve()
-
-    @property
-    def filename(self):
-        return Path(self.download_filepath).resolve()
-
-    @property
-    def _output_filename(self):
-        return Path(self.output_filepath).resolve()
-
-    @property
-    def record_locator(self) -> t.Optional[t.Dict[str, t.Any]]:
-        return {
-            "user_pname": self.connector_config.user_pname,
-            "server_relative_path": self.server_relative_path,
-        }
-
-    @SourceConnectionNetworkError.wrap
-    @requires_dependencies(["office365"], extras="onedrive")
-    def _fetch_file(self):
-        from office365.graph_client import GraphClient
-
-        client = GraphClient(self.connector_config.token_factory)
-        root = client.users[self.connector_config.user_pname].drive.get().execute_query().root
-        file = root.get_by_path(self.server_relative_path).get().execute_query()
-        return file
-
-    def update_source_metadata(self, **kwargs):
-        file = kwargs.get("file", self._fetch_file())
-        if file is None:
-            self.source_metadata = SourceMetadata(
-                exists=False,
-            )
-            return
-
-        version = None
-        if (n_versions := len(file.versions)) > 0:
-            version = file.versions[n_versions - 1].properties.get("id", None)
-
-        self.source_metadata = SourceMetadata(
-            date_created=ensure_isoformat_datetime(timestamp=file.created_datetime),
-            date_modified=ensure_isoformat_datetime(timestamp=file.last_modified_datetime),
-            version=version,
-            source_url=file.parent_reference.path + "/" + self.file_name,
-            exists=True,
-        )
-
-    @SourceConnectionError.wrap
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        file = self._fetch_file()
-        self.update_source_metadata(file=file)
-        if file is None:
-            raise ValueError(
-                f"Failed to retrieve file {self.file_path}/{self.file_name}",
-            )
-
-        fsize = file.get_property("size", 0)
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-        if not self.download_dir.is_dir():
-            logger.debug(f"creating directory: {self.download_dir}")
-            self.download_dir.mkdir(parents=True, exist_ok=True)
-
-        if fsize > MAX_MB_SIZE:
-            logger.info(f"downloading file with size: {fsize} bytes in chunks")
-            with self.filename.open(mode="wb") as f:
-                file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
-        else:
-            with self.filename.open(mode="wb") as f:
-                file.download(f).execute_query()
-        logger.info(f"file downloaded: {self.filename}")
-        return
-
-
-@dataclass
-class OneDriveSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    connector_config: SimpleOneDriveConfig
-    _client: t.Optional["GraphClient"] = field(init=False, default=None)
-
-    @property
-    def client(self) -> "GraphClient":
-        from office365.graph_client import GraphClient
-
-        if self._client is None:
-            self._client = GraphClient(self.connector_config.token_factory)
-        return self._client
-
-    @requires_dependencies(["office365"], extras="onedrive")
-    def initialize(self):
-        _ = self.client
-
-    @requires_dependencies(["office365"], extras="onedrive")
-    def check_connection(self):
-        try:
-            token_resp: dict = self.connector_config.token_factory()
-            if error := token_resp.get("error"):
-                raise SourceConnectionError(
-                    "{} ({})".format(error, token_resp.get("error_description"))
-                )
-            _ = self.client
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    def _list_objects(self, folder, recursive) -> t.List["DriveItem"]:
-        drive_items = folder.children.get().execute_query()
-        files = [d for d in drive_items if d.is_file]
-        if not recursive:
-            return files
-        folders = [d for d in drive_items if d.is_folder]
-        for f in folders:
-            files += self._list_objects(f, recursive)
-        return files
-
-    def _gen_ingest_doc(self, file: "DriveItem") -> OneDriveIngestDoc:
-        file_path = file.parent_reference.path.split(":")[-1]
-        file_path = file_path[1:] if file_path[0] == "/" else file_path
-        return OneDriveIngestDoc(
-            connector_config=self.connector_config,
-            processor_config=self.processor_config,
-            read_config=self.read_config,
-            file_name=file.name,
-            file_path=file_path,
-        )
-
-    def get_ingest_docs(self):
-        root = self.client.users[self.connector_config.user_pname].drive.get().execute_query().root
-        if fpath := self.connector_config.path:
-            root = root.get_by_path(fpath).get().execute_query()
-            if root is None or not root.is_folder:
-                raise ValueError(f"Unable to find directory, given: {fpath}")
-        files = self._list_objects(root, self.connector_config.recursive)
-        return [self._gen_ingest_doc(f) for f in files]

unstructured_ingest/connector/opensearch.py

@@ -1,218 +0,0 @@
-import typing as t
-from dataclasses import dataclass, field
-
-from dataclasses_json.core import Json
-
-from unstructured_ingest.connector.elasticsearch import (
-    ElasticsearchDestinationConnector,
-    ElasticsearchDocumentMeta,
-    ElasticsearchIngestDoc,
-    ElasticsearchIngestDocBatch,
-    ElasticsearchSourceConnector,
-    SimpleElasticsearchConfig,
-)
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.interfaces import AccessConfig, BaseSingleIngestDoc
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.data_prep import flatten_dict, generator_batching_wbytes
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-if t.TYPE_CHECKING:
-    from opensearchpy import OpenSearch
-
-"""Since the actual OpenSearch project is a fork of Elasticsearch, we are relying
-heavily on the Elasticsearch connector code, inheriting the functionality as much as possible."""
-
-
-@dataclass
-class OpenSearchAccessConfig(AccessConfig):
-    hosts: t.Optional[t.List[str]] = None
-    username: t.Optional[str] = None
-    password: t.Optional[str] = enhanced_field(default=None, sensitive=True)
-    use_ssl: bool = False
-    verify_certs: bool = False
-    ssl_show_warn: bool = False
-    ca_certs: t.Optional[str] = None
-    client_cert: t.Optional[str] = None
-    client_key: t.Optional[str] = None
-
-    def to_dict(self, **kwargs) -> t.Dict[str, Json]:
-        d = super().to_dict(**kwargs)
-        d["http_auth"] = (self.username, self.password)
-        return d
-
-
-@dataclass
-class SimpleOpenSearchConfig(SimpleElasticsearchConfig):
-    access_config: OpenSearchAccessConfig = None
-
-
-@dataclass
-class OpenSearchIngestDoc(ElasticsearchIngestDoc):
-    """Class encapsulating fetching a doc and writing processed results (but not
-    doing the processing!).
-
-    Current implementation creates a python OpenSearch client to fetch each doc,
-    rather than creating a client for each thread.
-    """
-
-    connector_config: SimpleOpenSearchConfig
-    registry_name: str = "opensearch"
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        pass
-
-
-@dataclass
-class OpenSearchIngestDocBatch(ElasticsearchIngestDocBatch):
-    connector_config: SimpleOpenSearchConfig
-    ingest_docs: t.List[OpenSearchIngestDoc] = field(default_factory=list)
-    registry_name: str = "opensearch_batch"
-
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    def _get_docs(self):
-        from opensearchpy import OpenSearch
-        from opensearchpy.helpers import scan
-
-        ops = OpenSearch(**self.connector_config.access_config.to_dict(apply_name_overload=False))
-        scan_query = {
-            "_source": self.connector_config.fields,
-            "version": True,
-            "query": {"ids": {"values": self.list_of_ids}},
-        }
-
-        result = scan(
-            ops,
-            query=scan_query,
-            scroll="1m",
-            index=self.connector_config.index_name,
-        )
-        return list(result)
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    def get_files(self):
-        documents = self._get_docs()
-        for doc in documents:
-            ingest_doc = OpenSearchIngestDoc(
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                connector_config=self.connector_config,
-                document=doc,
-                document_meta=ElasticsearchDocumentMeta(
-                    self.connector_config.index_name, doc["_id"]
-                ),
-            )
-            ingest_doc.update_source_metadata()
-            doc_body = doc["_source"]
-            filename = ingest_doc.filename
-            flattened_dict = flatten_dict(dictionary=doc_body)
-            str_values = [str(value) for value in flattened_dict.values()]
-            concatenated_values = "\n".join(str_values)
-
-            filename.parent.mkdir(parents=True, exist_ok=True)
-            with open(filename, "w", encoding="utf8") as f:
-                f.write(concatenated_values)
-            self.ingest_docs.append(ingest_doc)
-
-
-@dataclass
-class OpenSearchSourceConnector(ElasticsearchSourceConnector):
-    """Fetches particular fields from all documents in a given opensearch cluster and index"""
-
-    connector_config: SimpleOpenSearchConfig
-    _ops: t.Optional["OpenSearch"] = field(init=False, default=None)
-
-    @property
-    def ops(self):
-        from opensearchpy import OpenSearch
-
-        if self._ops is None:
-            self._ops = OpenSearch(
-                **self.connector_config.access_config.to_dict(apply_name_overload=False)
-            )
-        return self._ops
-
-    def check_connection(self):
-        try:
-            assert self.ops.ping()
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    def _get_doc_ids(self):
-        """Fetches all document ids in an index"""
-        from opensearchpy.helpers import scan
-
-        hits = scan(
-            self.ops,
-            query=self.scan_query,
-            scroll="1m",
-            index=self.connector_config.index_name,
-        )
-
-        return [hit["_id"] for hit in hits]
-
-    def get_ingest_docs(self):
-        """Fetches all documents in an index, using ids that are fetched with _get_doc_ids"""
-        ids = self._get_doc_ids()
-        id_batches = [
-            ids[
-                i
-                * self.connector_config.batch_size : (i + 1)  # noqa
-                * self.connector_config.batch_size
-            ]
-            for i in range(
-                (len(ids) + self.connector_config.batch_size - 1)
-                // self.connector_config.batch_size
-            )
-        ]
-        return [
-            OpenSearchIngestDocBatch(
-                connector_config=self.connector_config,
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                list_of_ids=batched_ids,
-            )
-            for batched_ids in id_batches
-        ]
-
-
-@dataclass
-class OpenSearchDestinationConnector(ElasticsearchDestinationConnector):
-    connector_config: SimpleOpenSearchConfig
-    _client: t.Optional["OpenSearch"] = field(init=False, default=None)
-
-    @DestinationConnectionError.wrap
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    def generate_client(self) -> "OpenSearch":
-        from opensearchpy import OpenSearch
-
-        return OpenSearch(**self.connector_config.access_config.to_dict(apply_name_overload=False))
-
-    @requires_dependencies(["opensearchpy"], extras="opensearch")
-    def write_dict(self, *args, elements_dict: t.List[t.Dict[str, t.Any]]) -> None:
-        logger.info(
-            f"writing document batches to destination"
-            f" index named {self.connector_config.index_name}"
-            f" at {self.connector_config.access_config.hosts}"
-            f" with batch size (in bytes) {self.write_config.batch_size_bytes}"
-            f" with {self.write_config.num_processes} (number of) processes"
-        )
-        from opensearchpy.helpers import parallel_bulk
-
-        for batch in generator_batching_wbytes(
-            elements_dict, batch_size_limit_bytes=self.write_config.batch_size_bytes
-        ):
-            for success, info in parallel_bulk(
-                self.client, batch, thread_count=self.write_config.num_processes
-            ):
-                if not success:
-                    logger.error(
-                        "upload failed for a batch in opensearch destination connector:", info
-                    )