unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/connector/notion/types/rich_text.py

@@ -0,0 +1,189 @@
+# https://developers.notion.com/reference/rich-text
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.attributes import Href, Style
+from htmlBuilder.tags import A, B, Code, Div, HtmlTag, I, S, Span, U
+from htmlBuilder.tags import Text as HtmlText
+
+from unstructured_ingest.connector.notion.interfaces import (
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.connector.notion.types.date import Date
+from unstructured_ingest.connector.notion.types.user import People
+
+
+@dataclass
+class Annotations(FromJSONMixin):
+    bold: bool
+    code: bool
+    italic: bool
+    strikethrough: bool
+    underline: bool
+    color: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class Equation(FromJSONMixin, GetHTMLMixin):
+    expression: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Code([], self.expression) if self.expression else None
+
+
+@dataclass
+class MentionDatabase(FromJSONMixin, GetHTMLMixin):
+    id: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.id) if self.id else None
+
+
+@dataclass
+class MentionLinkPreview(FromJSONMixin, GetHTMLMixin):
+    url: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return A([Href(self.url)], self.url) if self.url else None
+
+
+@dataclass
+class MentionPage(FromJSONMixin, GetHTMLMixin):
+    id: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.id) if self.id else None
+
+
+@dataclass
+class MentionTemplate(FromJSONMixin):
+    template_mention_date: Optional[str]
+    template_mention_user: Optional[str]
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class Mention(FromJSONMixin, GetHTMLMixin):
+    type: str
+    database: Optional[MentionDatabase] = None
+    date: Optional[Date] = None
+    link_preview: Optional[MentionLinkPreview] = None
+    page: Optional[MentionPage] = None
+    template_mention: Optional[MentionTemplate] = None
+    user: Optional[People] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        t = data["type"]
+        mention = cls(type=t)
+        if t == "date":
+            mention.date = Date.from_dict(data["date"])
+        elif t == "database":
+            mention.database = MentionDatabase.from_dict(data["database"])
+        elif t == "link_preview":
+            mention.link_preview = MentionLinkPreview.from_dict(data["link_preview"])
+        elif t == "page":
+            mention.page = MentionPage.from_dict(data["page"])
+        elif t == "template_mention":
+            mention.template_mention = MentionTemplate.from_dict(data["template_mention"])
+        elif t == "user":
+            mention.user = People.from_dict(data["user"])
+
+        return mention
+
+    def get_html(self) -> Optional[HtmlTag]:
+        t = self.type
+        if t == "date":
+            return self.date.get_html() if self.date else None
+        elif t == "database":
+            return self.database.get_html() if self.database else None
+        elif t == "link_preview":
+            return self.link_preview.get_html() if self.link_preview else None
+        elif t == "page":
+            return self.page.get_html() if self.page else None
+        elif t == "user":
+            return self.user.get_html() if self.user else None
+        return None
+
+
+@dataclass
+class Text(FromJSONMixin):
+    content: str
+    link: Optional[dict]
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class RichText(FromJSONMixin, GetHTMLMixin):
+    type: str
+    plain_text: str
+    annotations: Optional[Annotations] = None
+    href: Optional[str] = None
+    text: Optional[Text] = None
+    mention: Optional[Mention] = None
+    equation: Optional[Equation] = None
+
+    def get_html(self) -> Optional[HtmlTag]:
+        text = HtmlText(self.plain_text)
+        if self.href:
+            text = A([Href(self.href)], text)
+        if self.annotations:
+            annotations = self.annotations
+            if annotations.bold:
+                text = B([], text)
+            if annotations.code:
+                text = Code([], text)
+            if annotations.italic:
+                text = I([], text)
+            if annotations.strikethrough:
+                text = S([], text)
+            if annotations.underline:
+                text = U([], text)
+            if annotations.color and annotations.color != "default":
+                if isinstance(text, HtmlText):
+                    text = Span([], text)
+                text.attributes.append(Style(f"color:{annotations.color}"))
+        return text
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        t = data["type"]
+        rich_text = cls(
+            annotations=Annotations.from_dict(data.pop("annotations")),
+            **data,
+        )
+        if t == "text":
+            rich_text.text = Text.from_dict(data["text"])
+        elif t == "mention":
+            rich_text.mention = Mention.from_dict(data["mention"])
+        elif t == "equation":
+            rich_text.equation = Equation.from_dict(data["equation"])
+
+        return rich_text

unstructured_ingest/connector/notion/types/user.py

@@ -0,0 +1,76 @@
+# https://developers.notion.com/reference/user
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.attributes import Href
+from htmlBuilder.tags import A, Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import FromJSONMixin, GetHTMLMixin
+
+
+@dataclass
+class PartialUser(FromJSONMixin):
+    id: str
+    object: str = "user"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(id=data["id"])
+
+
+@dataclass
+class User(FromJSONMixin, GetHTMLMixin):
+    object: dict
+    id: str
+    type: Optional[str] = None
+    name: Optional[str] = None
+    avatar_url: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_text(self) -> Optional[str]:
+        text = self.name
+        if self.avatar_url:
+            text = f"[{text}]({self.avatar_url}"
+        return text
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.avatar_url:
+            return A([Href(self.avatar_url)], self.name)
+        else:
+            return Div([], self.name)
+
+
+@dataclass
+class People(User):
+    person: dict = field(default_factory=dict)
+
+
+@dataclass
+class Bots(FromJSONMixin, GetHTMLMixin):
+    object: dict
+    id: str
+    bot: dict
+    owner: dict
+    type: str
+    workspace_name: str
+    name: Optional[str] = None
+    avatar_url: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_text(self) -> Optional[str]:
+        text = self.name
+        if self.avatar_url:
+            text = f"[{text}]({self.avatar_url}"
+        return text
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.avatar_url:
+            return A([Href(self.avatar_url)], self.name)
+        else:
+            return Div([], self.name)

unstructured_ingest/connector/onedrive.py

@@ -0,0 +1,232 @@
+import typing as t
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    BaseConnectorConfig,
+    BaseSingleIngestDoc,
+    BaseSourceConnector,
+    IngestDocCleanupMixin,
+    SourceConnectorCleanupMixin,
+    SourceMetadata,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import ensure_isoformat_datetime
+
+if t.TYPE_CHECKING:
+    from office365.graph_client import GraphClient
+    from office365.onedrive.driveitems.driveItem import DriveItem
+MAX_MB_SIZE = 512_000_000
+
+
+@dataclass
+class OneDriveAccessConfig(AccessConfig):
+    client_credential: str = enhanced_field(repr=False, sensitive=True, overload_name="client_cred")
+
+
+@dataclass
+class SimpleOneDriveConfig(BaseConnectorConfig):
+    access_config: OneDriveAccessConfig
+    client_id: str
+    user_pname: str
+    tenant: str = field(repr=False)
+    authority_url: t.Optional[str] = field(repr=False, default="https://login.microsoftonline.com")
+    path: t.Optional[str] = field(default="")
+    recursive: bool = False
+
+    def __post_init__(self):
+        if not (self.client_id and self.access_config.client_credential and self.user_pname):
+            raise ValueError(
+                "Please provide all the following mandatory values:"
+                "\n-ms-client_id\n-ms-client_cred\n-ms-user-pname",
+            )
+        self.token_factory = self._acquire_token
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["msal"])
+    def _acquire_token(self):
+        from msal import ConfidentialClientApplication
+
+        try:
+            app = ConfidentialClientApplication(
+                authority=f"{self.authority_url}/{self.tenant}",
+                client_id=self.client_id,
+                client_credential=self.access_config.client_credential,
+            )
+            token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
+        except ValueError as exc:
+            logger.error("Couldn't set up credentials for OneDrive")
+            raise exc
+        return token
+
+
+@dataclass
+class OneDriveIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
+    connector_config: SimpleOneDriveConfig
+    file_name: str
+    file_path: str
+    registry_name: str = "onedrive"
+
+    def __post_init__(self):
+        self.ext = Path(self.file_name).suffix
+        if not self.ext:
+            raise ValueError("Unsupported file without extension.")
+
+        self.server_relative_path = self.file_path + "/" + self.file_name
+        self._set_download_paths()
+
+    def _set_download_paths(self) -> None:
+        """Parses the folder structure from the source and creates the download and output paths"""
+        download_path = Path(f"{self.read_config.download_dir}")
+        output_path = Path(f"{self.processor_config.output_dir}")
+
+        if parent_path := self.file_path:
+            download_path = (
+                download_path if parent_path == "" else (download_path / parent_path).resolve()
+            )
+            output_path = (
+                output_path if parent_path == "" else (output_path / parent_path).resolve()
+            )
+
+        self.download_dir = download_path
+        self.download_filepath = (download_path / self.file_name).resolve()
+        output_filename = output_filename = self.file_name + ".json"
+        self.output_dir = output_path
+        self.output_filepath = (output_path / output_filename).resolve()
+
+    @property
+    def filename(self):
+        return Path(self.download_filepath).resolve()
+
+    @property
+    def _output_filename(self):
+        return Path(self.output_filepath).resolve()
+
+    @property
+    def record_locator(self) -> t.Optional[t.Dict[str, t.Any]]:
+        return {
+            "user_pname": self.connector_config.user_pname,
+            "server_relative_path": self.server_relative_path,
+        }
+
+    @SourceConnectionNetworkError.wrap
+    @requires_dependencies(["office365"], extras="onedrive")
+    def _fetch_file(self):
+        from office365.graph_client import GraphClient
+
+        client = GraphClient(self.connector_config.token_factory)
+        root = client.users[self.connector_config.user_pname].drive.get().execute_query().root
+        file = root.get_by_path(self.server_relative_path).get().execute_query()
+        return file
+
+    def update_source_metadata(self, **kwargs):
+        file = kwargs.get("file", self._fetch_file())
+        if file is None:
+            self.source_metadata = SourceMetadata(
+                exists=False,
+            )
+            return
+
+        version = None
+        if (n_versions := len(file.versions)) > 0:
+            version = file.versions[n_versions - 1].properties.get("id", None)
+
+        self.source_metadata = SourceMetadata(
+            date_created=ensure_isoformat_datetime(timestamp=file.created_datetime),
+            date_modified=ensure_isoformat_datetime(timestamp=file.last_modified_datetime),
+            version=version,
+            source_url=file.parent_reference.path + "/" + self.file_name,
+            exists=True,
+        )
+
+    @SourceConnectionError.wrap
+    @BaseSingleIngestDoc.skip_if_file_exists
+    def get_file(self):
+        file = self._fetch_file()
+        self.update_source_metadata(file=file)
+        if file is None:
+            raise ValueError(
+                f"Failed to retrieve file {self.file_path}/{self.file_name}",
+            )
+
+        fsize = file.get_property("size", 0)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        if not self.download_dir.is_dir():
+            logger.debug(f"creating directory: {self.download_dir}")
+            self.download_dir.mkdir(parents=True, exist_ok=True)
+
+        if fsize > MAX_MB_SIZE:
+            logger.info(f"downloading file with size: {fsize} bytes in chunks")
+            with self.filename.open(mode="wb") as f:
+                file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
+        else:
+            with self.filename.open(mode="wb") as f:
+                file.download(f).execute_query()
+        logger.info(f"file downloaded: {self.filename}")
+        return
+
+
+@dataclass
+class OneDriveSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
+    connector_config: SimpleOneDriveConfig
+    _client: t.Optional["GraphClient"] = field(init=False, default=None)
+
+    @property
+    def client(self) -> "GraphClient":
+        from office365.graph_client import GraphClient
+
+        if self._client is None:
+            self._client = GraphClient(self.connector_config.token_factory)
+        return self._client
+
+    @requires_dependencies(["office365"], extras="onedrive")
+    def initialize(self):
+        _ = self.client
+
+    @requires_dependencies(["office365"], extras="onedrive")
+    def check_connection(self):
+        try:
+            token_resp: dict = self.connector_config.token_factory()
+            if error := token_resp.get("error"):
+                raise SourceConnectionError(
+                    "{} ({})".format(error, token_resp.get("error_description"))
+                )
+            _ = self.client
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def _list_objects(self, folder, recursive) -> t.List["DriveItem"]:
+        drive_items = folder.children.get().execute_query()
+        files = [d for d in drive_items if d.is_file]
+        if not recursive:
+            return files
+        folders = [d for d in drive_items if d.is_folder]
+        for f in folders:
+            files += self._list_objects(f, recursive)
+        return files
+
+    def _gen_ingest_doc(self, file: "DriveItem") -> OneDriveIngestDoc:
+        file_path = file.parent_reference.path.split(":")[-1]
+        file_path = file_path[1:] if file_path[0] == "/" else file_path
+        return OneDriveIngestDoc(
+            connector_config=self.connector_config,
+            processor_config=self.processor_config,
+            read_config=self.read_config,
+            file_name=file.name,
+            file_path=file_path,
+        )
+
+    def get_ingest_docs(self):
+        root = self.client.users[self.connector_config.user_pname].drive.get().execute_query().root
+        if fpath := self.connector_config.path:
+            root = root.get_by_path(fpath).get().execute_query()
+            if root is None or not root.is_folder:
+                raise ValueError(f"Unable to find directory, given: {fpath}")
+        files = self._list_objects(root, self.connector_config.recursive)
+        return [self._gen_ingest_doc(f) for f in files]