unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/notion/ingest_backoff/types.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class RetryStrategyConfig:
+    """
+    Contains all info needed for decorator to pull from `self` for backoff
+    and retry triggered by exception.
+
+    Args:
+        max_retries: The maximum number of attempts to make before giving
+            up. Once exhausted, the exception will be allowed to escape.
+            The default value of None means there is no limit to the
+            number of tries. If a callable is passed, it will be
+            evaluated at runtime and its return value used.
+        max_retry_time: The maximum total amount of time to try for before
+            giving up. Once expired, the exception will be allowed to
+            escape. If a callable is passed, it will be
+            evaluated at runtime and its return value used.
+    """
+
+    max_retries: Optional[int] = None
+    max_retry_time: Optional[float] = None

unstructured_ingest/processes/connectors/notion/interfaces.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+
+class FromJSONMixin(ABC):
+    @classmethod
+    @abstractmethod
+    def from_dict(cls, data: dict):
+        pass
+
+
+class GetHTMLMixin(ABC):
+    @abstractmethod
+    def get_html(self) -> Optional[HtmlTag]:
+        pass
+
+
+class BlockBase(FromJSONMixin, GetHTMLMixin):
+    @staticmethod
+    @abstractmethod
+    def can_have_children() -> bool:
+        pass
+
+
+class DBPropertyBase(FromJSONMixin):
+    pass
+
+
+class DBCellBase(FromJSONMixin, GetHTMLMixin):
+    pass

unstructured_ingest/processes/connectors/notion/types/block.py

@@ -0,0 +1,96 @@
+# https://developers.notion.com/reference/page
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    BlockBase,
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.processes.connectors.notion.types import blocks
+from unstructured_ingest.processes.connectors.notion.types.parent import Parent
+from unstructured_ingest.processes.connectors.notion.types.user import PartialUser
+
+block_type_mapping = {
+    "bookmark": blocks.Bookmark,
+    "breadcrumb": blocks.Breadcrumb,
+    "bulleted_list_item": blocks.BulletedListItem,
+    "callout": blocks.Callout,
+    "child_database": blocks.ChildDatabase,
+    "child_page": blocks.ChildPage,
+    "code": blocks.Code,
+    "column": blocks.Column,
+    "column_list": blocks.ColumnList,
+    "divider": blocks.Divider,
+    "heading_1": blocks.Heading,
+    "heading_2": blocks.Heading,
+    "heading_3": blocks.Heading,
+    "embed": blocks.Embed,
+    "equation": blocks.Equation,
+    "file": blocks.File,
+    "image": blocks.Image,
+    "link_preview": blocks.LinkPreview,
+    "link_to_page": blocks.LinkToPage,
+    "numbered_list_item": blocks.NumberedListItem,
+    "paragraph": blocks.Paragraph,
+    "pdf": blocks.PDF,
+    "quote": blocks.Quote,
+    "synced_block": blocks.SyncBlock,
+    "table": blocks.Table,
+    "table_of_contents": blocks.TableOfContents,
+    "table_row": blocks.TableRow,
+    "template": blocks.Template,
+    "to_do": blocks.ToDo,
+    "toggle": blocks.Toggle,
+    "unsupported": blocks.Unsupported,
+    "video": blocks.Video,
+}
+
+
+@dataclass
+class Block(FromJSONMixin, GetHTMLMixin):
+    id: str
+    type: str
+    created_time: str
+    created_by: PartialUser
+    last_edited_time: str
+    last_edited_by: PartialUser
+    archived: bool
+    in_trash: bool
+    has_children: bool
+    parent: Parent
+    block: BlockBase
+    object: str = "block"
+    request_id: Optional[str] = None
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(id={self.id}, type={self.type})"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        t = data["type"]
+        block_data = data.pop(t)
+        created_by = data.pop("created_by")
+        last_edited_by = data.pop("last_edited_by")
+        parent = data.pop("parent")
+        try:
+            block = cls(
+                created_by=PartialUser.from_dict(created_by),
+                last_edited_by=PartialUser.from_dict(last_edited_by),
+                parent=Parent.from_dict(parent),
+                block=block_type_mapping[t].from_dict(block_data),  # type: ignore
+                **data,
+            )
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated block type -> {t}: {block_data}") from ke
+        except TypeError as te:
+            raise TypeError(f"failed to map to associated block type -> {t}: {block_data}") from te
+
+        return block
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.block:
+            return self.block.get_html()
+        return None

unstructured_ingest/processes/connectors/notion/types/blocks/__init__.py

@@ -0,0 +1,63 @@
+from .bookmark import Bookmark
+from .breadcrumb import Breadcrumb
+from .bulleted_list_item import BulletedListItem
+from .callout import Callout
+from .child_database import ChildDatabase
+from .child_page import ChildPage
+from .code import Code
+from .column_list import Column, ColumnList
+from .divider import Divider
+from .embed import Embed
+from .equation import Equation
+from .file import File
+from .heading import Heading
+from .image import Image
+from .link_preview import LinkPreview
+from .link_to_page import LinkToPage
+from .numbered_list import NumberedListItem
+from .paragraph import Paragraph
+from .pdf import PDF
+from .quote import Quote
+from .synced_block import DuplicateSyncedBlock, OriginalSyncedBlock, SyncBlock
+from .table import Table, TableRow
+from .table_of_contents import TableOfContents
+from .template import Template
+from .todo import ToDo
+from .toggle import Toggle
+from .unsupported import Unsupported
+from .video import Video
+
+__all__ = [
+    "Bookmark",
+    "Breadcrumb",
+    "BulletedListItem",
+    "Callout",
+    "ChildDatabase",
+    "ChildPage",
+    "Code",
+    "Column",
+    "ColumnList",
+    "Divider",
+    "Embed",
+    "Equation",
+    "File",
+    "Heading",
+    "Image",
+    "LinkPreview",
+    "LinkToPage",
+    "NumberedListItem",
+    "Paragraph",
+    "PDF",
+    "Quote",
+    "SyncBlock",
+    "OriginalSyncedBlock",
+    "DuplicateSyncedBlock",
+    "Table",
+    "TableRow",
+    "TableOfContents",
+    "Template",
+    "ToDo",
+    "Toggle",
+    "Unsupported",
+    "Video",
+]

unstructured_ingest/processes/connectors/notion/types/blocks/bookmark.py

@@ -0,0 +1,40 @@
+# https://developers.notion.com/reference/block#bookmark
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Href
+from htmlBuilder.tags import A, Br, Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+
+
+@dataclass
+class Bookmark(BlockBase):
+    url: str
+    caption: List[RichText] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        captions = data.pop("caption", [])
+        return cls(
+            url=data["url"],
+            caption=[RichText.from_dict(c) for c in captions],
+        )
+
+    def get_html(self) -> Optional[HtmlTag]:
+        texts = []
+        if self.url:
+            texts.append(A([Href(self.url)], self.url))
+        if self.caption:
+            texts.append(Div([], [rt.get_html() for rt in self.caption]))
+        if not texts:
+            return None
+        joined = [Br()] * (len(texts) * 2 - 1)
+        joined[0::2] = texts
+
+        return Div([], joined)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False

unstructured_ingest/processes/connectors/notion/types/blocks/breadcrumb.py

@@ -0,0 +1,21 @@
+# https://developers.notion.com/reference/block#breadcrumb
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class Breadcrumb(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        pass

unstructured_ingest/processes/connectors/notion/types/blocks/bulleted_list_item.py

@@ -0,0 +1,31 @@
+# https://developers.notion.com/reference/block#bulleted-list-item
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag, Li
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+
+
+@dataclass
+class BulletedListItem(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        return cls(
+            color=data["color"],
+            children=data.get("children", []),
+            rich_text=[RichText.from_dict(rt) for rt in rich_text],
+        )
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Li([], [rt.get_html() for rt in self.rich_text])

unstructured_ingest/processes/connectors/notion/types/blocks/callout.py

@@ -0,0 +1,131 @@
+# https://developers.notion.com/reference/block#callout
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
+
+from htmlBuilder.attributes import Href, Style
+from htmlBuilder.tags import A, Div, HtmlTag, P
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    BlockBase,
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+
+
+@dataclass
+class EmojiIcon(FromJSONMixin, GetHTMLMixin):
+    emoji: str
+    type: str = "emoji"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return P([], self.emoji)
+
+
+@dataclass
+class ExternalIconContent(FromJSONMixin):
+    url: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class ExternalIcon(FromJSONMixin, GetHTMLMixin):
+    external: ExternalIconContent
+    type: str = "external"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(external=ExternalIconContent.from_dict(data=data.pop("external")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.external:
+            return A([Href(self.external.url)], [self.external.url])
+        else:
+            return None
+
+
+@dataclass
+class FileIconContent(FromJSONMixin):
+    url: str
+    expiry_time: Optional[str] = None  # Add expiry_time if needed
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        # Only include expiry_time if it exists in the dictionary
+        # Notion API might not always include it
+        init_data = {"url": data.get("url")}
+        if "expiry_time" in data:
+            init_data["expiry_time"] = data.get("expiry_time")
+        return cls(**init_data)
+
+
+@dataclass
+class FileIcon(FromJSONMixin, GetHTMLMixin):
+    file: FileIconContent
+    type: str = "file"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(file=FileIconContent.from_dict(data=data.pop("file")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        # Render the file URL, similar to how ExternalIcon is handled
+        if self.file:
+            # Could potentially render an <img> tag, but sticking to URL for consistency
+            return A([Href(self.file.url)], [f"[File Icon: {self.file.url}]"])
+        else:
+            return None
+
+
+class Icon(FromJSONMixin):
+    @classmethod
+    def from_dict(cls, data: dict) -> Union[EmojiIcon, ExternalIcon, FileIcon]:
+        t = data.get("type")
+        if t == "emoji":
+            return EmojiIcon.from_dict(data)
+        elif t == "external":
+            return ExternalIcon.from_dict(data)
+        elif t == "file":
+            return FileIcon.from_dict(data)
+        else:
+            raise ValueError(f"Unexpected icon type: {t} ({data})")
+
+
+@dataclass
+class Callout(BlockBase):
+    color: str
+    icon: Optional[Union[EmojiIcon, ExternalIcon, FileIcon]] = None
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        icon_data = data.pop("icon", None)
+        icon = Icon.from_dict(icon_data) if icon_data else None
+        return cls(
+            color=data["color"],
+            icon=icon,
+            rich_text=[RichText.from_dict(rt) for rt in rich_text],
+        )
+
+    def get_html(self) -> Optional[HtmlTag]:
+        elements = []
+        if self.icon and self.icon.get_html():
+            elements.append(self.icon.get_html())
+        if self.rich_text:
+            elements.extend([rt.get_html() for rt in self.rich_text])
+        attributes = []
+        if self.color:
+            attributes.append(Style(f"color:{self.color}"))
+        return Div(attributes, elements)

unstructured_ingest/processes/connectors/notion/types/blocks/child_database.py

@@ -0,0 +1,23 @@
+# https://developers.notion.com/reference/block#child-database
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag, P
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class ChildDatabase(BlockBase):
+    title: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return P([], self.title)

unstructured_ingest/processes/connectors/notion/types/blocks/child_page.py

@@ -0,0 +1,23 @@
+# https://developers.notion.com/reference/block#child-page
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag, P
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase, GetHTMLMixin
+
+
+@dataclass
+class ChildPage(BlockBase, GetHTMLMixin):
+    title: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return P([], self.title)

unstructured_ingest/processes/connectors/notion/types/blocks/code.py

@@ -0,0 +1,43 @@
+# https://developers.notion.com/reference/block#code
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import Br, Div, HtmlTag
+from htmlBuilder.tags import Code as HtmlCode
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+
+
+@dataclass
+class Code(BlockBase):
+    language: str
+    rich_text: List[RichText] = field(default_factory=list)
+    caption: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        caption = data.pop("caption", [])
+        return cls(
+            language=data["language"],
+            rich_text=[RichText.from_dict(rt) for rt in rich_text],
+            caption=[RichText.from_dict(c) for c in caption],
+        )
+
+    def get_html(self) -> Optional[HtmlTag]:
+        texts = []
+        if self.rich_text:
+            texts.append(HtmlCode([], [rt.get_html() for rt in self.rich_text]))
+        if self.caption:
+            texts.append(Div([], [rt.get_html() for rt in self.caption]))
+        if not texts:
+            return None
+        joined = [Br()] * (len(texts) * 2 - 1)
+        joined[0::2] = texts
+
+        return Div([], joined)

unstructured_ingest/processes/connectors/notion/types/blocks/column_list.py

@@ -0,0 +1,35 @@
+# https://developers.notion.com/reference/block#column-list-and-column
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class ColumnList(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class Column(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/processes/connectors/notion/types/blocks/divider.py

@@ -0,0 +1,22 @@
+# https://developers.notion.com/reference/block#divider
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.attributes import Style
+from htmlBuilder.tags import Hr, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class Divider(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Hr([Style("border-top: 3px solid #bbb")])

unstructured_ingest/processes/connectors/notion/types/blocks/embed.py

@@ -0,0 +1,36 @@
+# https://developers.notion.com/reference/block#embed
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Href
+from htmlBuilder.tags import A, Br, Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+
+
+@dataclass
+class Embed(BlockBase):
+    url: str
+    caption: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(caption=[RichText.from_dict(d) for d in data.pop("caption", [])], **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        texts = []
+        if self.url:
+            texts.append(A([Href(self.url)], self.url))
+        if self.caption:
+            texts.append(Div([], [rt.get_html() for rt in self.caption]))
+        if not texts:
+            return None
+        joined = [Br()] * (len(texts) * 2 - 1)
+        joined[0::2] = texts
+
+        return Div([], joined)

unstructured_ingest/processes/connectors/notion/types/blocks/equation.py

@@ -0,0 +1,23 @@
+# https://developers.notion.com/reference/block#equation
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import BlockBase
+
+
+@dataclass
+class Equation(BlockBase):
+    expression: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.expression)