unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/connector/notion/types/blocks/link_preview.py

@@ -0,0 +1,24 @@
+# https://developers.notion.com/reference/block#link-preview
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.attributes import Href
+from htmlBuilder.tags import A, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class LinkPreview(BlockBase):
+    url: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return A([Href(self.url)], self.url)

unstructured_ingest/connector/notion/types/blocks/link_to_page.py

@@ -0,0 +1,29 @@
+# https://developers.notion.com/reference/block#link-to-page
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class LinkToPage(BlockBase):
+    type: str
+    page_id: Optional[str] = None
+    database_id: Optional[str] = None
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if page_id := self.page_id:
+            return Div([], page_id)
+        if database_id := self.database_id:
+            return Div([], database_id)
+        return None

unstructured_ingest/connector/notion/types/blocks/numbered_list.py

@@ -0,0 +1,29 @@
+# https://developers.notion.com/reference/block#numbered-list-item
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag, Li
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class NumberedListItem(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        numbered_list = cls(**data)
+        numbered_list.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return numbered_list
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Li([], [rt.get_html() for rt in self.rich_text])

unstructured_ingest/connector/notion/types/blocks/paragraph.py

@@ -0,0 +1,31 @@
+# https://developers.notion.com/reference/block#paragraph
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import Br, Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Paragraph(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        paragraph = cls(**data)
+        paragraph.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return paragraph
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return Br()
+        return Div([], [rt.get_html() for rt in self.rich_text])

unstructured_ingest/connector/notion/types/blocks/pdf.py

@@ -0,0 +1,49 @@
+# https://developers.notion.com/reference/block#pdf
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Href
+from htmlBuilder.tags import A, Br, Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.file import External, File
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class PDF(BlockBase):
+    type: str
+    caption: List[RichText] = field(default_factory=list)
+    external: Optional[External] = None
+    file: Optional[File] = None
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        caption = data.pop("caption", [])
+        t = data["type"]
+        paragraph = cls(type=t)
+        paragraph.caption = [RichText.from_dict(c) for c in caption]
+        if t == "external":
+            paragraph.external = External.from_dict(data["external"])
+        elif t == "file":
+            paragraph.file = File.from_dict(data["file"])
+        return paragraph
+
+    def get_html(self) -> Optional[HtmlTag]:
+        texts = []
+        if self.external:
+            texts.append(A([Href(self.external.url)], self.external.url))
+        if self.file:
+            texts.append(A([Href(self.file.url)], self.file.url))
+        if self.caption:
+            texts.append(Div([], [rt.get_html() for rt in self.caption]))
+        if not texts:
+            return None
+        joined = [Br()] * (len(texts) * 2 - 1)
+        joined[0::2] = texts
+
+        return Div([], joined)

unstructured_ingest/connector/notion/types/blocks/quote.py

@@ -0,0 +1,37 @@
+# https://developers.notion.com/reference/block#quote
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Style
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Quote(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        quote = cls(**data)
+        quote.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return quote
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+
+        texts = [rt.get_html() for rt in self.rich_text]
+        attributes = []
+        if self.color and self.color != "default":
+            attributes.append(Style(f"color: {self.color}"))
+        return Div(attributes, texts)

unstructured_ingest/connector/notion/types/blocks/synced_block.py

@@ -0,0 +1,57 @@
+# https://developers.notion.com/reference/block#synced-block
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class OriginalSyncedBlock(BlockBase):
+    synced_from: Optional[str] = None
+    children: List[dict] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(children=data["children"])
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class DuplicateSyncedBlock(BlockBase):
+    type: str
+    block_id: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+class SyncBlock(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        if "synced_from" in data:
+            return OriginalSyncedBlock.from_dict(data)
+        else:
+            return DuplicateSyncedBlock.from_dict(data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/connector/notion/types/blocks/table.py

@@ -0,0 +1,63 @@
+# https://developers.notion.com/reference/block#table
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag, Td, Th, Tr
+
+from unstructured_ingest.connector.notion.interfaces import (
+    BlockBase,
+    FromJSONMixin,
+)
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Table(BlockBase):
+    table_width: int
+    has_column_header: bool
+    has_row_header: bool
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class TableCell(FromJSONMixin):
+    rich_texts: List[RichText]
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(rich_texts=[RichText.from_dict(rt) for rt in data.pop("rich_texts", [])])
+
+    def get_html(self, is_header: bool) -> Optional[HtmlTag]:
+        if is_header:
+            return Th([], [rt.get_html() for rt in self.rich_texts])
+        else:
+            return Td([], [rt.get_html() for rt in self.rich_texts])
+
+
+# https://developers.notion.com/reference/block#table-rows
+@dataclass
+class TableRow(BlockBase):
+    is_header: bool = False
+    cells: List[TableCell] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        cells = data.get("cells", [])
+        return cls(cells=[TableCell.from_dict({"rich_texts": c}) for c in cells])
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Tr([], [cell.get_html(is_header=self.is_header) for cell in self.cells])

unstructured_ingest/connector/notion/types/blocks/table_of_contents.py

@@ -0,0 +1,23 @@
+# https://developers.notion.com/reference/block#table-of-contents
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class TableOfContents(BlockBase):
+    color: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/connector/notion/types/blocks/template.py

@@ -0,0 +1,30 @@
+# https://developers.notion.com/reference/block#template
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Template(BlockBase):
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        template = cls(**data)
+        template.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return template
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+        return Div([], [rt.get_html() for rt in self.rich_text])

unstructured_ingest/connector/notion/types/blocks/todo.py

@@ -0,0 +1,42 @@
+# https://developers.notion.com/reference/block#to-do
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Checked, Style, Type
+from htmlBuilder.tags import Div, HtmlTag, Input
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class ToDo(BlockBase):
+    color: str
+    checked: bool = False
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        todo = cls(**data)
+        todo.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return todo
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+
+        elements = []
+        check_input_attributes = [Type("checkbox")]
+        if self.checked:
+            check_input_attributes.append(Checked(""))
+        elements.append(Input(check_input_attributes))
+        elements.extend([rt.get_html() for rt in self.rich_text])
+        attributes = []
+        if self.color and self.color != "default":
+            attributes.append(Style(f"color: {self.color}"))
+        return Div(attributes, elements)

unstructured_ingest/connector/notion/types/blocks/toggle.py

@@ -0,0 +1,37 @@
+# https://developers.notion.com/reference/block#toggle-blocks
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Style
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Toggle(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        toggle = cls(**data)
+        toggle.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return toggle
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+
+        texts = [rt.get_html() for rt in self.rich_text]
+        attributes = []
+        if self.color and self.color != "default":
+            attributes.append(Style(f"color: {self.color}"))
+        return Div(attributes, texts)

unstructured_ingest/connector/notion/types/blocks/unsupported.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class Unsupported(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/connector/notion/types/blocks/video.py

@@ -0,0 +1,22 @@
+# https://developers.notion.com/reference/block#image
+from typing import Optional
+
+from htmlBuilder.attributes import Src
+from htmlBuilder.tags import HtmlTag, Source
+from htmlBuilder.tags import Video as VideoHtml
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.file import FileObject
+
+
+class Video(BlockBase, FileObject):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.external:
+            return VideoHtml([], [Source([Src(self.external.url)], [self.external.url])])
+        if self.file:
+            return VideoHtml([], [Source([Src(self.file.url)], [self.file.url])])
+        return None

unstructured_ingest/connector/notion/types/database.py

@@ -0,0 +1,73 @@
+# https://developers.notion.com/reference/database
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag, Span
+
+from unstructured_ingest.connector.notion.interfaces import (
+    DBPropertyBase,
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.connector.notion.types.database_properties import (
+    map_properties,
+)
+from unstructured_ingest.connector.notion.types.file import FileObject
+from unstructured_ingest.connector.notion.types.parent import Parent
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+from unstructured_ingest.connector.notion.types.user import PartialUser
+
+
+@dataclass
+class Database(FromJSONMixin, GetHTMLMixin):
+    id: str
+    created_time: str
+    created_by: PartialUser
+    last_edited_time: str
+    last_edited_by: PartialUser
+    archived: bool
+    in_trash: bool
+    parent: Parent
+    url: str
+    is_inline: bool
+    public_url: str
+    request_id: Optional[str] = None
+    properties: Dict[str, DBPropertyBase] = field(default_factory=dict)
+    title: List[RichText] = field(default_factory=list)
+    description: List[RichText] = field(default_factory=list)
+    icon: Optional[FileObject] = None
+    cover: Optional[FileObject] = None
+    object: str = "database"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        created_by = data.pop("created_by")
+        last_edited_by = data.pop("last_edited_by")
+        icon = data.pop("icon")
+        cover = data.pop("cover")
+        parent = data.pop("parent")
+        title = data.pop("title")
+        description = data.pop("description")
+        page = cls(
+            properties=map_properties(data.pop("properties", {})),
+            created_by=PartialUser.from_dict(created_by),
+            last_edited_by=PartialUser.from_dict(last_edited_by),
+            icon=FileObject.from_dict(icon) if icon else None,
+            cover=FileObject.from_dict(cover) if cover else None,
+            parent=Parent.from_dict(parent),
+            title=[RichText.from_dict(data=r) for r in title],
+            description=[RichText.from_dict(data=r) for r in description],
+            **data,
+        )
+
+        return page
+
+    def get_html(self) -> Optional[HtmlTag]:
+        spans = []
+        if title := self.title:
+            spans.append(Span([], [rt.get_html() for rt in title]))
+        if description := self.description:
+            spans.append(Span([], [rt.get_html() for rt in description]))
+        if spans:
+            return Div([], spans)
+        return None

unstructured_ingest/connector/notion/types/database_properties/__init__.py

@@ -0,0 +1,106 @@
+from typing import Dict
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+
+from .checkbox import Checkbox, CheckboxCell
+from .created_by import CreatedBy, CreatedByCell
+from .created_time import CreatedTime, CreatedTimeCell
+from .date import Date, DateCell
+from .email import Email, EmailCell
+from .files import Files, FilesCell
+from .formula import Formula, FormulaCell
+from .last_edited_by import LastEditedBy, LastEditedByCell
+from .last_edited_time import LastEditedTime, LastEditedTimeCell
+from .multiselect import MultiSelect, MultiSelectCell
+from .number import Number, NumberCell
+from .people import People, PeopleCell
+from .phone_number import PhoneNumber, PhoneNumberCell
+from .relation import Relation, RelationCell
+from .rich_text import RichText, RichTextCell
+from .rollup import Rollup, RollupCell
+from .select import Select, SelectCell
+from .status import Status, StatusCell
+from .title import Title, TitleCell
+from .unique_id import UniqueID, UniqueIDCell
+from .url import URL, URLCell
+from .verification import Verification, VerificationCell
+
+db_prop_type_mapping = {
+    "checkbox": Checkbox,
+    "created_by": CreatedBy,
+    "created_time": CreatedTime,
+    "date": Date,
+    "email": Email,
+    "files": Files,
+    "formula": Formula,
+    "last_edited_by": LastEditedBy,
+    "last_edited_time": LastEditedTime,
+    "multi_select": MultiSelect,
+    "number": Number,
+    "people": People,
+    "phone_number": PhoneNumber,
+    "relation": Relation,
+    "rich_text": RichText,
+    "rollup": Rollup,
+    "select": Select,
+    "status": Status,
+    "title": Title,
+    "unique_id": UniqueID,
+    "url": URL,
+    "verification": Verification,
+}
+
+
+def map_properties(props: Dict[str, dict]) -> Dict[str, DBPropertyBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            mapped_dict[k] = db_prop_type_mapping[v["type"]].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+db_cell_type_mapping = {
+    "checkbox": CheckboxCell,
+    "created_by": CreatedByCell,
+    "created_time": CreatedTimeCell,
+    "date": DateCell,
+    "email": EmailCell,
+    "files": FilesCell,
+    "formula": FormulaCell,
+    "last_edited_by": LastEditedByCell,
+    "last_edited_time": LastEditedTimeCell,
+    "multi_select": MultiSelectCell,
+    "number": NumberCell,
+    "people": PeopleCell,
+    "phone_number": PhoneNumberCell,
+    "relation": RelationCell,
+    "rich_text": RichTextCell,
+    "rollup": RollupCell,
+    "select": SelectCell,
+    "status": StatusCell,
+    "title": TitleCell,
+    "unique_id": UniqueIDCell,
+    "url": URLCell,
+    "verification": VerificationCell,
+}
+
+
+def map_cells(props: Dict[str, dict]) -> Dict[str, DBCellBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            t = v["type"]
+            mapped_dict[k] = db_cell_type_mapping[t].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+__all__ = [
+    "map_properties",
+    "map_cells",
+]