unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/notion/types/blocks/synced_block.py

@@ -0,0 +1,57 @@
+# https://developers.notion.com/reference/block#synced-block
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class OriginalSyncedBlock(BlockBase):
+    synced_from: Optional[str] = None
+    children: List[dict] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(children=data["children"])
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class DuplicateSyncedBlock(BlockBase):
+    type: str
+    block_id: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+class SyncBlock(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        if "synced_from" in data:
+            return OriginalSyncedBlock.from_dict(data)
+        else:
+            return DuplicateSyncedBlock.from_dict(data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/blocks/table.py

@@ -0,0 +1,63 @@
+# https://developers.notion.com/reference/block#table
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import HtmlTag, Td, Th, Tr
+
+from unstructured_ingest.connector.notion.interfaces import (
+    BlockBase,
+    FromJSONMixin,
+)
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Table(BlockBase):
+    table_width: int
+    has_column_header: bool
+    has_row_header: bool
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None
+
+
+@dataclass
+class TableCell(FromJSONMixin):
+    rich_texts: List[RichText]
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(rich_texts=[RichText.from_dict(rt) for rt in data.pop("rich_texts", [])])
+
+    def get_html(self, is_header: bool) -> Optional[HtmlTag]:
+        if is_header:
+            return Th([], [rt.get_html() for rt in self.rich_texts])
+        else:
+            return Td([], [rt.get_html() for rt in self.rich_texts])
+
+
+# https://developers.notion.com/reference/block#table-rows
+@dataclass
+class TableRow(BlockBase):
+    is_header: bool = False
+    cells: List[TableCell] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        cells = data.get("cells", [])
+        return cls(cells=[TableCell.from_dict({"rich_texts": c}) for c in cells])
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Tr([], [cell.get_html(is_header=self.is_header) for cell in self.cells])

unstructured_ingest/v2/processes/connectors/notion/types/blocks/table_of_contents.py

@@ -0,0 +1,23 @@
+# https://developers.notion.com/reference/block#table-of-contents
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class TableOfContents(BlockBase):
+    color: str
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/blocks/template.py

@@ -0,0 +1,30 @@
+# https://developers.notion.com/reference/block#template
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Template(BlockBase):
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        template = cls(**data)
+        template.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return template
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+        return Div([], [rt.get_html() for rt in self.rich_text])

unstructured_ingest/v2/processes/connectors/notion/types/blocks/todo.py

@@ -0,0 +1,42 @@
+# https://developers.notion.com/reference/block#to-do
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Checked, Style, Type
+from htmlBuilder.tags import Div, HtmlTag, Input
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class ToDo(BlockBase):
+    color: str
+    checked: bool = False
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        todo = cls(**data)
+        todo.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return todo
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+
+        elements = []
+        check_input_attributes = [Type("checkbox")]
+        if self.checked:
+            check_input_attributes.append(Checked(""))
+        elements.append(Input(check_input_attributes))
+        elements.extend([rt.get_html() for rt in self.rich_text])
+        attributes = []
+        if self.color and self.color != "default":
+            attributes.append(Style(f"color: {self.color}"))
+        return Div(attributes, elements)

unstructured_ingest/v2/processes/connectors/notion/types/blocks/toggle.py

@@ -0,0 +1,37 @@
+# https://developers.notion.com/reference/block#toggle-blocks
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Style
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.rich_text import RichText
+
+
+@dataclass
+class Toggle(BlockBase):
+    color: str
+    children: List[dict] = field(default_factory=list)
+    rich_text: List[RichText] = field(default_factory=list)
+
+    @staticmethod
+    def can_have_children() -> bool:
+        return True
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        rich_text = data.pop("rich_text", [])
+        toggle = cls(**data)
+        toggle.rich_text = [RichText.from_dict(rt) for rt in rich_text]
+        return toggle
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.rich_text:
+            return None
+
+        texts = [rt.get_html() for rt in self.rich_text]
+        attributes = []
+        if self.color and self.color != "default":
+            attributes.append(Style(f"color: {self.color}"))
+        return Div(attributes, texts)

unstructured_ingest/v2/processes/connectors/notion/types/blocks/unsupported.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+
+
+@dataclass
+class Unsupported(BlockBase):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/blocks/video.py

@@ -0,0 +1,22 @@
+# https://developers.notion.com/reference/block#image
+from typing import Optional
+
+from htmlBuilder.attributes import Src
+from htmlBuilder.tags import HtmlTag, Source
+from htmlBuilder.tags import Video as VideoHtml
+
+from unstructured_ingest.connector.notion.interfaces import BlockBase
+from unstructured_ingest.connector.notion.types.file import FileObject
+
+
+class Video(BlockBase, FileObject):
+    @staticmethod
+    def can_have_children() -> bool:
+        return False
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if self.external:
+            return VideoHtml([], [Source([Src(self.external.url)], [self.external.url])])
+        if self.file:
+            return VideoHtml([], [Source([Src(self.file.url)], [self.file.url])])
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/database.py

@@ -0,0 +1,73 @@
+# https://developers.notion.com/reference/database
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag, Span
+
+from unstructured_ingest.v2.processes.connectors.notion.interfaces import (
+    DBPropertyBase,
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.v2.processes.connectors.notion.types.database_properties import (
+    map_properties,
+)
+from unstructured_ingest.v2.processes.connectors.notion.types.file import FileObject
+from unstructured_ingest.v2.processes.connectors.notion.types.parent import Parent
+from unstructured_ingest.v2.processes.connectors.notion.types.rich_text import RichText
+from unstructured_ingest.v2.processes.connectors.notion.types.user import PartialUser
+
+
+@dataclass
+class Database(FromJSONMixin, GetHTMLMixin):
+    id: str
+    created_time: str
+    created_by: PartialUser
+    last_edited_time: str
+    last_edited_by: PartialUser
+    archived: bool
+    in_trash: bool
+    parent: Parent
+    url: str
+    is_inline: bool
+    public_url: str
+    request_id: Optional[str] = None
+    properties: Dict[str, DBPropertyBase] = field(default_factory=dict)
+    title: List[RichText] = field(default_factory=list)
+    description: List[RichText] = field(default_factory=list)
+    icon: Optional[FileObject] = None
+    cover: Optional[FileObject] = None
+    object: str = "database"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        created_by = data.pop("created_by")
+        last_edited_by = data.pop("last_edited_by")
+        icon = data.pop("icon")
+        cover = data.pop("cover")
+        parent = data.pop("parent")
+        title = data.pop("title")
+        description = data.pop("description")
+        page = cls(
+            properties=map_properties(data.pop("properties", {})),
+            created_by=PartialUser.from_dict(created_by),
+            last_edited_by=PartialUser.from_dict(last_edited_by),
+            icon=FileObject.from_dict(icon) if icon else None,
+            cover=FileObject.from_dict(cover) if cover else None,
+            parent=Parent.from_dict(parent),
+            title=[RichText.from_dict(data=r) for r in title],
+            description=[RichText.from_dict(data=r) for r in description],
+            **data,
+        )
+
+        return page
+
+    def get_html(self) -> Optional[HtmlTag]:
+        spans = []
+        if title := self.title:
+            spans.append(Span([], [rt.get_html() for rt in title]))
+        if description := self.description:
+            spans.append(Span([], [rt.get_html() for rt in description]))
+        if spans:
+            return Div([], spans)
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/__init__.py

@@ -0,0 +1,106 @@
+from typing import Dict
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+
+from .checkbox import Checkbox, CheckboxCell
+from .created_by import CreatedBy, CreatedByCell
+from .created_time import CreatedTime, CreatedTimeCell
+from .date import Date, DateCell
+from .email import Email, EmailCell
+from .files import Files, FilesCell
+from .formula import Formula, FormulaCell
+from .last_edited_by import LastEditedBy, LastEditedByCell
+from .last_edited_time import LastEditedTime, LastEditedTimeCell
+from .multiselect import MultiSelect, MultiSelectCell
+from .number import Number, NumberCell
+from .people import People, PeopleCell
+from .phone_number import PhoneNumber, PhoneNumberCell
+from .relation import Relation, RelationCell
+from .rich_text import RichText, RichTextCell
+from .rollup import Rollup, RollupCell
+from .select import Select, SelectCell
+from .status import Status, StatusCell
+from .title import Title, TitleCell
+from .unique_id import UniqueID, UniqueIDCell
+from .url import URL, URLCell
+from .verification import Verification, VerificationCell
+
+db_prop_type_mapping = {
+    "checkbox": Checkbox,
+    "created_by": CreatedBy,
+    "created_time": CreatedTime,
+    "date": Date,
+    "email": Email,
+    "files": Files,
+    "formula": Formula,
+    "last_edited_by": LastEditedBy,
+    "last_edited_time": LastEditedTime,
+    "multi_select": MultiSelect,
+    "number": Number,
+    "people": People,
+    "phone_number": PhoneNumber,
+    "relation": Relation,
+    "rich_text": RichText,
+    "rollup": Rollup,
+    "select": Select,
+    "status": Status,
+    "title": Title,
+    "unique_id": UniqueID,
+    "url": URL,
+    "verification": Verification,
+}
+
+
+def map_properties(props: Dict[str, dict]) -> Dict[str, DBPropertyBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            mapped_dict[k] = db_prop_type_mapping[v["type"]].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+db_cell_type_mapping = {
+    "checkbox": CheckboxCell,
+    "created_by": CreatedByCell,
+    "created_time": CreatedTimeCell,
+    "date": DateCell,
+    "email": EmailCell,
+    "files": FilesCell,
+    "formula": FormulaCell,
+    "last_edited_by": LastEditedByCell,
+    "last_edited_time": LastEditedTimeCell,
+    "multi_select": MultiSelectCell,
+    "number": NumberCell,
+    "people": PeopleCell,
+    "phone_number": PhoneNumberCell,
+    "relation": RelationCell,
+    "rich_text": RichTextCell,
+    "rollup": RollupCell,
+    "select": SelectCell,
+    "status": StatusCell,
+    "title": TitleCell,
+    "unique_id": UniqueIDCell,
+    "url": URLCell,
+    "verification": VerificationCell,
+}
+
+
+def map_cells(props: Dict[str, dict]) -> Dict[str, DBCellBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            t = v["type"]
+            mapped_dict[k] = db_cell_type_mapping[t].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+__all__ = [
+    "map_properties",
+    "map_cells",
+]

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/checkbox.py

@@ -0,0 +1,38 @@
+# https://developers.notion.com/reference/property-object#checkbox
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.attributes import Checked, Type
+from htmlBuilder.tags import Div, HtmlTag, Input
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class Checkbox(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "checkbox"
+    checkbox: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CheckboxCell(DBCellBase):
+    id: str
+    checkbox: bool
+    name: Optional[str] = None
+    type: str = "checkbox"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        check_input_attributes = [Type("checkbox")]
+        if self.checkbox:
+            check_input_attributes.append(Checked(""))
+        return Div([], Input(check_input_attributes))

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/created_by.py

@@ -0,0 +1,35 @@
+# https://developers.notion.com/reference/property-object#created-by
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.connector.notion.types.user import People
+
+
+@dataclass
+class CreatedBy(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "created_by"
+    created_by: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CreatedByCell(DBCellBase):
+    id: str
+    created_by: People
+    type: str = "created_by"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(created_by=People.from_dict(data.pop("created_by")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return self.created_by.get_html()

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/created_time.py

@@ -0,0 +1,34 @@
+# https://developers.notion.com/reference/property-object#created-time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class CreatedTime(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "created_time"
+    created_time: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CreatedTimeCell(DBCellBase):
+    id: str
+    created_time: str
+    type: str = "created_time"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.created_time)

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/date.py

@@ -0,0 +1,41 @@
+# https://developers.notion.com/reference/property-object#date
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.connector.notion.types.date import Date as DateType
+
+
+@dataclass
+class Date(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "date"
+    date: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class DateCell(DBCellBase):
+    id: str
+    date: Optional[DateType] = None
+    name: Optional[str] = None
+    type: str = "date"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        date = None
+        date_data = data.pop("date")
+        if date_data:
+            date = DateType.from_dict(date_data)
+        return cls(date=date, **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if date := self.date:
+            return date.get_html()
+        return None

unstructured_ingest/v2/processes/connectors/notion/types/database_properties/email.py

@@ -0,0 +1,36 @@
+# https://developers.notion.com/reference/property-object#email
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.connector.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class Email(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "email"
+    email: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class EmailCell(DBCellBase):
+    id: str
+    email: str
+    name: Optional[str] = None
+    type: str = "email"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if email := self.email:
+            return Div([], email)
+        return None