unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/notion/types/database.py

@@ -0,0 +1,73 @@
+# https://developers.notion.com/reference/database
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag, Span
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    DBPropertyBase,
+    FromJSONMixin,
+    GetHTMLMixin,
+)
+from unstructured_ingest.processes.connectors.notion.types.database_properties import (
+    map_properties,
+)
+from unstructured_ingest.processes.connectors.notion.types.file import FileObject
+from unstructured_ingest.processes.connectors.notion.types.parent import Parent
+from unstructured_ingest.processes.connectors.notion.types.rich_text import RichText
+from unstructured_ingest.processes.connectors.notion.types.user import PartialUser
+
+
+@dataclass
+class Database(FromJSONMixin, GetHTMLMixin):
+    id: str
+    created_time: str
+    created_by: PartialUser
+    last_edited_time: str
+    last_edited_by: PartialUser
+    archived: bool
+    in_trash: bool
+    parent: Parent
+    url: str
+    is_inline: bool
+    public_url: str
+    request_id: Optional[str] = None
+    properties: Dict[str, DBPropertyBase] = field(default_factory=dict)
+    title: List[RichText] = field(default_factory=list)
+    description: List[RichText] = field(default_factory=list)
+    icon: Optional[FileObject] = None
+    cover: Optional[FileObject] = None
+    object: str = "database"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        created_by = data.pop("created_by")
+        last_edited_by = data.pop("last_edited_by")
+        icon = data.pop("icon")
+        cover = data.pop("cover")
+        parent = data.pop("parent")
+        title = data.pop("title")
+        description = data.pop("description")
+        page = cls(
+            properties=map_properties(data.pop("properties", {})),
+            created_by=PartialUser.from_dict(created_by),
+            last_edited_by=PartialUser.from_dict(last_edited_by),
+            icon=FileObject.from_dict(icon) if icon else None,
+            cover=FileObject.from_dict(cover) if cover else None,
+            parent=Parent.from_dict(parent),
+            title=[RichText.from_dict(data=r) for r in title],
+            description=[RichText.from_dict(data=r) for r in description],
+            **data,
+        )
+
+        return page
+
+    def get_html(self) -> Optional[HtmlTag]:
+        spans = []
+        if title := self.title:
+            spans.append(Span([], [rt.get_html() for rt in title]))
+        if description := self.description:
+            spans.append(Span([], [rt.get_html() for rt in description]))
+        if spans:
+            return Div([], spans)
+        return None

unstructured_ingest/processes/connectors/notion/types/database_properties/__init__.py

@@ -0,0 +1,125 @@
+from typing import Dict
+
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+
+from .checkbox import Checkbox, CheckboxCell
+from .created_by import CreatedBy, CreatedByCell
+from .created_time import CreatedTime, CreatedTimeCell
+from .date import Date, DateCell
+from .email import Email, EmailCell
+from .files import Files, FilesCell
+from .formula import Formula, FormulaCell
+from .last_edited_by import LastEditedBy, LastEditedByCell
+from .last_edited_time import LastEditedTime, LastEditedTimeCell
+from .multiselect import MultiSelect, MultiSelectCell
+from .number import Number, NumberCell
+from .people import People, PeopleCell
+from .phone_number import PhoneNumber, PhoneNumberCell
+from .relation import Relation, RelationCell
+from .rich_text import RichText, RichTextCell
+from .rollup import Rollup, RollupCell
+from .select import Select, SelectCell
+from .status import Status, StatusCell
+from .title import Title, TitleCell
+from .unique_id import UniqueID, UniqueIDCell
+from .url import URL, URLCell
+from .verification import Verification, VerificationCell
+
+# It's possible to add 'button' property to Notion database.
+# However, current Notion API documentation doesn't mention it.
+# Buttons are only functional inside Notion UI. We can simply
+# ignore them so that the we don't throw an error when trying
+# to map 'button' properties.
+unsupported_db_prop_types = ["button"]
+
+db_prop_type_mapping = {
+    "checkbox": Checkbox,
+    "created_by": CreatedBy,
+    "created_time": CreatedTime,
+    "date": Date,
+    "email": Email,
+    "files": Files,
+    "formula": Formula,
+    "last_edited_by": LastEditedBy,
+    "last_edited_time": LastEditedTime,
+    "multi_select": MultiSelect,
+    "number": Number,
+    "people": People,
+    "phone_number": PhoneNumber,
+    "relation": Relation,
+    "rich_text": RichText,
+    "rollup": Rollup,
+    "select": Select,
+    "status": Status,
+    "title": Title,
+    "unique_id": UniqueID,
+    "url": URL,
+    "verification": Verification,
+}
+
+
+def map_properties(props: Dict[str, dict]) -> Dict[str, DBPropertyBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_prop_type_mapping[property_type].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+db_cell_type_mapping = {
+    "checkbox": CheckboxCell,
+    "created_by": CreatedByCell,
+    "created_time": CreatedTimeCell,
+    "date": DateCell,
+    "email": EmailCell,
+    "files": FilesCell,
+    "formula": FormulaCell,
+    "last_edited_by": LastEditedByCell,
+    "last_edited_time": LastEditedTimeCell,
+    "multi_select": MultiSelectCell,
+    "number": NumberCell,
+    "people": PeopleCell,
+    "phone_number": PhoneNumberCell,
+    "relation": RelationCell,
+    "rich_text": RichTextCell,
+    "rollup": RollupCell,
+    "select": SelectCell,
+    "status": StatusCell,
+    "title": TitleCell,
+    "unique_id": UniqueIDCell,
+    "url": URLCell,
+    "verification": VerificationCell,
+}
+
+
+def map_cells(props: Dict[str, dict]) -> Dict[str, DBCellBase]:
+    mapped_dict = {}
+    for k, v in props.items():
+        try:
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_cell_type_mapping[property_type].from_dict(v)  # type: ignore
+        except KeyError as ke:
+            raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
+
+    return mapped_dict
+
+
+__all__ = [
+    "map_properties",
+    "map_cells",
+]

unstructured_ingest/processes/connectors/notion/types/database_properties/checkbox.py

@@ -0,0 +1,39 @@
+# https://developers.notion.com/reference/property-object#checkbox
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.attributes import Checked, Type
+from htmlBuilder.tags import Div, HtmlTag, Input
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class Checkbox(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "checkbox"
+    description: Optional[str] = None
+    checkbox: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CheckboxCell(DBCellBase):
+    id: str
+    checkbox: bool
+    name: Optional[str] = None
+    type: str = "checkbox"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        check_input_attributes = [Type("checkbox")]
+        if self.checkbox:
+            check_input_attributes.append(Checked(""))
+        return Div([], Input(check_input_attributes))

unstructured_ingest/processes/connectors/notion/types/database_properties/created_by.py

@@ -0,0 +1,36 @@
+# https://developers.notion.com/reference/property-object#created-by
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.processes.connectors.notion.types.user import People
+
+
+@dataclass
+class CreatedBy(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "created_by"
+    description: Optional[str] = None
+    created_by: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CreatedByCell(DBCellBase):
+    id: str
+    created_by: People
+    type: str = "created_by"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(created_by=People.from_dict(data.pop("created_by")), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return self.created_by.get_html()

unstructured_ingest/processes/connectors/notion/types/database_properties/created_time.py

@@ -0,0 +1,35 @@
+# https://developers.notion.com/reference/property-object#created-time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class CreatedTime(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "created_time"
+    description: Optional[str] = None
+    created_time: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class CreatedTimeCell(DBCellBase):
+    id: str
+    created_time: str
+    type: str = "created_time"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.created_time)

unstructured_ingest/processes/connectors/notion/types/database_properties/date.py

@@ -0,0 +1,42 @@
+# https://developers.notion.com/reference/property-object#date
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.processes.connectors.notion.types.date import Date as DateType
+
+
+@dataclass
+class Date(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "date"
+    description: Optional[str] = None
+    date: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class DateCell(DBCellBase):
+    id: str
+    date: Optional[DateType] = None
+    name: Optional[str] = None
+    type: str = "date"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        date = None
+        date_data = data.pop("date")
+        if date_data:
+            date = DateType.from_dict(date_data)
+        return cls(date=date, **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if date := self.date:
+            return date.get_html()
+        return None

unstructured_ingest/processes/connectors/notion/types/database_properties/email.py

@@ -0,0 +1,37 @@
+# https://developers.notion.com/reference/property-object#email
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class Email(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "email"
+    description: Optional[str] = None
+    email: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class EmailCell(DBCellBase):
+    id: str
+    email: str
+    name: Optional[str] = None
+    type: str = "email"
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if email := self.email:
+            return Div([], email)
+        return None

unstructured_ingest/processes/connectors/notion/types/database_properties/files.py

@@ -0,0 +1,38 @@
+# https://developers.notion.com/reference/property-object#files
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.processes.connectors.notion.types.file import FileObject
+
+
+@dataclass
+class Files(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "files"
+    description: Optional[str] = None
+    files: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class FilesCell(DBCellBase):
+    id: str
+    files: List[FileObject]
+    type: str = "files"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(files=[FileObject.from_dict(f) for f in data.pop("files", [])], **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.files:
+            return None
+        return Div([], [f.get_html() for f in self.files])

unstructured_ingest/processes/connectors/notion/types/database_properties/formula.py

@@ -0,0 +1,50 @@
+# https://developers.notion.com/reference/property-object#formula
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    DBCellBase,
+    DBPropertyBase,
+    FromJSONMixin,
+)
+
+
+@dataclass
+class FormulaProp(FromJSONMixin):
+    expression: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class Formula(DBPropertyBase):
+    id: str
+    name: str
+    formula: FormulaProp
+    type: str = "formula"
+    description: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(formula=FormulaProp.from_dict(data.pop("formula", {})), **data)
+
+
+@dataclass
+class FormulaCell(DBCellBase):
+    id: str
+    formula: dict
+    type: str = "formula"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        formula = self.formula
+        t = formula.get("type")
+        return Div([], str(formula[t]))

unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_by.py

@@ -0,0 +1,34 @@
+# https://developers.notion.com/reference/property-object#last-edited-by
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.processes.connectors.notion.types.user import People
+
+
+@dataclass
+class LastEditedBy(DBPropertyBase):
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls()
+
+    def get_text(self) -> Optional[str]:
+        return None
+
+
+@dataclass
+class LastEditedByCell(DBCellBase):
+    id: str
+    last_edited_by: People
+    type: str = "last_edited_by"
+    description: Optional[str] = None
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(last_edited_by=People.from_dict(data.pop("last_edited_by", {})), **data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return self.last_edited_by.get_html()

unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_time.py

@@ -0,0 +1,35 @@
+# https://developers.notion.com/reference/property-object#last-edited-time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+
+
+@dataclass
+class LastEditedTime(DBPropertyBase):
+    id: str
+    name: str
+    type: str = "last_edited_time"
+    description: Optional[str] = None
+    last_edited_time: dict = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class LastEditedTimeCell(DBCellBase):
+    id: str
+    last_edited_time: str
+    type: str = "last_edited_time"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        return Div([], self.last_edited_time)

unstructured_ingest/processes/connectors/notion/types/database_properties/multiselect.py

@@ -0,0 +1,74 @@
+# https://developers.notion.com/reference/property-object#multi-select
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from htmlBuilder.attributes import Style
+from htmlBuilder.tags import Div, HtmlTag, Span
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    DBCellBase,
+    DBPropertyBase,
+    FromJSONMixin,
+)
+
+
+@dataclass
+class MultiSelectOption(FromJSONMixin):
+    color: str
+    id: str
+    name: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class MultiSelectProp(FromJSONMixin):
+    options: List[MultiSelectOption] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(options=[MultiSelectOption.from_dict(o) for o in data.get("options", [])])
+
+
+@dataclass
+class MultiSelect(DBPropertyBase):
+    id: str
+    name: str
+    multi_select: MultiSelectProp
+    type: str = "multi_select"
+    description: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            multi_select=data.pop("multi_select", {}),
+            **data,
+        )
+
+
+@dataclass
+class MultiSelectCell(DBCellBase):
+    id: str
+    multi_select: List[MultiSelectOption]
+    type: str = "multi_select"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            multi_select=[MultiSelectOption.from_dict(o) for o in data.pop("multi_select", [])],
+            **data,
+        )
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if not self.multi_select:
+            return None
+        option_spans = []
+        for option in self.multi_select:
+            option_attributes = []
+            if option.color and option.color != "default":
+                option_attributes.append(Style(f"color: {option.color}"))
+            option_spans.append(Span(option_attributes, option.name))
+        return Div([], option_spans)

unstructured_ingest/processes/connectors/notion/types/database_properties/number.py

@@ -0,0 +1,50 @@
+# https://developers.notion.com/reference/property-object#number
+from dataclasses import dataclass
+from typing import Optional
+
+from htmlBuilder.tags import Div, HtmlTag
+
+from unstructured_ingest.processes.connectors.notion.interfaces import (
+    DBCellBase,
+    DBPropertyBase,
+    FromJSONMixin,
+)
+
+
+@dataclass
+class NumberProp(FromJSONMixin):
+    format: str
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+
+@dataclass
+class Number(DBPropertyBase):
+    id: str
+    name: str
+    number: NumberProp
+    type: str = "number"
+    description: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(number=NumberProp.from_dict(data.pop("number")), **data)
+
+
+@dataclass
+class NumberCell(DBCellBase):
+    id: str
+    number: Optional[int] = None
+    type: str = "number"
+    name: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(**data)
+
+    def get_html(self) -> Optional[HtmlTag]:
+        if number := self.number:
+            return Div([], str(number))
+        return None