unstructured-ingest 1.0.21__py3-none-any.whl → 1.0.23__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.21"  # pragma: no cover
+__version__ = "1.0.23"  # pragma: no cover

unstructured_ingest/embed/mixedbreadai.py

@@ -19,8 +19,7 @@ TRUNCATION_STRATEGY = "end"
 
 
 if TYPE_CHECKING:
-    from mixedbread_ai.client import AsyncMixedbreadAI, MixedbreadAI
-    from mixedbread_ai.core import RequestOptions
+    from mixedbread import AsyncMixedbread, Mixedbread
 
 
 class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
@@ -44,31 +43,33 @@ class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
     )
 
     @requires_dependencies(
-        ["mixedbread_ai"],
-        extras="mixedbreadai",
+        ["mixedbread"],
+        extras="embed-mixedbreadai",
     )
-    def get_client(self) -> "MixedbreadAI":
+    def get_client(self) -> "Mixedbread":
         """
         Create the Mixedbread AI client.
 
         Returns:
-            MixedbreadAI: Initialized client.
+            Mixedbread: Initialized client.
         """
-        from mixedbread_ai.client import MixedbreadAI
+        from mixedbread import Mixedbread
 
-        return MixedbreadAI(
+        return Mixedbread(
             api_key=self.api_key.get_secret_value(),
+            max_retries=MAX_RETRIES,
         )
 
     @requires_dependencies(
-        ["mixedbread_ai"],
-        extras="mixedbreadai",
+        ["mixedbread"],
+        extras="embed-mixedbreadai",
     )
-    def get_async_client(self) -> "AsyncMixedbreadAI":
-        from mixedbread_ai.client import AsyncMixedbreadAI
+    def get_async_client(self) -> "AsyncMixedbread":
+        from mixedbread import AsyncMixedbread
 
-        return AsyncMixedbreadAI(
+        return AsyncMixedbread(
             api_key=self.api_key.get_secret_value(),
+            max_retries=MAX_RETRIES,
         )
 
 
@@ -88,29 +89,20 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
         return self.embed_query(query="Q")
 
     @requires_dependencies(
-        ["mixedbread_ai"],
+        ["mixedbread"],
         extras="embed-mixedbreadai",
     )
-    def get_request_options(self) -> "RequestOptions":
-        from mixedbread_ai.core import RequestOptions
-
-        return RequestOptions(
-            max_retries=MAX_RETRIES,
-            timeout_in_seconds=TIMEOUT,
-            additional_headers={"User-Agent": USER_AGENT},
-        )
-
-    def get_client(self) -> "MixedbreadAI":
+    def get_client(self) -> "Mixedbread":
         return self.config.get_client()
 
-    def embed_batch(self, client: "MixedbreadAI", batch: list[str]) -> list[list[float]]:
-        response = client.embeddings(
+    def embed_batch(self, client: "Mixedbread", batch: list[str]) -> list[list[float]]:
+        response = client.embed(
             model=self.config.embedder_model_name,
+            input=batch,
             normalized=True,
             encoding_format=ENCODING_FORMAT,
-            truncation_strategy=TRUNCATION_STRATEGY,
-            request_options=self.get_request_options(),
-            input=batch,
+            extra_headers={"User-Agent": USER_AGENT},
+            timeout=TIMEOUT,
         )
         return [datum.embedding for datum in response.data]
 
@@ -124,28 +116,19 @@ class AsyncMixedbreadAIEmbeddingEncoder(AsyncBaseEmbeddingEncoder):
         return await self.embed_query(query="Q")
 
     @requires_dependencies(
-        ["mixedbread_ai"],
+        ["mixedbread"],
         extras="embed-mixedbreadai",
     )
-    def get_request_options(self) -> "RequestOptions":
-        from mixedbread_ai.core import RequestOptions
-
-        return RequestOptions(
-            max_retries=MAX_RETRIES,
-            timeout_in_seconds=TIMEOUT,
-            additional_headers={"User-Agent": USER_AGENT},
-        )
-
-    def get_client(self) -> "AsyncMixedbreadAI":
+    def get_client(self) -> "AsyncMixedbread":
         return self.config.get_async_client()
 
-    async def embed_batch(self, client: "AsyncMixedbreadAI", batch: list[str]) -> list[list[float]]:
-        response = await client.embeddings(
+    async def embed_batch(self, client: "AsyncMixedbread", batch: list[str]) -> list[list[float]]:
+        response = await client.embed(
             model=self.config.embedder_model_name,
+            input=batch,
             normalized=True,
             encoding_format=ENCODING_FORMAT,
-            truncation_strategy=TRUNCATION_STRATEGY,
-            request_options=self.get_request_options(),
-            input=batch,
+            extra_headers={"User-Agent": USER_AGENT},
+            timeout=TIMEOUT,
         )
         return [datum.embedding for datum in response.data]

unstructured_ingest/processes/connectors/jira.py

@@ -1,11 +1,11 @@
-import math
 from collections import abc
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Union, cast
+from time import time
+from typing import TYPE_CHECKING, Any, Callable, Generator, List, Optional, Union, cast
 
-from pydantic import Field, Secret
+from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest.data_types.file_data import (
     FileData,
@@ -21,6 +21,7 @@ from unstructured_ingest.interfaces import (
     DownloadResponse,
     Indexer,
     IndexerConfig,
+    download_responses,
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
@@ -37,24 +38,13 @@ DEFAULT_C_SEP = " " * 5
 DEFAULT_R_SEP = "\n"
 
 
-@dataclass
-class JiraIssueMetadata:
+class JiraIssueMetadata(BaseModel):
     id: str
     key: str
-    board_id: Optional[str] = None
 
-    @property
-    def project_id(self) -> str:
+    def get_project_id(self) -> str:
         return self.key.split("-")[0]
 
-    def to_dict(self) -> Dict[str, Union[str, None]]:
-        return {
-            "id": self.id,
-            "key": self.key,
-            "board_id": self.board_id,
-            "project_id": self.project_id,
-        }
-
 
 class FieldGetter(dict):
     def __getitem__(self, key):
@@ -77,52 +67,32 @@ def nested_object_to_field_getter(obj: dict) -> Union[FieldGetter, dict]:
         return obj
 
 
-def issues_fetcher_wrapper(func, results_key="results", number_of_issues_to_fetch: int = 100):
-    """
-    A decorator function that wraps around a function to fetch issues from Jira API in a paginated
-    manner. This is required because the Jira API has a limit of 100 issues per request.
-
-    Args:
-        func (callable): The function to be wrapped. This function should accept `limit` and `start`
-        as keyword arguments.
-        results_key (str, optional): The key in the response dictionary that contains the list of
-        results. Defaults to "results".
-        number_of_issues_to_fetch (int, optional): The total number of issues to fetch. Defaults to
-        100.
-
-    Returns:
-        list: A list of all fetched issues.
-
-    Raises:
-        KeyError: If the response dictionary does not contain the specified `results_key`.
-        TypeError: If the response type from the Jira API is neither list nor dict.
-    """
-
-    def wrapper(*args, **kwargs) -> list:
-        kwargs["limit"] = min(100, number_of_issues_to_fetch)
-        kwargs["start"] = kwargs.get("start", 0)
-
-        all_results = []
-        num_iterations = math.ceil(number_of_issues_to_fetch / kwargs["limit"])
-
-        for _ in range(num_iterations):
-            response = func(*args, **kwargs)
-            if isinstance(response, list):
-                all_results += response
-            elif isinstance(response, dict):
-                if results_key not in response:
-                    raise KeyError(f'Response object is missing "{results_key}" key.')
-                all_results += response[results_key]
-            else:
-                raise TypeError(
-                    f"""Unexpected response type from Jira API.
-                    Response type has to be either list or dict, got: {type(response).__name__}."""
-                )
-            kwargs["start"] += kwargs["limit"]
-
-        return all_results
-
-    return wrapper
+def api_token_based_generator(
+    fn: Callable, key: str = "issues", **kwargs
+) -> Generator[dict, None, None]:
+    nextPageToken = kwargs.pop("nextPageToken", None)
+    while True:
+        resp = fn(nextPageToken=nextPageToken, **kwargs)
+        issues = resp.get(key, [])
+        for issue in issues:
+            yield issue
+        nextPageToken = resp.get("nextPageToken")
+        if not nextPageToken:
+            break
+
+
+def api_page_based_generator(
+    fn: Callable, key: str = "issues", **kwargs
+) -> Generator[dict, None, None]:
+    start = kwargs.pop("start", 0)
+    while True:
+        resp = fn(start=start, **kwargs)
+        issues = resp.get(key, [])
+        if not issues:
+            break
+        for issue in issues:
+            yield issue
+        start += len(issues)
 
 
 class JiraAccessConfig(AccessConfig):
@@ -201,9 +171,17 @@ class JiraConnectionConfig(ConnectionConfig):
 
 
 class JiraIndexerConfig(IndexerConfig):
-    projects: Optional[List[str]] = Field(None, description="List of project keys")
-    boards: Optional[List[str]] = Field(None, description="List of board IDs")
-    issues: Optional[List[str]] = Field(None, description="List of issue keys or IDs")
+    projects: Optional[list[str]] = Field(None, description="List of project keys")
+    boards: Optional[list[str]] = Field(None, description="List of board IDs")
+    issues: Optional[list[str]] = Field(None, description="List of issue keys or IDs")
+    status_filters: Optional[list[str]] = Field(
+        default=None,
+        description="List of status filters, if provided will only return issues that have these statuses",  # noqa: E501
+    )
+
+    def model_post_init(self, context: Any, /) -> None:
+        if not self.projects and not self.boards and not self.issues:
+            raise ValueError("At least one of projects, boards, or issues must be provided.")
 
 
 @dataclass
@@ -228,122 +206,103 @@ class JiraIndexer(Indexer):
             )
         logger.info("Connection to Jira successful.")
 
-    def _get_issues_within_single_project(self, project_key: str) -> List[JiraIssueMetadata]:
+    def _get_issues_within_projects(self) -> Generator[JiraIssueMetadata, None, None]:
         with self.connection_config.get_client() as client:
-            number_of_issues_to_fetch = client.get_project_issues_count(project=project_key)
-            if isinstance(number_of_issues_to_fetch, dict):
-                if "total" not in number_of_issues_to_fetch:
-                    raise KeyError('Response object is missing "total" key.')
-                number_of_issues_to_fetch = number_of_issues_to_fetch["total"]
-            if not number_of_issues_to_fetch:
-                logger.warning(f"No issues found in project: {project_key}. Skipping!")
-                return []
-            get_project_issues = issues_fetcher_wrapper(
-                client.get_all_project_issues,
-                results_key="issues",
-                number_of_issues_to_fetch=number_of_issues_to_fetch,
-            )
-            issues = get_project_issues(project=project_key, fields=["key", "id"])
-            logger.debug(f"Found {len(issues)} issues in project: {project_key}")
-            return [JiraIssueMetadata(id=issue["id"], key=issue["key"]) for issue in issues]
-
-    def _get_issues_within_projects(self) -> List[JiraIssueMetadata]:
-        project_keys = self.index_config.projects
-        if not project_keys:
-            # for when a component list is provided, without any projects
-            if self.index_config.boards or self.index_config.issues:
-                return []
-            # for when no components are provided. all projects will be ingested
-            else:
-                with self.connection_config.get_client() as client:
-                    project_keys = [project["key"] for project in client.projects()]
-        return [
-            issue
-            for project_key in project_keys
-            for issue in self._get_issues_within_single_project(project_key)
-        ]
+            fields = ["key", "id"]
+            jql = "project in ({})".format(", ".join(self.index_config.projects))
+            jql = self._update_jql(jql)
+            for issue in api_token_based_generator(client.enhanced_jql, jql=jql, fields=fields):
+                yield JiraIssueMetadata.model_validate(issue)
 
     def _get_issues_within_single_board(self, board_id: str) -> List[JiraIssueMetadata]:
         with self.connection_config.get_client() as client:
-            get_board_issues = issues_fetcher_wrapper(
-                client.get_issues_for_board,
-                results_key="issues",
-            )
-            issues = get_board_issues(board_id=board_id, fields=["key", "id"], jql=None)
-            logger.debug(f"Found {len(issues)} issues in board: {board_id}")
-            return [
-                JiraIssueMetadata(id=issue["id"], key=issue["key"], board_id=board_id)
-                for issue in issues
-            ]
-
-    def _get_issues_within_boards(self) -> List[JiraIssueMetadata]:
+            fields = ["key", "id"]
+            if self.index_config.status_filters:
+                jql = "status in ({}) ORDER BY id".format(
+                    ", ".join([f'"{s}"' for s in self.index_config.status_filters])
+                )
+            else:
+                jql = "ORDER BY id"
+            for issue in api_page_based_generator(
+                fn=client.get_issues_for_board, board_id=board_id, fields=fields, jql=jql
+            ):
+                yield JiraIssueMetadata.model_validate(issue)
+
+    def _get_issues_within_boards(self) -> Generator[JiraIssueMetadata, None, None]:
         if not self.index_config.boards:
-            return []
-        return [
-            issue
-            for board_id in self.index_config.boards
-            for issue in self._get_issues_within_single_board(board_id)
-        ]
-
-    def _get_issues(self) -> List[JiraIssueMetadata]:
-        with self.connection_config.get_client() as client:
-            issues = [
-                client.get_issue(issue_id_or_key=issue_key, fields=["key", "id"])
-                for issue_key in self.index_config.issues or []
-            ]
-        return [JiraIssueMetadata(id=issue["id"], key=issue["key"]) for issue in issues]
-
-    def get_issues(self) -> List[JiraIssueMetadata]:
-        issues = [
-            *self._get_issues_within_boards(),
-            *self._get_issues_within_projects(),
-            *self._get_issues(),
-        ]
-        # Select unique issues by issue 'id'.
-        # Since boards issues are fetched first,
-        # if there are duplicates, the board issues will be kept,
-        # in order to keep issue 'board_id' information.
-        seen = set()
-        unique_issues: List[JiraIssueMetadata] = []
-        for issue in issues:
-            if issue.id not in seen:
-                unique_issues.append(issue)
-            seen.add(issue.id)
-        return unique_issues
+            yield
+        for board_id in self.index_config.boards:
+            for issue in self._get_issues_within_single_board(board_id=board_id):
+                yield issue
+
+    def _update_jql(self, jql: str) -> str:
+        if self.index_config.status_filters:
+            jql += " and status in ({})".format(
+                ", ".join([f'"{s}"' for s in self.index_config.status_filters])
+            )
+        jql = jql + " ORDER BY id"
+        return jql
 
-    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        from time import time
+    def _get_issues_by_keys(self) -> Generator[JiraIssueMetadata, None, None]:
+        with self.connection_config.get_client() as client:
+            fields = ["key", "id"]
+            jql = "key in ({})".format(", ".join(self.index_config.issues))
+            jql = self._update_jql(jql)
+            for issue in api_token_based_generator(client.enhanced_jql, jql=jql, fields=fields):
+                yield JiraIssueMetadata.model_validate(issue)
+
+    def _create_file_data_from_issue(self, issue: JiraIssueMetadata) -> FileData:
+        # Build metadata
+        metadata = FileDataSourceMetadata(
+            date_processed=str(time()),
+            record_locator=issue.model_dump(),
+        )
 
-        issues = self.get_issues()
-        for issue in issues:
-            # Build metadata
-            metadata = FileDataSourceMetadata(
-                date_processed=str(time()),
-                record_locator=issue.to_dict(),
-            )
+        # Construct relative path and filename
+        filename = f"{issue.id}.txt"
+        relative_path = str(Path(issue.get_project_id()) / filename)
 
-            # Construct relative path and filename
-            filename = f"{issue.id}.txt"
-            relative_path = str(Path(issue.project_id) / filename)
+        source_identifiers = SourceIdentifiers(
+            filename=filename,
+            fullpath=relative_path,
+            rel_path=relative_path,
+        )
 
-            source_identifiers = SourceIdentifiers(
-                filename=filename,
-                fullpath=relative_path,
-                rel_path=relative_path,
-            )
+        file_data = FileData(
+            identifier=issue.id,
+            connector_type=self.connector_type,
+            metadata=metadata,
+            additional_metadata=issue.model_dump(),
+            source_identifiers=source_identifiers,
+        )
+        return file_data
+
+    def get_generators(self) -> List[Callable]:
+        generators = []
+        if self.index_config.boards:
+            generators.append(self._get_issues_within_boards)
+        if self.index_config.issues:
+            generators.append(self._get_issues_by_keys)
+        if self.index_config.projects:
+            generators.append(self._get_issues_within_projects)
+        return generators
 
-            file_data = FileData(
-                identifier=issue.id,
-                connector_type=self.connector_type,
-                metadata=metadata,
-                additional_metadata=issue.to_dict(),
-                source_identifiers=source_identifiers,
-            )
-            yield file_data
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        seen_keys = []
+        for gen in self.get_generators():
+            for issue in gen():
+                if not issue:
+                    continue
+                if issue.key in seen_keys:
+                    continue
+                seen_keys.append(issue.key)
+                yield self._create_file_data_from_issue(issue=issue)
 
 
 class JiraDownloaderConfig(DownloaderConfig):
-    pass
+    download_attachments: bool = Field(
+        default=False, description="If True, will download any attachments and process as well"
+    )
 
 
 @dataclass
@@ -448,7 +407,56 @@ class JiraDownloader(Downloader):
             logger.error(f"Failed to fetch issue with key: {issue_key}: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to fetch issue with key: {issue_key}: {e}")
 
-    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+    def generate_attachment_file_data(
+        self, attachment_dict: dict, parent_filedata: FileData
+    ) -> FileData:
+        new_filedata = parent_filedata.model_copy(deep=True)
+        if new_filedata.metadata.record_locator is None:
+            new_filedata.metadata.record_locator = {}
+        new_filedata.metadata.record_locator["parent_issue"] = (
+            parent_filedata.metadata.record_locator["id"]
+        )
+        # Append an identifier for attachment to not conflict with issue ids
+        new_filedata.identifier = "{}a".format(attachment_dict["id"])
+        filename = attachment_dict["filename"]
+        new_filedata.metadata.filesize_bytes = attachment_dict.pop("size", None)
+        new_filedata.metadata.date_created = attachment_dict.pop("created", None)
+        new_filedata.metadata.url = attachment_dict.pop("self", None)
+        new_filedata.metadata.record_locator = attachment_dict
+        new_filedata.source_identifiers = SourceIdentifiers(
+            filename=filename,
+            fullpath=(Path(str(attachment_dict["id"])) / Path(filename)).as_posix(),
+        )
+        return new_filedata
+
+    def process_attachments(
+        self, file_data: FileData, attachments: list[dict]
+    ) -> list[DownloadResponse]:
+        with self.connection_config.get_client() as client:
+            download_path = self.get_download_path(file_data)
+            attachment_download_dir = download_path.parent / "attachments"
+            attachment_download_dir.mkdir(parents=True, exist_ok=True)
+            download_responses = []
+            for attachment in attachments:
+                attachment_filename = Path(attachment["filename"])
+                attachment_id = attachment["id"]
+                attachment_download_path = attachment_download_dir / Path(
+                    attachment_id
+                ).with_suffix(attachment_filename.suffix)
+                resp = client.get_attachment_content(attachment_id=attachment_id)
+                with open(attachment_download_path, "wb") as f:
+                    f.write(resp)
+                attachment_filedata = self.generate_attachment_file_data(
+                    attachment_dict=attachment, parent_filedata=file_data
+                )
+                download_responses.append(
+                    self.generate_download_response(
+                        file_data=attachment_filedata, download_path=attachment_download_path
+                    )
+                )
+        return download_responses
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
         issue_key = file_data.additional_metadata.get("key")
         if not issue_key:
             raise ValueError("Issue key not found in metadata.")
@@ -463,7 +471,17 @@ class JiraDownloader(Downloader):
         with open(download_path, "w") as f:
             f.write(issue_str)
         self.update_file_data(file_data, issue)
-        return self.generate_download_response(file_data=file_data, download_path=download_path)
+        download_response = self.generate_download_response(
+            file_data=file_data, download_path=download_path
+        )
+        if self.download_config.download_attachments and (
+            attachments := issue.get("fields", {}).get("attachment")
+        ):
+            attachment_responses = self.process_attachments(
+                file_data=file_data, attachments=attachments
+            )
+            download_response = [download_response] + attachment_responses
+        return download_response
 
 
 jira_source_entry = SourceRegistryEntry(

unstructured_ingest/processes/connectors/notion/types/database_properties/__init__.py

@@ -1,5 +1,6 @@
 from typing import Dict
 
+from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
 
 from .checkbox import Checkbox, CheckboxCell
@@ -25,6 +26,13 @@ from .unique_id import UniqueID, UniqueIDCell
 from .url import URL, URLCell
 from .verification import Verification, VerificationCell
 
+# It's possible to add 'button' property to Notion database.
+# However, current Notion API documentation doesn't mention it.
+# Buttons are only functional inside Notion UI. We can simply
+# ignore them so that the we don't throw an error when trying
+# to map 'button' properties.
+unsupported_db_prop_types = ["button"]
+
 db_prop_type_mapping = {
     "checkbox": Checkbox,
     "created_by": CreatedBy,
@@ -55,7 +63,13 @@ def map_properties(props: Dict[str, dict]) -> Dict[str, DBPropertyBase]:
     mapped_dict = {}
     for k, v in props.items():
         try:
-            mapped_dict[k] = db_prop_type_mapping[v["type"]].from_dict(v)  # type: ignore
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_prop_type_mapping[property_type].from_dict(v)  # type: ignore
         except KeyError as ke:
             raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
 
@@ -92,8 +106,13 @@ def map_cells(props: Dict[str, dict]) -> Dict[str, DBCellBase]:
     mapped_dict = {}
     for k, v in props.items():
         try:
-            t = v["type"]
-            mapped_dict[k] = db_cell_type_mapping[t].from_dict(v)  # type: ignore
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_cell_type_mapping[property_type].from_dict(v)  # type: ignore
         except KeyError as ke:
             raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
 

unstructured_ingest/processes/connectors/notion/types/database_properties/checkbox.py

@@ -13,6 +13,7 @@ class Checkbox(DBPropertyBase):
     id: str
     name: str
     type: str = "checkbox"
+    description: Optional[str] = None
     checkbox: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/created_by.py

@@ -13,6 +13,7 @@ class CreatedBy(DBPropertyBase):
     id: str
     name: str
     type: str = "created_by"
+    description: Optional[str] = None
     created_by: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/created_time.py

@@ -12,6 +12,7 @@ class CreatedTime(DBPropertyBase):
     id: str
     name: str
     type: str = "created_time"
+    description: Optional[str] = None
     created_time: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/date.py

@@ -13,6 +13,7 @@ class Date(DBPropertyBase):
     id: str
     name: str
     type: str = "date"
+    description: Optional[str] = None
     date: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/email.py

@@ -12,6 +12,7 @@ class Email(DBPropertyBase):
     id: str
     name: str
     type: str = "email"
+    description: Optional[str] = None
     email: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/files.py

@@ -13,6 +13,7 @@ class Files(DBPropertyBase):
     id: str
     name: str
     type: str = "files"
+    description: Optional[str] = None
     files: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/formula.py

@@ -26,6 +26,7 @@ class Formula(DBPropertyBase):
     name: str
     formula: FormulaProp
     type: str = "formula"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_by.py

@@ -23,7 +23,7 @@ class LastEditedByCell(DBCellBase):
     id: str
     last_edited_by: People
     type: str = "last_edited_by"
-
+    description: Optional[str] = None
     name: Optional[str] = None
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_time.py

@@ -12,6 +12,7 @@ class LastEditedTime(DBPropertyBase):
     id: str
     name: str
     type: str = "last_edited_time"
+    description: Optional[str] = None
     last_edited_time: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/multiselect.py

@@ -38,6 +38,7 @@ class MultiSelect(DBPropertyBase):
     name: str
     multi_select: MultiSelectProp
     type: str = "multi_select"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/number.py

@@ -26,6 +26,7 @@ class Number(DBPropertyBase):
     name: str
     number: NumberProp
     type: str = "number"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/people.py

@@ -14,6 +14,7 @@ class People(DBPropertyBase):
     name: str
     description: Optional[str] = None
     type: str = "people"
+    description: Optional[str] = None
     people: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/phone_number.py

@@ -12,6 +12,7 @@ class PhoneNumber(DBPropertyBase):
     id: str
     name: str
     type: str = "phone_number"
+    description: Optional[str] = None
     phone_number: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/relation.py

@@ -45,6 +45,7 @@ class Relation(DBPropertyBase):
     name: str
     relation: RelationProp
     type: str = "relation"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/rich_text.py

@@ -15,6 +15,7 @@ class RichText(DBPropertyBase):
     id: str
     name: str
     type: str = "rich_text"
+    description: Optional[str] = None
     rich_text: dict = field(default_factory=dict)
 
     @classmethod

unstructured_ingest/processes/connectors/notion/types/database_properties/rollup.py

@@ -30,6 +30,7 @@ class Rollup(DBPropertyBase):
     name: str
     rollup: RollupProp
     type: str = "rollup"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/select.py

@@ -38,8 +38,8 @@ class Select(DBPropertyBase):
     id: str
     name: str
     select: SelectProp
-    description: Optional[str] = None
     type: str = "select"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/status.py

@@ -55,6 +55,7 @@ class Status(DBPropertyBase):
     name: str
     status: StatusProp
     type: str = "status"
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/title.py

@@ -14,6 +14,7 @@ class Title(DBPropertyBase):
     name: str
     type: str = "title"
     title: dict = field(default_factory=dict)
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/unique_id.py

@@ -17,6 +17,7 @@ class UniqueID(DBPropertyBase):
     name: str
     type: str = "unique_id"
     unique_id: dict = field(default_factory=dict)
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/url.py

@@ -14,6 +14,7 @@ class URL(DBPropertyBase):
     name: str
     type: str = "url"
     url: dict = field(default_factory=dict)
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

unstructured_ingest/processes/connectors/notion/types/database_properties/verification.py

@@ -20,6 +20,7 @@ class Verification(DBPropertyBase):
     name: str
     type: str = "verification"
     verification: dict = field(default_factory=dict)
+    description: Optional[str] = None
 
     @classmethod
     def from_dict(cls, data: dict):

{unstructured_ingest-1.0.21.dist-info → unstructured_ingest-1.0.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.21
+Version: 1.0.23
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -114,7 +114,7 @@ Requires-Dist: unstructured[md]; extra == 'md'
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == 'milvus'
 Provides-Extra: mixedbreadai
-Requires-Dist: mixedbread-ai; extra == 'mixedbreadai'
+Requires-Dist: mixedbread; extra == 'mixedbreadai'
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == 'mongodb'
 Provides-Extra: msg

{unstructured_ingest-1.0.21.dist-info → unstructured_ingest-1.0.23.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=_fAo4tbdJV7k_s1lgXUPPmLFVpxbTy7HhoN9KbPxQ4Y,43
+unstructured_ingest/__version__.py,sha256=xbdPxvOGZJUW_s_LZYTaPijNvLNKSjZuHlwNDGHpDjE,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -26,7 +26,7 @@ unstructured_ingest/embed/azure_openai.py,sha256=fk9yTG-Xr1TSu4n4l8O3DQo9-oceVL9
 unstructured_ingest/embed/bedrock.py,sha256=dzfCsatB0i8hUp1YnXmoImoxgvUdZ4srKI6eSvn-lYM,9132
 unstructured_ingest/embed/huggingface.py,sha256=6Gx9L3xa3cv9fX4AMuLsePJQF4T_jwkKjovfqF5X1NM,2435
 unstructured_ingest/embed/interfaces.py,sha256=Y3PLhgWnMDmtpugE37hlAiBIbC8izrFFXXkrPVby-HY,5137
-unstructured_ingest/embed/mixedbreadai.py,sha256=pmpGQ0E-bfkkg4rvPvsFxL6Oc7H5f0mJGguHtfL7oLc,4592
+unstructured_ingest/embed/mixedbreadai.py,sha256=uKTqzoi4M_WeYZu-qc_TSxwJONOESzxVbBLUbD1Wbns,3922
 unstructured_ingest/embed/octoai.py,sha256=yZuD7R4mEKS4Jjyae_IrNWogMPOFFS8gW5oUllj3ROU,4540
 unstructured_ingest/embed/openai.py,sha256=TMEOPVfm_OSs4tb3Ymd6q5J49R_-YKvO4TOqCHb3bwk,4647
 unstructured_ingest/embed/togetherai.py,sha256=EehrzTRx4sd_P6AG9JkHAGwTG-o93GMaV5ufmJaxKWs,3629
@@ -73,7 +73,7 @@ unstructured_ingest/processes/connectors/discord.py,sha256=6yEJ_agfKUqsV43wFsbMk
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=6h1CdqznJmzeWxGfXrFLdNdT23PExGnUMMX7usK_4Kk,10013
 unstructured_ingest/processes/connectors/google_drive.py,sha256=BIFBZGp26JlBBOcXy5Gq0UoNzWv6pwRKhEAHMVMI2_M,25050
-unstructured_ingest/processes/connectors/jira.py,sha256=alnwUYyID-mUIlGq1xh5QGEw2iZ2RwbOIyptev3dI6Q,18011
+unstructured_ingest/processes/connectors/jira.py,sha256=Hw07c2HT2vA2l2wpoYWXPNtLbnWreXCIRimAxm0Gfpw,19055
 unstructured_ingest/processes/connectors/kdbai.py,sha256=XhxYpKSAoFPBsDQWwNuLX03DCxOVr7yquj9VYM55Rtc,5174
 unstructured_ingest/processes/connectors/local.py,sha256=LluTLKv4g7FbJb4A6vuSxI9VhzKZuuQUpDS-cVNAQ2g,7426
 unstructured_ingest/processes/connectors/milvus.py,sha256=Jr9cul7By03tGAPFnFBoqncnNWwbhKd-qbmkuqnin8U,8908
@@ -174,29 +174,29 @@ unstructured_ingest/processes/connectors/notion/types/blocks/todo.py,sha256=Kiga
 unstructured_ingest/processes/connectors/notion/types/blocks/toggle.py,sha256=6ae_eR3SOfUgTw-XO_F3JRBaczSp8UZfLBFMRMO5NHo,1188
 unstructured_ingest/processes/connectors/notion/types/blocks/unsupported.py,sha256=q_p9XH8sQB8xwFqi9yEl6Fvur3fTLdeVdQCh0gSju58,442
 unstructured_ingest/processes/connectors/notion/types/blocks/video.py,sha256=XK-O7XPs5ejTUWrg2FTLvbOZajs-yDtVhR79HSEcxvo,779
-unstructured_ingest/processes/connectors/notion/types/database_properties/__init__.py,sha256=iUdtLGlHe52daXBWVlGghXcGSxCOCDiFASsuKb4_UAM,3225
-unstructured_ingest/processes/connectors/notion/types/database_properties/checkbox.py,sha256=1GiebXsYYoQmM1GZJBrzv9dnM9P9dtQJ-dwbQDo6PdI,1010
-unstructured_ingest/processes/connectors/notion/types/database_properties/created_by.py,sha256=NvOuM_1SVBdn-6acYxKJ1ThMuWJ935aYtaKV1TOBTFQ,949
-unstructured_ingest/processes/connectors/notion/types/database_properties/created_time.py,sha256=HPf6Dp7wWWRQ-j7AGFWMm-wkakuJ4R8_rfz9RPNZuec,834
-unstructured_ingest/processes/connectors/notion/types/database_properties/date.py,sha256=5nYyh8S9HzFyZb2bZZttSZj8CE0Q2dc73cA27m-H1k4,1067
-unstructured_ingest/processes/connectors/notion/types/database_properties/email.py,sha256=0powD6-ZftD0WSHxjTHMQ388RK-WfWLaYuEJIT19mdM,831
-unstructured_ingest/processes/connectors/notion/types/database_properties/files.py,sha256=1-Jnx1YyWptNW3737oOzr7WGItS7aNj68BYaWyBRljo,1020
-unstructured_ingest/processes/connectors/notion/types/database_properties/formula.py,sha256=LV9d6NRLGu7eIr6wpfnkLxgQug107LGAFC1Z9ooc3xw,1069
-unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_by.py,sha256=1TMFWA5GIb6pJbYAF_Q2ZPOWNa9y0p8ELk6UzLMZ2FY,926
-unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_time.py,sha256=MvZgXTBx1uS-vn9vITFCIRPFnV26_rcao1YhBiZu5bU,864
-unstructured_ingest/processes/connectors/notion/types/database_properties/multiselect.py,sha256=WnEJyDmEaI39ETopeoXwFp6Iog-rF2TYWG3o-DRv2ic,1917
-unstructured_ingest/processes/connectors/notion/types/database_properties/number.py,sha256=LbrTryg5z8Q1dLekxwXi_qCvOwH6n_LD4iVYsVBNGB4,1056
-unstructured_ingest/processes/connectors/notion/types/database_properties/people.py,sha256=LXNbhBo53Lf7jM_bLYgarwLao7ymEJjZhJ-U5xMBbLc,1184
-unstructured_ingest/processes/connectors/notion/types/database_properties/phone_number.py,sha256=NoIVkd31mvvZYa0DOkE4JibJdhJjIEL8C78jNxcxtVo,909
-unstructured_ingest/processes/connectors/notion/types/database_properties/relation.py,sha256=e4zQmjtP2w1RNfKDaWu2-AOhwjM_QP_TrjyxAQUnpN4,1528
-unstructured_ingest/processes/connectors/notion/types/database_properties/rich_text.py,sha256=EeWvzLx9zYT3crH_eshAOf1L60k5gMqJUqQGiI-is7c,1171
-unstructured_ingest/processes/connectors/notion/types/database_properties/rollup.py,sha256=xyaA2yIvDfe-tu_TXi6ZjCz2jCawFS2DVKvfSr-nxjQ,1277
-unstructured_ingest/processes/connectors/notion/types/database_properties/select.py,sha256=Qy-t9kmJq5cZKohxNX2DbvIftX95CvDX6JMDVWzVYl4,1794
-unstructured_ingest/processes/connectors/notion/types/database_properties/status.py,sha256=GneqbmGaBqOecoApH6I4YWf5flvuVk2m5wMJrRw7Nvo,2044
-unstructured_ingest/processes/connectors/notion/types/database_properties/title.py,sha256=4Tnfxxx_Nr-Wkwd7urotp1xTGO9itWe8qI3OnfooLrE,1023
-unstructured_ingest/processes/connectors/notion/types/database_properties/unique_id.py,sha256=i95msSj2-l3yG8tw02dIChADDEy695MWmNWXJq0jGD4,1173
-unstructured_ingest/processes/connectors/notion/types/database_properties/url.py,sha256=azWD7VwkYlI4vFYgsuMvr2618skr4MCtagSTuJ509bg,873
-unstructured_ingest/processes/connectors/notion/types/database_properties/verification.py,sha256=bge_vjWUYh9-CJ_gmVg7ESVLAxHeq_wicp-6vlqNRwQ,2367
+unstructured_ingest/processes/connectors/notion/types/database_properties/__init__.py,sha256=6kUXmCI58R1e50b1U-_xqrrPw3g2Mqtbt02aC7DVAxw,4118
+unstructured_ingest/processes/connectors/notion/types/database_properties/checkbox.py,sha256=uS2B4nQ-ISt8QGxw7nNwst8MX5xRTecSvqokZ23DKyA,1048
+unstructured_ingest/processes/connectors/notion/types/database_properties/created_by.py,sha256=RcpjFijEwyuGrPhSjrXT1nxaLoX2mnCvjveZ0f5Ke3c,987
+unstructured_ingest/processes/connectors/notion/types/database_properties/created_time.py,sha256=eJjkK1nKb0-Ohi4lpCplbrUTkCgf4D2gWbFxEhDI_G8,872
+unstructured_ingest/processes/connectors/notion/types/database_properties/date.py,sha256=DECAkkhR6qQ-WKsOzQf2VPdYGcyrnAJNk4y4JHDVDuc,1105
+unstructured_ingest/processes/connectors/notion/types/database_properties/email.py,sha256=yOHIV_fpF9xzqcrkRIC4cF_aC8C7RsJJvRtEgSn30a8,869
+unstructured_ingest/processes/connectors/notion/types/database_properties/files.py,sha256=5_6FpFWoKNriaBRLtNDRUxu1ZO1UTvAFeu4H55VNY68,1058
+unstructured_ingest/processes/connectors/notion/types/database_properties/formula.py,sha256=zcTeVuXpmuMNh4FHJHW5zgKWAqo0Wx7s9UsSEvA_wR8,1107
+unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_by.py,sha256=WpWlXz9AwS1rugpvoDoVOo055dVEAt3XmvudD17HJu8,963
+unstructured_ingest/processes/connectors/notion/types/database_properties/last_edited_time.py,sha256=kvmMmEXj3WndR9BG9MHwuM40luA4XhGfnF6rKDpYiF0,902
+unstructured_ingest/processes/connectors/notion/types/database_properties/multiselect.py,sha256=LSDfUXpgUox7Z77_TKIlKHqYPUgO8Y06lVgvju6NXx8,1955
+unstructured_ingest/processes/connectors/notion/types/database_properties/number.py,sha256=y4ocq8_yX_yKkAtM3qcqIueM9y96-47gshM2mra_tgw,1094
+unstructured_ingest/processes/connectors/notion/types/database_properties/people.py,sha256=HBTbqw1L1h8XHKuuS0e0aoz0dAZXSLDy7zRwM1_rRps,1222
+unstructured_ingest/processes/connectors/notion/types/database_properties/phone_number.py,sha256=mBEcwNLCI-FLU6t2FqR_tNTvrJFIQ7hqYeTB51HavBc,947
+unstructured_ingest/processes/connectors/notion/types/database_properties/relation.py,sha256=fYYyzBhi1fmXrhdxu6W6uMr2e6HaDCfrvY7yZIFvgmM,1566
+unstructured_ingest/processes/connectors/notion/types/database_properties/rich_text.py,sha256=D2y-98U7MP-YmsrAPeT4vqG3m7HB4zoOzMMhhYN8VHY,1209
+unstructured_ingest/processes/connectors/notion/types/database_properties/rollup.py,sha256=r8UXCW7Y-eE5W6RpXiyyMszCMRDtiwBmYOmYHZ_9-VY,1315
+unstructured_ingest/processes/connectors/notion/types/database_properties/select.py,sha256=-UAIuddoyKol45epuOYNlS8dchuwL0wMGwash4BwuH4,1794
+unstructured_ingest/processes/connectors/notion/types/database_properties/status.py,sha256=kUFZsWGQZAApEbs5qI37t8LPN0vUM5vcu4pPbEvIGkE,2082
+unstructured_ingest/processes/connectors/notion/types/database_properties/title.py,sha256=yd1vPbCBgGIbtUsC3zOu3-Cdpcst0dEkuFVdtS97hxA,1061
+unstructured_ingest/processes/connectors/notion/types/database_properties/unique_id.py,sha256=e8WslwVD6ccf4x_3NihX4BWtH7y4zMAFH7Ur4jS3dH8,1211
+unstructured_ingest/processes/connectors/notion/types/database_properties/url.py,sha256=Bxu8x7mmH28l2nQSaAmygal8dZdUdHEFbUYIk75B0iQ,911
+unstructured_ingest/processes/connectors/notion/types/database_properties/verification.py,sha256=B9J4zXHVJzzIcn_QD04Z9eibEij1ornw5RhZ2qmUdDU,2405
 unstructured_ingest/processes/connectors/qdrant/__init__.py,sha256=7WN_3M3qQ0O7pUJSXIKtPqAvKX2tQ_WxClCHbFeqPfc,757
 unstructured_ingest/processes/connectors/qdrant/cloud.py,sha256=H5Plp2xqFheESLertj56o78CL4exyCQhBDE1TGAzcWU,1618
 unstructured_ingest/processes/connectors/qdrant/local.py,sha256=3b43kSVoGMcFWTRiIHMPcctKyVBdsaLi8KXloAwq76o,1582
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.21.dist-info/METADATA,sha256=lYMmxWJ0ySauI_NWrAQo4YZQ7pXAK4bZ0dX0XIsgacE,8694
-unstructured_ingest-1.0.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.21.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.21.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.21.dist-info/RECORD,,
+unstructured_ingest-1.0.23.dist-info/METADATA,sha256=b0LZ3XzhlhUgDZd4mEUPxxhOT-lqKAOnDfiQeJhCgoA,8691
+unstructured_ingest-1.0.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.23.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.23.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.23.dist-info/RECORD,,