deepsights-api 0.2.0__py3-none-any.whl

deepsights/documents/__init__.py

@@ -0,0 +1,44 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to retrieve documents from the DeepSights API.
+"""
+
+from deepsights.documents._cache import (
+    set_document,
+    has_document,
+    get_document,
+    remove_document,
+    get_document_cache_size,
+    set_document_page,
+    has_document_page,
+    get_document_page,
+    remove_document_page,
+    get_document_page_cache_size,
+)
+from deepsights.documents.model import (
+    Document,
+    DocumentPage,
+    DocumentPageSearchResult,
+    DocumentSearchResult,
+)
+from deepsights.documents.upload import document_upload, document_wait_for_processing
+from deepsights.documents.download import document_download
+from deepsights.documents.delete import documents_delete, document_wait_for_deletion
+from deepsights.documents.load import (
+    documents_load,
+    document_pages_load,
+)
+from deepsights.documents.search import documents_search, document_pages_search

deepsights/documents/_cache.py

@@ -0,0 +1,39 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to cache documents and document pages.
+"""
+
+from deepsights.utils import create_global_lru_cache
+
+#############################################
+# a global static LRU cache for 1k docs
+(
+    set_document,
+    has_document,
+    get_document,
+    remove_document,
+    get_document_cache_size,
+) = create_global_lru_cache(1000)
+
+#############################################
+# a global static LRU cache for 100k pages
+(
+    set_document_page,
+    has_document_page,
+    get_document_page,
+    remove_document_page,
+    get_document_page_cache_size,
+) = create_global_lru_cache(100000)

deepsights/documents/_segmenter.py

@@ -0,0 +1,113 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to segment the content of a document.
+"""
+
+import re
+from typing import List, Dict
+
+
+#############################################
+def _parse_page(content):
+    """
+    Construct segments as semantic units from the given page structure, guessing from font sizes.
+    Generates markdown for headings.
+
+    Args:
+
+        content (list): The list of tuples representing the page structure, where each tuple contains
+                        the font size and the corresponding text.
+
+    Returns:
+
+        list: The list of segments as semantic units generated from the page structure.
+    """
+
+    # no more content?
+    if len(content) == 0:
+        return []
+
+    # we want to find the next segment now and then recurse on the tail of the content
+    next_segment = ""
+
+    # track what font size we had last
+    last_font_size = None
+
+    # inspect next item
+    while len(content) > 0:
+        (next_font_size, next_text) = next_item = content.pop(0)
+
+        # check font size
+        if last_font_size is not None:
+            # compare to last
+            if next_font_size > 1.4 * last_font_size:
+                # new section
+                content.insert(0, next_item)
+                break
+
+        last_font_size = next_font_size
+
+        # if we have sth to append...
+        if next_text is not None:
+            # append text & remember page
+            next_segment += next_text + "\n"
+
+    # get rid of duplicate whitespace (except LF)
+    segment_text = re.sub(r"[^\S\n]+", " ", next_segment)
+
+    # get rid of 3+ LFs
+    segment_text = re.sub(r"\n{3,}", "\n\n", segment_text, re.MULTILINE)
+
+    # recurse on tail
+    segments = [segment_text]
+    segments.extend(_parse_page(content))
+
+    return segments
+
+
+#############################################
+def segment_landscape_page(page_structure: Dict) -> List[Dict]:
+    """Segmentation strategy that follows pages
+
+    Args:
+
+        page_structure (Dict): The structure of the page.
+
+    Returns:
+
+        List[Dict]: A list of segmented sections.
+
+    """
+    # collect font / text tuples by page
+    content = []
+    if page_structure["semantic_sections"] is not None:
+        for section in page_structure["semantic_sections"]:
+            for element in section["semantic_section_elements"]:
+                text = element["text_value"]["text"].strip()
+
+                font_size = 2 * int(
+                    500
+                    * float(element["normalized_layout"]["height"])
+                    / len(text.split("\n"))
+                )
+
+                content.append((font_size, text))
+
+    # collect segments
+    segments = list(filter(lambda s: len(s) > 40, _parse_page(content)))
+
+    # now merge into one
+    return "\n\n".join(segments)

deepsights/documents/delete.py

@@ -0,0 +1,82 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to delete documents from the DeepSights API.
+"""
+
+import time
+from typing import List
+import requests
+from deepsights.api import DeepSights
+from deepsights.documents._cache import remove_document
+
+
+#################################################
+def documents_delete(api: DeepSights, document_ids: List):
+    """
+    Delete documents from the DeepSights API.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        document_ids (List): A list of document IDs to be deleted.
+    """
+
+    # delete documents one by one
+    for document_id in document_ids:
+        api.delete(
+            f"/artifact-service/artifacts/{document_id}",
+        )
+
+        # remove from cache
+        remove_document(document_id)
+
+
+#################################################
+def document_wait_for_deletion(api: DeepSights, document_id: str, timeout: int = 60):
+    """
+    Wait for the document to be deleted.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        document_id (str): The ID of the document to wait for.
+        timeout (int, optional): The maximum time to wait for the document to be deleted, in seconds. Defaults to 60.
+
+    Raises:
+
+        TimeoutError: If the document fails to delete within the specified timeout.
+        ValueError: If the document deletion fails with an error message.
+    """
+    # wait for completion
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            response = api.get(f"/artifact-service/artifacts/{document_id}")
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                remove_document(document_id)
+                return
+            
+            raise e
+
+        if response["status"] in ("DELETING", "SCHEDULED_FOR_DELETING"):
+            time.sleep(2)
+        elif response["status"].startswith("FAILED"):
+            raise ValueError(
+                f"Document {document_id} failed to delete: {response['error_message']}"
+            )
+
+    raise TimeoutError(f"Document {document_id} failed to delete in {timeout} seconds.")

deepsights/documents/download.py

@@ -0,0 +1,63 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to download documents to the DeepSights API.
+"""
+
+import os
+import urllib.request
+import tempfile
+from deepsights.api import DeepSights
+from deepsights.documents.load import documents_load
+
+
+#################################################
+def document_download(api: DeepSights, document_id: str, local_path: str):
+    """
+    Download a document from the DeepSights API.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        document_id (str): The ID of the document to download.
+        local_path (str): The local path to save the downloaded document in.
+
+    Raises:
+
+        FileNotFoundError: If the local directory does not exist.
+        ValueError: If the document fails to download.
+
+    Returns:
+
+        Tuple[str, str]: A tuple containing the file name and the local path of the downloaded document.
+    """
+    # check if local path exists
+    if not os.path.exists(local_path):
+        raise FileNotFoundError(f"Local directory {local_path} does not exist.")
+
+    # obtain real filename
+    document = documents_load(api, [document_id])[0]
+
+    # obtain download link
+    response = api.get(
+        f"/artifact-service/artifacts/{document_id}/gcs-object-link",
+    )
+
+    # download document to temp file
+    local_filename = tempfile.mktemp(dir=local_path)
+    urllib.request.urlretrieve(response["signed_link"], local_filename)
+
+    # return the filename
+    return document.file_name, local_filename

deepsights/documents/load.py

@@ -0,0 +1,172 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to load documents from the DeepSights API.
+"""
+
+from typing import List
+from deepsights.api import DeepSights
+from deepsights.utils import run_in_parallel
+from deepsights.documents._cache import (
+    get_document,
+    get_document_cache_size,
+    has_document,
+    set_document,
+    get_document_page,
+    get_document_page_cache_size,
+    has_document_page,
+    set_document_page,
+)
+from deepsights.documents.model import Document, DocumentPage
+from deepsights.documents._segmenter import segment_landscape_page
+
+
+#################################################
+def document_pages_load(api: DeepSights, page_ids: List[str]):
+    """
+    Load document pages from the cache or fetch them from the API if not cached.
+
+    Args:
+
+        api (DeepSights): The DeepSights API object.
+        page_ids (List[str]): A list of page IDs to load.
+
+    Returns:
+
+        List[DocumentPage]: A list of loaded document pages.
+    """
+
+    assert (
+        len(page_ids) < get_document_page_cache_size()
+    ), "Cannot load more document pages than the cache size."
+
+    # touch cached document pages
+    for page_id in page_ids:
+        get_document_page(page_id)
+
+    # filter uncached document pages
+    uncached_document_page_ids = [
+        page_id for page_id in page_ids if not has_document_page(page_id)
+    ]
+
+    # load uncached document pages
+    def _load_document_page(page_id: str):
+        result = api.get(f"/artifact-service/pages/{page_id}", timeout=5)
+
+        # map the document page
+        return DocumentPage(
+            id=result["id"],
+            page_number=result["number"],
+            text=segment_landscape_page(result),
+        )
+
+    uncached_document_pages = run_in_parallel(
+        _load_document_page, uncached_document_page_ids, max_workers=5
+    )
+
+    # set in cache
+    for page in uncached_document_pages:
+        set_document_page(page.id, page)
+
+    # collect results
+    return [get_document_page(page_id) for page_id in page_ids]
+
+
+#################################################
+def documents_load(
+    api: DeepSights,
+    document_ids: List[str],
+    force_load: bool = False,
+    load_pages: bool = False,
+):
+    """
+    Load documents from the DeepSights API.
+
+    Args:
+
+        api (DeepSights): The DeepSights API object.
+        document_ids (List[str]): A list of document IDs to load.
+        force_load (bool, optional): Whether to force load the documents, even if in cache. Defaults to False.
+        load_pages (bool, optional): Whether to load the pages of the documents. Defaults to False.
+
+    Returns:
+
+        List[Document]: A list of loaded documents.
+    """
+    assert (
+        len(document_ids) < get_document_cache_size()
+    ), "Cannot load more documents than the cache size."
+
+    # touch cached documents
+    if not force_load:
+        for doc_id in document_ids:
+            get_document(doc_id)
+
+    # filter uncached documents
+    uncached_document_ids = [
+        doc_id for doc_id in document_ids if force_load or not has_document(doc_id)
+    ]
+
+    # load uncached documents
+    def _load_document(document_id: str):
+        result = api.get(f"/artifact-service/artifacts/{document_id}", timeout=5)
+
+        # capitalze the first letter of the summary
+        result["summary"] = result["summary"][0].upper() + result["summary"][1:]
+
+        # map the document
+        return Document.model_validate(result)
+
+    uncached_documents = run_in_parallel(
+        _load_document, uncached_document_ids, max_workers=5
+    )
+
+    # set in cache
+    for doc in uncached_documents:
+        set_document(doc.id, doc)
+
+    # load pages if desired
+    if load_pages:
+        # collect docs that need page loading
+        document_ids_to_load_pages = [
+            doc_id for doc_id in document_ids if not get_document(doc_id).page_ids
+        ]
+
+        # load pages
+        def _load_pages(document_id: str):
+            document = get_document(document_id)
+
+            # load page ids
+            result = api.get(
+                f"/artifact-service/artifacts/{document_id}/page-ids", timeout=5
+            )
+
+            # set page ids
+            document.page_ids = result["ids"]
+
+            return document.page_ids
+
+        page_ids = run_in_parallel(
+            _load_pages, document_ids_to_load_pages, max_workers=5
+        )
+
+        # flatten page ids
+        page_ids = [page_id for page_ids in page_ids for page_id in page_ids]
+
+        # now load actual pages
+        document_pages_load(api, page_ids)
+
+    # collect results
+    return [get_document(doc_id) for doc_id in document_ids]

deepsights/documents/model.py

@@ -0,0 +1,161 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the model classes for documents.
+"""
+
+from typing import List, Optional
+from datetime import datetime
+from pydantic import Field
+from deepsights.utils import DeepSightsIdModel, DeepSightsIdTitleModel
+from deepsights.documents._cache import get_document_page, get_document
+
+
+#################################################
+class DocumentPage(DeepSightsIdModel):
+    """
+    Represents a document page.
+
+    Attributes:
+
+        page_number (Optional[int], optional): The number of the page.
+        text (Optional[str]): The text content of the page (optional).
+    """
+
+    page_number: Optional[int] = Field(
+        default=None, description="The number of the page (one-based)."
+    )
+    text: Optional[str] = Field(
+        default=None, description="The text content of the page."
+    )
+
+
+#################################################
+class Document(DeepSightsIdTitleModel):
+    """
+    Represents a document.
+
+    Attributes:
+
+        status (str, optional): The status of the document.
+        source (str, optional): The source of the document.
+        file_name (str, optional): The name of the file.
+        file_size (int, optional): The size of the file.
+        description (str, optional): The description of the document.
+        timestamp (datetime, optional): The timestamp of the document.
+        page_ids (List[DocumentPage], optional): The list of page IDs in the document.
+        number_of_pages (int, optional): The total number of pages in the document.
+    """
+
+    status: Optional[str] = Field(description="The processing status of the document.")
+    source: Optional[str] = Field(
+        alias="ai_generated_source",
+        default="n/a",
+        description="The human-readable source of the document.",
+    )
+    file_name: Optional[str] = Field(default=None, description="The name of the file.")
+    file_size: Optional[int] = Field(
+        default=None, description="The size of the file in bytes."
+    )
+    description: Optional[str] = Field(
+        alias="summary", description="The human-readable summary of the document."
+    )
+    timestamp: Optional[datetime] = Field(
+        alias="publication_date",
+        default=None,
+        description="The publication timestamp of the document.",
+    )
+    page_ids: List[DocumentPage] = Field(
+        default=None, description="The list of page IDs in the document."
+    )
+    number_of_pages: Optional[int] = Field(
+        alias="total_pages",
+        default=None,
+        description="The total number of pages in the document.",
+    )
+
+    @property
+    def pages(self) -> List[DocumentPage]:
+        return [get_document_page(page_id) for page_id in self.page_ids]
+
+
+#################################################
+class DocumentPageSearchResult(DeepSightsIdModel):
+    """
+    Represents a search result for a page.
+
+    Attributes:
+
+        document_id (str): The ID of the document.
+        score (float): The score of the search result.
+    """
+
+    document_id: str = Field(
+        description="The ID of the document to which the page belongs."
+    )
+    score: float = Field(description="The score of the search result.")
+
+    @property
+    def page_number(self) -> DocumentPage:
+        page = get_document_page(self.id)
+        return page.page_number if page else None
+
+    @property
+    def text(self) -> str:
+        page = get_document_page(self.id)
+        return page.text if page else None
+
+
+#################################################
+class DocumentSearchResult(DeepSightsIdModel):
+    """
+    Represents the search result for a document.
+
+    Attributes:
+
+        page_matches (List[DocumentPageSearchResult]): The search results for each page of the document.
+        score_rank (Optional[int]): The rank of the document based on the score.
+        age_rank (Optional[int]): The rank of the document based on the age.
+        rank (Optional[int]): The overall rank of the document.
+    """
+
+    page_matches: List[DocumentPageSearchResult] = Field(
+        default=[], description="The matching page search results for the document."
+    )
+    rank: Optional[int] = Field(
+        default=None, description="The final rank of the item in the search results."
+    )
+    score_rank: Optional[int] = Field(
+        default=None, description="The rank of the item based on its score."
+    )
+    age_rank: Optional[int] = Field(
+        default=None, description="The rank of the item based on its age; may be None."
+    )
+
+    @property
+    def document(self) -> Document:
+        return get_document(self.id)
+
+    @property
+    def timestamp(self) -> datetime:
+        document = self.document
+        return document.timestamp if document else None
+
+    #############################################
+    def __repr__(self) -> str:
+        if self.document is not None:
+            return f"{self.__class__.__name__}@{self.id}: {self.document.title}"
+
+        return super().__repr__()