deepsights-api 0.2.0__py3-none-any.whl

deepsights/documents/search.py

@@ -0,0 +1,182 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to search for documents and document pages based on their vector embeddings.
+"""
+
+from typing import List
+from deepsights.api import DeepSights
+from deepsights.utils import rerank_by_recency, promote_exact_matches
+from deepsights.documents.model import DocumentPageSearchResult, DocumentSearchResult
+from deepsights.documents.load import documents_load, document_pages_load
+
+
+#################################################
+def document_pages_search(
+    api: DeepSights,
+    query_embedding: List,
+    min_score: float = 0.7,
+    max_results: int = 50,
+    load_pages=False,
+):
+    """
+    Searches for document pages based on their vector embeddings.
+
+    Args:
+
+        api (ds.DeepSights): The DeepSights API instance.
+        query_embedding (List): The query vector embedding.
+        min_score (float, optional): The minimum score threshold for search results. Defaults to 0.7.
+        max_results (int, optional): The maximum number of search results to return. Defaults to 50.
+        load_pages (bool, optional): Whether to load the pages associated with the search results. Defaults to False.
+
+    Returns:
+
+        List[DocumentPageSearchResult]: The list of DocumentPageSearchResult objects representing the search results.
+    """
+    assert query_embedding, "The 'query_embedding' argument is required."
+    assert len(query_embedding) == 1536, "The 'query_embedding' must be of length 1536."
+    assert 0 <= min_score <= 1, "The 'min_score' must be between 0 and 1."
+    assert 0 < max_results <= 100, "Maximum results must be between 1 and 100."
+
+    body = {
+        "embeddings": query_embedding,
+        "min_score": min_score,
+        "limit": max_results,
+    }
+    params = {"ai_model": "ADA", "search_model": "PAGE"}
+    response = api.post(
+        "vector-search-service/vectors/_search", params=params, body=body
+    )
+
+    # parse
+    results = [
+        DocumentPageSearchResult(
+            document_id=d["artifact_id"], id=p["part_id"], score=p["score"]
+        )
+        for d in response["results"]
+        for p in d["result_parts"]
+    ]
+
+    # make sure we are sorted by score
+    results.sort(key=lambda x: x.score, reverse=True)
+
+    # load pages if requested
+    if load_pages:
+        # make sure pages are loaded
+        document_pages_load(api, page_ids=[r.id for r in results])
+
+    return results
+
+
+#################################################
+def documents_search(
+    api: DeepSights,
+    query: str = None,
+    query_embedding: List = None,
+    min_score: float = 0.7,
+    max_results: int = 50,
+    recency_weight: float = None,
+    promote_exact_match: bool = False,
+    load_documents=False,
+):
+    """
+    Searches for document based on their vector embeddings.
+
+    Args:
+
+        api (ds.DeepSights): The DeepSights API instance.
+        query (str): The search query; currently only used for promoting exact matches.
+        query_embedding (List): The query vector embedding.
+        min_score (float, optional): The minimum score threshold for document matches. Defaults to 0.7.
+        max_results (int, optional): The maximum number of document matches to return. Defaults to 50.
+        recency_weight (float, optional): The weight to apply to the recency factor in ranking. Defaults to None, i.e. no recency weighting.
+        promote_exact_match (bool, optional): Whether to promote exact matches to the top of the search results. Defaults to False.
+        load_documents (bool, optional): Whether to load documents and matching pages associated with the search results. Defaults to False.
+
+    Returns:
+
+        List: The DocumentSearchResults.
+    """
+    assert query_embedding, "The 'query_embedding' argument is required."
+    assert len(query_embedding) == 1536, "The 'query_embedding' must be of length 1536."
+    assert 0 <= min_score <= 1, "The 'min_score' must be between 0 and 1."
+    assert 0 < max_results <= 100, "Maximum results must be between 1 and 100."
+    assert (
+        recency_weight is None or 0 <= recency_weight <= 1
+    ), "Recency weight must be between 0 and 1."
+    assert (
+        query is None or promote_exact_matches
+    ), "The 'query' argument is only used when 'promote_exact_match' is set to True."
+
+    # get the page matches
+    page_matches = document_pages_search(
+        api,
+        query_embedding,
+        min_score=min_score,
+        max_results=max_results,
+        load_pages=load_documents,
+    )
+
+    # calculate aggregated document rank score
+    document_rank_score = {}
+    for rank, page in enumerate(page_matches):
+        document_rank_score[page.document_id] = document_rank_score.get(
+            page.document_id, 0
+        ) + 1.0 / (rank + max_results / 2)
+
+    document_rank_score = {
+        k: v
+        for k, v in sorted(
+            document_rank_score.items(), key=lambda item: item[1], reverse=True
+        )
+    }
+
+    # now construct the document matches in rank order
+    results = [
+        DocumentSearchResult(
+            id=document_id,
+            score_rank=rank + 1,
+            page_matches=[p for p in page_matches if p.document_id == document_id],
+        )
+        for rank, document_id in enumerate(document_rank_score)
+    ]
+
+    # load documents if requested
+    if load_documents or recency_weight:
+        # make sure the documents are loaded
+        documents_load(api, document_ids=[r.id for r in results])
+
+        # load pages if requested
+        if load_documents:
+            page_ids = [p.id for r in results for p in r.page_matches]
+            document_pages_load(api, page_ids=page_ids)
+
+            # order pages by their number
+            for r in results:
+                r.page_matches.sort(key=lambda x: x.page_number)
+
+        # apply recency weight
+        results = rerank_by_recency(results, recency_weight=recency_weight)
+
+    # pull exact matches to the top
+    if promote_exact_match and query:
+        results = promote_exact_matches(query, results)
+
+    # record rank
+    for rank, result in enumerate(results):
+        result.rank = rank + 1
+
+    return results

deepsights/documents/upload.py

@@ -0,0 +1,130 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to upload documents to the DeepSights API.
+"""
+
+import os
+import time
+import requests
+from deepsights.api import DeepSights
+from deepsights.documents.load import documents_load
+
+
+#################################################
+def document_upload(api: DeepSights, document_filename: str):
+    """
+    Upload a document to the DeepSights API.
+
+    Args:
+
+        document_filename (str): The filename of the document to upload. Must be PDF, PPT(X), DOC(X)
+    """
+    # check if document exists
+    if not os.path.exists(document_filename):
+        raise FileNotFoundError(f"Document {document_filename} does not exist.")
+
+    # MIME map
+    mime_map = {
+        "pdf": "application/pdf",
+        "ppt": "application/vnd.ms-powerpoint",
+        "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        "doc": "application/msword",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    }
+
+    # check proper file extension: must be PDF, PPT, PPTX, DOC, DOCX
+    if not any (document_filename.lower().endswith(extension) for extension in mime_map):
+        raise ValueError(
+            f"Document {document_filename} is not a valid file type. Only supporting {', '.join(mime_map)}."
+        )
+
+    # get file basename
+    document_basename = os.path.basename(document_filename)
+
+    # obtain upload link
+    response = api.post(
+        "/artifact-service/document-upload-links/_generate",
+        body={
+            "file_name": document_basename,
+            "file_type": document_filename.split(".")[-1].upper(),
+        },
+    )
+    upload_link = response["signed_link"]
+    gcs_object_id = response["gcs_object_id"]
+
+    # upload document
+    with open(document_filename, "rb") as f:
+        headers = {
+            "Content-Type": mime_map[document_filename.split(".")[-1].lower()],
+            "x-goog-if-generation-match": "0",
+        }
+        response = requests.put(upload_link, headers=headers, data=f, timeout=30)
+
+    # check response
+    if response.status_code != 200:
+        raise ValueError(
+            f"Document {document_filename} failed to upload: {response.text}"
+        )
+
+    # create artifact
+    response = api.post(
+        "/artifact-service/artifacts",
+        body={"gcs_object_id": gcs_object_id},
+    )
+
+    # return artifact ID
+    return response["id"]
+
+
+#################################################
+def document_wait_for_processing(api: DeepSights, document_id: str, timeout: int = 300):
+    """
+    Wait for the document to be processed and completed.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        document_id (str): The ID of the document to wait for.
+        timeout (int, optional): The maximum time to wait for the document to be processed, in seconds. Defaults to 300.
+
+    Raises:
+
+        ValueError: If the document fails to process.
+        TimeoutError: If the document fails to process within the specified timeout.
+
+    """
+    # wait for completion
+    start = time.time()
+    while time.time() - start < timeout:
+        response = api.get(f"/artifact-service/artifacts/{document_id}")
+        if response["status"] == "COMPLETED":
+            break
+        
+        if response["status"].startswith("FAILED"):
+            raise ValueError(
+                f"Document {document_id} failed to process: {response['error_message']}"
+            )
+        
+        time.sleep(2)
+
+    # timeout?
+    if time.time() - start >= timeout:
+        raise TimeoutError(
+            f"Document {document_id} failed to process in {timeout} seconds."
+        )
+
+    # now load into cache
+    documents_load(api, [document_id])

deepsights/minions/_minions.py

@@ -0,0 +1,59 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains functions to interact with minions.
+"""
+
+import time
+from deepsights.api import DeepSights
+
+
+#################################################
+def minion_wait_for_completion(
+    api: DeepSights, minion_name: str, minion_job_id: str, timeout: int
+):
+    """
+    Waits for the completion of a minion job.
+
+    Args:
+
+        api (DeepSights): The DeepSights API instance.
+        minion_name (str): The name of the minion.
+        minion_job_id (str): The ID of the answer set.
+        timeout (int, optional): The maximum time to wait for the answer set to complete, in seconds.
+
+    Raises:
+
+        ValueError: If the answer set fails to complete.
+    """
+    # wait for completion
+    start = time.time()
+    while time.time() - start < timeout:
+        response = api.get(f"/minion-commander-service/{minion_name}/{minion_job_id}")[
+            "minion_job"
+        ]
+
+        if response["status"] in ("CREATED", "STARTED"):
+            time.sleep(2)
+        elif response["status"].startswith("FAILED"):
+            raise ValueError(
+                f"Minion {minion_job_id} failed to complete: {response['error_reason']}"
+            )
+        else:
+            return
+
+    raise ValueError(
+        f"Minion {minion_job_id} failed to complete within {timeout} seconds."
+    )

deepsights/reports/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This module contains the functions to retrieve reports from the DeepSights API.
+"""
+
+from deepsights.reports.report import (
+    report_create,
+    report_wait_for_completion,
+    report_get,
+)

deepsights/reports/model.py

@@ -0,0 +1,141 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+This module contains the model classes for reports.
+"""
+
+from typing import Optional, List
+from datetime import datetime
+from pydantic import Field
+from deepsights.utils import DeepSightsIdModel, DeepSightsIdTitleModel
+
+
+#################################################
+class ReportEvidence(DeepSightsIdTitleModel):
+    """
+    Represents evidence related to a report.
+
+    Attributes:
+
+        summary (Optional[str]): The summary of the evidence with respect to the report's question.
+        publication_date (Optional[datetime]): The publication date of the evidence.
+        rating (int): The rating of the evidence's relevance on a 1-5 scale with respect to the report's question.
+        reference (Optional[str]): The quotation reference code of the evidence.
+    """
+
+    evidence_summary: Optional[str] = Field(
+        alias="summary",
+        description="The summary of the evidence with respect to the report's question.",
+    )
+    publication_date: Optional[datetime] = Field(
+        description="The publication date of the evidence."
+    )
+    rating: int = Field(
+        description="The rating of the evidence's relevance on a 1-5 scale with respect to the report's question."
+    )
+    reference: Optional[str] = Field(
+        alias="reference_id",
+        description="The quotation reference code of the evidence.",
+    )
+
+
+#################################################
+class DocumentPageReportEvidence(DeepSightsIdModel):
+    """
+    Represents a document page in a report evidence.
+
+    Attributes:
+
+        page_number (Optional[int], optional): The number of the page.
+    """
+
+    page_number: Optional[int] = Field(
+        default=None, description="The number of the page (one-based)."
+    )
+
+
+#################################################
+class DocumentReportEvidence(ReportEvidence):
+    """
+    Represents evidence found in a document for a report.
+
+    Attributes:
+
+        description (Optional[str]): The human-readable summary of the document.
+        pages (List[DocumentPageSearchResult]): The list of pages in the document where the evidence is found.
+    """
+
+    description: Optional[str] = Field(
+        alias="artifact_summary",
+        description="The human-readable summary of the document.",
+    )
+    pages: List[DocumentPageReportEvidence] = Field(
+        alias="page_references",
+        description="The list of pages in the document where the evidence is found.",
+        default=[],
+    )
+
+
+#################################################
+class NewsReportEvidence(ReportEvidence):
+    """
+    Represents evidence for a news report.
+
+    Attributes:
+
+        source (Optional[str]): The name of the item's source; may be None.
+        description (Optional[str]): The human-readable summary of the news item.
+    """
+
+    source: Optional[str] = Field(
+        description="The name of the item's source; may be None.", default=None
+    )
+    description: Optional[str] = Field(
+        alias="summary",
+        description="The human-readable summary of the news item.",
+    )
+
+
+#################################################
+class Report(DeepSightsIdModel):
+    """
+    Represents a report generated by the DeepSights API.
+
+    Attributes:
+    
+        status (str): The processing status of the report.
+        question (str): The question that the report is answering.
+        topic (str, optional): The AI-generated topic of the report.
+        summary (str, optional): The summary text of the report in markdown format.
+        document_sources (List[ReportEvidence]): List of evidence from documents used in the report.
+        news_sources (List[ReportEvidence]): List of evidence from news sources used in the report.
+    """
+
+    status: str = Field(description="The processing status of the report.")
+    question: str = Field(description="The question that the report is answering.")
+    language: str = Field(description="The language of the report.")
+    topic: Optional[str] = Field(
+        default=None, description="The AI-generated topic of the report."
+    )
+    summary: Optional[str] = Field(
+        default=None, description="The summary text of the report in markdown format."
+    )
+    document_sources: List[DocumentReportEvidence] = Field(
+        default=[], description="List of evidence from documents used in the report."
+    )
+    news_sources: List[NewsReportEvidence] = Field(
+        default=[], description="List of evidence from news sources used in the report."
+    )

deepsights/reports/report.py

@@ -0,0 +1,95 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the functions to retrieve reports from the DeepSights API.
+"""
+
+from ratelimit import sleep_and_retry, limits
+from deepsights.api import DeepSights
+from deepsights.minions._minions import minion_wait_for_completion
+from deepsights.reports.model import Report
+
+
+#################################################
+@sleep_and_retry
+@limits(calls=3, period=60)
+def report_create(api: DeepSights, question: str) -> str:
+    """
+    Creates a new report by submitting a question to the DeepSights API.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        question (str): The question to be submitted for the report.
+
+    Returns:
+
+        str: The ID of the created report's minion job.
+    """
+
+    body = {"input": question}
+    response = api.post(
+        "/minion-commander-service/desk-researches", body=body, timeout=5
+    )
+
+    return response["minion_job"]["id"]
+
+
+#################################################
+def report_wait_for_completion(api: DeepSights, report_id: str, timeout=600):
+    """
+    Waits for the completion of a report.
+
+    Args:
+
+        api (DeepSights): The DeepSights API instance.
+        report_id (str): The ID of the report.
+        timeout (int, optional): The maximum time to wait for the report to complete, in seconds. Defaults to 600.
+
+    Raises:
+
+        ValueError: If the report fails to complete.
+    """
+    return minion_wait_for_completion(api, "desk-researches", report_id, timeout)
+
+
+#################################################
+def report_get(api: DeepSights, report_id: str) -> Report:
+    """
+    Loads a report from the DeepSights API.
+
+    Args:
+
+        api (DeepSights): An instance of the DeepSights API client.
+        report_id (str): The ID of the report to load.
+
+    Returns:
+
+        Report: The loaded report.
+    """
+    response = api.get(f"/minion-commander-service/desk-researches/{report_id}")
+
+    return Report(
+        **dict(
+            id=response["minion_job"]["id"],
+            status=response["minion_job"]["status"],
+            question=response["context"]["input"],
+            language=response["context"]["input_language"],
+            topic=response["context"]["topic"],
+            summary=response["context"]["summary"],
+            document_sources=response["context"]["artifact_vector_search_results"],
+            news_sources=response["context"]["scs_news_search_results"],
+        )
+    )

deepsights/utils/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains utility functions and classes used by the DeepSights API.
+"""
+
+from deepsights.utils._utils import run_in_parallel
+from deepsights.utils._cache import create_global_lru_cache
+from deepsights.utils._ranking import (
+    rrf_merge_multi,
+    rrf_merge_single,
+    rerank_by_recency,
+    promote_exact_matches,
+)
+from deepsights.utils.model import (
+    DeepSightsBaseModel,
+    DeepSightsIdModel,
+    DeepSightsIdTitleModel,
+)