deepsights-api 1.2.2__tar.gz → 1.2.4__tar.gz

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 <!--next-version-placeholder-->
 
+## v1.2.4 (16-Arp-2025)
+
+- Sort order in SCS text search
+
+## v1.2.3 (25-Jan-2025)
+
+- Support item text download from cont
+
 ## v1.2.2 (12-Jan-2025)
 
 - Add AI generated title to model 

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepsights-api
-Version: 1.2.2
+Version: 1.2.4
 Summary: Python library for the DeepSights APIs
 Project-URL: Documentation, https://marketlogicsoftware.github.io/deepsights-api/
 Project-URL: Repository, https://github.com/marketlogicsoftware/deepsights-api

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/deepsights/api/api.py

@@ -16,18 +16,19 @@
 This module contains base API client classes.
 """
 
-import os
 import logging
+import os
 from typing import Dict
+
+from ratelimit import limits, sleep_and_retry
+from requests import Session
+from requests.exceptions import Timeout
 from tenacity import (
     retry,
+    retry_if_exception_type,
     stop_after_attempt,
     wait_random_exponential,
-    retry_if_exception_type,
 )
-from requests import Session
-from requests.exceptions import Timeout
-from ratelimit import limits, sleep_and_retry
 
 
 #################################################
@@ -107,6 +108,50 @@ class API:
 
         return response.json()
 
+    #######################################
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_random_exponential(max=5),
+        retry=retry_if_exception_type(Timeout),
+    )
+    @sleep_and_retry
+    @limits(calls=1000, period=60)
+    def get_content(
+        self, path: str, params: Dict = None, timeout=15, expected_statuscodes=[]
+    ) -> bytes:
+        """
+        Sends a GET request to the specified path and returns the raw response content.
+
+        This method is similar to get() but returns the raw response content instead of parsing it as JSON.
+        Useful for downloading binary data like files or images.
+
+        Args:
+            path (str): The path to send the GET request to.
+            params (Dict, optional): Optional parameters to include in the request. Defaults to None.
+            timeout (int, optional): The timeout in seconds for the request. Defaults to 15.
+            expected_statuscodes (List[int], optional): List of expected status codes. Defaults to an empty list.
+
+        Returns:
+            bytes: The raw content of the server's response.
+
+        Raises:
+            HTTPError: If the GET request fails with a non-200 status code and not in the expected_statuscodes list.
+        """
+        response = self._session.get(
+            self._endpoint(path), params=params, timeout=timeout
+        )
+
+        if (
+            response.status_code != 200
+            and not response.status_code in expected_statuscodes
+        ):
+            logging.error(
+                "GET %s failed with status code %s", path, response.status_code
+            )
+            response.raise_for_status()
+
+        return response.content.decode("utf-8")
+
     #######################################
     @retry(
         stop=stop_after_attempt(3),
@@ -208,9 +253,9 @@ class APIKeyAPI(API):
         super().__init__(endpoint_base)
 
         # set api key
-        assert (
-            api_key or api_key_env_var
-        ), "Must provide either API key or environment variable"
+        assert api_key or api_key_env_var, (
+            "Must provide either API key or environment variable"
+        )
         if not api_key:
             api_key = os.environ.get(api_key_env_var)
         self._api_key = api_key

deepsights_api-1.2.4/deepsights/contentstore/resources/_download.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Market Logic Software AG. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the base functions to download content from the ContentStore.
+"""
+
+import os
+import tempfile
+import urllib.request
+
+from deepsights.api import APIResource
+
+
+#################################################
+def contentstore_download(resource: APIResource, item_id: str) -> str:
+    """
+    Downloads and returns the extracted text content of an item from the ContentStore.
+
+    This function retrieves a signed download link for the specified item's extracted text content
+    from the ContentStore API, downloads the content to a temporary file, reads the text content,
+    and returns it as a string. The temporary file is automatically cleaned up after use.
+
+    Args:
+        resource (APIResource): An instance of the DeepSights API resource used to make API calls
+            to the ContentStore service.
+        item_id (str): The unique identifier of the item whose content should be downloaded.
+
+    Raises:
+        ValueError: If the download fails for any reason, such as network errors, invalid item ID,
+            or issues reading the downloaded content.
+
+    Returns:
+        str: The extracted text content of the downloaded item as a string.
+    """
+    return str(
+        resource.api.get_content(
+            f"/item-service/items/{item_id}/_download-content-extracted-text",
+        )
+    )

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/deepsights/contentstore/resources/_search.py

@@ -16,9 +16,11 @@
 This module contains the base functions to search the ContentStore.
 """
 
-from typing import List
 from datetime import datetime
+from typing import List
+
 from pydantic import BaseModel
+
 from deepsights.api import API
 from deepsights.utils import (
     promote_exact_matches,
@@ -96,9 +98,9 @@ def contentstore_hybrid_search(
     assert 0 <= vector_fraction <= 1, "Vector fraction must be between 0 and 1"
     assert 0 <= vector_weight <= 1, "Vector weught must be between 0 and 1"
     assert 0 < max_results <= 250, "Maximum results must be between 1 and 250."
-    assert (
-        recency_weight is None or 0 <= recency_weight <= 1
-    ), "Recency weight must be between 0 and 1."
+    assert recency_weight is None or 0 <= recency_weight <= 1, (
+        "Recency weight must be between 0 and 1."
+    )
 
     body = {
         "query": query,
@@ -171,9 +173,9 @@ def contentstore_vector_search(
     assert len(query_embedding) == 1536, "The 'query_embedding' must be of length 1536."
     assert 0 <= min_score <= 1, "Minimum score must be between 0 and 1."
     assert 0 < max_results <= 100, "Maximum results must be between 1 and 100."
-    assert (
-        recency_weight is None or 0 <= recency_weight <= 1
-    ), "Recency weight must be between 0 and 1."
+    assert recency_weight is None or 0 <= recency_weight <= 1, (
+        "Recency weight must be between 0 and 1."
+    )
 
     body = {
         "vector": query_embedding,
@@ -206,7 +208,7 @@ def contentstore_text_search(
     max_results: int = 50,
     offset: int = 0,
     languages: List[str] = None,
-    recency_weight: float = None,
+    sort_descending: bool = True,
     search_from_timestamp: datetime = None,
     search_to_timestamp: datetime = None,
     search_only_ai_allowed_content: bool = True,
@@ -223,7 +225,7 @@ def contentstore_text_search(
         max_results (int, optional): The maximum number of results to return. Defaults to 50.
         offset (int, optional): The offset to start the search from. Defaults to 0.
         languages (List[str], optional): The languages to search for. Defaults to None.
-        recency_weight (float, optional): The weight assigned to recency in result ranking. Defaults to None.
+        most_recent_first (bool, optional): Whether to sort results by most recent first. Defaults to True.
         search_from_timestamp (datetime, optional): The start timestamp for the search. Defaults to None.
         search_to_timestamp (datetime, optional): The end timestamp for the search. Defaults to None.
         search_only_ai_allowed_content (bool, optional): Whether to search only AI-allowed content. Defaults to True.
@@ -233,21 +235,24 @@ def contentstore_text_search(
         List[BaseModel]: The re-ranked search results.
     """
     assert 0 < max_results <= 100, "Maximum results must be between 1 and 100."
-    assert (
-        recency_weight is None or 0 <= recency_weight <= 1
-    ), "Recency weight must be between 0 and 1."
 
     # force proper empty search
     if query is not None and len(query.strip()) == 0:
         query = None
 
+    # determine sort order
+    if query is not None:
+        sort_order = "RELEVANCY_DESC" if sort_descending else "RELEVANCY_ASC"
+    else:
+        sort_order = "PUBLISHED_AT_DESC" if sort_descending else "PUBLISHED_AT_ASC"
+
     body = {
         "query": query,
         "source_items_type": item_type,
         "content_restrictions": "NONE",
         "limit": max_results,
         "offset": offset,
-        "sort": "RELEVANCY_DESC" if query is not None else "PUBLISHED_AT_DESC",
+        "sort": sort_order,
         "languages": languages,
         "published_at": _get_time_filter(search_from_timestamp, search_to_timestamp),
         "content_restrictions": (
@@ -259,5 +264,8 @@ def contentstore_text_search(
     # parse
     results = [search_result(i) for i in response["items"]]
 
-    # re-rank
-    return rerank_by_recency(results, recency_weight=recency_weight)
+    # record rank
+    for rank, result in enumerate(results):
+        result.rank = rank + 1
+
+    return results

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/deepsights/contentstore/resources/news.py

@@ -16,19 +16,19 @@
 This module defines the resource to retrieve news content from the DeepSights API.
 """
 
-from typing import List
 from datetime import datetime
+from typing import List
 
 from deepsights.api import APIResource
+from deepsights.contentstore.resources._download import contentstore_download
 from deepsights.contentstore.resources._model import ContentStoreSearchResult
 from deepsights.contentstore.resources._search import (
+    contentstore_hybrid_search,
     contentstore_text_search,
     contentstore_vector_search,
-    contentstore_hybrid_search,
 )
 
 
-
 #################################################
 class NewsSearchResult(ContentStoreSearchResult):
     """
@@ -42,6 +42,12 @@ class NewsResource(APIResource):
     Represents a resource for retrieving news articles from the DeepSights content store.
     """
 
+    def download(self, item_id: str) -> str:
+        """
+        Download the extracted text content of an item from the ContentStore.
+        """
+        return contentstore_download(self, item_id)
+
     #################################################
     def vector_search(
         self,
@@ -93,7 +99,7 @@ class NewsResource(APIResource):
         max_results: int = 30,
         offset: int = 0,
         languages: List[str] = None,
-        recency_weight: float = None,
+        sort_descending: bool = True,
         search_from_timestamp: datetime = None,
         search_to_timestamp: datetime = None,
     ):
@@ -106,7 +112,7 @@ class NewsResource(APIResource):
             max_results (int, optional): The maximum number of search results to return. Defaults to 30.
             offset (int, optional): The offset to start the search from. Defaults to 0.
             languages (List[str], optional): The list of languages to search for. Defaults to None.
-            recency_weight (float, optional): The weight to assign to recency in the search ranking. Defaults to None.
+            sort_descending (bool, optional): Whether to sort the results in descending order. Defaults to True.
             search_from_timestamp (datetime, optional): The start timestamp for the search. Defaults to None.
             search_to_timestamp (datetime, optional): The end timestamp for the search. Defaults to None.
 
@@ -122,7 +128,7 @@ class NewsResource(APIResource):
             ),
             query=query,
             max_results=max_results,
-            recency_weight=recency_weight,
+            sort_descending=sort_descending,
             languages=languages,
             offset=offset,
             search_from_timestamp=search_from_timestamp,

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/deepsights/contentstore/resources/secondary.py

@@ -16,15 +16,16 @@
 This module contains the resource to retrieve secondary reports from the DeepSights content store.
 """
 
-from typing import List
 from datetime import datetime
+from typing import List
 
 from deepsights.api import APIResource
+from deepsights.contentstore.resources._download import contentstore_download
 from deepsights.contentstore.resources._model import ContentStoreSearchResult
 from deepsights.contentstore.resources._search import (
+    contentstore_hybrid_search,
     contentstore_text_search,
     contentstore_vector_search,
-    contentstore_hybrid_search,
 )
 
 
@@ -41,6 +42,12 @@ class SecondaryResource(APIResource):
     Represents a resource for retrieving secondary reports from the DeepSights content store.
     """
 
+    def download(self, item_id: str) -> str:
+        """
+        Download the extracted text content of an item from the ContentStore.
+        """
+        return contentstore_download(self, item_id)
+
     #################################################
     def vector_search(
         self,
@@ -92,7 +99,7 @@ class SecondaryResource(APIResource):
         max_results: int = 50,
         offset: int = 0,
         languages: List[str] = None,
-        recency_weight: float = None,
+        sort_descending: bool = True,
         search_from_timestamp: datetime = None,
         search_to_timestamp: datetime = None,
     ):
@@ -105,7 +112,7 @@ class SecondaryResource(APIResource):
             max_results (int, optional): The maximum number of search results to return. Defaults to 50.
             offset (int, optional): The offset to start the search from. Defaults to 0.
             languages (List[str], optional): The languages to search for. Defaults to None.
-            recency_weight (float, optional): The weight to assign to recency in the search ranking. Defaults to None.
+            sort_descending (bool, optional): Whether to sort the results in descending order. Defaults to True.
             search_from_timestamp (datetime, optional): The start timestamp for the search. Defaults to None.
             search_to_timestamp (datetime, optional): The end timestamp for the search. Defaults to None.
 
@@ -123,7 +130,7 @@ class SecondaryResource(APIResource):
             max_results=max_results,
             offset=offset,
             languages=languages,
-            recency_weight=recency_weight,
+            sort_descending=sort_descending,
             search_from_timestamp=search_from_timestamp,
             search_to_timestamp=search_to_timestamp,
         )

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "deepsights-api"
-version = "1.2.2"
+version = "1.2.4"
 description = "Python library for the DeepSights APIs"
 readme = "README.md"
 requires-python = ">=3.10"

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/tests/contentstore/test_news_search.py

@@ -79,6 +79,17 @@ def test_news_text_empty_search():
             assert result.publication_date <= results[ix - 1].publication_date
 
 
+def test_news_download():
+    """ """
+    results = ds.contentstore.news.text_search(
+        query="",
+        max_results=1,
+    )
+
+    content = ds.contentstore.news.download(results[0].id)
+    assert len(content) > 0
+
+
 def test_news_text_search_offset():
     """
     Test the news text search function with an offset.
@@ -152,51 +163,43 @@ def test_news_text_search_with_date():
         assert result.publication_date <= end and result.publication_date >= start
 
 
-def test_news_text_search_with_recency_low():
+def test_news_text_search_descending():
     """
-    Test the news text search function with a low recency weight.
-
-    This test verifies that the news text search function returns results
-    when the recency weight is set to a low value. It checks that the
-    returned results have valid IDs, ranks, score ranks, and age ranks,
-    and that the final rank is identical to the score rank.
-
-    Note: This test assumes the existence of a `test_query` variable.
+    Test news text search with descending publication date sort order.
+    Verifies results have valid IDs, sequential ranks, and descending dates.
     """
-
     results = ds.contentstore.news.text_search(
-        query=test_query,
+        query=None,
         max_results=10,
-        recency_weight=0.00001,
+        sort_descending=True,
     )
 
     assert len(results) > 0
     for ix, result in enumerate(results):
         assert result.id is not None
         assert result.rank == ix + 1
+        if ix < len(results) - 1:
+            assert result.publication_date >= results[ix + 1].publication_date
 
 
-def test_news_text_search_with_recency_high():
+def test_news_text_search_ascending():
     """
-    Test the news text search function with a high recency weight.
-
-    This test verifies that the news text search function returns results
-    when the recency weight is set to a high value. It checks that the
-    returned results have valid IDs, ranks, score ranks, and age ranks,
-    and that the final rank is identical to the age rank.
-
-    Note: This test assumes the existence of a `test_query` variable.
+    Test news text search with ascending publication date sort order.
+    Verifies results have valid IDs, sequential ranks, and ascending dates.
     """
+
     results = ds.contentstore.news.text_search(
-        query=test_query,
+        query=None,
         max_results=10,
-        recency_weight=0.99999,
+        sort_descending=False,
     )
 
     assert len(results) > 0
     for ix, result in enumerate(results):
         assert result.id is not None
         assert result.rank == ix + 1
+        if ix < len(results) - 1:
+            assert result.publication_date <= results[ix + 1].publication_date
 
 
 def test_news_vector_search():
@@ -349,9 +352,7 @@ def test_news_hybrid_search_only_text():
     )
 
     text_results = ds.contentstore.news.text_search(
-        query=test_query,
-        max_results=results,
-        recency_weight=0.0,
+        query=test_query, max_results=results
     )
 
     assert len(hybrid_results) == results
@@ -439,19 +440,19 @@ def test_news_hybrid_search_with_vector_high():
     """
     hybrid_results = ds.contentstore.news.search(
         query=test_query,
-        max_results=10,
-        vector_weight=0.99999,
+        max_results=5,
+        vector_weight=0.9999999999,
         vector_fraction=1.0,
         recency_weight=0.0,
     )
 
     vector_results = ds.contentstore.news.vector_search(
         query_embedding=test_embedding,
-        max_results=10,
+        max_results=5,
         recency_weight=0.0,
     )
 
-    assert len(hybrid_results) == 10
+    assert len(hybrid_results) == 5
     for ix, result in enumerate(hybrid_results):
         assert equal_results(result, vector_results[ix])
 
@@ -476,7 +477,6 @@ def test_news_hybrid_search_with_vector_low():
     text_results = ds.contentstore.news.text_search(
         query=test_query,
         max_results=results,
-        recency_weight=0.0,
     )
 
     assert len(hybrid_results) == results

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/tests/contentstore/test_secondary_search.py

@@ -80,6 +80,17 @@ def test_secondary_text_empty_search():
             assert result.publication_date <= results[ix - 1].publication_date
 
 
+def test_news_download():
+    """ """
+    results = ds.contentstore.secondary.text_search(
+        query="",
+        max_results=1,
+    )
+
+    content = ds.contentstore.secondary.download(results[0].id)
+    assert len(content) > 0
+
+
 def test_secondary_text_search_offset():
     """
     Test the secondary text search function with an offset.
@@ -153,51 +164,43 @@ def test_secondary_text_search_with_date():
         assert result.publication_date <= end and result.publication_date >= start
 
 
-def test_secondary_text_search_with_recency_low():
+def test_secondary_text_search_descending():
     """
-    Test the secondary text search function with a low recency weight.
-
-    This test verifies that the secondary text search function returns results
-    when the recency weight is set to a low value. It checks that the
-    returned results have valid IDs, ranks, score ranks, and age ranks,
-    and that the final rank is identical to the score rank.
-
-    Note: This test assumes the existence of a `test_query` variable.
+    Test secondary text search with descending publication date sort order.
+    Verifies results have valid IDs, sequential ranks, and descending dates.
     """
-
     results = ds.contentstore.secondary.text_search(
-        query=test_query,
+        query=None,
         max_results=10,
-        recency_weight=0.00001,
+        sort_descending=True,
     )
 
     assert len(results) > 0
     for ix, result in enumerate(results):
         assert result.id is not None
         assert result.rank == ix + 1
+        if ix < len(results) - 1:
+            assert result.publication_date >= results[ix + 1].publication_date
 
 
-def test_secondary_text_search_with_recency_high():
+def test_secondary_text_search_ascending():
     """
-    Test the secondary text search function with a high recency weight.
-
-    This test verifies that the secondary text search function returns results
-    when the recency weight is set to a high value. It checks that the
-    returned results have valid IDs, ranks, score ranks, and age ranks,
-    and that the final rank is identical to the age rank.
-
-    Note: This test assumes the existence of a `test_query` variable.
+    Test secondary text search with ascending publication date sort order.
+    Verifies results have valid IDs, sequential ranks, and ascending dates.
     """
+
     results = ds.contentstore.secondary.text_search(
-        query=test_query,
+        query=None,
         max_results=10,
-        recency_weight=0.99999,
+        sort_descending=False,
     )
 
     assert len(results) > 0
     for ix, result in enumerate(results):
         assert result.id is not None
         assert result.rank == ix + 1
+        if ix < len(results) - 1:
+            assert result.publication_date <= results[ix + 1].publication_date
 
 
 def test_secondary_vector_search():
@@ -348,7 +351,6 @@ def test_secondary_hybrid_search_only_text():
     text_results = ds.contentstore.secondary.text_search(
         query=test_query,
         max_results=5,
-        recency_weight=0.0,
     )
 
     assert len(hybrid_results) == 5
@@ -480,7 +482,7 @@ def test_secondary_text_search_with_title_promotion():
     This test case verifies that the hybrid search function returns the expected results
     when searching for secondary articles based on a query with title promotion enabled.
     """
-    query = "consumer spending"
+    query = "shopping consumer"
 
     # first find top 10 results without title promotion and hard recency weight
     hybrid_results_no_promotion = ds.contentstore.secondary.search(

{deepsights_api-1.2.2 → deepsights_api-1.2.4}/tests/data/test_data.json

@@ -1,9 +1,9 @@
 {
     "comment": "text-ada-002 embedding for 'What are emerging food moments with Gen Z consumers?'",
     "question": "What are emerging food moments with Gen Z consumers?",
-    "document_id": "900aa7c7-60f4-4b2e-bc33-9dba20f4508f",
-    "document_page_id": "3d861e4e-37cd-4a7b-b4cf-6986556f9544",
-    "report_id": "7e9943f3-d9b8-4b36-86b0-c5ef939fcc71",
+    "document_id": "b3d728a6-6ef8-4c31-9949-dcc49439f4f1",
+    "document_page_id": "319eb5ab-e701-402e-881f-be3ccede65b3",
+    "report_id": "44126fc2-2aff-47c0-ac3f-d86bd6afcca2",
     "answerV2_id": "7511f5dd-bb5d-4082-8799-0d1733d61afc",
     "embedding": [
         -0.0012380050029605627,