unique_toolkit 0.7.32__tar.gz → 0.7.34__tar.gz

{unique_toolkit-0.7.32 → unique_toolkit-0.7.34}/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.34] - 2025-05-30
+- Fix incorrect mapping in `ContentService` for the `search_content` function when mapping into `ContentChunk` object
+
+## [0.7.33] - 2025-06-25
+- Update reference post-processing
+
 ## [0.7.32] - 2025-06-24
 - Create `classmethod` for `LanguageModelMessages` to load raw messages to root
 

{unique_toolkit-0.7.32 → unique_toolkit-0.7.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.7.32
+Version: 0.7.34
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -111,6 +111,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.34] - 2025-05-30
+- Fix incorrect mapping in `ContentService` for the `search_content` function when mapping into `ContentChunk` object
+
+## [0.7.33] - 2025-06-25
+- Update reference post-processing
+
 ## [0.7.32] - 2025-06-24
 - Create `classmethod` for `LanguageModelMessages` to load raw messages to root
 

{unique_toolkit-0.7.32 → unique_toolkit-0.7.34}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "0.7.32"
+version = "0.7.34"
 description = ""
 authors = [
     "Martin Fadler <martin.fadler@unique.ch>",

{unique_toolkit-0.7.32 → unique_toolkit-0.7.34}/unique_toolkit/content/utils.py

@@ -190,9 +190,11 @@ def count_tokens(text: str, encoding_model="cl100k_base") -> int:
     return len(encoding.encode(text))
 
 
-def map_content_chunk(content_chunk: dict):
+def map_content_chunk(content_id: str, content_key: str, content_chunk: dict):
     return ContentChunk(
-        id=content_chunk["id"],
+        id=content_id,
+        key=content_key,
+        chunk_id=content_chunk["id"],
         text=content_chunk["text"],
         start_page=content_chunk["startPage"],
         end_page=content_chunk["endPage"],
@@ -206,7 +208,10 @@ def map_content(content: dict):
         key=content["key"],
         title=content["title"],
         url=content["url"],
-        chunks=[map_content_chunk(chunk) for chunk in content["chunks"]],
+        chunks=[
+            map_content_chunk(content["id"], content["key"], chunk)
+            for chunk in content["chunks"]
+        ],
         created_at=content["createdAt"],
         updated_at=content["updatedAt"],
     )

{unique_toolkit-0.7.32 → unique_toolkit-0.7.34}/unique_toolkit/language_model/reference.py

@@ -51,9 +51,9 @@ def _add_references(
     # Only reference a source once, even if it is mentioned multiple times in the text.
     with_footnotes = _add_footnotes_to_text(text=text, references=references)
 
-    # Gemini 2.5 flash model has tendency to add multiple references for the same fact
+    # Gemini 2.5 models have tendency to add multiple references for the same fact
     # This is a workaround to limit the number of references to 5
-    if model and model.startswith("litellm:gemini-2-5-flash"):
+    if model and model.startswith("litellm:gemini-2-5"):
         reduced_text = _limit_consecutive_source_references(with_footnotes)
 
         # Get the references that remain after reduction
@@ -92,8 +92,11 @@ def _preprocess_message(text: str) -> str:
     # Replace XML format '[<source XX>]', '[<sourceXX>]' and '[\<sourceXX>]' with [XX]
     text = re.sub(r"\[(\\)?<source[\s]?(\d+)>\]", r"[\2]", text)
 
-    # Replace format '[source XX]' and '[sourceXX]' with [XX]
-    text = re.sub(r"\[source[\s]?(\d+)\]", r"[\1]", text)
+    # Replace format 'source XX', 'source_X' and 'sourceXX' references with XX, where XX is a number
+    text = re.sub(r"source[\s_]?(\d+)", r"[\1]", text)
+
+    # Replace 'source_number="X"' with X, where X is a number
+    text = re.sub(r"source_number=\"(\d+)\"", r"[\1]", text)
 
     # Make all references non-bold
     text = re.sub(r"\[\*\*(\d+)\*\*\]", r"[\1]", text)
@@ -107,13 +110,19 @@ def _preprocess_message(text: str) -> str:
     # Replace '[<[XX]>]' and '[\<[XX]>]' with [XX]
     text = re.sub(r"\[(\\)?\[?<\[(\d+)\]?\]>\]", r"[\2]", text)
 
-    # Replace '[[A], [B], ...]' or '[[A], B, C, ...]' with [A][B][C]...
+    # Replace '[source: X, Y, Z]' with [X][Y][Z], where X,Y,Z are numbers
+    def replace_source_colon(match):
+        numbers = re.findall(r"\d+", match.group(0))
+        return "".join(f"[{n}]" for n in numbers)
+    text = re.sub(r"\[source:\s*([\d,\s]+)\]", replace_source_colon, text)
+
+    # Replace '[[A], [B], ...]', '[[A], B, C, ...]', and '[X, Y, Z]' with [A][B][C]... where A,B,C are numbers
     def replace_combined_brackets(match):
         numbers = re.findall(r"\d+", match.group(0))
         return "".join(f"[{n}]" for n in numbers)
 
     text = re.sub(
-        r"\[\[(\d+)\](?:,\s*(?:\[)?\d+(?:\])?)*\]", replace_combined_brackets, text
+        r"(?:\[\[(\d+)\](?:,\s*(?:\[)?\d+(?:\])?)*\]|\[([\d,\s]+)\])", replace_combined_brackets, text
     )
 
     return text