unique_toolkit 0.7.32__py3-none-any.whl → 0.7.34__py3-none-any.whl

unique_toolkit/content/utils.py

@@ -190,9 +190,11 @@ def count_tokens(text: str, encoding_model="cl100k_base") -> int:
     return len(encoding.encode(text))
 
 
-def map_content_chunk(content_chunk: dict):
+def map_content_chunk(content_id: str, content_key: str, content_chunk: dict):
     return ContentChunk(
-        id=content_chunk["id"],
+        id=content_id,
+        key=content_key,
+        chunk_id=content_chunk["id"],
         text=content_chunk["text"],
         start_page=content_chunk["startPage"],
         end_page=content_chunk["endPage"],
@@ -206,7 +208,10 @@ def map_content(content: dict):
         key=content["key"],
         title=content["title"],
         url=content["url"],
-        chunks=[map_content_chunk(chunk) for chunk in content["chunks"]],
+        chunks=[
+            map_content_chunk(content["id"], content["key"], chunk)
+            for chunk in content["chunks"]
+        ],
         created_at=content["createdAt"],
         updated_at=content["updatedAt"],
     )

unique_toolkit/language_model/reference.py

@@ -51,9 +51,9 @@ def _add_references(
     # Only reference a source once, even if it is mentioned multiple times in the text.
     with_footnotes = _add_footnotes_to_text(text=text, references=references)
 
-    # Gemini 2.5 flash model has tendency to add multiple references for the same fact
+    # Gemini 2.5 models have tendency to add multiple references for the same fact
     # This is a workaround to limit the number of references to 5
-    if model and model.startswith("litellm:gemini-2-5-flash"):
+    if model and model.startswith("litellm:gemini-2-5"):
         reduced_text = _limit_consecutive_source_references(with_footnotes)
 
         # Get the references that remain after reduction
@@ -92,8 +92,11 @@ def _preprocess_message(text: str) -> str:
     # Replace XML format '[<source XX>]', '[<sourceXX>]' and '[\<sourceXX>]' with [XX]
     text = re.sub(r"\[(\\)?<source[\s]?(\d+)>\]", r"[\2]", text)
 
-    # Replace format '[source XX]' and '[sourceXX]' with [XX]
-    text = re.sub(r"\[source[\s]?(\d+)\]", r"[\1]", text)
+    # Replace format 'source XX', 'source_X' and 'sourceXX' references with XX, where XX is a number
+    text = re.sub(r"source[\s_]?(\d+)", r"[\1]", text)
+
+    # Replace 'source_number="X"' with X, where X is a number
+    text = re.sub(r"source_number=\"(\d+)\"", r"[\1]", text)
 
     # Make all references non-bold
     text = re.sub(r"\[\*\*(\d+)\*\*\]", r"[\1]", text)
@@ -107,13 +110,19 @@ def _preprocess_message(text: str) -> str:
     # Replace '[<[XX]>]' and '[\<[XX]>]' with [XX]
     text = re.sub(r"\[(\\)?\[?<\[(\d+)\]?\]>\]", r"[\2]", text)
 
-    # Replace '[[A], [B], ...]' or '[[A], B, C, ...]' with [A][B][C]...
+    # Replace '[source: X, Y, Z]' with [X][Y][Z], where X,Y,Z are numbers
+    def replace_source_colon(match):
+        numbers = re.findall(r"\d+", match.group(0))
+        return "".join(f"[{n}]" for n in numbers)
+    text = re.sub(r"\[source:\s*([\d,\s]+)\]", replace_source_colon, text)
+
+    # Replace '[[A], [B], ...]', '[[A], B, C, ...]', and '[X, Y, Z]' with [A][B][C]... where A,B,C are numbers
     def replace_combined_brackets(match):
         numbers = re.findall(r"\d+", match.group(0))
         return "".join(f"[{n}]" for n in numbers)
 
     text = re.sub(
-        r"\[\[(\d+)\](?:,\s*(?:\[)?\d+(?:\])?)*\]", replace_combined_brackets, text
+        r"(?:\[\[(\d+)\](?:,\s*(?:\[)?\d+(?:\])?)*\]|\[([\d,\s]+)\])", replace_combined_brackets, text
     )
 
     return text

{unique_toolkit-0.7.32.dist-info → unique_toolkit-0.7.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.7.32
+Version: 0.7.34
 Summary: 
 License: Proprietary
 Author: Martin Fadler
@@ -111,6 +111,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.34] - 2025-05-30
+- Fix incorrect mapping in `ContentService` for the `search_content` function when mapping into `ContentChunk` object
+
+## [0.7.33] - 2025-06-25
+- Update reference post-processing
+
 ## [0.7.32] - 2025-06-24
 - Create `classmethod` for `LanguageModelMessages` to load raw messages to root
 

{unique_toolkit-0.7.32.dist-info → unique_toolkit-0.7.34.dist-info}/RECORD

@@ -23,7 +23,7 @@ unique_toolkit/content/constants.py,sha256=1iy4Y67xobl5VTnJB6SxSyuoBWbdLl9244xfV
 unique_toolkit/content/functions.py,sha256=0ELepm3_sl0SD_SYzvQVQ-jTdrcUqK5mVJZv0nQBuAw,18367
 unique_toolkit/content/schemas.py,sha256=KJ604BOx0vBh2AwlTCZkOo55aHsI6yj8vxDAARKKqEo,2995
 unique_toolkit/content/service.py,sha256=jGRTIt0JQKL6qRyZ9-_njWJejKKvIvDOzEQrf8CdMaU,19491
-unique_toolkit/content/utils.py,sha256=GUVPrkZfMoAj4MRoBs5BD_7vSuLZTZx69hyWzYFrI50,7747
+unique_toolkit/content/utils.py,sha256=qNVmHTuETaPNGqheg7TbgPr1_1jbNHDc09N5RrmUIyo,7901
 unique_toolkit/embedding/__init__.py,sha256=uUyzjonPvuDCYsvXCIt7ErQXopLggpzX-MEQd3_e2kE,250
 unique_toolkit/embedding/constants.py,sha256=Lj8-Lcy1FvuC31PM9Exq7vaFuxQV4pEI1huUMFX-J2M,52
 unique_toolkit/embedding/functions.py,sha256=3qp-BfuMAbnp8YB04rh3xH8vsJuCBPizoy-JeaBFtoQ,1944
@@ -50,7 +50,7 @@ unique_toolkit/language_model/constants.py,sha256=B-topqW0r83dkC_25DeQfnPk3n53qz
 unique_toolkit/language_model/functions.py,sha256=WhgHbJgz4Z2aZt9TLdOpI0PGyYWA5R90tdwkwdDeT8c,11987
 unique_toolkit/language_model/infos.py,sha256=w5__BVG-IiiEYKG1FwM838wzqNbYI3eCCEDocKezc0I,34801
 unique_toolkit/language_model/prompt.py,sha256=JSawaLjQg3VR-E2fK8engFyJnNdk21zaO8pPIodzN4Q,3991
-unique_toolkit/language_model/reference.py,sha256=ntoWdJ_hly8FntTgg1qpj0ta55SOCCVsC5L49VqOiAQ,8067
+unique_toolkit/language_model/reference.py,sha256=jd-JPuqLwEJDs56im56o-AdiT73pdwNtSzcW38LZ53o,8565
 unique_toolkit/language_model/schemas.py,sha256=AeuDRJFblGzEYcEMyrlxpOPk12Di3J45I9rT2xZrhEU,14332
 unique_toolkit/language_model/service.py,sha256=VRkUk2XbijqGlnTTvqU7uCue6qtT7lpLd_Y8f3bWv1I,10486
 unique_toolkit/language_model/utils.py,sha256=bPQ4l6_YO71w-zaIPanUUmtbXC1_hCvLK0tAFc3VCRc,1902
@@ -62,7 +62,7 @@ unique_toolkit/short_term_memory/schemas.py,sha256=OhfcXyF6ACdwIXW45sKzjtZX_gkcJ
 unique_toolkit/short_term_memory/service.py,sha256=cqpXA0nMbi4PhFweg-Cql3u0RvaTi5c8Xjv0uHMiSGc,8112
 unique_toolkit/smart_rules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unique_toolkit/smart_rules/compile.py,sha256=cxWjb2dxEI2HGsakKdVCkSNi7VK9mr08w5sDcFCQyWI,9553
-unique_toolkit-0.7.32.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-0.7.32.dist-info/METADATA,sha256=qmPxpLockST_2FR2k05nfwgNDtI7C1pBK9fbNwCoKGE,24713
-unique_toolkit-0.7.32.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-0.7.32.dist-info/RECORD,,
+unique_toolkit-0.7.34.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-0.7.34.dist-info/METADATA,sha256=1OxeaVaHJorHRvzuJ8TXg6cbe-4PqT9iUMF-FZ5rG1Q,24918
+unique_toolkit-0.7.34.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-0.7.34.dist-info/RECORD,,