datasourcelib 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

datasourcelib/datasources/azure_devops_source.py

@@ -300,7 +300,6 @@ class AzureDevOpsSource(DataSourceBase):
 
                         # Display name and url
                         display_name = page.get("name") or page.get("pageName") or page_path.strip("/") or "/"
-                        new_display_name = self.sanitize(display_name.replace(" ", "_").strip()),
                         url = (
                             page.get("remoteUrl")
                             or page.get("url")
@@ -343,7 +342,7 @@ class AzureDevOpsSource(DataSourceBase):
                          # Construct a 'full' description string using available pieces
                         content_text = BeautifulSoup(content_text or "", "html.parser").get_text(),
                         parts = []
-                        if new_display_name:
+                        if display_name:
                             parts.append(f"Wiki Page Name is {display_name}. Page has information  about {display_name}")
                         if project_name:
                             parts.append(f"This page is documented by for Project '{project_name}' and by the team '{project_name}'")
@@ -354,7 +353,8 @@ class AzureDevOpsSource(DataSourceBase):
 
                         index_content = ". ".join(parts)
                         results.append({
-                            "display_name": new_display_name,
+                            "display_name": self.sanitize(display_name.replace(" ", "_").strip()),
+                            "page_name": display_name,
                             "url": url,
                             "content": index_content,
                             "project": project_name

datasourcelib/utils/aggregation.py

@@ -2,6 +2,7 @@
 import pandas as pd
 from string import Formatter
 from typing import Iterable, Any, Dict, List, Optional, Union
+import regex as re
 
 def _placeholders(fmt: str) -> List[str]:
     """
@@ -12,6 +13,12 @@ def _placeholders(fmt: str) -> List[str]:
 def _safe_str(x) -> str:
     return "" if pd.isna(x) else str(x).strip()
 
+@staticmethod
+def sanitize(s: str) -> str:
+    """Keep only A-Z a-z 0-9 underscore/dash/equals in a safe way."""
+    # using the `regex` import already present as `re`
+    return re.sub(r'[^A-Za-z0-9_\-=]', '', s)
+
 def generate_grouped_summaries(
     df: pd.DataFrame,
     aggregation_field: str,
@@ -146,7 +153,7 @@ def generate_grouped_summaries(
 
         content = header + " " + " ".join(lines)
         summaries.append(
-            {"content" : content, "id": group_value}
+            {"content" : content, "id": sanitize(group_value.replace(" ", "_").strip())}
             )
 
     return summaries

{datasourcelib-0.1.12.dist-info → datasourcelib-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.12
+Version: 0.1.14
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.12.dist-info → datasourcelib-0.1.14.dist-info}/RECORD

@@ -5,7 +5,7 @@ datasourcelib/core/sync_manager.py,sha256=pfnvWv4AwmlJJUIsfxNNxYDBOsa7juTIxgFJIE
 datasourcelib/core/sync_types.py,sha256=KVZB7PkfkFTzghoe--U8jLeAU8XAfba9qMRIVcUjuMc,297
 datasourcelib/datasources/__init__.py,sha256=lZtgs0vT-2gub5UZo8BUnREZl3K_-_xYqUP8mjf8vhM,436
 datasourcelib/datasources/azure_devops_source copy.py,sha256=g-IOCq5vGwwteU21jZPWW_GggMu1_myVJkP0_BmSdGY,7282
-datasourcelib/datasources/azure_devops_source.py,sha256=o-rl090HxbBA_Sl6WHazIDoA1NhjybIrmyQCU0SwzqA,19649
+datasourcelib/datasources/azure_devops_source.py,sha256=jWey-FbL5DnzVFFhFaw5QdVUv0t81ATLM3ASKMozGhc,19639
 datasourcelib/datasources/azure_devops_source10dec.py,sha256=J48E78AEfqkS-eBq7sesA48zmSiZ9oSfJkQjL7RAbyA,7928
 datasourcelib/datasources/blob_source.py,sha256=Qk61_ulqUSPYDaiMzqgvJAu43c4AjTlDRdfFg4VwgDU,3574
 datasourcelib/datasources/datasource_base.py,sha256=N8fOGvTl8oWWAiydLI0Joz66luq73a5yovO0XA9Q3jk,1068
@@ -24,14 +24,14 @@ datasourcelib/strategies/incremental_load.py,sha256=CY1tAyXwjZLoq5zMLwB5i5qmT_L8
 datasourcelib/strategies/ondemand_load.py,sha256=MgenKJbJePLeErdEkXKsz1h7RuR8yT0RV_X523G7UUs,1304
 datasourcelib/strategies/timerange_load.py,sha256=W_sSZg059Lw2o9tmdGKM9D5-z1pph7AN1ftalXhuyjo,1557
 datasourcelib/utils/__init__.py,sha256=9pSIpaK-kdmNuDzwl0Z7QU-_lV3cZE-iwOEPh3RBBTs,298
-datasourcelib/utils/aggregation.py,sha256=5aOBcxay4eTyY-S4BRafNgSi37AY-JXERzcCv055E8w,6060
+datasourcelib/utils/aggregation.py,sha256=_XzTxdGIc-nc0w1FE1NfPA6J1PmAKiSpz0sYU7yEU6s,6337
 datasourcelib/utils/byte_reader.py,sha256=GaoPXwJa2YTWG1Kim0K6JG20eVSaWkZJd1o9bswxHmc,9082
 datasourcelib/utils/exceptions.py,sha256=mgcDaW1k3VndgpMOwSm7NqgyRTvvE2a5ehn3x4fYQww,369
 datasourcelib/utils/file_reader.py,sha256=Zr0rwNTRWE6KeVJEXgTOPS1_JI74LiUSiX5-6qojmN0,7301
 datasourcelib/utils/logger.py,sha256=Sl6lNlvubxtK9ztzyq7vjGVyA8_-pZ_ixpk5jfVsh6U,424
 datasourcelib/utils/validators.py,sha256=fLgmRAb5OZSdMVlHu_n0RKJUDl-G8dI8JsRSfxIquh8,205
-datasourcelib-0.1.12.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
-datasourcelib-0.1.12.dist-info/METADATA,sha256=Rvu5r33TNr6s-ph4bH6MCcwOx_jELup4C3KNnmTZA8Y,1200
-datasourcelib-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datasourcelib-0.1.12.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
-datasourcelib-0.1.12.dist-info/RECORD,,
+datasourcelib-0.1.14.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
+datasourcelib-0.1.14.dist-info/METADATA,sha256=EyIP4yk74vb4-yhOWh--E9aXo8E1gt8RDVkYVoexfNI,1200
+datasourcelib-0.1.14.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+datasourcelib-0.1.14.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
+datasourcelib-0.1.14.dist-info/RECORD,,

{datasourcelib-0.1.12.dist-info → datasourcelib-0.1.14.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any