datasourcelib 0.1.12__tar.gz → 0.1.14__tar.gz

{datasourcelib-0.1.12 → datasourcelib-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.12
+Version: 0.1.14
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.12 → datasourcelib-0.1.14}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="datasourcelib",
-    version="0.1.12",
+    version="0.1.14",
     packages=find_packages(where="src", exclude=["tests.*", "tests", "examples.*", "examples"]),
     package_dir={"": "src"},
     install_requires=[

{datasourcelib-0.1.12 → datasourcelib-0.1.14}/src/datasourcelib/datasources/azure_devops_source.py

@@ -300,7 +300,6 @@ class AzureDevOpsSource(DataSourceBase):
 
                         # Display name and url
                         display_name = page.get("name") or page.get("pageName") or page_path.strip("/") or "/"
-                        new_display_name = self.sanitize(display_name.replace(" ", "_").strip()),
                         url = (
                             page.get("remoteUrl")
                             or page.get("url")
@@ -343,7 +342,7 @@ class AzureDevOpsSource(DataSourceBase):
                          # Construct a 'full' description string using available pieces
                         content_text = BeautifulSoup(content_text or "", "html.parser").get_text(),
                         parts = []
-                        if new_display_name:
+                        if display_name:
                             parts.append(f"Wiki Page Name is {display_name}. Page has information  about {display_name}")
                         if project_name:
                             parts.append(f"This page is documented by for Project '{project_name}' and by the team '{project_name}'")
@@ -354,7 +353,8 @@ class AzureDevOpsSource(DataSourceBase):
 
                         index_content = ". ".join(parts)
                         results.append({
-                            "display_name": new_display_name,
+                            "display_name": self.sanitize(display_name.replace(" ", "_").strip()),
+                            "page_name": display_name,
                             "url": url,
                             "content": index_content,
                             "project": project_name

{datasourcelib-0.1.12 → datasourcelib-0.1.14}/src/datasourcelib/utils/aggregation.py

@@ -2,6 +2,7 @@
 import pandas as pd
 from string import Formatter
 from typing import Iterable, Any, Dict, List, Optional, Union
+import regex as re
 
 def _placeholders(fmt: str) -> List[str]:
     """
@@ -12,6 +13,12 @@ def _placeholders(fmt: str) -> List[str]:
 def _safe_str(x) -> str:
     return "" if pd.isna(x) else str(x).strip()
 
+@staticmethod
+def sanitize(s: str) -> str:
+    """Keep only A-Z a-z 0-9 underscore/dash/equals in a safe way."""
+    # using the `regex` import already present as `re`
+    return re.sub(r'[^A-Za-z0-9_\-=]', '', s)
+
 def generate_grouped_summaries(
     df: pd.DataFrame,
     aggregation_field: str,
@@ -146,7 +153,7 @@ def generate_grouped_summaries(
 
         content = header + " " + " ".join(lines)
         summaries.append(
-            {"content" : content, "id": group_value}
+            {"content" : content, "id": sanitize(group_value.replace(" ", "_").strip())}
             )
 
     return summaries

{datasourcelib-0.1.12 → datasourcelib-0.1.14}/src/datasourcelib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.12
+Version: 0.1.14
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya