datasourcelib 0.1.8__tar.gz → 0.1.10__tar.gz

{datasourcelib-0.1.8 → datasourcelib-0.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.8
+Version: 0.1.10
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.8 → datasourcelib-0.1.10}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="datasourcelib",
-    version="0.1.8",
+    version="0.1.10",
     packages=find_packages(where="src", exclude=["tests.*", "tests", "examples.*", "examples"]),
     package_dir={"": "src"},
     install_requires=[

{datasourcelib-0.1.8 → datasourcelib-0.1.10}/src/datasourcelib/datasources/azure_devops_source.py

@@ -176,8 +176,7 @@ class AzureDevOpsSource(DataSourceBase):
                 "release_type": rtype,
                 "target_date": target_date,
                 "description": c_desc,
-                "full": fullfeature,
-                "fields": norm_fields  # full field set for this work item
+                "full": fullfeature
             }
             work_item_details.append(entry)
 

{datasourcelib-0.1.8 → datasourcelib-0.1.10}/src/datasourcelib/datasources/dataverse_source.py

@@ -3,6 +3,7 @@ from datasourcelib.datasources.datasource_base import DataSourceBase
 from datasourcelib.utils.logger import get_logger
 from datasourcelib.utils.validators import require_keys
 from datasourcelib.utils.aggregation import generate_grouped_summaries
+from azure.identity import DefaultAzureCredential
 import pyodbc
 import time
 import pandas as pd
@@ -26,12 +27,26 @@ class DataverseSource(DataSourceBase):
         self._max_retries = int(self.config.get("dv_max_retries", 3))
 
     def validate_config(self) -> bool:
+
         """
         Validate required keys depending on selected dv_mode.
         - tds: requires either 'tds_connection_string' OR ('dataverse_server' and 'dataverse_database')
-        - webapi: requires 'webapi_url','client_id','client_secret','tenant_id' (or 'resource')
+        - webapi: 
+            * client credentials: 'dv_webapi_url','dv_webapi_client_id','dv_webapi_client_secret','dv_webapi_tenant_id'
+            * managed identity:    'dv_webapi_url' + dv_webapi_managed_identity_auth=True
         """
-        try:
+
+        try:            
+            if self._mode == "webapi":
+                use_mi = bool(self.config.get("dv_webapi_managed_identity_auth", False))
+                require_keys(self.config, ["dv_webapi_url"])
+                if not use_mi:
+                    require_keys(
+                        self.config,
+                        ["dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"]
+                    )
+                return True
+
             if self._mode == "webapi":
                 require_keys(self.config, ["dv_webapi_url", "dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"])
             else:
@@ -123,11 +138,44 @@ class DataverseSource(DataSourceBase):
         return conn_str
 
     def _obtain_webapi_token(self) -> Tuple[str, Dict[str, str]]:
+
         """
-        Acquire OAuth2 token using client credentials flow.
+        Acquire OAuth2 token for Dataverse Web API.
         Returns (access_token, headers)
-        Config expected keys: tenant_id, client_id, client_secret, optional resource
+
+        Paths:
+            - Managed Identity (dv_webapi_managed_identity_auth=True): uses DefaultAzureCredential and scope '<webapi_url>/.default'
+            - Client Credentials (dv_webapi_managed_identity_auth=False): uses OAuth2 client credentials with tenant/client/secret
         """
+
+        webapi_url = self.config["dv_webapi_url"].rstrip("/")
+        # Scope for MSAL-style request
+        scope = f"{webapi_url}/.default"
+
+        use_mi = bool(self.config.get("dv_webapi_managed_identity_auth", False))
+
+        # --- Managed Identity / DefaultAzureCredential path ---
+        if use_mi:
+            if DefaultAzureCredential is None:
+                raise RuntimeError(
+                    "azure-identity package required for managed identity auth (pip install azure-identity)"
+                )
+            # DefaultAzureCredential works with system/user-assigned MI in Azure,
+            # and falls back to developer credentials locally (Azure CLI/VS Code).
+            credential = DefaultAzureCredential()
+            # Obtain token for Dataverse scope
+            token_obj = credential.get_token(scope)
+            token = token_obj.token
+            if not token:
+                raise RuntimeError("Failed to obtain Managed Identity token for Dataverse Web API")
+            headers = {
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/json",
+                "OData-MaxVersion": "4.0",
+                "OData-Version": "4.0"
+            }
+            return token, headers
+
         if requests is None:
             raise RuntimeError("requests package required for Dataverse Web API mode")
         tenant = self.config["dv_webapi_tenant_id"]
@@ -166,7 +214,10 @@ class DataverseSource(DataSourceBase):
                 self._access_token = token
                 self._headers = headers
                 self._connected = True
-                logger.info("DataverseSource connected (webapi mode) to %s", self.config.get("dv_webapi_url"))
+                
+                auth_mode = "managed identity" if bool(self.config.get("dv_webapi_managed_identity_auth", False)) else "client credentials"
+                logger.info("DataverseSource connected (webapi, %s) to %s", auth_mode, self.config.get("dv_webapi_url"))
+
                 return True
             # else TDS mode
             conn_str = self._build_tds_conn_str()
@@ -251,9 +302,19 @@ class DataverseSource(DataSourceBase):
                         # exclude SharePoint metadata columns (start with '__' or prefixed with '@')
                         cols_to_keep = [c for c in df.columns if not str(c).startswith("__") and not str(c).startswith("@")]
                     df = df[cols_to_keep]
+                    summaries = generate_grouped_summaries(
+                        df=df,
+                        aggregation_field=self.config.get("dv_webapi_aggregation_field"),
+                        row_format=self.config.get("dv_webapi_row_format"),
+                        constants={"title": ""},
+                        header_format=self.config.get("dv_webapi_header_format"),
+                        sort_by=self.config.get("dv_webapi_sort_by"),     # or a column/list if you want ordering
+                        validate=True     # ensures all placeholders exist
+                    )
                     
-                    results = df.to_dict("records")
-                    return results
+                    return summaries
+                    #results = df.to_dict("records")
+                    #return results
                 # else TDS mode
                 sql = query or self.config.get("dv_tds_query") or self.config.get("dv_sql_query")
                 if not sql:

{datasourcelib-0.1.8 → datasourcelib-0.1.10}/src/datasourcelib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.8
+Version: 0.1.10
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya