datasourcelib 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

datasourcelib/datasources/dataverse_source.py

@@ -3,6 +3,7 @@ from datasourcelib.datasources.datasource_base import DataSourceBase
 from datasourcelib.utils.logger import get_logger
 from datasourcelib.utils.validators import require_keys
 from datasourcelib.utils.aggregation import generate_grouped_summaries
+from azure.identity import DefaultAzureCredential
 import pyodbc
 import time
 import pandas as pd
@@ -26,12 +27,26 @@ class DataverseSource(DataSourceBase):
         self._max_retries = int(self.config.get("dv_max_retries", 3))
 
     def validate_config(self) -> bool:
+
         """
         Validate required keys depending on selected dv_mode.
         - tds: requires either 'tds_connection_string' OR ('dataverse_server' and 'dataverse_database')
-        - webapi: requires 'webapi_url','client_id','client_secret','tenant_id' (or 'resource')
+        - webapi: 
+            * client credentials: 'dv_webapi_url','dv_webapi_client_id','dv_webapi_client_secret','dv_webapi_tenant_id'
+            * managed identity:    'dv_webapi_url' + dv_webapi_managed_identity_auth=True
         """
-        try:
+
+        try:            
+            if self._mode == "webapi":
+                use_mi = bool(self.config.get("dv_webapi_managed_identity_auth", False))
+                require_keys(self.config, ["dv_webapi_url"])
+                if not use_mi:
+                    require_keys(
+                        self.config,
+                        ["dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"]
+                    )
+                return True
+
             if self._mode == "webapi":
                 require_keys(self.config, ["dv_webapi_url", "dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"])
             else:
@@ -123,11 +138,44 @@ class DataverseSource(DataSourceBase):
         return conn_str
 
     def _obtain_webapi_token(self) -> Tuple[str, Dict[str, str]]:
+
         """
-        Acquire OAuth2 token using client credentials flow.
+        Acquire OAuth2 token for Dataverse Web API.
         Returns (access_token, headers)
-        Config expected keys: tenant_id, client_id, client_secret, optional resource
+
+        Paths:
+            - Managed Identity (dv_webapi_managed_identity_auth=True): uses DefaultAzureCredential and scope '<webapi_url>/.default'
+            - Client Credentials (dv_webapi_managed_identity_auth=False): uses OAuth2 client credentials with tenant/client/secret
         """
+
+        webapi_url = self.config["dv_webapi_url"].rstrip("/")
+        # Scope for MSAL-style request
+        scope = f"{webapi_url}/.default"
+
+        use_mi = bool(self.config.get("dv_webapi_managed_identity_auth", False))
+
+        # --- Managed Identity / DefaultAzureCredential path ---
+        if use_mi:
+            if DefaultAzureCredential is None:
+                raise RuntimeError(
+                    "azure-identity package required for managed identity auth (pip install azure-identity)"
+                )
+            # DefaultAzureCredential works with system/user-assigned MI in Azure,
+            # and falls back to developer credentials locally (Azure CLI/VS Code).
+            credential = DefaultAzureCredential()
+            # Obtain token for Dataverse scope
+            token_obj = credential.get_token(scope)
+            token = token_obj.token
+            if not token:
+                raise RuntimeError("Failed to obtain Managed Identity token for Dataverse Web API")
+            headers = {
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/json",
+                "OData-MaxVersion": "4.0",
+                "OData-Version": "4.0"
+            }
+            return token, headers
+
         if requests is None:
             raise RuntimeError("requests package required for Dataverse Web API mode")
         tenant = self.config["dv_webapi_tenant_id"]
@@ -166,7 +214,10 @@ class DataverseSource(DataSourceBase):
                 self._access_token = token
                 self._headers = headers
                 self._connected = True
-                logger.info("DataverseSource connected (webapi mode) to %s", self.config.get("dv_webapi_url"))
+                
+                auth_mode = "managed identity" if bool(self.config.get("dv_webapi_managed_identity_auth", False)) else "client credentials"
+                logger.info("DataverseSource connected (webapi, %s) to %s", auth_mode, self.config.get("dv_webapi_url"))
+
                 return True
             # else TDS mode
             conn_str = self._build_tds_conn_str()
@@ -251,9 +302,19 @@ class DataverseSource(DataSourceBase):
                         # exclude SharePoint metadata columns (start with '__' or prefixed with '@')
                         cols_to_keep = [c for c in df.columns if not str(c).startswith("__") and not str(c).startswith("@")]
                     df = df[cols_to_keep]
+                    summaries = generate_grouped_summaries(
+                        df=df,
+                        aggregation_field=self.config.get("dv_webapi_aggregation_field"),
+                        row_format=self.config.get("dv_webapi_row_format"),
+                        constants={"title": ""},
+                        header_format=self.config.get("dv_webapi_header_format"),
+                        sort_by=self.config.get("dv_webapi_sort_by"),     # or a column/list if you want ordering
+                        validate=True     # ensures all placeholders exist
+                    )
                     
-                    results = df.to_dict("records")
-                    return results
+                    return summaries
+                    #results = df.to_dict("records")
+                    #return results
                 # else TDS mode
                 sql = query or self.config.get("dv_tds_query") or self.config.get("dv_sql_query")
                 if not sql:

datasourcelib/datasources/sql_source.py

@@ -2,9 +2,10 @@ from typing import Any, Dict, List, Optional
 from datasourcelib.datasources.datasource_base import DataSourceBase
 from datasourcelib.utils.logger import get_logger
 from datasourcelib.utils.validators import require_keys
+from datasourcelib.utils.aggregation import generate_grouped_summaries
 import os
 import pyodbc
-
+import pandas as pd
 
 logger = get_logger(__name__)
 
@@ -121,7 +122,20 @@ class SQLDataSource(DataSourceBase):
                     results: List[Dict[str, Any]] = []
                     for r in rows:
                         results.append({cols[i]: r[i] for i in range(len(cols))})
-                    return results
+
+                    df = pd.DataFrame(results)
+                    summaries = generate_grouped_summaries(
+                        df=df,
+                        aggregation_field=self.config.get("sql_aggregation_field"),
+                        row_format=self.config.get("sql_aggregation_row_format"),
+                        constants={"title": ""},
+                        header_format=self.config.get("sql_aggregation_header_format"),
+                        sort_by=self.config.get("sql_aggregation_sort_by"),     # or a column/list if you want ordering
+                        validate=True     # ensures all placeholders exist
+                    )
+                    
+                    return summaries
+
                 finally:
                     try:
                         cur.close()

{datasourcelib-0.1.7.dist-info → datasourcelib-0.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.7
+Version: 0.1.9
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.7.dist-info → datasourcelib-0.1.9.dist-info}/RECORD

@@ -9,10 +9,10 @@ datasourcelib/datasources/azure_devops_source.py,sha256=3hyZIrUdgwZEQNjb2iZGDMJc
 datasourcelib/datasources/blob_source.py,sha256=Qk61_ulqUSPYDaiMzqgvJAu43c4AjTlDRdfFg4VwgDU,3574
 datasourcelib/datasources/datasource_base.py,sha256=N8fOGvTl8oWWAiydLI0Joz66luq73a5yovO0XA9Q3jk,1068
 datasourcelib/datasources/datasource_types.py,sha256=jpm4f9n1l7X9aBD58Pbr9evXiCHHEhRCLojGwchUD7A,205
-datasourcelib/datasources/dataverse_source.py,sha256=PTIWArl_rRMap5QfH8ST5kCewE0Ax1xPZ1vgSxeujpU,14080
+datasourcelib/datasources/dataverse_source.py,sha256=FRsdFdkTTXkSC_sY8IujAFBGRKCqG6xcRfres48IhaM,17021
 datasourcelib/datasources/sharepoint_source - Copy.py,sha256=7V1c-zyvTo4IuPN_YMrKwLZFgbtipbP-mtunmXjOLJQ,17664
 datasourcelib/datasources/sharepoint_source.py,sha256=t3rly2mVEI2qEDuUVqstck5ktkZW0BnF16Bke_NjPLI,23126
-datasourcelib/datasources/sql_source.py,sha256=ntZjiFXpa7V797x7mAATJV0LH-g878VHuRw-QTxEe28,6372
+datasourcelib/datasources/sql_source.py,sha256=pXs5UDAxRyRYuvw-zMNieJAZSqDndh6LlJy9GS6GoiY,7159
 datasourcelib/datasources/sql_source_bkup.py,sha256=ntZjiFXpa7V797x7mAATJV0LH-g878VHuRw-QTxEe28,6372
 datasourcelib/indexes/__init__.py,sha256=S8dz-lyxy1BTuDuLGRJNLrZD_1ku_FIUnDEm6HhMyT0,94
 datasourcelib/indexes/azure_search_index.py,sha256=kznAz06UXgyT1Clqj6gRhnBQ5HFw40ZQHJElRFIcbRo,22115
@@ -29,8 +29,8 @@ datasourcelib/utils/exceptions.py,sha256=mgcDaW1k3VndgpMOwSm7NqgyRTvvE2a5ehn3x4f
 datasourcelib/utils/file_reader.py,sha256=Zr0rwNTRWE6KeVJEXgTOPS1_JI74LiUSiX5-6qojmN0,7301
 datasourcelib/utils/logger.py,sha256=Sl6lNlvubxtK9ztzyq7vjGVyA8_-pZ_ixpk5jfVsh6U,424
 datasourcelib/utils/validators.py,sha256=fLgmRAb5OZSdMVlHu_n0RKJUDl-G8dI8JsRSfxIquh8,205
-datasourcelib-0.1.7.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
-datasourcelib-0.1.7.dist-info/METADATA,sha256=Jo1RgpvptXpS-FxA6g9-7rVkknZDfzUrOpMQVFxG-9Y,1199
-datasourcelib-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datasourcelib-0.1.7.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
-datasourcelib-0.1.7.dist-info/RECORD,,
+datasourcelib-0.1.9.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
+datasourcelib-0.1.9.dist-info/METADATA,sha256=e5FeHitCJ3JZaYoBST4sE4awseM3Sl7-kBVTAwVEXfk,1199
+datasourcelib-0.1.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datasourcelib-0.1.9.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
+datasourcelib-0.1.9.dist-info/RECORD,,