datasourcelib 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

datasourcelib/core/sync_base.py

@@ -14,6 +14,6 @@ class SyncBase(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def sync(self, **kwargs) -> bool:
+    def sync(self, **kwargs) -> Dict[str, Any]:
         """Execute sync operation. Returns True on success, False otherwise."""
         raise NotImplementedError

datasourcelib/core/sync_manager.py

@@ -10,6 +10,7 @@ from ..datasources.sql_source import SQLDataSource
 from ..datasources.azure_devops_source import AzureDevOpsSource
 from ..datasources.sharepoint_source import SharePointSource
 from ..datasources.blob_source import BlobStorageSource
+from ..datasources.dataverse_source import DataverseSource
 
 # concrete strategies
 from datasourcelib.strategies.full_load import FullLoadStrategy
@@ -35,11 +36,12 @@ class SyncManager:
         DataSourceType.SQL: SQLDataSource,
         DataSourceType.AZURE_DEVOPS: AzureDevOpsSource, 
         DataSourceType.SHAREPOINT: SharePointSource,
-        DataSourceType.BLOB_STORAGE: BlobStorageSource
+        DataSourceType.BLOB_STORAGE: BlobStorageSource,
+        DataSourceType.Dataverse: DataverseSource
     }
 
-    def execute_sync(self, sync_type: SyncType, 
-                    source_type: DataSourceType,
+    def execute_sync(self, sync_type: str, 
+                    source_type: str,
                     source_config: Dict[str, Any],
                     vector_db_config: Dict[str, Any], 
                     **kwargs) -> Dict[str, Any]:
@@ -47,6 +49,33 @@ class SyncManager:
         logger.info(f"Execute {sync_type} sync using {source_type} source")
         
         try:
+            # validate and convert sync_type and source_type to their Enum members
+            def _to_enum(enum_cls, val, label):
+                if isinstance(val, enum_cls):
+                    return val
+                s = str(val)
+                # case-insensitive name match
+                for member in enum_cls:
+                    if member.name.lower() == s.lower():
+                        return member
+                # try by value
+                try:
+                    return enum_cls(val)
+                except Exception:
+                    names = ", ".join([m.name for m in enum_cls])
+                    values = ", ".join([str(m.value) for m in enum_cls])
+                    raise ValueError(f"Invalid {label}. Permitted names: {names}. Permitted values: {values}")
+
+            try:
+                sync_type = _to_enum(SyncType, sync_type, "sync_type")
+                source_type = _to_enum(DataSourceType, source_type, "source_type")
+            except ValueError as ex:
+                logger.error(str(ex))
+                return {
+                    "status": SyncStatus.FAILED,
+                    "message": str(ex),
+                    "started_at": start
+                }
             # Get data source class
             source_cls = self._datasource_map.get(source_type)
             if not source_cls:
@@ -76,15 +105,7 @@ class SyncManager:
                 }
 
             # Execute sync
-            success = strategy.sync(**kwargs)
-            status = SyncStatus.SUCCESS if success else SyncStatus.FAILED
-            
-            return {
-                "status": status,
-                "message": f"{sync_type} completed" if success else f"{sync_type} failed",
-                "started_at": start,
-                "finished_at": datetime.utcnow()
-            }
+            return strategy.sync(**kwargs)
 
         except Exception as ex:
             logger.exception("SyncManager.execute_sync failed")

datasourcelib/datasources/datasource_types.py

@@ -4,4 +4,5 @@ class DataSourceType(str, Enum):
     SQL = "sql"
     AZURE_DEVOPS = "azure_devops"
     SHAREPOINT = "sharepoint" 
-    BLOB_STORAGE = "blob_storage"
+    BLOB_STORAGE = "blob_storage"
+    Dataverse = "Dataverse"

datasourcelib/datasources/dataverse_source.py

@@ -0,0 +1,305 @@
+from typing import Any, Dict, List, Optional, Tuple
+from datasourcelib.datasources.datasource_base import DataSourceBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.utils.validators import require_keys
+from datasourcelib.utils.aggregation import generate_grouped_summaries
+import pyodbc
+import time
+import pandas as pd
+
+# optional requests import (webapi mode)
+try:
+    import requests  # type: ignore
+except Exception:
+    requests = None  # lazy import
+
+logger = get_logger(__name__)
+
+class DataverseSource(DataSourceBase):
+    
+    def __init__(self, config: Dict[str, Any]):
+        super().__init__(config)
+        self._conn = None
+        self._mode = (self.config.get("dv_mode") or "tds").lower()  # "tds" or "webapi"
+        self._access_token: Optional[str] = None
+        self._headers: Dict[str, str] = {}
+        self._max_retries = int(self.config.get("dv_max_retries", 3))
+
+    def validate_config(self) -> bool:
+        """
+        Validate required keys depending on selected dv_mode.
+        - tds: requires either 'tds_connection_string' OR ('dataverse_server' and 'dataverse_database')
+        - webapi: requires 'webapi_url','client_id','client_secret','tenant_id' (or 'resource')
+        """
+        try:
+            if self._mode == "webapi":
+                require_keys(self.config, ["dv_webapi_url", "dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"])
+            else:
+                # TDS mode (ODBC)
+                if "dv_tds_connection_string" in self.config:
+                    return True
+                # otherwise require components
+                require_keys(self.config, ["dv_tds_server", "dv_tds_database"])
+                # if not using integrated auth require creds
+                if not bool(self.config.get("dv_tds_windows_auth", False)):
+                    require_keys(self.config, ["dv_tds_username", "dv_tds_password"])
+            return True
+        except Exception as ex:
+            logger.error("DataverseSource.validate_config failed: %s", ex)
+            return False
+
+    # -------------------------
+    # Connection helpers
+    # -------------------------
+    def _get_available_driver(self) -> str:
+        """Return first suitable ODBC driver for SQL/Dataverse TDS access."""
+        preferred_drivers = [
+            "ODBC Driver 18 for SQL Server",
+            "ODBC Driver 17 for SQL Server",
+            "SQL Server Native Client 11.0",
+            "SQL Server"
+        ]
+        try:
+            drivers = pyodbc.drivers()
+            logger.info("Available ODBC drivers: %s", drivers)
+            
+            for d in preferred_drivers:
+                if d in drivers:
+                    logger.info("Using ODBC driver: %s", d)
+                    return d
+            
+            # fallback to first available
+            if drivers:
+                logger.warning("No preferred driver found. Using: %s", drivers[0])
+                return drivers[0]
+            raise RuntimeError("No ODBC drivers available")
+        except Exception as ex:
+            logger.error("DataverseSource._get_available_driver failed: %s", ex)
+            raise
+
+    def _build_tds_conn_str(self) -> str:
+        """Build valid connection string with proper parameter names."""
+        if "dv_tds_connection_string" in self.config:
+            return self.config["dv_tds_connection_string"]
+        
+        driver = self._get_available_driver()
+        # Fix: use correct config key names (dv_tds_server, not dv_tds_dataverse_server)
+        server = self.config.get("dv_tds_server", "").strip()
+        database = self.config.get("dv_tds_database", "").strip()
+        
+        if not server:
+            raise ValueError("dv_tds_server are required")
+        
+        logger.info("Building TDS connection (driver=%s, server=%s, database=%s)", driver, server, database)
+        
+        # Use curly braces for driver name (handles spaces in driver names)
+        parts = [f"DRIVER={{{driver}}}"]
+        parts.append(f"Server={server}")
+        parts.append(f"Database={database}")
+        password = None
+        if bool(self.config.get("dv_tds_windows_auth", False)):
+            parts.append("Trusted_Connection=yes")
+            logger.info("Using Windows authentication")
+        else:
+            username = self.config.get("dv_tds_username", "").strip()
+            password = self.config.get("dv_tds_password", "").strip()
+            
+            if not username or not password:
+                raise ValueError("dv_tds_username and dv_tds_password required when Windows auth disabled")
+            
+            parts.append(f"UID={username}")
+            parts.append(f"PWD={password}")
+            parts.append("Authentication=ActiveDirectoryInteractive")
+        # Encryption settings
+        if not bool(self.config.get("dv_tds_is_onprem", False)):
+            parts.append("Encrypt=yes")
+            parts.append("TrustServerCertificate=no")
+        else:
+            parts.append("Encrypt=optional")
+            parts.append("TrustServerCertificate=yes")
+        
+        conn_str = ";".join(parts)
+        logger.debug("Connection string: %s", conn_str.replace(password or "", "***") if password else conn_str)
+        return conn_str
+
+    def _obtain_webapi_token(self) -> Tuple[str, Dict[str, str]]:
+        """
+        Acquire OAuth2 token using client credentials flow.
+        Returns (access_token, headers)
+        Config expected keys: tenant_id, client_id, client_secret, optional resource
+        """
+        if requests is None:
+            raise RuntimeError("requests package required for Dataverse Web API mode")
+        tenant = self.config["dv_webapi_tenant_id"]
+        client_id = self.config["dv_webapi_client_id"]
+        client_secret = self.config["dv_webapi_client_secret"]
+        # resource or scope: prefer explicit resource, else fallback to webapi_url host
+        resource = self.config.get("dv_webapi_resource")
+        if not resource:
+            # infer resource from webapi_url e.g. https://<org>.crm.dynamics.com
+            webapi_url = self.config["dv_webapi_url"].rstrip("/")
+            resource = webapi_url.split("://")[-1]
+            resource = f"https://{resource}"  # as resource
+        token_url = f"https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token"
+        data = {
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "scope": f"{resource}/.default"
+        }
+        resp = requests.post(token_url, data=data, timeout=30)
+        resp.raise_for_status()
+        j = resp.json()
+        token = j.get("access_token")
+        if not token:
+            raise RuntimeError("Failed to obtain access token for Dataverse webapi")
+        headers = {"Authorization": f"Bearer {token}", "Accept": "application/json", "OData-MaxVersion": "4.0", "OData-Version": "4.0"}
+        return token, headers
+
+    # -------------------------
+    # Public connection API
+    # -------------------------
+    def connect(self) -> bool:
+        try:
+            if self._mode == "webapi":
+                token, headers = self._obtain_webapi_token()
+                self._access_token = token
+                self._headers = headers
+                self._connected = True
+                logger.info("DataverseSource connected (webapi mode) to %s", self.config.get("dv_webapi_url"))
+                return True
+            # else TDS mode
+            conn_str = self._build_tds_conn_str()
+            self._conn = pyodbc.connect(conn_str, timeout=int(self.config.get("dv_tds_timeout", 30)))
+            self._connected = True
+            logger.info("DataverseSource connected (dv_tds mode) to %s/%s", self.config.get("dv_server"), self.config.get("dv_database"))
+            return True
+        except pyodbc.Error as ex:
+            logger.error("DataverseSource.connect failed - ODBC Error: %s", ex)
+            self._connected = False
+            return False
+        except requests.RequestException as ex:
+            logger.error("DataverseSource.connect failed - HTTP Error: %s", ex)
+            self._connected = False
+            return False
+        except Exception as ex:
+            logger.exception("DataverseSource.connect failed")
+            self._connected = False
+            return False
+
+    def disconnect(self) -> None:
+        try:
+            if self._conn:
+                try:
+                    self._conn.close()
+                except Exception:
+                    pass
+            self._conn = None
+            self._access_token = None
+            self._headers = {}
+        finally:
+            self._connected = False
+            logger.info("DataverseSource disconnected")
+
+    # -------------------------
+    # Data fetch
+    # -------------------------
+    def fetch_data(self, query: Optional[str] = None, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Fetch rows from Dataverse.
+        - TDS mode: executes SQL query (config key 'tds_query' or provided 'query')
+        - WebAPI mode: calls Dataverse Web API path fragment (e.g. 'accounts?$select=name') or uses 'entity_set' + query params
+        Returns list[dict].
+        """
+        attempt = 0
+        while attempt < self._max_retries:
+            try:
+                if not getattr(self, "_connected", False):
+                    ok = self.connect()
+                    if not ok:
+                        raise RuntimeError("DataverseSource: cannot connect")
+
+                if self._mode == "webapi":
+                    if requests is None:
+                        raise RuntimeError("requests package required for webapi mode")
+                    webapi_url = self.config["dv_webapi_url"].rstrip("/")
+                    # if query provided, treat it as path fragment; else use entity_set from config
+                    path_fragment = query or self.config.get("dv_webapi_entity_set")
+                    if not path_fragment:
+                        raise ValueError("DataverseSource.fetch_data requires a webapi 'query' or 'entity_set' in config")
+                    url = f"{webapi_url}/api/data/v9.1/{path_fragment.lstrip('/')}"
+                    params = kwargs.get("params")
+                    resp = requests.get(url, headers=self._headers, params=params, timeout=60)
+                    resp.raise_for_status()
+                    j = resp.json()
+                    items: Any = []
+                    # Dataverse OData responses typically use 'value' for collections
+                    if isinstance(j, dict) and "value" in j:
+                        items = j["value"]
+                    # otherwise return the raw json wrapped in a list or as-is
+                    elif isinstance(j, list):
+                        items= j
+                    else:
+                        items= [j]
+
+                    df = pd.DataFrame(items)
+                    # filter columns if configured
+                    keep = self.config.get("dv_webapi_columns_to_keep")
+                    if isinstance(keep, list) and keep:
+                        cols_to_keep = [c for c in df.columns if c in keep]
+                    else:
+                        # exclude SharePoint metadata columns (start with '__' or prefixed with '@')
+                        cols_to_keep = [c for c in df.columns if not str(c).startswith("__") and not str(c).startswith("@")]
+                    df = df[cols_to_keep]
+                    
+                    results = df.to_dict("records")
+                    return results
+                # else TDS mode
+                sql = query or self.config.get("dv_tds_query") or self.config.get("dv_sql_query")
+                if not sql:
+                    raise ValueError("DataverseSource.fetch_data requires a SQL query (tds mode)")
+
+                cur = self._conn.cursor()
+                try:
+                    cur.execute(sql)
+                    cols = [c[0] for c in (cur.description or [])]
+                    rows = cur.fetchall()
+                    results: List[Dict[str, Any]] = []
+                    for r in rows:
+                        results.append({cols[i]: r[i] for i in range(len(cols))})
+                    
+                    df = pd.DataFrame(results)
+                    summaries = generate_grouped_summaries(
+                        df=df,
+                        aggregation_field=self.config.get("dv_tds_aggregation_field"),
+                        row_format=self.config.get("dv_tds_row_format"),
+                        constants={"title": ""},
+                        header_format=self.config.get("dv_tds_header_format"),
+                        sort_by=self.config.get("dv_tds_sort_by"),     # or a column/list if you want ordering
+                        validate=True     # ensures all placeholders exist
+                    )
+                    
+                    return summaries
+                finally:
+                    try:
+                        cur.close()
+                    except Exception:
+                        pass
+
+            except Exception as ex:
+                attempt += 1
+                logger.warning("DataverseSource.fetch_data attempt %d/%d failed: %s", attempt, self._max_retries, ex)
+                # transient retry for network/connection errors
+                if attempt >= self._max_retries:
+                    logger.exception("DataverseSource.fetch_data final failure")
+                    raise
+                # backoff
+                time.sleep(min(2 ** attempt, 10))
+                # try reconnect for next attempt
+                try:
+                    self.disconnect()
+                except Exception:
+                    pass
+
+        # unreachable; defensive
+        return []

datasourcelib/datasources/sql_source_bkup.py

@@ -0,0 +1,159 @@
+from typing import Any, Dict, List, Optional
+from datasourcelib.datasources.datasource_base import DataSourceBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.utils.validators import require_keys
+import os
+import pyodbc
+
+
+logger = get_logger(__name__)
+
+class SQLDataSource(DataSourceBase):
+    
+    def __init__(self, config: Dict[str, Any]):
+        super().__init__(config)
+        self._conn = None
+        self._is_sqlite = False
+
+    def validate_config(self) -> bool:
+        """
+        Validate config. If sql_windows_auth is True then sql_username/sql_password are optional.
+        Otherwise require sql_username and sql_password.
+        """
+        try:
+            # Always require server/database at minimum
+            require_keys(self.config, ["sql_server", "sql_database"])
+            # If not using Windows authentication, require credentials
+            if not bool(self.config.get("sql_windows_auth", False)):
+                require_keys(self.config, ["sql_username", "sql_password"])
+            return True
+        except Exception as ex:
+            logger.error("SQLDataSource.validate_config: %s", ex)
+            return False
+
+    def connect(self) -> bool:
+        try:
+            sql_server = self.config.get("sql_server", "")
+            sql_database = self.config.get("sql_database", "")
+            sql_is_onprem = self.config.get("sql_is_onprem", False)
+
+            # Determine auth mode: sql_windows_auth (Trusted Connection) overrides username/password
+            sql_windows_auth = bool(self.config.get("sql_windows_auth", False))
+
+            # Get available driver
+            sql_driver = self._get_available_driver()
+
+            # Build connection string
+            conn_params = [
+                f'DRIVER={sql_driver}',
+                f'SERVER={sql_server}',
+                f'DATABASE={sql_database}',
+            ]
+
+            if sql_windows_auth:
+                # Use integrated Windows authentication (Trusted Connection)
+                # This will use the current process credentials / kerberos ticket.
+                conn_params.append('Trusted_Connection=yes')
+                logger.info("SQLDataSource using Windows (integrated) authentication")
+            else:
+                sql_username = self.config.get("sql_username", "")
+                sql_password = self.config.get("sql_password", "")
+                conn_params.extend([f'UID={sql_username}', f'PWD={sql_password}'])
+
+            # Add encryption settings based on environment
+            if not sql_is_onprem:
+                conn_params.extend([
+                    'Encrypt=yes',
+                    'TrustServerCertificate=no'
+                ])
+            else:
+                conn_params.extend([
+                    'Encrypt=optional',
+                    'TrustServerCertificate=yes'
+                ])
+
+            conn_str = ';'.join(conn_params)
+
+            # Attempt connection with timeout
+            self._conn = pyodbc.connect(conn_str, timeout=30)
+            self._connected = True
+            logger.info("SQLDataSource connected to %s using driver %s (sql_windows_auth=%s)", sql_server, sql_driver, sql_windows_auth)
+            return True
+
+        except pyodbc.Error as ex:
+            logger.error("SQLDataSource.connect failed - ODBC Error: %s", ex)
+            self._connected = False
+            return False
+        except Exception as ex:
+            logger.error("SQLDataSource.connect failed - Unexpected Error: %s", ex)
+            self._connected = False
+            return False
+
+    def disconnect(self) -> None:
+        try:
+            if self._conn:
+                self._conn.close()
+        finally:
+            self._conn = None
+            self._connected = False
+            logger.info("SQLDataSource disconnected")
+
+    def fetch_data(self, query: Optional[str] = None, **kwargs) -> List[Dict[str, Any]]:
+        max_retries = 3
+        retry_count = 0
+        
+        while retry_count < max_retries:
+            try:
+                if not self._connected:
+                    ok = self.connect()
+                    if not ok:
+                        raise RuntimeError("SQLDataSource: not connected and cannot connect")
+
+                query = self.config.get("sql_query")
+                if not query:
+                    raise ValueError("SQLDataSource.fetch_data requires a query")
+
+                cur = self._conn.cursor()
+                try:
+                    cur.execute(query)
+                    cols = [d[0] if hasattr(d, "__len__") else d[0] for d in (cur.description or [])]
+                    rows = cur.fetchall()
+                    results: List[Dict[str, Any]] = []
+                    for r in rows:
+                        results.append({cols[i]: r[i] for i in range(len(cols))})
+                    return results
+                finally:
+                    try:
+                        cur.close()
+                    except Exception:
+                        pass
+                        
+            except pyodbc.OperationalError as ex:
+                # Handle connection lost
+                retry_count += 1
+                logger.warning("Connection lost, attempt %d of %d: %s", retry_count, max_retries, ex)
+                self.disconnect()
+                if retry_count >= max_retries:
+                    raise
+            except Exception as ex:
+                logger.error("Query execution failed: %s", ex)
+                raise
+
+    def _get_available_driver(self) -> str:
+        """Get first available SQL Server driver from preferred list."""
+        preferred_drivers = [
+            'ODBC Driver 18 for SQL Server',
+            'ODBC Driver 17 for SQL Server',
+            'SQL Server Native Client 11.0',
+            'SQL Server'
+        ]
+        
+        try:
+            available_drivers = pyodbc.drivers()
+            for driver in preferred_drivers:
+                if driver in available_drivers:
+                    return driver
+            raise RuntimeError(f"No suitable SQL Server driver found. Available drivers: {available_drivers}")
+        except Exception as ex:
+            logger.error("Failed to get SQL drivers: %s", ex)
+            raise

datasourcelib/strategies/daily_load.py

@@ -1,22 +1,47 @@
 from datasourcelib.core.sync_base import SyncBase
 from datasourcelib.utils.logger import get_logger
-from datetime import datetime, timedelta
+from datetime import datetime, timezone
+from typing import Dict, Any, Optional
 
 logger = get_logger(__name__)
 
 class DailyLoadStrategy(SyncBase):
-    """Daily scheduled load (wraps incremental)."""
+    """Daily scheduled load strategy (wraps incremental sync)."""
 
     def validate(self) -> bool:
+        """Validate strategy preconditions."""
         return True
 
-    def sync(self, run_date: str = None, **kwargs) -> bool:
+    def sync(self, run_date: Optional[str] = None, **kwargs) -> Dict[str, Any]:
+        """
+        Execute daily load for the given run_date (ISO date string).
+        If run_date is None, today's UTC date is used.
+
+        Returns a dict with status, message and ISO timestamps.
+        """
+        # Ensure run_date and started_at exist even if exceptions occur early
+        run_date = run_date
+        started_at = datetime.now(timezone.utc).isoformat()
         try:
-            run_date = run_date or datetime.utcnow().date().isoformat()
-            logger.info("Starting daily load for %s", run_date)
-            # Typically call incremental with last_sync = previous day midnight
-            # TODO implement scheduling integration externally; the strategy here is idempotent
-            return True
-        except Exception:
+            run_date = run_date or datetime.now(timezone.utc).date().isoformat()
+            logger.info("Starting daily load for %s (requested run_date=%s)", started_at, run_date)
+
+            # TODO: call incremental sync / processing here, for example:
+            # result = self.incremental_sync(last_sync=..., **kwargs)
+
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "success",
+                "message": f"Daily load completed for {run_date}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }
+        except Exception as ex:
             logger.exception("DailyLoadStrategy.sync failed")
-            return False
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "failure",
+                "message": f"Exception: {ex}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }

datasourcelib/strategies/full_load.py

@@ -1,38 +1,99 @@
+from typing import Dict, Any
+from datetime import datetime, timezone
+
 from datasourcelib.core.sync_base import SyncBase
 from datasourcelib.utils.logger import get_logger
 from datasourcelib.indexes.azure_search_index import AzureSearchIndexer
+
 logger = get_logger(__name__)
 
+
 class FullLoadStrategy(SyncBase):
     """Full load: replace or reload entire source into vector DB."""
 
     def validate(self) -> bool:
-        # Minimal validation: required keys exist
-        dsok = self.data_source.validate_config()
-        return dsok
+        # Minimal validation: required keys exist on datasource
+        try:
+            return bool(self.data_source and self.data_source.validate_config())
+        except Exception:
+            logger.exception("FullLoadStrategy.validate failed")
+            return False
 
-    def sync(self, **kwargs) -> bool:
+    def sync(self, **kwargs) -> Dict[str, Any]:
+        """
+        Execute full load: read data from data_source and index into vector DB (Azure Search).
+        Returns a dict with status, message and ISO timestamps.
+        """
+        started_at = datetime.now(timezone.utc).isoformat()
         try:
-            logger.info("Running full data load")
+            logger.info("Running full data load (started_at=%s)", started_at)
+
+            # Fetch data from configured data source
             data = self.data_source.fetch_data(**kwargs)
-            for key, value in kwargs.items():
-                print(f"{key} = {value}")
-            # Implement real extract -> transform -> load to vector DB
-            # Example pseudocode:
-            # vector_client.upsert_batch(self.vector_db_config, rows)
-            # New: use AzureSearchIndexer to create index and upload documents if requested
-            if isinstance(data, list) and data:                
-                indexer = AzureSearchIndexer(self.vector_db_config or {})
-                if not indexer.validate_config():
-                    logger.error("Vector DB config invalid for Azure Search indexer")
-                    return False
-                ok = indexer.index(data)
-                if not ok:
-                    logger.error("Indexing data to Azure Search failed")
-                    return False
-
-            logger.info("Full data load finished successfully")
-            return True
-        except Exception:
+
+            # Log kwargs for debugging at debug level
+            if kwargs:
+                logger.debug("FullLoadStrategy.sync kwargs: %s", kwargs)
+
+            # If no data returned, finish gracefully
+            total_records = len(data) if isinstance(data, (list, tuple)) else (1 if data is not None else 0)
+            if total_records == 0:
+                finished_at = datetime.now(timezone.utc).isoformat()
+                msg = "No records returned from data source"
+                logger.info(msg)
+                return {
+                    "status": "success",
+                    "message": msg,
+                    "started_at": started_at,
+                    "finished_at": finished_at,
+                    "loaded_records": 0
+                }
+
+            # Use AzureSearchIndexer to create index and upload documents if requested
+            indexer = AzureSearchIndexer(self.vector_db_config or {})
+            if not indexer.validate_config():
+                finished_at = datetime.now(timezone.utc).isoformat()
+                msg = "Vector DB config invalid for Azure Search indexer"
+                logger.error(msg)
+                return {
+                    "status": "failure",
+                    "message": msg,
+                    "started_at": started_at,
+                    "finished_at": finished_at,
+                    "loaded_records": 0
+                }
+
+            ok = indexer.index(data)
+            if not ok:
+                finished_at = datetime.now(timezone.utc).isoformat()
+                msg = "Indexing data to Azure Search failed"
+                logger.error(msg)
+                return {
+                    "status": "failure",
+                    "message": msg,
+                    "started_at": started_at,
+                    "finished_at": finished_at,
+                    "loaded_records": total_records
+                }
+
+            finished_at = datetime.now(timezone.utc).isoformat()
+            msg = f"Full load completed. Loaded {total_records} records."
+            logger.info("Full data load finished successfully (%s)", msg)
+            return {
+                "status": "success",
+                "message": msg,
+                "started_at": started_at,
+                "finished_at": finished_at,
+                "loaded_records": total_records
+            }
+
+        except Exception as ex:
             logger.exception("FullLoadStrategy.sync failed")
-            return False
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "failure",
+                "message": f"Exception: {ex}",
+                "started_at": started_at,
+                "finished_at": finished_at,
+                "loaded_records": 0
+            }

datasourcelib/strategies/incremental_load.py

@@ -1,7 +1,7 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from datasourcelib.core.sync_base import SyncBase
 from datasourcelib.utils.logger import get_logger
-
+from typing import Dict, Any
 logger = get_logger(__name__)
 
 class IncrementalLoadStrategy(SyncBase):
@@ -14,14 +14,27 @@ class IncrementalLoadStrategy(SyncBase):
             return False
         return True
 
-    def sync(self, last_sync: str = None, **kwargs) -> bool:
+    def sync(self, last_sync: str = None, **kwargs) -> Dict[str, Any]:
         try:
+            started_at = datetime.now(timezone.utc).isoformat()
             last = last_sync or self.source_config.get("last_sync")
             logger.info("Running incremental load since %s", last)
             # TODO: fetch delta rows since 'last' and upsert to vector DB
             # After successful run store new last_sync timestamp
             logger.info("Incremental load completed")
-            return True
-        except Exception:
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "success",
+                "message": f"Incremental load completed since {last}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }
+        except Exception as ex:
             logger.exception("IncrementalLoadStrategy.sync failed")
-            return False
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "failure",
+                "message": f"Exception: {ex}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }

datasourcelib/strategies/ondemand_load.py

@@ -1,6 +1,7 @@
 from datasourcelib.core.sync_base import SyncBase
 from datasourcelib.utils.logger import get_logger
-
+from typing import Dict, Any
+from datetime import datetime, timezone
 logger = get_logger(__name__)
 
 class OnDemandLoadStrategy(SyncBase):
@@ -9,11 +10,24 @@ class OnDemandLoadStrategy(SyncBase):
     def validate(self) -> bool:
         return True
 
-    def sync(self, **kwargs) -> bool:
+    def sync(self, **kwargs) -> Dict[str, Any]:
         try:
+            started_at = datetime.now(timezone.utc).isoformat()
             logger.info("On-demand sync invoked with params: %s", kwargs)
             # Use kwargs to drive partial loads, filters, ids etc.
-            return True
-        except Exception:
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "success",
+                "message": f"Ondemand load completed.",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }
+        except Exception as ex:
             logger.exception("OnDemandLoadStrategy.sync failed")
-            return False
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "failure",
+                "message": f"Exception: {ex}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }

datasourcelib/strategies/timerange_load.py

@@ -1,6 +1,7 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from datasourcelib.core.sync_base import SyncBase
 from datasourcelib.utils.logger import get_logger
+from typing import Dict, Any
 
 logger = get_logger(__name__)
 
@@ -11,14 +12,27 @@ class TimeRangeLoadStrategy(SyncBase):
         # rely on params at runtime; minimal validation OK
         return True
 
-    def sync(self, start: str = None, end: str = None, **kwargs) -> bool:
+    def sync(self, start: str = None, end: str = None, **kwargs) -> Dict[str, Any]:
         try:
+            started_at = datetime.now(timezone.utc).isoformat()
             if not start or not end:
                 logger.error("TimeRangeLoadStrategy requires 'start' and 'end'")
                 return False
             logger.info("Time range load between %s and %s", start, end)
             # TODO: query source for timeframe and upsert
-            return True
-        except Exception:
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "success",
+                "message": f"TimeRange load completed between {start} and {end}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }
+        except Exception as ex:
             logger.exception("TimeRangeLoadStrategy.sync failed")
-            return False
+            finished_at = datetime.now(timezone.utc).isoformat()
+            return {
+                "status": "failure",
+                "message": f"Exception: {ex}",
+                "started_at": started_at,
+                "finished_at": finished_at
+            }

datasourcelib/utils/aggregation.py

@@ -0,0 +1,152 @@
+
+import pandas as pd
+from string import Formatter
+from typing import Iterable, Any, Dict, List, Optional, Union
+
+def _placeholders(fmt: str) -> List[str]:
+    """
+    Extract top-level placeholder names from a format string.
+    e.g., 'Number {i} is {fname}' -> ['i', ' """
+    return [field_name for _, field_name, _, _ in Formatter().parse(fmt) if field_name]
+
+def _safe_str(x) -> str:
+    return "" if pd.isna(x) else str(x).strip()
+
+def generate_grouped_summaries(
+    df: pd.DataFrame,
+    aggregation_field: str,
+    row_format: str,
+    *,
+    header_format: str = "{group_value} has {count} record{plural}.",
+    constants: Optional[Dict[str, Union[str, int, float]]] = None,
+    drop_empty_groups: bool = True,
+    sort_by: Optional[Union[str, Iterable[str]]] = None,
+    validate: bool = True
+) -> List[Dict[str, Any]]:
+    """
+    Build grouped summaries strictly when `aggregation_field` exists in `df` and is non-empty.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Source dataset.
+    aggregation_field : str
+        Column name to group by. Must exist in `df`.
+    row_format : str
+        Format string applied per row within a group.
+        You may use placeholders for any df columns, plus:
+          - {i}: 1-based sequence number within group
+          - constants you provide (e.g., {title_prefix})
+    headertr, optional
+        Format string for group headers. Available placeholders:
+          - {group_value}: the group key
+          - {count}: number of rows in the group
+          - {plural}: '' when count==1 else 's'
+        Default: "{group_value} has {count} record{plural}."
+    constants : dict, optional
+        Additional fixed values to be merged into each row's format context.
+        Example: {"title_prefix": "Mr"}
+    drop_empty_groups : bool, optional
+        If True, rows with blank/empty group values are discarded before grouping.
+    sort_by : str | Iterable[str] | None, optional
+        If provided, sorts rows within each group by these columns before formatting.
+    validate : bool, optional
+        If True, checks that all placeholders used in `row_format` and `header_format`
+        are available (in df columns or computed context). Raises ValueError if missing.
+
+    Returns
+    -------
+    List[str]
+        One formatted string per group (header + row lines joined with spaces).
+
+    Raises
+    ------
+    ValueError
+        - If `aggregation_field` is missing or empty
+        - If no non-empty values exist for `aggregation_field` (with drop_empty_groups=True)
+        - If required placeholders are missing when `validate=True`
+    KeyError
+        - If columns referenced in `sort_by` are missing
+    """
+    # Basic checks
+    if df.empty:
+        return []
+
+    agg_field = (aggregation_field or "").strip()
+    if not agg_field:
+        return df.to_dict("records")
+    if agg_field not in df.columns:
+        raise ValueError(f"aggregation_field '{agg_field}' not found in DataFrame columns: {list(df.columns)}")
+
+    # Prepare working frame
+    working = df.copy()
+    working[agg_field] = working[agg_field].astype(str).str.strip()
+
+    if drop_empty_groups:
+        working = working[working[agg_field].astype(bool)]
+
+    if working.empty:
+        raise ValueError(f"No rows with non-empty values found for aggregation_field '{agg_field}'.")
+
+    # Optional sort within groups
+    if sort_by is not None:
+        sort_cols = [sort_by] if isinstance(sort_by, str) else list(sort_by)
+        missing_sort = [c for c in sort_cols if c not in working.columns]
+        if missing_sort:
+            raise KeyError(f"sort_by columns not found in DataFrame: {missing_sort}")
+        working = working.sort_values(sort_cols, kind="stable")
+
+    # Validation of placeholders (if requested)
+    if validate:
+        df_cols = set(working.columns)
+        row_keys = set(_placeholders(row_format))
+        header_keys = set(_placeholders(header_format))
+        # Context keys provided by the function
+        provided_keys = {"i", "group_value", "count", "plural"}
+        constant_keys = set((constants or {}).keys())
+
+        missing_row = [k for k in row_keys if k not in df_cols and k not in constant_keys and k not in provided_keys]
+        missing_header = [k for k in header_keys if k not in provided_keys and k not in constant_keys and k not in df_cols]
+        if missing_row:
+            raise ValueError(
+                f"row_format references missing keys: {missing_row}. "
+                f"Ensure these are either df columns or in `constants`."
+            )
+        if missing_header:
+            raise ValueError(
+                f"header_format references missing keys: {missing_header}. "
+                f"Use only {{group_value}}, {{count}}, {{plural}} or provide constants."
+            )
+
+    # Build summaries per group
+    summaries = []
+    for group_value, group_df in working.groupby(agg_field, sort=True):
+        group_df = group_df.reset_index(drop=True)
+        count = len(group_df)
+        plural = "" if count == 1 else "s"
+
+        header_ctx = {
+            "group_value": _safe_str(group_value),
+            "count": count,
+            "plural": plural,
+            **(constants or {}),
+        }
+        header = header_format.format(**header_ctx)
+
+        lines = []
+        for i, row in enumerate(group_df.to_dict(orient="records"), start=1):
+            # Row context = df row + sequence + constants (constants override df if same key)
+            row_ctx = {k: _safe_str(v) for k, v in row.items()}
+            row_ctx.update({"i": i})
+            if constants:
+                # Constants override row values with same keys
+                row_ctx.update(constants)
+
+            lines.append(row_format.format(**row_ctx))
+
+        content = header + " " + " ".join(lines)
+        summaries.append(
+            {"content" : content, "id": group_value}
+            )
+
+    return summaries

{datasourcelib-0.1.5.dist-info → datasourcelib-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.5
+Version: 0.1.7
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.5.dist-info → datasourcelib-0.1.7.dist-info}/RECORD

@@ -1,33 +1,36 @@
 datasourcelib/__init__.py,sha256=I7JTSZ1J6ULg_TfdMEgFcd1regkCHuyKdZT4DcPtoyQ,78
 datasourcelib/core/__init__.py,sha256=nsXojDd97T7eMqqtCsZr1qSYLBitvKydSZRb9Dg7hqU,462
-datasourcelib/core/sync_base.py,sha256=AfwwaV3rJOFKVmKKpSj-BwznnCDCaeuT4LLNDfA3NAY,716
-datasourcelib/core/sync_manager.py,sha256=lj070S3PwSNcB0UL_ZDzDAm6uJ9G38TY491vQZ1dL3o,3849
+datasourcelib/core/sync_base.py,sha256=fKbsJYtPIV0ow7sGH7O7GmAEeeSefvD16LBOz0dP4TU,726
+datasourcelib/core/sync_manager.py,sha256=pfnvWv4AwmlJJUIsfxNNxYDBOsa7juTIxgFJIEZ5bIM,4842
 datasourcelib/core/sync_types.py,sha256=KVZB7PkfkFTzghoe--U8jLeAU8XAfba9qMRIVcUjuMc,297
 datasourcelib/datasources/__init__.py,sha256=lZtgs0vT-2gub5UZo8BUnREZl3K_-_xYqUP8mjf8vhM,436
 datasourcelib/datasources/azure_devops_source copy.py,sha256=g-IOCq5vGwwteU21jZPWW_GggMu1_myVJkP0_BmSdGY,7282
 datasourcelib/datasources/azure_devops_source.py,sha256=3hyZIrUdgwZEQNjb2iZGDMJcAw3Z6r7oV0hWAq_zMsg,8005
 datasourcelib/datasources/blob_source.py,sha256=Qk61_ulqUSPYDaiMzqgvJAu43c4AjTlDRdfFg4VwgDU,3574
 datasourcelib/datasources/datasource_base.py,sha256=N8fOGvTl8oWWAiydLI0Joz66luq73a5yovO0XA9Q3jk,1068
-datasourcelib/datasources/datasource_types.py,sha256=eEiWymYS05X_TxwuB7P3MpphPG1En67h3kRiSGeHjQ0,176
+datasourcelib/datasources/datasource_types.py,sha256=jpm4f9n1l7X9aBD58Pbr9evXiCHHEhRCLojGwchUD7A,205
+datasourcelib/datasources/dataverse_source.py,sha256=PTIWArl_rRMap5QfH8ST5kCewE0Ax1xPZ1vgSxeujpU,14080
 datasourcelib/datasources/sharepoint_source - Copy.py,sha256=7V1c-zyvTo4IuPN_YMrKwLZFgbtipbP-mtunmXjOLJQ,17664
 datasourcelib/datasources/sharepoint_source.py,sha256=t3rly2mVEI2qEDuUVqstck5ktkZW0BnF16Bke_NjPLI,23126
 datasourcelib/datasources/sql_source.py,sha256=ntZjiFXpa7V797x7mAATJV0LH-g878VHuRw-QTxEe28,6372
+datasourcelib/datasources/sql_source_bkup.py,sha256=ntZjiFXpa7V797x7mAATJV0LH-g878VHuRw-QTxEe28,6372
 datasourcelib/indexes/__init__.py,sha256=S8dz-lyxy1BTuDuLGRJNLrZD_1ku_FIUnDEm6HhMyT0,94
 datasourcelib/indexes/azure_search_index.py,sha256=kznAz06UXgyT1Clqj6gRhnBQ5HFw40ZQHJElRFIcbRo,22115
 datasourcelib/strategies/__init__.py,sha256=kot3u62KIAqYBg9M-KRE4mkMII_zwrDBZNf8Dj1vmX8,399
-datasourcelib/strategies/daily_load.py,sha256=Rh-veUhxKYsplwHTyko_Zp9C6NkUJV5VAGtg-p7Iy34,856
-datasourcelib/strategies/full_load.py,sha256=U1a9wO_ZLRnMInvU0IRW-ZKnhu0Cv437VcNMKIYuzMA,1691
-datasourcelib/strategies/incremental_load.py,sha256=TVqmDLu3m571nqGvzo_69i36QtYe4sBpllFwfPNL0TE,1178
-datasourcelib/strategies/ondemand_load.py,sha256=VxzAYgrW2ebTOC3xm61CerL2AFehZUJLnKrqtGRGJoE,644
-datasourcelib/strategies/timerange_load.py,sha256=c62BN2yXwVFaA_dQV54qenP4vrb4rcFqbx6m-nqhaTA,900
+datasourcelib/strategies/daily_load.py,sha256=A9BnPqPfbPO8UeBy-jtS53eORK7QWWqLOWHrtyFLbl4,1909
+datasourcelib/strategies/full_load.py,sha256=4BS_g4loR28OVqSDwXBCH2jCKbJLZxx6354KCOi_Qjk,4020
+datasourcelib/strategies/incremental_load.py,sha256=CY1tAyXwjZLoq5zMLwB5i5qmT_L8JBaiBxDy9hx8QkQ,1822
+datasourcelib/strategies/ondemand_load.py,sha256=MgenKJbJePLeErdEkXKsz1h7RuR8yT0RV_X523G7UUs,1304
+datasourcelib/strategies/timerange_load.py,sha256=W_sSZg059Lw2o9tmdGKM9D5-z1pph7AN1ftalXhuyjo,1557
 datasourcelib/utils/__init__.py,sha256=9pSIpaK-kdmNuDzwl0Z7QU-_lV3cZE-iwOEPh3RBBTs,298
+datasourcelib/utils/aggregation.py,sha256=5aOBcxay4eTyY-S4BRafNgSi37AY-JXERzcCv055E8w,6060
 datasourcelib/utils/byte_reader.py,sha256=GaoPXwJa2YTWG1Kim0K6JG20eVSaWkZJd1o9bswxHmc,9082
 datasourcelib/utils/exceptions.py,sha256=mgcDaW1k3VndgpMOwSm7NqgyRTvvE2a5ehn3x4fYQww,369
 datasourcelib/utils/file_reader.py,sha256=Zr0rwNTRWE6KeVJEXgTOPS1_JI74LiUSiX5-6qojmN0,7301
 datasourcelib/utils/logger.py,sha256=Sl6lNlvubxtK9ztzyq7vjGVyA8_-pZ_ixpk5jfVsh6U,424
 datasourcelib/utils/validators.py,sha256=fLgmRAb5OZSdMVlHu_n0RKJUDl-G8dI8JsRSfxIquh8,205
-datasourcelib-0.1.5.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
-datasourcelib-0.1.5.dist-info/METADATA,sha256=jDGgTdya-zt_go_TpEOJNfTQUI7CsbjM4m-Fg51XdqU,1199
-datasourcelib-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datasourcelib-0.1.5.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
-datasourcelib-0.1.5.dist-info/RECORD,,
+datasourcelib-0.1.7.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
+datasourcelib-0.1.7.dist-info/METADATA,sha256=Jo1RgpvptXpS-FxA6g9-7rVkknZDfzUrOpMQVFxG-9Y,1199
+datasourcelib-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datasourcelib-0.1.7.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
+datasourcelib-0.1.7.dist-info/RECORD,,