datasourcelib 0.1.5__tar.gz → 0.1.6__tar.gz

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.5
+Version: 0.1.6
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/setup.py

@@ -2,8 +2,8 @@ from setuptools import setup, find_packages
 
 setup(
     name="datasourcelib",
-    version="0.1.5",
-    packages=find_packages(where="src", exclude=["tests","examples"]),
+    version="0.1.6",
+    packages=find_packages(where="src", exclude=["tests.*", "tests", "examples.*", "examples"]),
     package_dir={"": "src"},
     install_requires=[
         "fastapi>=0.68.0",

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/src/datasourcelib/core/sync_manager.py

@@ -10,6 +10,7 @@ from ..datasources.sql_source import SQLDataSource
 from ..datasources.azure_devops_source import AzureDevOpsSource
 from ..datasources.sharepoint_source import SharePointSource
 from ..datasources.blob_source import BlobStorageSource
+from ..datasources.dataverse_source import DataverseSource
 
 # concrete strategies
 from datasourcelib.strategies.full_load import FullLoadStrategy
@@ -35,7 +36,8 @@ class SyncManager:
         DataSourceType.SQL: SQLDataSource,
         DataSourceType.AZURE_DEVOPS: AzureDevOpsSource, 
         DataSourceType.SHAREPOINT: SharePointSource,
-        DataSourceType.BLOB_STORAGE: BlobStorageSource
+        DataSourceType.BLOB_STORAGE: BlobStorageSource,
+        DataSourceType.Dataverse: DataverseSource
     }
 
     def execute_sync(self, sync_type: SyncType, 

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/src/datasourcelib/datasources/datasource_types.py

@@ -4,4 +4,5 @@ class DataSourceType(str, Enum):
     SQL = "sql"
     AZURE_DEVOPS = "azure_devops"
     SHAREPOINT = "sharepoint" 
-    BLOB_STORAGE = "blob_storage"
+    BLOB_STORAGE = "blob_storage"
+    Dataverse = "Dataverse"

datasourcelib-0.1.6/src/datasourcelib/datasources/dataverse_source.py

@@ -0,0 +1,291 @@
+from typing import Any, Dict, List, Optional, Tuple
+from datasourcelib.datasources.datasource_base import DataSourceBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.utils.validators import require_keys
+import pyodbc
+import time
+import pandas as pd
+
+# optional requests import (webapi mode)
+try:
+    import requests  # type: ignore
+except Exception:
+    requests = None  # lazy import
+
+logger = get_logger(__name__)
+
+class DataverseSource(DataSourceBase):
+    
+    def __init__(self, config: Dict[str, Any]):
+        super().__init__(config)
+        self._conn = None
+        self._mode = (self.config.get("dv_mode") or "tds").lower()  # "tds" or "webapi"
+        self._access_token: Optional[str] = None
+        self._headers: Dict[str, str] = {}
+        self._max_retries = int(self.config.get("dv_max_retries", 3))
+
+    def validate_config(self) -> bool:
+        """
+        Validate required keys depending on selected dv_mode.
+        - tds: requires either 'tds_connection_string' OR ('dataverse_server' and 'dataverse_database')
+        - webapi: requires 'webapi_url','client_id','client_secret','tenant_id' (or 'resource')
+        """
+        try:
+            if self._mode == "webapi":
+                require_keys(self.config, ["dv_webapi_url", "dv_webapi_client_id", "dv_webapi_client_secret", "dv_webapi_tenant_id"])
+            else:
+                # TDS mode (ODBC)
+                if "dv_tds_connection_string" in self.config:
+                    return True
+                # otherwise require components
+                require_keys(self.config, ["dv_tds_server", "dv_tds_database"])
+                # if not using integrated auth require creds
+                if not bool(self.config.get("dv_tds_windows_auth", False)):
+                    require_keys(self.config, ["dv_tds_username", "dv_tds_password"])
+            return True
+        except Exception as ex:
+            logger.error("DataverseSource.validate_config failed: %s", ex)
+            return False
+
+    # -------------------------
+    # Connection helpers
+    # -------------------------
+    def _get_available_driver(self) -> str:
+        """Return first suitable ODBC driver for SQL/Dataverse TDS access."""
+        preferred_drivers = [
+            "ODBC Driver 18 for SQL Server",
+            "ODBC Driver 17 for SQL Server",
+            "SQL Server Native Client 11.0",
+            "SQL Server"
+        ]
+        try:
+            drivers = pyodbc.drivers()
+            logger.info("Available ODBC drivers: %s", drivers)
+            
+            for d in preferred_drivers:
+                if d in drivers:
+                    logger.info("Using ODBC driver: %s", d)
+                    return d
+            
+            # fallback to first available
+            if drivers:
+                logger.warning("No preferred driver found. Using: %s", drivers[0])
+                return drivers[0]
+            raise RuntimeError("No ODBC drivers available")
+        except Exception as ex:
+            logger.error("DataverseSource._get_available_driver failed: %s", ex)
+            raise
+
+    def _build_tds_conn_str(self) -> str:
+        """Build valid connection string with proper parameter names."""
+        if "dv_tds_connection_string" in self.config:
+            return self.config["dv_tds_connection_string"]
+        
+        driver = self._get_available_driver()
+        # Fix: use correct config key names (dv_tds_server, not dv_tds_dataverse_server)
+        server = self.config.get("dv_tds_server", "").strip()
+        database = self.config.get("dv_tds_database", "").strip()
+        
+        if not server:
+            raise ValueError("dv_tds_server are required")
+        
+        logger.info("Building TDS connection (driver=%s, server=%s, database=%s)", driver, server, database)
+        
+        # Use curly braces for driver name (handles spaces in driver names)
+        parts = [f"DRIVER={{{driver}}}"]
+        parts.append(f"Server={server}")
+        parts.append(f"Database={database}")
+        password = None
+        if bool(self.config.get("dv_tds_windows_auth", False)):
+            parts.append("Trusted_Connection=yes")
+            logger.info("Using Windows authentication")
+        else:
+            username = self.config.get("dv_tds_username", "").strip()
+            password = self.config.get("dv_tds_password", "").strip()
+            
+            if not username or not password:
+                raise ValueError("dv_tds_username and dv_tds_password required when Windows auth disabled")
+            
+            parts.append(f"UID={username}")
+            parts.append(f"PWD={password}")
+            parts.append("Authentication=ActiveDirectoryInteractive")
+        # Encryption settings
+        if not bool(self.config.get("dv_tds_is_onprem", False)):
+            parts.append("Encrypt=yes")
+            parts.append("TrustServerCertificate=no")
+        else:
+            parts.append("Encrypt=optional")
+            parts.append("TrustServerCertificate=yes")
+        
+        conn_str = ";".join(parts)
+        logger.debug("Connection string: %s", conn_str.replace(password or "", "***") if password else conn_str)
+        return conn_str
+
+    def _obtain_webapi_token(self) -> Tuple[str, Dict[str, str]]:
+        """
+        Acquire OAuth2 token using client credentials flow.
+        Returns (access_token, headers)
+        Config expected keys: tenant_id, client_id, client_secret, optional resource
+        """
+        if requests is None:
+            raise RuntimeError("requests package required for Dataverse Web API mode")
+        tenant = self.config["dv_webapi_tenant_id"]
+        client_id = self.config["dv_webapi_client_id"]
+        client_secret = self.config["dv_webapi_client_secret"]
+        # resource or scope: prefer explicit resource, else fallback to webapi_url host
+        resource = self.config.get("dv_webapi_resource")
+        if not resource:
+            # infer resource from webapi_url e.g. https://<org>.crm.dynamics.com
+            webapi_url = self.config["dv_webapi_url"].rstrip("/")
+            resource = webapi_url.split("://")[-1]
+            resource = f"https://{resource}"  # as resource
+        token_url = f"https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token"
+        data = {
+            "grant_type": "client_credentials",
+            "client_id": client_id,
+            "client_secret": client_secret,
+            "scope": f"{resource}/.default"
+        }
+        resp = requests.post(token_url, data=data, timeout=30)
+        resp.raise_for_status()
+        j = resp.json()
+        token = j.get("access_token")
+        if not token:
+            raise RuntimeError("Failed to obtain access token for Dataverse webapi")
+        headers = {"Authorization": f"Bearer {token}", "Accept": "application/json", "OData-MaxVersion": "4.0", "OData-Version": "4.0"}
+        return token, headers
+
+    # -------------------------
+    # Public connection API
+    # -------------------------
+    def connect(self) -> bool:
+        try:
+            if self._mode == "webapi":
+                token, headers = self._obtain_webapi_token()
+                self._access_token = token
+                self._headers = headers
+                self._connected = True
+                logger.info("DataverseSource connected (webapi mode) to %s", self.config.get("dv_webapi_url"))
+                return True
+            # else TDS mode
+            conn_str = self._build_tds_conn_str()
+            self._conn = pyodbc.connect(conn_str, timeout=int(self.config.get("dv_tds_timeout", 30)))
+            self._connected = True
+            logger.info("DataverseSource connected (dv_tds mode) to %s/%s", self.config.get("dv_server"), self.config.get("dv_database"))
+            return True
+        except pyodbc.Error as ex:
+            logger.error("DataverseSource.connect failed - ODBC Error: %s", ex)
+            self._connected = False
+            return False
+        except requests.RequestException as ex:
+            logger.error("DataverseSource.connect failed - HTTP Error: %s", ex)
+            self._connected = False
+            return False
+        except Exception as ex:
+            logger.exception("DataverseSource.connect failed")
+            self._connected = False
+            return False
+
+    def disconnect(self) -> None:
+        try:
+            if self._conn:
+                try:
+                    self._conn.close()
+                except Exception:
+                    pass
+            self._conn = None
+            self._access_token = None
+            self._headers = {}
+        finally:
+            self._connected = False
+            logger.info("DataverseSource disconnected")
+
+    # -------------------------
+    # Data fetch
+    # -------------------------
+    def fetch_data(self, query: Optional[str] = None, **kwargs) -> List[Dict[str, Any]]:
+        """
+        Fetch rows from Dataverse.
+        - TDS mode: executes SQL query (config key 'tds_query' or provided 'query')
+        - WebAPI mode: calls Dataverse Web API path fragment (e.g. 'accounts?$select=name') or uses 'entity_set' + query params
+        Returns list[dict].
+        """
+        attempt = 0
+        while attempt < self._max_retries:
+            try:
+                if not getattr(self, "_connected", False):
+                    ok = self.connect()
+                    if not ok:
+                        raise RuntimeError("DataverseSource: cannot connect")
+
+                if self._mode == "webapi":
+                    if requests is None:
+                        raise RuntimeError("requests package required for webapi mode")
+                    webapi_url = self.config["dv_webapi_url"].rstrip("/")
+                    # if query provided, treat it as path fragment; else use entity_set from config
+                    path_fragment = query or self.config.get("dv_webapi_entity_set")
+                    if not path_fragment:
+                        raise ValueError("DataverseSource.fetch_data requires a webapi 'query' or 'entity_set' in config")
+                    url = f"{webapi_url}/api/data/v9.1/{path_fragment.lstrip('/')}"
+                    params = kwargs.get("params")
+                    resp = requests.get(url, headers=self._headers, params=params, timeout=60)
+                    resp.raise_for_status()
+                    j = resp.json()
+                    items: Any = []
+                    # Dataverse OData responses typically use 'value' for collections
+                    if isinstance(j, dict) and "value" in j:
+                        items = j["value"]
+                    # otherwise return the raw json wrapped in a list or as-is
+                    elif isinstance(j, list):
+                        items= j
+                    else:
+                        items= [j]
+
+                    df = pd.DataFrame(items)
+                    # filter columns if configured
+                    keep = self.config.get("dv_webapi_columns_to_keep")
+                    if isinstance(keep, list) and keep:
+                        cols_to_keep = [c for c in df.columns if c in keep]
+                    else:
+                        # exclude SharePoint metadata columns (start with '__' or prefixed with '@')
+                        cols_to_keep = [c for c in df.columns if not str(c).startswith("__") and not str(c).startswith("@")]
+                    df = df[cols_to_keep]
+                    results = df.to_dict("records")
+                    return results
+                # else TDS mode
+                sql = query or self.config.get("dv_tds_query") or self.config.get("dv_sql_query")
+                if not sql:
+                    raise ValueError("DataverseSource.fetch_data requires a SQL query (tds mode)")
+
+                cur = self._conn.cursor()
+                try:
+                    cur.execute(sql)
+                    cols = [c[0] for c in (cur.description or [])]
+                    rows = cur.fetchall()
+                    results: List[Dict[str, Any]] = []
+                    for r in rows:
+                        results.append({cols[i]: r[i] for i in range(len(cols))})
+                    return results
+                finally:
+                    try:
+                        cur.close()
+                    except Exception:
+                        pass
+
+            except Exception as ex:
+                attempt += 1
+                logger.warning("DataverseSource.fetch_data attempt %d/%d failed: %s", attempt, self._max_retries, ex)
+                # transient retry for network/connection errors
+                if attempt >= self._max_retries:
+                    logger.exception("DataverseSource.fetch_data final failure")
+                    raise
+                # backoff
+                time.sleep(min(2 ** attempt, 10))
+                # try reconnect for next attempt
+                try:
+                    self.disconnect()
+                except Exception:
+                    pass
+
+        # unreachable; defensive
+        return []

datasourcelib-0.1.6/src/datasourcelib/datasources/sql_source_bkup.py

@@ -0,0 +1,159 @@
+from typing import Any, Dict, List, Optional
+from datasourcelib.datasources.datasource_base import DataSourceBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.utils.validators import require_keys
+import os
+import pyodbc
+
+
+logger = get_logger(__name__)
+
+class SQLDataSource(DataSourceBase):
+    
+    def __init__(self, config: Dict[str, Any]):
+        super().__init__(config)
+        self._conn = None
+        self._is_sqlite = False
+
+    def validate_config(self) -> bool:
+        """
+        Validate config. If sql_windows_auth is True then sql_username/sql_password are optional.
+        Otherwise require sql_username and sql_password.
+        """
+        try:
+            # Always require server/database at minimum
+            require_keys(self.config, ["sql_server", "sql_database"])
+            # If not using Windows authentication, require credentials
+            if not bool(self.config.get("sql_windows_auth", False)):
+                require_keys(self.config, ["sql_username", "sql_password"])
+            return True
+        except Exception as ex:
+            logger.error("SQLDataSource.validate_config: %s", ex)
+            return False
+
+    def connect(self) -> bool:
+        try:
+            sql_server = self.config.get("sql_server", "")
+            sql_database = self.config.get("sql_database", "")
+            sql_is_onprem = self.config.get("sql_is_onprem", False)
+
+            # Determine auth mode: sql_windows_auth (Trusted Connection) overrides username/password
+            sql_windows_auth = bool(self.config.get("sql_windows_auth", False))
+
+            # Get available driver
+            sql_driver = self._get_available_driver()
+
+            # Build connection string
+            conn_params = [
+                f'DRIVER={sql_driver}',
+                f'SERVER={sql_server}',
+                f'DATABASE={sql_database}',
+            ]
+
+            if sql_windows_auth:
+                # Use integrated Windows authentication (Trusted Connection)
+                # This will use the current process credentials / kerberos ticket.
+                conn_params.append('Trusted_Connection=yes')
+                logger.info("SQLDataSource using Windows (integrated) authentication")
+            else:
+                sql_username = self.config.get("sql_username", "")
+                sql_password = self.config.get("sql_password", "")
+                conn_params.extend([f'UID={sql_username}', f'PWD={sql_password}'])
+
+            # Add encryption settings based on environment
+            if not sql_is_onprem:
+                conn_params.extend([
+                    'Encrypt=yes',
+                    'TrustServerCertificate=no'
+                ])
+            else:
+                conn_params.extend([
+                    'Encrypt=optional',
+                    'TrustServerCertificate=yes'
+                ])
+
+            conn_str = ';'.join(conn_params)
+
+            # Attempt connection with timeout
+            self._conn = pyodbc.connect(conn_str, timeout=30)
+            self._connected = True
+            logger.info("SQLDataSource connected to %s using driver %s (sql_windows_auth=%s)", sql_server, sql_driver, sql_windows_auth)
+            return True
+
+        except pyodbc.Error as ex:
+            logger.error("SQLDataSource.connect failed - ODBC Error: %s", ex)
+            self._connected = False
+            return False
+        except Exception as ex:
+            logger.error("SQLDataSource.connect failed - Unexpected Error: %s", ex)
+            self._connected = False
+            return False
+
+    def disconnect(self) -> None:
+        try:
+            if self._conn:
+                self._conn.close()
+        finally:
+            self._conn = None
+            self._connected = False
+            logger.info("SQLDataSource disconnected")
+
+    def fetch_data(self, query: Optional[str] = None, **kwargs) -> List[Dict[str, Any]]:
+        max_retries = 3
+        retry_count = 0
+        
+        while retry_count < max_retries:
+            try:
+                if not self._connected:
+                    ok = self.connect()
+                    if not ok:
+                        raise RuntimeError("SQLDataSource: not connected and cannot connect")
+
+                query = self.config.get("sql_query")
+                if not query:
+                    raise ValueError("SQLDataSource.fetch_data requires a query")
+
+                cur = self._conn.cursor()
+                try:
+                    cur.execute(query)
+                    cols = [d[0] if hasattr(d, "__len__") else d[0] for d in (cur.description or [])]
+                    rows = cur.fetchall()
+                    results: List[Dict[str, Any]] = []
+                    for r in rows:
+                        results.append({cols[i]: r[i] for i in range(len(cols))})
+                    return results
+                finally:
+                    try:
+                        cur.close()
+                    except Exception:
+                        pass
+                        
+            except pyodbc.OperationalError as ex:
+                # Handle connection lost
+                retry_count += 1
+                logger.warning("Connection lost, attempt %d of %d: %s", retry_count, max_retries, ex)
+                self.disconnect()
+                if retry_count >= max_retries:
+                    raise
+            except Exception as ex:
+                logger.error("Query execution failed: %s", ex)
+                raise
+
+    def _get_available_driver(self) -> str:
+        """Get first available SQL Server driver from preferred list."""
+        preferred_drivers = [
+            'ODBC Driver 18 for SQL Server',
+            'ODBC Driver 17 for SQL Server',
+            'SQL Server Native Client 11.0',
+            'SQL Server'
+        ]
+        
+        try:
+            available_drivers = pyodbc.drivers()
+            for driver in preferred_drivers:
+                if driver in available_drivers:
+                    return driver
+            raise RuntimeError(f"No suitable SQL Server driver found. Available drivers: {available_drivers}")
+        except Exception as ex:
+            logger.error("Failed to get SQL drivers: %s", ex)
+            raise

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/src/datasourcelib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.5
+Version: 0.1.6
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/akashmaurya0217/datasourcelib
 Author: Akash Kumar Maurya

{datasourcelib-0.1.5 → datasourcelib-0.1.6}/src/datasourcelib.egg-info/SOURCES.txt

@@ -18,9 +18,11 @@ src/datasourcelib/datasources/azure_devops_source.py
 src/datasourcelib/datasources/blob_source.py
 src/datasourcelib/datasources/datasource_base.py
 src/datasourcelib/datasources/datasource_types.py
+src/datasourcelib/datasources/dataverse_source.py
 src/datasourcelib/datasources/sharepoint_source - Copy.py
 src/datasourcelib/datasources/sharepoint_source.py
 src/datasourcelib/datasources/sql_source.py
+src/datasourcelib/datasources/sql_source_bkup.py
 src/datasourcelib/indexes/__init__.py
 src/datasourcelib/indexes/azure_search_index.py
 src/datasourcelib/strategies/__init__.py