datasourcelib 0.1.2__tar.gz → 0.1.4__tar.gz

{datasourcelib-0.1.2 → datasourcelib-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.2
+Version: 0.1.4
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/jaiprakash0217/datasourcelib
 Author: Jai Prakash

{datasourcelib-0.1.2 → datasourcelib-0.1.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="datasourcelib",
-    version="0.1.2",
+    version="0.1.4",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     install_requires=[

{datasourcelib-0.1.2 → datasourcelib-0.1.4}/src/datasourcelib/indexes/azure_search_index.py

@@ -110,7 +110,7 @@ class AzureSearchIndexer:
             logger.exception(f"Failed to get embeddings for text: {text[:100]}...")
             raise
 
-    def _build_vector_search_config(self):
+    def _build_vector_search_config_old(self):
         AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration, SemanticSearch, SemanticField, SemanticConfiguration, SemanticPrioritizedFields = self._ensure_sdk()
         vector_config = self.config.get("vector_config", {})
         dimensions = vector_config.get("dimensions", 1536)
@@ -121,6 +121,107 @@ class AzureSearchIndexer:
             )
 
         return vector_search, dimensions
+    
+    def _build_vector_search_config(self):
+        AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration, SemanticSearch, SemanticField, SemanticConfiguration, SemanticPrioritizedFields = self._ensure_sdk()
+
+        vector_config = self.config.get("vector_config", {})
+        dimensions = vector_config.get("dimensions", 1536)
+        algorithm = vector_config.get("algorithm", "hnsw").lower()
+
+        # Build algorithm configuration (SDK model if available)
+        alg_cfg = HnswAlgorithmConfiguration(name="algorithms-config-1")
+
+        # Build vectorizer settings using Azure OpenAI config from vector_db_config
+        deployment = self.config.get("embedding_deployment")
+        endpoint = self.config.get("embedding_endpoint")
+        api_key = self.config.get("embedding_key")
+        # modelName required for API version 2025-09-01 — prefer explicit embedding_model, fall back to deployment
+        model_name = self.config.get("embedding_model") or deployment
+        content_field = self.config.get("content_field", "content")
+        vector_field = self.config.get("vector_field", "contentVector")
+
+        if not model_name:
+            raise RuntimeError("Vectorizer configuration requires 'embedding_model' or 'embedding_deployment' in vector_db_config")
+
+        # Define vectorizer with explicit name and required azureOpenAIParameters including modelName
+        vectorizer_name = "azure-openai-vectorizer"
+        vectorizer = {
+            "name": vectorizer_name,
+            "kind": "azureOpenAI",
+            "azureOpenAIParameters": {
+                "resourceUri": endpoint.rstrip('/') if endpoint else None,
+                # include both modelName (required) and deploymentId (if provided)
+                "modelName": model_name,
+                **({"deploymentId": deployment} if deployment else {}),
+                "apiKey": api_key
+            },
+            "options": {
+                "fieldMapping": [
+                    {
+                        "sourceContext": f"/document/{content_field}",
+                        "outputs": [
+                            {
+                                "targetContext": f"/document/{vector_field}",
+                                "targetDimensions": dimensions
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+
+        profile_name = "vector-profile-1"
+        try:
+            # Create profile with vectorizer reference (SDK may expect vectorizer_name or vectorizer depending on version)
+            try:
+                profile = VectorSearchProfile(
+                    name=profile_name,
+                    algorithm_configuration_name="algorithms-config-1",
+                    vectorizer_name=vectorizer_name
+                )
+            except TypeError:
+                # fallback if SDK constructor uses different parameter names
+                profile = VectorSearchProfile(name=profile_name, algorithm_configuration_name="algorithms-config-1")
+                try:
+                    setattr(profile, "vectorizer_name", vectorizer_name)
+                except Exception:
+                    pass
+
+            try:
+                # Construct full vector search config with both profile and vectorizer
+                vector_search = VectorSearch(
+                    profiles=[profile],
+                    algorithms=[alg_cfg],
+                    vectorizers=[vectorizer]
+                )
+            except Exception:
+                # Fallback to dict if SDK constructor differs
+                vector_search = {
+                    "profiles": [{
+                        "name": profile_name,
+                        "algorithmConfigurationName": "algorithms-config-1",
+                        "vectorizerName": vectorizer_name
+                    }],
+                    "algorithms": [{"name": "algorithms-config-1"}],
+                    "vectorizers": [vectorizer]
+                }
+        except Exception:
+            # Full dict fallback
+            vector_search = {
+                "profiles": [{
+                    "name": profile_name,
+                    "algorithmConfigurationName": "algorithms-config-1",
+                    "vectorizerName": vectorizer_name
+                }],
+                "algorithms": [{"name": "algorithms-config-1"}],
+                "vectorizers": [vectorizer]
+            }
+
+        logger.info("Built vector_search config (dimensions=%s, model=%s, vectorizer=%s)",
+                    dimensions, model_name, vectorizer_name)
+        return vector_search, dimensions
+
 
     def _build_semantic_settings(self):
         """

datasourcelib-0.1.4/src/datasourcelib/strategies/__init__.py

@@ -0,0 +1,14 @@
+from .daily_load import DailyLoadStrategy
+from .full_load import FullLoadStrategy
+from .incremental_load import IncrementalLoadStrategy
+from .ondemand_load import OnDemandLoadStrategy
+from .timerange_load import TimeRangeLoadStrategy
+
+
+__all__ = [
+    "DailyLoadStrategy",
+    "FullLoadStrategy", 
+    "IncrementalLoadStrategy",
+    "OnDemandLoadStrategy",
+    "TimeRangeLoadStrategy"
+]

datasourcelib-0.1.4/src/datasourcelib/strategies/daily_load.py

@@ -0,0 +1,22 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+from datetime import datetime, timedelta
+
+logger = get_logger(__name__)
+
+class DailyLoadStrategy(SyncBase):
+    """Daily scheduled load (wraps incremental)."""
+
+    def validate(self) -> bool:
+        return True
+
+    def sync(self, run_date: str = None, **kwargs) -> bool:
+        try:
+            run_date = run_date or datetime.utcnow().date().isoformat()
+            logger.info("Starting daily load for %s", run_date)
+            # Typically call incremental with last_sync = previous day midnight
+            # TODO implement scheduling integration externally; the strategy here is idempotent
+            return True
+        except Exception:
+            logger.exception("DailyLoadStrategy.sync failed")
+            return False

datasourcelib-0.1.4/src/datasourcelib/strategies/full_load.py

@@ -0,0 +1,38 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.indexes.azure_search_index import AzureSearchIndexer
+logger = get_logger(__name__)
+
+class FullLoadStrategy(SyncBase):
+    """Full load: replace or reload entire source into vector DB."""
+
+    def validate(self) -> bool:
+        # Minimal validation: required keys exist
+        dsok = self.data_source.validate_config()
+        return dsok
+
+    def sync(self, **kwargs) -> bool:
+        try:
+            logger.info("Running full data load")
+            data = self.data_source.fetch_data(**kwargs)
+            for key, value in kwargs.items():
+                print(f"{key} = {value}")
+            # Implement real extract -> transform -> load to vector DB
+            # Example pseudocode:
+            # vector_client.upsert_batch(self.vector_db_config, rows)
+            # New: use AzureSearchIndexer to create index and upload documents if requested
+            if isinstance(data, list) and data:                
+                indexer = AzureSearchIndexer(self.vector_db_config or {})
+                if not indexer.validate_config():
+                    logger.error("Vector DB config invalid for Azure Search indexer")
+                    return False
+                ok = indexer.index(data)
+                if not ok:
+                    logger.error("Indexing data to Azure Search failed")
+                    return False
+
+            logger.info("Full data load finished successfully")
+            return True
+        except Exception:
+            logger.exception("FullLoadStrategy.sync failed")
+            return False

datasourcelib-0.1.4/src/datasourcelib/strategies/incremental_load.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class IncrementalLoadStrategy(SyncBase):
+    """Incremental load using last_sync timestamp or cursor."""
+
+    def validate(self) -> bool:
+        # require source to support incremental field or cursor
+        if "cursor_field" not in self.source_config and "last_sync" not in self.source_config:
+            logger.error("IncrementalLoadStrategy missing cursor_field or last_sync in source_config")
+            return False
+        return True
+
+    def sync(self, last_sync: str = None, **kwargs) -> bool:
+        try:
+            last = last_sync or self.source_config.get("last_sync")
+            logger.info("Running incremental load since %s", last)
+            # TODO: fetch delta rows since 'last' and upsert to vector DB
+            # After successful run store new last_sync timestamp
+            logger.info("Incremental load completed")
+            return True
+        except Exception:
+            logger.exception("IncrementalLoadStrategy.sync failed")
+            return False

datasourcelib-0.1.4/src/datasourcelib/strategies/ondemand_load.py

@@ -0,0 +1,19 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class OnDemandLoadStrategy(SyncBase):
+    """On demand load triggered by user request (arbitrary params)."""
+
+    def validate(self) -> bool:
+        return True
+
+    def sync(self, **kwargs) -> bool:
+        try:
+            logger.info("On-demand sync invoked with params: %s", kwargs)
+            # Use kwargs to drive partial loads, filters, ids etc.
+            return True
+        except Exception:
+            logger.exception("OnDemandLoadStrategy.sync failed")
+            return False

datasourcelib-0.1.4/src/datasourcelib/strategies/timerange_load.py

@@ -0,0 +1,24 @@
+from datetime import datetime
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class TimeRangeLoadStrategy(SyncBase):
+    """Load records between a start and end timestamp."""
+
+    def validate(self) -> bool:
+        # rely on params at runtime; minimal validation OK
+        return True
+
+    def sync(self, start: str = None, end: str = None, **kwargs) -> bool:
+        try:
+            if not start or not end:
+                logger.error("TimeRangeLoadStrategy requires 'start' and 'end'")
+                return False
+            logger.info("Time range load between %s and %s", start, end)
+            # TODO: query source for timeframe and upsert
+            return True
+        except Exception:
+            logger.exception("TimeRangeLoadStrategy.sync failed")
+            return False

datasourcelib-0.1.4/src/datasourcelib/utils/__init__.py

@@ -0,0 +1,12 @@
+from .byte_reader import ByteReader
+from .exceptions import DatasourceLibError, SyncStrategyNotFound, DataSourceNotFound    
+from .file_reader import FileReader
+
+
+__all__ = [
+    "ByteReader",
+    "FileReader",
+    "DatasourceLibError",
+    "SyncStrategyNotFound",
+    "SourceNotFound"
+]

datasourcelib-0.1.4/src/datasourcelib/utils/byte_reader.py

@@ -0,0 +1,256 @@
+from pathlib import Path
+from typing import Optional, Union, List
+import io
+import pandas as pd
+
+# --- Optional helpers ---
+from charset_normalizer import from_bytes as cn_from_bytes
+
+# DOCX
+from docx import Document as DocxDocument
+
+# PDF
+import fitz  # pymupdf
+import pdfplumber
+
+# PPTX
+from pptx import Presentation
+
+# YAML / XML
+import yaml
+from lxml import etree
+import json
+
+
+class ByteReader:
+    """
+    Unified reader for common file types.
+    - read_text(path): file path -> text
+    - read_table(path): file path -> DataFrame
+    - read_text_from_bytes(data, ext): bytes -> text
+    - read_table_from_bytes(data, ext): bytes -> DataFrame
+    """
+
+    TEXT_EXTS = {".txt", ".log", ".md"}
+    TABLE_EXTS = {".csv", ".tsv", ".xlsx", ".xls"}
+    DOCX_EXTS = {".docx"}
+    PDF_EXTS = {".pdf"}
+    PPTX_EXTS = {".pptx"}
+    JSON_EXTS = {".json"}
+    YAML_EXTS = {".yaml", ".yml"}
+    INI_EXTS = {".ini", ".cfg"}
+    XML_EXTS = {".xml"}
+
+    def __init__(self, default_encoding: str = "utf-8", errors: str = "replace"):
+        self.default_encoding = default_encoding
+        self.errors = errors
+
+    # -----------------------
+    # Public API (paths)
+    # -----------------------
+    def read_text(self, path: Union[str, Path]) -> str:
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext in self.TEXT_EXTS:
+            return path.read_text(encoding=self.default_encoding, errors=self.errors)
+
+        if ext in self.PDF_EXTS:
+            return self._read_pdf_text_path(path)
+
+        if ext in self.DOCX_EXTS:
+            return self._read_docx_text_fp(open(path, "rb"))
+
+        if ext in self.PPTX_EXTS:
+            return self._read_pptx_text_fp(open(path, "rb"))
+
+        if ext in self.JSON_EXTS:
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                obj = json.load(f)
+            return json.dumps(obj, indent=2, ensure_ascii=False)
+
+        if ext in self.YAML_EXTS:
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                obj = yaml.safe_load(f)
+            return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)
+
+        if ext in self.INI_EXTS:
+            import configparser
+            parser = configparser.ConfigParser()
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                parser.read_file(f)
+            output = io.StringIO()
+            parser.write(output)
+            return output.getvalue()
+
+        if ext in self.XML_EXTS:
+            tree = etree.parse(str(path))
+            return etree.tostring(tree, pretty_print=True, encoding="unicode")
+
+        if ext in self.TABLE_EXTS:
+            df = self.read_table(path)
+            return df.to_csv(index=False)
+
+        raise ValueError(f"Unsupported file extension for text extraction: {ext}")
+
+    def read_table(self, path: Union[str, Path]) -> pd.DataFrame:
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext == ".csv":
+            return pd.read_csv(path)
+        if ext == ".tsv":
+            return pd.read_csv(path, sep="\t")
+        if ext == ".xlsx":
+            return pd.read_excel(path, engine="openpyxl")
+        if ext == ".xls":
+            return pd.read_excel(path, engine="xlrd")
+
+        # Fallback: attempt CSV read if unknown
+        try:
+            return pd.read_csv(path)
+        except Exception as e:
+            raise ValueError(f"Unsupported file extension for tables: {ext}") from e
+
+    # -----------------------
+    # Public API (bytes)
+    # -----------------------
+    def read_text_from_bytes(self, data: bytes, ext: str) -> str:
+        """
+        Extract text from in-memory bytes.
+        ext: file extension (e.g., '.pdf', '.docx', '.txt', '.pptx', '.json', '.yaml', '.xml', '.csv', '.xlsx')
+        """
+        ext = self._normalize_ext(ext)
+
+        if ext in self.TEXT_EXTS:
+            # Robust encoding detection
+            res = cn_from_bytes(data).best()
+            return str(res) if res else data.decode(self.default_encoding, errors=self.errors)
+
+        if ext in self.PDF_EXTS:
+            return self._read_pdf_text_bytes(data)
+
+        if ext in self.DOCX_EXTS:
+            return self._read_docx_text_fp(io.BytesIO(data))
+
+        if ext in self.PPTX_EXTS:
+            return self._read_pptx_text_fp(io.BytesIO(data))
+
+        if ext in self.JSON_EXTS:
+            obj = json.loads(data.decode(self.default_encoding, errors=self.errors))
+            return json.dumps(obj, indent=2, ensure_ascii=False)
+
+        if ext in self.YAML_EXTS:
+            obj = yaml.safe_load(data.decode(self.default_encoding, errors=self.errors))
+            return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)
+
+        if ext in self.INI_EXTS:
+            import configparser
+            parser = configparser.ConfigParser()
+            parser.read_string(data.decode(self.default_encoding, errors=self.errors))
+            output = io.StringIO()
+            parser.write(output)
+            return output.getvalue()
+
+        if ext in self.XML_EXTS:
+            tree = etree.parse(io.BytesIO(data))
+            return etree.tostring(tree, pretty_print=True, encoding="unicode")
+
+        if ext in self.TABLE_EXTS:
+            df = self.read_table_from_bytes(data, ext)
+            return df.to_csv(index=False)
+
+        raise ValueError(f"Unsupported extension for text extraction from bytes: {ext}")
+
+    def read_table_from_bytes(self, data: bytes, ext: str) -> pd.DataFrame:
+        """
+        Load tabular data from in-memory bytes into a DataFrame.
+        """
+        ext = self._normalize_ext(ext)
+
+        if ext == ".csv":
+            return pd.read_csv(io.BytesIO(data))
+        if ext == ".tsv":
+            return pd.read_csv(io.BytesIO(data), sep="\t")
+        if ext == ".xlsx":
+            return pd.read_excel(io.BytesIO(data), engine="openpyxl")
+        if ext == ".xls":
+            return pd.read_excel(io.BytesIO(data), engine="xlrd")
+
+        # Opportunistic fallback: try CSV
+        try:
+            return pd.read_csv(io.BytesIO(data))
+        except Exception as e:
+            raise ValueError(f"Unsupported extension for table reading from bytes: {ext}") from e
+
+    # -----------------------
+    # Internal helpers
+    # -----------------------
+    def _normalize_ext(self, ext: str) -> str:
+        ext = (ext or "").strip().lower()
+        if not ext.startswith("."):
+            ext = "." + ext
+        return ext
+
+    def _read_pdf_text_path(self, path: Path) -> str:
+        # Prefer PyMuPDF
+        try:
+            parts: List[str] = []
+            with fitz.open(str(path)) as doc:
+                if doc.is_encrypted and not doc.authenticate(""):
+                    raise RuntimeError("Encrypted PDF requires a password.")
+                for page in doc:
+                    parts.append(page.get_text("text"))
+            text = "\n\n".join(parts).strip()
+            if text:
+                return text
+        except Exception:
+            pass
+
+        # Fallback: pdfplumber
+        with pdfplumber.open(str(path)) as pdf:
+            return "\n\n".join([(p.extract_text() or "") for p in pdf.pages]).strip()
+
+    def _read_pdf_text_bytes(self, data: bytes) -> str:
+        # PyMuPDF can open from bytes
+        try:
+            doc = fitz.open(stream=data, filetype="pdf")
+            parts: List[str] = []
+            if doc.is_encrypted and not doc.authenticate(""):
+                raise RuntimeError("Encrypted PDF requires a password.")
+            for page in doc:
+                parts.append(page.get_text("text"))
+            doc.close()
+            text = "\n\n".join(parts).strip()
+            if text:
+                return text
+        except Exception:
+            pass
+
+        # Fallback to pdfplumber from BytesIO
+        with pdfplumber.open(io.BytesIO(data)) as pdf:
+            return "\n\n".join([(p.extract_text() or "") for p in pdf.pages]).strip()
+
+    def _read_docx_text_fp(self, fp) -> str:
+        doc = DocxDocument(fp)
+        chunks = []
+        for p in doc.paragraphs:
+            if p.text:
+                chunks.append(p.text)
+        for table in doc.tables:
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells]
+                if any(cells):
+                    chunks.append("\t".join(cells))
+        return "\n".join(chunks).strip()
+
+    def _read_pptx_text_fp(self, fp) -> str:
+        prs = Presentation(fp)
+        chunks = []
+        for slide in prs.slides:
+            for shape in slide.shapes:
+                if hasattr(shape, "has_text_frame") and shape.has_text_frame:
+                    text = shape.text or ""
+                    if text:
+                        chunks.append(text)
+        return "\n".join(chunks).strip()

datasourcelib-0.1.4/src/datasourcelib/utils/exceptions.py

@@ -0,0 +1,9 @@
+class DatasourceLibError(Exception):
+    """Base exception for datasourcelib."""
+
+class SyncStrategyNotFound(DatasourceLibError):
+    """Raised when a strategy is not found."""
+
+# Added: DataSourceNotFound to represent missing/unknown data sources
+class DataSourceNotFound(DatasourceLibError):
+    """Raised when a data source is not found or not registered."""