datasourcelib 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

datasourcelib/indexes/azure_search_index.py

@@ -110,7 +110,7 @@ class AzureSearchIndexer:
             logger.exception(f"Failed to get embeddings for text: {text[:100]}...")
             raise
 
-    def _build_vector_search_config(self):
+    def _build_vector_search_config_old(self):
         AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration, SemanticSearch, SemanticField, SemanticConfiguration, SemanticPrioritizedFields = self._ensure_sdk()
         vector_config = self.config.get("vector_config", {})
         dimensions = vector_config.get("dimensions", 1536)
@@ -121,6 +121,107 @@ class AzureSearchIndexer:
             )
 
         return vector_search, dimensions
+    
+    def _build_vector_search_config(self):
+        AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration, SemanticSearch, SemanticField, SemanticConfiguration, SemanticPrioritizedFields = self._ensure_sdk()
+
+        vector_config = self.config.get("vector_config", {})
+        dimensions = vector_config.get("dimensions", 1536)
+        algorithm = vector_config.get("algorithm", "hnsw").lower()
+
+        # Build algorithm configuration (SDK model if available)
+        alg_cfg = HnswAlgorithmConfiguration(name="algorithms-config-1")
+
+        # Build vectorizer settings using Azure OpenAI config from vector_db_config
+        deployment = self.config.get("embedding_deployment")
+        endpoint = self.config.get("embedding_endpoint")
+        api_key = self.config.get("embedding_key")
+        # modelName required for API version 2025-09-01 — prefer explicit embedding_model, fall back to deployment
+        model_name = self.config.get("embedding_model") or deployment
+        content_field = self.config.get("content_field", "content")
+        vector_field = self.config.get("vector_field", "contentVector")
+
+        if not model_name:
+            raise RuntimeError("Vectorizer configuration requires 'embedding_model' or 'embedding_deployment' in vector_db_config")
+
+        # Define vectorizer with explicit name and required azureOpenAIParameters including modelName
+        vectorizer_name = "azure-openai-vectorizer"
+        vectorizer = {
+            "name": vectorizer_name,
+            "kind": "azureOpenAI",
+            "azureOpenAIParameters": {
+                "resourceUri": endpoint.rstrip('/') if endpoint else None,
+                # include both modelName (required) and deploymentId (if provided)
+                "modelName": model_name,
+                **({"deploymentId": deployment} if deployment else {}),
+                "apiKey": api_key
+            },
+            "options": {
+                "fieldMapping": [
+                    {
+                        "sourceContext": f"/document/{content_field}",
+                        "outputs": [
+                            {
+                                "targetContext": f"/document/{vector_field}",
+                                "targetDimensions": dimensions
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+
+        profile_name = "vector-profile-1"
+        try:
+            # Create profile with vectorizer reference (SDK may expect vectorizer_name or vectorizer depending on version)
+            try:
+                profile = VectorSearchProfile(
+                    name=profile_name,
+                    algorithm_configuration_name="algorithms-config-1",
+                    vectorizer_name=vectorizer_name
+                )
+            except TypeError:
+                # fallback if SDK constructor uses different parameter names
+                profile = VectorSearchProfile(name=profile_name, algorithm_configuration_name="algorithms-config-1")
+                try:
+                    setattr(profile, "vectorizer_name", vectorizer_name)
+                except Exception:
+                    pass
+
+            try:
+                # Construct full vector search config with both profile and vectorizer
+                vector_search = VectorSearch(
+                    profiles=[profile],
+                    algorithms=[alg_cfg],
+                    vectorizers=[vectorizer]
+                )
+            except Exception:
+                # Fallback to dict if SDK constructor differs
+                vector_search = {
+                    "profiles": [{
+                        "name": profile_name,
+                        "algorithmConfigurationName": "algorithms-config-1",
+                        "vectorizerName": vectorizer_name
+                    }],
+                    "algorithms": [{"name": "algorithms-config-1"}],
+                    "vectorizers": [vectorizer]
+                }
+        except Exception:
+            # Full dict fallback
+            vector_search = {
+                "profiles": [{
+                    "name": profile_name,
+                    "algorithmConfigurationName": "algorithms-config-1",
+                    "vectorizerName": vectorizer_name
+                }],
+                "algorithms": [{"name": "algorithms-config-1"}],
+                "vectorizers": [vectorizer]
+            }
+
+        logger.info("Built vector_search config (dimensions=%s, model=%s, vectorizer=%s)",
+                    dimensions, model_name, vectorizer_name)
+        return vector_search, dimensions
+
 
     def _build_semantic_settings(self):
         """

datasourcelib/strategies/__init__.py

@@ -0,0 +1,14 @@
+from .daily_load import DailyLoadStrategy
+from .full_load import FullLoadStrategy
+from .incremental_load import IncrementalLoadStrategy
+from .ondemand_load import OnDemandLoadStrategy
+from .timerange_load import TimeRangeLoadStrategy
+
+
+__all__ = [
+    "DailyLoadStrategy",
+    "FullLoadStrategy", 
+    "IncrementalLoadStrategy",
+    "OnDemandLoadStrategy",
+    "TimeRangeLoadStrategy"
+]

datasourcelib/strategies/daily_load.py

@@ -0,0 +1,22 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+from datetime import datetime, timedelta
+
+logger = get_logger(__name__)
+
+class DailyLoadStrategy(SyncBase):
+    """Daily scheduled load (wraps incremental)."""
+
+    def validate(self) -> bool:
+        return True
+
+    def sync(self, run_date: str = None, **kwargs) -> bool:
+        try:
+            run_date = run_date or datetime.utcnow().date().isoformat()
+            logger.info("Starting daily load for %s", run_date)
+            # Typically call incremental with last_sync = previous day midnight
+            # TODO implement scheduling integration externally; the strategy here is idempotent
+            return True
+        except Exception:
+            logger.exception("DailyLoadStrategy.sync failed")
+            return False

datasourcelib/strategies/full_load.py

@@ -0,0 +1,38 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+from datasourcelib.indexes.azure_search_index import AzureSearchIndexer
+logger = get_logger(__name__)
+
+class FullLoadStrategy(SyncBase):
+    """Full load: replace or reload entire source into vector DB."""
+
+    def validate(self) -> bool:
+        # Minimal validation: required keys exist
+        dsok = self.data_source.validate_config()
+        return dsok
+
+    def sync(self, **kwargs) -> bool:
+        try:
+            logger.info("Running full data load")
+            data = self.data_source.fetch_data(**kwargs)
+            for key, value in kwargs.items():
+                print(f"{key} = {value}")
+            # Implement real extract -> transform -> load to vector DB
+            # Example pseudocode:
+            # vector_client.upsert_batch(self.vector_db_config, rows)
+            # New: use AzureSearchIndexer to create index and upload documents if requested
+            if isinstance(data, list) and data:                
+                indexer = AzureSearchIndexer(self.vector_db_config or {})
+                if not indexer.validate_config():
+                    logger.error("Vector DB config invalid for Azure Search indexer")
+                    return False
+                ok = indexer.index(data)
+                if not ok:
+                    logger.error("Indexing data to Azure Search failed")
+                    return False
+
+            logger.info("Full data load finished successfully")
+            return True
+        except Exception:
+            logger.exception("FullLoadStrategy.sync failed")
+            return False

datasourcelib/strategies/incremental_load.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class IncrementalLoadStrategy(SyncBase):
+    """Incremental load using last_sync timestamp or cursor."""
+
+    def validate(self) -> bool:
+        # require source to support incremental field or cursor
+        if "cursor_field" not in self.source_config and "last_sync" not in self.source_config:
+            logger.error("IncrementalLoadStrategy missing cursor_field or last_sync in source_config")
+            return False
+        return True
+
+    def sync(self, last_sync: str = None, **kwargs) -> bool:
+        try:
+            last = last_sync or self.source_config.get("last_sync")
+            logger.info("Running incremental load since %s", last)
+            # TODO: fetch delta rows since 'last' and upsert to vector DB
+            # After successful run store new last_sync timestamp
+            logger.info("Incremental load completed")
+            return True
+        except Exception:
+            logger.exception("IncrementalLoadStrategy.sync failed")
+            return False

datasourcelib/strategies/ondemand_load.py

@@ -0,0 +1,19 @@
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class OnDemandLoadStrategy(SyncBase):
+    """On demand load triggered by user request (arbitrary params)."""
+
+    def validate(self) -> bool:
+        return True
+
+    def sync(self, **kwargs) -> bool:
+        try:
+            logger.info("On-demand sync invoked with params: %s", kwargs)
+            # Use kwargs to drive partial loads, filters, ids etc.
+            return True
+        except Exception:
+            logger.exception("OnDemandLoadStrategy.sync failed")
+            return False

datasourcelib/strategies/timerange_load.py

@@ -0,0 +1,24 @@
+from datetime import datetime
+from datasourcelib.core.sync_base import SyncBase
+from datasourcelib.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+class TimeRangeLoadStrategy(SyncBase):
+    """Load records between a start and end timestamp."""
+
+    def validate(self) -> bool:
+        # rely on params at runtime; minimal validation OK
+        return True
+
+    def sync(self, start: str = None, end: str = None, **kwargs) -> bool:
+        try:
+            if not start or not end:
+                logger.error("TimeRangeLoadStrategy requires 'start' and 'end'")
+                return False
+            logger.info("Time range load between %s and %s", start, end)
+            # TODO: query source for timeframe and upsert
+            return True
+        except Exception:
+            logger.exception("TimeRangeLoadStrategy.sync failed")
+            return False

datasourcelib/utils/__init__.py

@@ -0,0 +1,12 @@
+from .byte_reader import ByteReader
+from .exceptions import DatasourceLibError, SyncStrategyNotFound, DataSourceNotFound    
+from .file_reader import FileReader
+
+
+__all__ = [
+    "ByteReader",
+    "FileReader",
+    "DatasourceLibError",
+    "SyncStrategyNotFound",
+    "SourceNotFound"
+]

datasourcelib/utils/byte_reader.py

@@ -0,0 +1,256 @@
+from pathlib import Path
+from typing import Optional, Union, List
+import io
+import pandas as pd
+
+# --- Optional helpers ---
+from charset_normalizer import from_bytes as cn_from_bytes
+
+# DOCX
+from docx import Document as DocxDocument
+
+# PDF
+import fitz  # pymupdf
+import pdfplumber
+
+# PPTX
+from pptx import Presentation
+
+# YAML / XML
+import yaml
+from lxml import etree
+import json
+
+
+class ByteReader:
+    """
+    Unified reader for common file types.
+    - read_text(path): file path -> text
+    - read_table(path): file path -> DataFrame
+    - read_text_from_bytes(data, ext): bytes -> text
+    - read_table_from_bytes(data, ext): bytes -> DataFrame
+    """
+
+    TEXT_EXTS = {".txt", ".log", ".md"}
+    TABLE_EXTS = {".csv", ".tsv", ".xlsx", ".xls"}
+    DOCX_EXTS = {".docx"}
+    PDF_EXTS = {".pdf"}
+    PPTX_EXTS = {".pptx"}
+    JSON_EXTS = {".json"}
+    YAML_EXTS = {".yaml", ".yml"}
+    INI_EXTS = {".ini", ".cfg"}
+    XML_EXTS = {".xml"}
+
+    def __init__(self, default_encoding: str = "utf-8", errors: str = "replace"):
+        self.default_encoding = default_encoding
+        self.errors = errors
+
+    # -----------------------
+    # Public API (paths)
+    # -----------------------
+    def read_text(self, path: Union[str, Path]) -> str:
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext in self.TEXT_EXTS:
+            return path.read_text(encoding=self.default_encoding, errors=self.errors)
+
+        if ext in self.PDF_EXTS:
+            return self._read_pdf_text_path(path)
+
+        if ext in self.DOCX_EXTS:
+            return self._read_docx_text_fp(open(path, "rb"))
+
+        if ext in self.PPTX_EXTS:
+            return self._read_pptx_text_fp(open(path, "rb"))
+
+        if ext in self.JSON_EXTS:
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                obj = json.load(f)
+            return json.dumps(obj, indent=2, ensure_ascii=False)
+
+        if ext in self.YAML_EXTS:
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                obj = yaml.safe_load(f)
+            return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)
+
+        if ext in self.INI_EXTS:
+            import configparser
+            parser = configparser.ConfigParser()
+            with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+                parser.read_file(f)
+            output = io.StringIO()
+            parser.write(output)
+            return output.getvalue()
+
+        if ext in self.XML_EXTS:
+            tree = etree.parse(str(path))
+            return etree.tostring(tree, pretty_print=True, encoding="unicode")
+
+        if ext in self.TABLE_EXTS:
+            df = self.read_table(path)
+            return df.to_csv(index=False)
+
+        raise ValueError(f"Unsupported file extension for text extraction: {ext}")
+
+    def read_table(self, path: Union[str, Path]) -> pd.DataFrame:
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext == ".csv":
+            return pd.read_csv(path)
+        if ext == ".tsv":
+            return pd.read_csv(path, sep="\t")
+        if ext == ".xlsx":
+            return pd.read_excel(path, engine="openpyxl")
+        if ext == ".xls":
+            return pd.read_excel(path, engine="xlrd")
+
+        # Fallback: attempt CSV read if unknown
+        try:
+            return pd.read_csv(path)
+        except Exception as e:
+            raise ValueError(f"Unsupported file extension for tables: {ext}") from e
+
+    # -----------------------
+    # Public API (bytes)
+    # -----------------------
+    def read_text_from_bytes(self, data: bytes, ext: str) -> str:
+        """
+        Extract text from in-memory bytes.
+        ext: file extension (e.g., '.pdf', '.docx', '.txt', '.pptx', '.json', '.yaml', '.xml', '.csv', '.xlsx')
+        """
+        ext = self._normalize_ext(ext)
+
+        if ext in self.TEXT_EXTS:
+            # Robust encoding detection
+            res = cn_from_bytes(data).best()
+            return str(res) if res else data.decode(self.default_encoding, errors=self.errors)
+
+        if ext in self.PDF_EXTS:
+            return self._read_pdf_text_bytes(data)
+
+        if ext in self.DOCX_EXTS:
+            return self._read_docx_text_fp(io.BytesIO(data))
+
+        if ext in self.PPTX_EXTS:
+            return self._read_pptx_text_fp(io.BytesIO(data))
+
+        if ext in self.JSON_EXTS:
+            obj = json.loads(data.decode(self.default_encoding, errors=self.errors))
+            return json.dumps(obj, indent=2, ensure_ascii=False)
+
+        if ext in self.YAML_EXTS:
+            obj = yaml.safe_load(data.decode(self.default_encoding, errors=self.errors))
+            return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)
+
+        if ext in self.INI_EXTS:
+            import configparser
+            parser = configparser.ConfigParser()
+            parser.read_string(data.decode(self.default_encoding, errors=self.errors))
+            output = io.StringIO()
+            parser.write(output)
+            return output.getvalue()
+
+        if ext in self.XML_EXTS:
+            tree = etree.parse(io.BytesIO(data))
+            return etree.tostring(tree, pretty_print=True, encoding="unicode")
+
+        if ext in self.TABLE_EXTS:
+            df = self.read_table_from_bytes(data, ext)
+            return df.to_csv(index=False)
+
+        raise ValueError(f"Unsupported extension for text extraction from bytes: {ext}")
+
+    def read_table_from_bytes(self, data: bytes, ext: str) -> pd.DataFrame:
+        """
+        Load tabular data from in-memory bytes into a DataFrame.
+        """
+        ext = self._normalize_ext(ext)
+
+        if ext == ".csv":
+            return pd.read_csv(io.BytesIO(data))
+        if ext == ".tsv":
+            return pd.read_csv(io.BytesIO(data), sep="\t")
+        if ext == ".xlsx":
+            return pd.read_excel(io.BytesIO(data), engine="openpyxl")
+        if ext == ".xls":
+            return pd.read_excel(io.BytesIO(data), engine="xlrd")
+
+        # Opportunistic fallback: try CSV
+        try:
+            return pd.read_csv(io.BytesIO(data))
+        except Exception as e:
+            raise ValueError(f"Unsupported extension for table reading from bytes: {ext}") from e
+
+    # -----------------------
+    # Internal helpers
+    # -----------------------
+    def _normalize_ext(self, ext: str) -> str:
+        ext = (ext or "").strip().lower()
+        if not ext.startswith("."):
+            ext = "." + ext
+        return ext
+
+    def _read_pdf_text_path(self, path: Path) -> str:
+        # Prefer PyMuPDF
+        try:
+            parts: List[str] = []
+            with fitz.open(str(path)) as doc:
+                if doc.is_encrypted and not doc.authenticate(""):
+                    raise RuntimeError("Encrypted PDF requires a password.")
+                for page in doc:
+                    parts.append(page.get_text("text"))
+            text = "\n\n".join(parts).strip()
+            if text:
+                return text
+        except Exception:
+            pass
+
+        # Fallback: pdfplumber
+        with pdfplumber.open(str(path)) as pdf:
+            return "\n\n".join([(p.extract_text() or "") for p in pdf.pages]).strip()
+
+    def _read_pdf_text_bytes(self, data: bytes) -> str:
+        # PyMuPDF can open from bytes
+        try:
+            doc = fitz.open(stream=data, filetype="pdf")
+            parts: List[str] = []
+            if doc.is_encrypted and not doc.authenticate(""):
+                raise RuntimeError("Encrypted PDF requires a password.")
+            for page in doc:
+                parts.append(page.get_text("text"))
+            doc.close()
+            text = "\n\n".join(parts).strip()
+            if text:
+                return text
+        except Exception:
+            pass
+
+        # Fallback to pdfplumber from BytesIO
+        with pdfplumber.open(io.BytesIO(data)) as pdf:
+            return "\n\n".join([(p.extract_text() or "") for p in pdf.pages]).strip()
+
+    def _read_docx_text_fp(self, fp) -> str:
+        doc = DocxDocument(fp)
+        chunks = []
+        for p in doc.paragraphs:
+            if p.text:
+                chunks.append(p.text)
+        for table in doc.tables:
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells]
+                if any(cells):
+                    chunks.append("\t".join(cells))
+        return "\n".join(chunks).strip()
+
+    def _read_pptx_text_fp(self, fp) -> str:
+        prs = Presentation(fp)
+        chunks = []
+        for slide in prs.slides:
+            for shape in slide.shapes:
+                if hasattr(shape, "has_text_frame") and shape.has_text_frame:
+                    text = shape.text or ""
+                    if text:
+                        chunks.append(text)
+        return "\n".join(chunks).strip()

datasourcelib/utils/exceptions.py

@@ -0,0 +1,9 @@
+class DatasourceLibError(Exception):
+    """Base exception for datasourcelib."""
+
+class SyncStrategyNotFound(DatasourceLibError):
+    """Raised when a strategy is not found."""
+
+# Added: DataSourceNotFound to represent missing/unknown data sources
+class DataSourceNotFound(DatasourceLibError):
+    """Raised when a data source is not found or not registered."""

datasourcelib/utils/file_reader.py

@@ -0,0 +1,217 @@
+from pathlib import Path
+from typing import Optional, Union, List
+import io
+import pandas as pd
+
+# --- Optional helpers ---
+from charset_normalizer import from_path as cn_from_path
+
+# DOCX
+from docx import Document as DocxDocument
+
+# PDF
+import fitz  # pymupdf
+import pdfplumber
+
+# PPTX
+from pptx import Presentation
+
+# YAML / XML
+import yaml
+from lxml import etree
+
+
+class FileReader:
+    """
+    A unified reader for common file types.
+    - read_text(path): extract text from txt, pdf, docx, pptx, json, yaml, ini, xml
+    - read_table(path): load tabular data from csv, tsv, xlsx/xls
+    """
+
+    TEXT_EXTS = {".txt", ".log", ".md"}
+    TABLE_EXTS = {".csv", ".tsv", ".xlsx", ".xls"}
+    DOCX_EXTS = {".docx"}
+    PDF_EXTS = {".pdf"}
+    PPTX_EXTS = {".pptx"}
+    JSON_EXTS = {".json"}
+    YAML_EXTS = {".yaml", ".yml"}
+    INI_EXTS = {".ini", ".cfg"}
+    XML_EXTS = {".xml"}
+
+    def __init__(self, default_encoding: str = "utf-8", errors: str = "replace"):
+        self.default_encoding = default_encoding
+        self.errors = errors
+
+    # -----------------------
+    # Public API
+    # -----------------------
+    def read_text(self, path: Union[str, Path]) -> str:
+        """
+        Extract best-effort text from a given file based on extension.
+        """
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext in self.TEXT_EXTS:
+            return self._read_plain_text(path)
+
+        if ext in self.PDF_EXTS:
+            return self._read_pdf_text(path)
+
+        if ext in self.DOCX_EXTS:
+            return self._read_docx_text(path)
+
+        if ext in self.PPTX_EXTS:
+            return self._read_pptx_text(path)
+
+        if ext in self.JSON_EXTS:
+            return self._read_json_text(path)
+
+        if ext in self.YAML_EXTS:
+            return self._read_yaml_text(path)
+
+        if ext in self.INI_EXTS:
+            return self._read_ini_text(path)
+
+        if ext in self.XML_EXTS:
+            return self._read_xml_text(path)
+
+        if ext in self.TABLE_EXTS:
+            # For tabular files, provide a quick text representation
+            df = self.read_table(path)
+            return df.to_csv(index=False)
+
+        raise ValueError(f"Unsupported file extension for text extraction: {ext}")
+
+    def read_table(self, path: Union[str, Path]) -> pd.DataFrame:
+        """
+        Load tabular data from CSV/TSV/Excel, returning a DataFrame.
+        """
+        path = Path(path)
+        ext = path.suffix.lower()
+
+        if ext == ".csv":
+            return pd.read_csv(path)
+        if ext == ".tsv":
+            return pd.read_csv(path, sep="\t")
+        if ext == ".xlsx":
+            return pd.read_excel(path, engine="openpyxl")
+        if ext == ".xls":
+            return pd.read_excel(path, engine="xlrd")
+
+        # Fallback: attempt CSV read if unknown
+        try:
+            return pd.read_csv(path)
+        except Exception as e:
+            raise ValueError(f"Unsupported file extension for tables: {ext}") from e
+
+    # -----------------------
+    # Text readers
+    # -----------------------
+    def _read_plain_text(self, path: Path) -> str:
+        # Detect encoding for robustness
+        res = cn_from_path(str(path)).best()
+        if res:
+            return str(res)
+        # Fallback to configured defaults
+        return path.read_text(encoding=self.default_encoding, errors=self.errors)
+
+    def _read_pdf_text(self, path: Path) -> str:
+        # Try PyMuPDF (fast, layout-aware)
+        try:
+            text_parts: List[str] = []
+            with fitz.open(str(path)) as doc:
+                if doc.can_save and doc.is_encrypted:
+                    # If encrypted and requires a password, this will fail to extract text.
+                    if not doc.authenticate(""):
+                        raise RuntimeError("Encrypted PDF requires a password.")
+                for page in doc:
+                    text_parts.append(page.get_text("text"))
+            text = "\n".join(text_parts).strip()
+            if text:
+                return text
+        except Exception:
+            pass
+
+        # Fallback to pdfplumber (good for tables/structured text)
+        try:
+            text_parts = []
+            with pdfplumber.open(str(path)) as pdf:
+                for page in pdf.pages:
+                    t = page.extract_text() or ""
+                    text_parts.append(t)
+            return "\n".join(text_parts).strip()
+        except Exception as e:
+            raise RuntimeError(f"Failed to read PDF: {e}") from e
+
+    def _read_docx_text(self, path: Path) -> str:
+        doc = DocxDocument(str(path))
+        chunks = []
+        # Paragraphs
+        for p in doc.paragraphs:
+            if p.text:
+                chunks.append(p.text)
+        # Tables (optional: include)
+        for table in doc.tables:
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells]
+                if any(cells):
+                    chunks.append("\t".join(cells))
+        return "\n".join(chunks).strip()
+
+    def _read_pptx_text(self, path: Path) -> str:
+        prs = Presentation(str(path))
+        chunks = []
+        for slide in prs.slides:
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.has_text_frame:
+                    text = shape.text if hasattr(shape, "text") else ""
+                    if text:
+                        chunks.append(text)
+        return "\n".join(chunks).strip()
+
+    def _read_json_text(self, path: Path) -> str:
+        import json
+        with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+            obj = json.load(f)
+        # Pretty-print
+        return json.dumps(obj, indent=2, ensure_ascii=False)
+
+    def _read_yaml_text(self, path: Path) -> str:
+        with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+            obj = yaml.safe_load(f)
+        return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)
+
+    def _read_ini_text(self, path: Path) -> str:
+        import configparser
+        parser = configparser.ConfigParser()
+        with path.open("r", encoding=self.default_encoding, errors=self.errors) as f:
+            # INI files might have duplicate keys; defaults handle many cases
+            parser.read_file(f)
+        output = io.StringIO()
+        parser.write(output)
+        return output.getvalue()
+
+    def _read_xml_text(self, path: Path) -> str:
+        # Pretty-print XML
+        tree = etree.parse(str(path))
+        return etree.tostring(tree, pretty_print=True, encoding="unicode")
+
+
+# -----------------------
+# Example usage
+# -----------------------
+#if __name__ == "__main__":
+#    reader = FileReader()
+
+    # 1) Extract text
+    # print(reader.read_text("document.pdf"))
+    # print(reader.read_text("report.docx"))
+    # print(reader.read_text("slides.pptx"))
+    # print(reader.read_text("notes.txt"))
+    # print(reader.read_text("config.yaml"))
+    # print(reader.read_text("data.xml"))
+
+    # 2) Load tabular data
+    # df = reader.read_table("data.xlsx")
+    # print(df.head())

datasourcelib/utils/logger.py

@@ -0,0 +1,12 @@
+import logging
+from typing import Optional
+
+def get_logger(name: Optional[str] = None) -> logging.Logger:
+    logger = logging.getLogger(name or __name__)
+    if not logger.handlers:
+        h = logging.StreamHandler()
+        fmt = "%(asctime)s %(levelname)s %(name)s %(message)s"
+        h.setFormatter(logging.Formatter(fmt))
+        logger.addHandler(h)
+        logger.setLevel(logging.INFO)
+    return logger

datasourcelib/utils/validators.py

@@ -0,0 +1,7 @@
+from typing import Dict
+
+def require_keys(cfg: Dict, keys):
+    missing = [k for k in keys if k not in cfg]
+    if missing:
+        raise KeyError(f"Missing required keys: {missing}")
+    return True

{datasourcelib-0.1.2.dist-info → datasourcelib-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasourcelib
-Version: 0.1.2
+Version: 0.1.4
 Summary: Data source sync strategies for vector DBs
 Home-page: https://github.com/jaiprakash0217/datasourcelib
 Author: Jai Prakash

datasourcelib-0.1.4.dist-info/RECORD

@@ -0,0 +1,32 @@
+datasourcelib/__init__.py,sha256=I7JTSZ1J6ULg_TfdMEgFcd1regkCHuyKdZT4DcPtoyQ,78
+datasourcelib/core/__init__.py,sha256=nsXojDd97T7eMqqtCsZr1qSYLBitvKydSZRb9Dg7hqU,462
+datasourcelib/core/sync_base.py,sha256=AfwwaV3rJOFKVmKKpSj-BwznnCDCaeuT4LLNDfA3NAY,716
+datasourcelib/core/sync_manager.py,sha256=lj070S3PwSNcB0UL_ZDzDAm6uJ9G38TY491vQZ1dL3o,3849
+datasourcelib/core/sync_types.py,sha256=KVZB7PkfkFTzghoe--U8jLeAU8XAfba9qMRIVcUjuMc,297
+datasourcelib/datasources/__init__.py,sha256=lZtgs0vT-2gub5UZo8BUnREZl3K_-_xYqUP8mjf8vhM,436
+datasourcelib/datasources/azure_devops_source.py,sha256=g-IOCq5vGwwteU21jZPWW_GggMu1_myVJkP0_BmSdGY,7282
+datasourcelib/datasources/blob_source.py,sha256=Qk61_ulqUSPYDaiMzqgvJAu43c4AjTlDRdfFg4VwgDU,3574
+datasourcelib/datasources/datasource_base.py,sha256=N8fOGvTl8oWWAiydLI0Joz66luq73a5yovO0XA9Q3jk,1068
+datasourcelib/datasources/datasource_types.py,sha256=eEiWymYS05X_TxwuB7P3MpphPG1En67h3kRiSGeHjQ0,176
+datasourcelib/datasources/sharepoint_source - Copy.py,sha256=7V1c-zyvTo4IuPN_YMrKwLZFgbtipbP-mtunmXjOLJQ,17664
+datasourcelib/datasources/sharepoint_source.py,sha256=Pv9735Gu2FylVeeT9e_cZlCvgGUwxn-pVRRZQe2PHU8,20196
+datasourcelib/datasources/sql_source.py,sha256=sCYHrmeD82fQVcdQjL9Y2TTTjaqlv2v8B5noAng3Bl4,5450
+datasourcelib/indexes/__init__.py,sha256=S8dz-lyxy1BTuDuLGRJNLrZD_1ku_FIUnDEm6HhMyT0,94
+datasourcelib/indexes/azure_search_index.py,sha256=kznAz06UXgyT1Clqj6gRhnBQ5HFw40ZQHJElRFIcbRo,22115
+datasourcelib/strategies/__init__.py,sha256=kot3u62KIAqYBg9M-KRE4mkMII_zwrDBZNf8Dj1vmX8,399
+datasourcelib/strategies/daily_load.py,sha256=Rh-veUhxKYsplwHTyko_Zp9C6NkUJV5VAGtg-p7Iy34,856
+datasourcelib/strategies/full_load.py,sha256=U1a9wO_ZLRnMInvU0IRW-ZKnhu0Cv437VcNMKIYuzMA,1691
+datasourcelib/strategies/incremental_load.py,sha256=TVqmDLu3m571nqGvzo_69i36QtYe4sBpllFwfPNL0TE,1178
+datasourcelib/strategies/ondemand_load.py,sha256=VxzAYgrW2ebTOC3xm61CerL2AFehZUJLnKrqtGRGJoE,644
+datasourcelib/strategies/timerange_load.py,sha256=c62BN2yXwVFaA_dQV54qenP4vrb4rcFqbx6m-nqhaTA,900
+datasourcelib/utils/__init__.py,sha256=9pSIpaK-kdmNuDzwl0Z7QU-_lV3cZE-iwOEPh3RBBTs,298
+datasourcelib/utils/byte_reader.py,sha256=GaoPXwJa2YTWG1Kim0K6JG20eVSaWkZJd1o9bswxHmc,9082
+datasourcelib/utils/exceptions.py,sha256=mgcDaW1k3VndgpMOwSm7NqgyRTvvE2a5ehn3x4fYQww,369
+datasourcelib/utils/file_reader.py,sha256=Zr0rwNTRWE6KeVJEXgTOPS1_JI74LiUSiX5-6qojmN0,7301
+datasourcelib/utils/logger.py,sha256=Sl6lNlvubxtK9ztzyq7vjGVyA8_-pZ_ixpk5jfVsh6U,424
+datasourcelib/utils/validators.py,sha256=fLgmRAb5OZSdMVlHu_n0RKJUDl-G8dI8JsRSfxIquh8,205
+datasourcelib-0.1.4.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
+datasourcelib-0.1.4.dist-info/METADATA,sha256=LR3db7O_rnbTmF_owLl-lH06xAfP-iZu4aXPtmjVtRo,1185
+datasourcelib-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datasourcelib-0.1.4.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
+datasourcelib-0.1.4.dist-info/RECORD,,

datasourcelib/indexes/azure_search_index_only.py

@@ -1,162 +0,0 @@
-from typing import List, Dict, Any, Optional
-from datasourcelib.utils.logger import get_logger
-
-logger = get_logger(__name__)
-
-class AzureSearchIndexer:
-    """
-    Minimal Azure Cognitive Search indexer wrapper.
-    Expects vector_db_config with:
-      - service_endpoint: str
-      - index_name: str
-      - api_key: str
-    Optional:
-      - key_field: name of unique key in documents (default 'id')
-    """
-
-    def __init__(self, vector_db_config: Dict[str, Any]):
-        self.config = vector_db_config or {}
-        self._client = None
-        self._index_client = None
-
-    def validate_config(self) -> bool:
-        required = ("aisearch_endpoint", "aisearch_index_name", "aisearch_api_key")
-        missing = [k for k in required if k not in self.config]
-        if missing:
-            logger.error("AzureSearchIndexer.validate_config missing: %s", missing)
-            return False
-        return True
-    
-    def _ensure_sdk(self):
-        try:
-            from azure.core.credentials import AzureKeyCredential  # type: ignore
-            from azure.search.documents import SearchClient  # type: ignore
-            from azure.search.documents.indexes import SearchIndexClient  # type: ignore
-            from azure.search.documents.indexes.models import (
-                SearchIndex,
-                SimpleField,
-                SearchableField,
-                SearchFieldDataType,
-            )  # type: ignore
-        except Exception as e:
-            raise RuntimeError("azure-search-documents package is required: install azure-search-documents") from e
-
-        return AzureKeyCredential, SearchClient, SearchIndexClient, SearchIndex, SimpleField, SearchableField, SearchFieldDataType
-    
-    def _infer_field_type(self, value) -> Any:
-        """
-        Map Python types to SearchFieldDataType
-        """
-        *_, SearchFieldDataType = self._ensure_sdk()
-        if value is None:
-            return SearchFieldDataType.String
-        t = type(value)
-        if t is str:
-            return SearchFieldDataType.String
-        if t is bool:
-            return SearchFieldDataType.Boolean
-        if t is int:
-            return SearchFieldDataType.Int32
-        if t is float:
-            return SearchFieldDataType.Double
-        # fallback to string
-        return SearchFieldDataType.String
-
-    def _build_fields(self, sample: Dict[str, Any], key_field: str):
-        AzureKeyCredential, SearchClient, SearchIndexClient, SearchIndex, SimpleField, SearchableField, SearchFieldDataType = self._ensure_sdk()
-        
-        fields = []
-        # ensure key field present
-        if key_field not in sample:
-            # we'll create a string key, uploader will populate unique ids
-            fields.append(SimpleField(name=key_field, type=SearchFieldDataType.String, key=True))
-        else:
-            typ = self._infer_field_type(sample[key_field])
-            fields.append(SimpleField(name=key_field, type=SearchFieldDataType.String, key=True))
-
-        for k, v in sample.items():
-            logger.info(f"================={k}============")
-            if k == key_field:
-                continue
-            typ = self._infer_field_type(v)
-            # for strings use SearchableField so full text queries work
-            if typ == SearchFieldDataType.String:
-                fields.append(SearchableField(name=k, type=SearchFieldDataType.String))
-            else:
-                fields.append(SimpleField(name=k, type=typ))
-        return fields
-
-    def create_index(self, sample: Dict[str, Any]) -> bool:
-        try:
-            AzureKeyCredential, SearchClient, SearchIndexClient, SearchIndex, SimpleField, SearchableField, SearchFieldDataType = self._ensure_sdk()
-            endpoint = self.config["aisearch_endpoint"]
-            api_key = self.config["aisearch_api_key"]
-            index_name = self.config["aisearch_index_name"]
-            key_field = self.config.get("key_field", "id")
-
-            index_client = SearchIndexClient(endpoint, AzureKeyCredential(api_key))
-            fields = self._build_fields(sample, key_field)
-            logger.info("=================Creating Index============")
-            index = SearchIndex(name=index_name, fields=fields)
-            # create or update index
-            index_client.create_or_update_index(index)
-            logger.info("Azure Search index '%s' created/updated", index_name)
-            return True
-        except Exception as ex:
-            logger.exception("AzureSearchIndexer.create_index failed")
-            return False
-
-    def upload_documents(self, docs: List[Dict[str, Any]]) -> bool:
-        try:
-            AzureKeyCredential, SearchClient, SearchIndexClient, SearchIndex, SimpleField, SearchableField, SearchFieldDataType = self._ensure_sdk()
-            endpoint = self.config["aisearch_endpoint"]
-            api_key = self.config["aisearch_api_key"]
-            index_name = self.config["aisearch_index_name"]
-            key_field = self.config.get("key_field", "id")
-
-            # ensure each doc has key_field
-            from uuid import uuid4
-            for d in docs:
-                if key_field not in d:
-                    d[key_field] = str(uuid4())
-            # ensure each doc has key_field is of string type
-            for d in docs:
-                if key_field in d:                    
-                    typ = self._infer_field_type(d[key_field])
-                    if typ != SearchFieldDataType.String:
-                        d[key_field] = str(d[key_field])
-
-            client = SearchClient(endpoint=endpoint, index_name=index_name, credential=AzureKeyCredential(api_key))
-            logger.info("Uploading %d documents to index %s", len(docs), index_name)
-            result = client.upload_documents(documents=docs)
-            # Check results for failures
-            failed = [r for r in result if not r.succeeded]
-            if failed:
-                logger.error("Some documents failed to upload: %s", failed)
-                return False
-            logger.info("Uploaded documents successfully")
-            return True
-        except Exception:
-            logger.exception("AzureSearchIndexer.upload_documents failed")
-            return False
-
-    def index(self, rows: List[Dict[str, Any]]) -> bool:
-        """
-        High level: create index (based on first row) and upload all rows.
-        """
-        if not rows:
-            logger.error("AzureSearchIndexer.index called with empty rows")
-            return False
-        try:
-            if not self.validate_config():
-                return False
-            sample = rows[0]
-            logger.info(f"================={sample}============")
-            ok = self.create_index(sample)
-            if not ok:
-                return False
-            ok2 = self.upload_documents(rows)
-            return ok2
-        except Exception:
-            logger.exception("AzureSearchIndexer.index failed")
-            return False

datasourcelib/indexes/azure_search_index_vector.py

@@ -1,286 +0,0 @@
-from typing import List, Dict, Any, Optional
-from datasourcelib.utils.logger import get_logger
-
-logger = get_logger(__name__)
-
-class AzureSearchIndexer:
-    """
-    Azure Cognitive Search indexer with vector search support.
-    Required vector_db_config:
-      - aisearch_endpoint: str
-      - aisearch_index_name: str
-      - aisearch_api_key
-    
-    Optional vector search config:
-      - vectorization: bool (enable vector search)
-      - vector_config: dict
-        - dimensions: int (default 1024)
-        - algorithm: str ('hnsw' or 'flat', default 'hnsw')
-        - metric: str ('cosine', 'euclidean', 'dotProduct', default 'cosine')
-      - key_field: str (default 'id')
-      - vector_field: str (default 'contentVector')
-      - embedding_endpoint: str (Azure OpenAI endpoint for embeddings)
-      - embedding_key: str (Azure OpenAI API key)
-      - embedding_deployment: str (Azure OpenAI model deployment name)
-    """
-
-    def __init__(self, vector_db_config: Dict[str, Any]):
-        self.config = vector_db_config or {}
-        self._client = None
-        self._index_client = None
-        self._embedding_client = None
-
-    def validate_config(self) -> bool:
-        required = ("aisearch_endpoint", "aisearch_index_name", "aisearch_api_key")
-        missing = [k for k in required if k not in self.config]
-        
-        # Check vector search requirements if enabled
-        if self.config.get("vectorization", False):
-            vector_required = ("embedding_endpoint", "embedding_key", "embedding_deployment")
-            missing.extend([k for k in vector_required if k not in self.config])
-        
-        if missing:
-            logger.error("AzureSearchIndexer.validate_config missing: %s", missing)
-            return False
-        return True
-
-    def _ensure_sdk(self):
-        try:
-            from azure.core.credentials import AzureKeyCredential  # type: ignore
-            from azure.search.documents import SearchClient  # type: ignore
-            from azure.search.documents.indexes import SearchIndexClient  # type: ignore
-            from openai import AzureOpenAI # type: ignore
-            from azure.search.documents.indexes.models import (
-                SearchIndex,
-                SearchField,
-                SearchFieldDataType,
-                SimpleField,
-                SearchableField,
-                VectorSearch,
-                VectorSearchProfile,
-                HnswAlgorithmConfiguration
-            ) # type: ignore
-            
-        except Exception as e:
-            raise RuntimeError("Required packages missing. Install: azure-search-documents openai") from e
-
-        return (
-            AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration
-        )
-
-    def _setup_embedding_client(self):
-        if not self._embedding_client and self.config.get("vectorization"):
-            try:
-                AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-                self._embedding_client = AzureOpenAI(
-                    api_version=self.config["embedding_api_version"],
-                    azure_endpoint=self.config["embedding_endpoint"],
-                    api_key=self.config["embedding_key"],
-                )
-                logger.info("Azure OpenAI embedding client initialized")
-            except Exception as ex:
-                logger.exception("Failed to initialize embedding client")
-                raise
-
-    def _get_embeddings(self, text: str) -> List[float]:
-        try:
-            self._setup_embedding_client()
-            response = self._embedding_client.embeddings.create(
-                model=self.config["embedding_deployment"],
-                input=text
-            )
-            return response.data[0].embedding
-        except Exception as ex:
-            logger.exception(f"Failed to get embeddings for text: {text[:100]}...")
-            raise
-
-    def _build_vector_search_config(self):
-        AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-        vector_config = self.config.get("vector_config", {})
-        dimensions = vector_config.get("dimensions", 1536)
-             
-        vector_search = VectorSearch(
-                profiles=[VectorSearchProfile(name="vector-profile-1", algorithm_configuration_name="algorithms-config-1")],
-                algorithms=[HnswAlgorithmConfiguration(name="algorithms-config-1")]
-            )
-
-        return vector_search, dimensions
-    
-    def _infer_field_type(self, value) -> Any:
-        #Map Python types to SearchFieldDataType, including collections
-
-        AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-        
-        if value is None:
-            return SearchFieldDataType.String
-            
-        t = type(value)
-        
-        # Handle list/array types as Collections
-        if t in (list, tuple):
-            # If empty list, default to Collection of Double
-            if not value:
-                return SearchFieldDataType.Collection(SearchFieldDataType.Double)
-            # Get type of first element for non-empty lists
-            element_type = self._infer_field_type(value[0])
-            return SearchFieldDataType.Collection(element_type)
-        # Handle vector embeddings (list or tuple of floats)
-        if type(value) in (list, tuple) and all(isinstance(x, (int, float)) for x in value):
-            return SearchFieldDataType.Collection(SearchFieldDataType.Single)
-            
-        # Handle basic types
-        logger.info(f"######## Infer field type for value:[ {value} ] of type [ {t} ]")
-        if t is bool:
-            return SearchFieldDataType.Boolean
-        if t is int:
-            return SearchFieldDataType.Int32
-        if t is float:
-            return SearchFieldDataType.Double
-        print(f"############## Infer field type for value: {value} of type {t}")
-        print(t is str)
-        if t is str:
-            return SearchFieldDataType.String
-        # fallback to string
-        logger.warning(f"Falling back to string type for value: {value} of type {t}")
-        return SearchFieldDataType.String
-        
-    def _build_fields(self, sample: Dict[str, Any], key_field: str):
-        AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-
-        fields = []
-        # Add key field
-        if key_field not in sample:
-            fields.append(SimpleField(name=key_field, type=SearchFieldDataType.String, key=True))
-        else:
-            fields.append(SimpleField(name=key_field, type=SearchFieldDataType.String, key=True))
-
-        # Add regular fields
-        for k, v in sample.items():
-            logger.info(f"================={k}============")
-            if k == key_field:
-                continue
-            logger.info(f"#### Infer field type for field: {k}")
-            typ = self._infer_field_type(v)
-            logger.info(f"#### Inferred type for field {k}: {typ}")
-            if typ == SearchFieldDataType.String:
-                fields.append(SearchableField(name=k, type=SearchFieldDataType.String))
-            else:
-                fields.append(SimpleField(name=k, type=typ))
-
-        # Add vector field if vectorization is enabled
-        if self.config.get("vectorization"):
-            vector_field = self.config.get("vector_field", "contentVector")
-            _, dimensions = self._build_vector_search_config()
-            fields.append(
-                SearchField(
-                    name=vector_field,
-                    type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
-                    searchable=True,
-                    vector_search_dimensions=dimensions,
-                    vector_search_profile_name="vector-profile-1"
-                )
-            )
-
-        return fields
-
-    def create_index(self, sample: Dict[str, Any]) -> bool:
-        try:
-            AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-            
-            endpoint = self.config["aisearch_endpoint"]
-            api_key = self.config["aisearch_api_key"]
-            index_name = self.config["aisearch_index_name"]
-            key_field = self.config.get("key_field", "id")
-
-            index_client = SearchIndexClient(endpoint, AzureKeyCredential(api_key))
-            fields = self._build_fields(sample, key_field)
-
-            # Create index with vector search if enabled
-            if self.config.get("vectorization"):
-                vector_search, _ = self._build_vector_search_config()
-                index = SearchIndex(
-                    name=index_name,
-                    fields=fields,
-                    vector_search=vector_search
-                )
-            else:
-                index = SearchIndex(name=index_name, fields=fields)
-
-            index_client.create_or_update_index(index)
-            logger.info(f"Azure Search index '{index_name}' created/updated with vectorization={self.config.get('vectorization', False)}")
-            return True
-        except Exception as ex:
-            logger.exception("AzureSearchIndexer.create_index failed")
-            return False
-
-    def upload_documents(self, docs: List[Dict[str, Any]]) -> bool:
-        try:
-            AzureKeyCredential, SearchClient, SearchIndexClient, AzureOpenAI, SearchIndex, SearchField, SearchFieldDataType, SimpleField, SearchableField, VectorSearch, VectorSearchProfile, HnswAlgorithmConfiguration = self._ensure_sdk()
-            endpoint = self.config["aisearch_endpoint"]
-            api_key = self.config["aisearch_api_key"]
-            index_name = self.config["aisearch_index_name"]
-            key_field = self.config.get("key_field", "id")
-            
-            # Add IDs if missing
-            from uuid import uuid4
-            for d in docs:
-                if key_field not in d:
-                    d[key_field] = str(uuid4())
-                elif not isinstance(d[key_field], str):
-                    d[key_field] = str(d[key_field])
-
-            # Add vector embeddings if enabled
-            if self.config.get("vectorization"):
-                vector_field = self.config.get("vector_field", "contentVector")
-                content_field = self.config.get("content_field", "content")
-                
-                for doc in docs:
-                    if content_field in doc:
-                        try:
-                            embedding = self._get_embeddings(str(doc[content_field]))
-                            doc[vector_field] = embedding
-                        except Exception as e:
-                            logger.error(f"Failed to get embedding for document {doc.get(key_field)}: {str(e)}")
-                            continue
-
-            client = SearchClient(endpoint=endpoint, index_name=index_name, 
-                               credential=AzureKeyCredential(api_key))
-            
-            logger.info(f"Uploading {len(docs)} documents to index {index_name}")
-            result = client.upload_documents(documents=docs)
-            
-            failed = [r for r in result if not r.succeeded]
-            if failed:
-                logger.error(f"Some documents failed to upload: {failed}")
-                return False
-                
-            logger.info("Documents uploaded successfully")
-            return True
-            
-        except Exception:
-            logger.exception("AzureSearchIndexer.upload_documents failed")
-            return False
-
-    def index(self, rows: List[Dict[str, Any]]) -> bool:
-        """High level: create index (based on first row) and upload all rows."""
-        if not rows:
-            logger.error("AzureSearchIndexer.index called with empty rows")
-            return False
-            
-        try:
-            if not self.validate_config():
-                return False
-                
-            sample = rows[0]
-            logger.info(f"Creating/updating index with sample: {sample}")
-            
-            ok = self.create_index(sample)
-            if not ok:
-                return False
-                
-            ok2 = self.upload_documents(rows)
-            return ok2
-            
-        except Exception:
-            logger.exception("AzureSearchIndexer.index failed")
-            return False

datasourcelib-0.1.2.dist-info/RECORD

@@ -1,22 +0,0 @@
-datasourcelib/__init__.py,sha256=I7JTSZ1J6ULg_TfdMEgFcd1regkCHuyKdZT4DcPtoyQ,78
-datasourcelib/core/__init__.py,sha256=nsXojDd97T7eMqqtCsZr1qSYLBitvKydSZRb9Dg7hqU,462
-datasourcelib/core/sync_base.py,sha256=AfwwaV3rJOFKVmKKpSj-BwznnCDCaeuT4LLNDfA3NAY,716
-datasourcelib/core/sync_manager.py,sha256=lj070S3PwSNcB0UL_ZDzDAm6uJ9G38TY491vQZ1dL3o,3849
-datasourcelib/core/sync_types.py,sha256=KVZB7PkfkFTzghoe--U8jLeAU8XAfba9qMRIVcUjuMc,297
-datasourcelib/datasources/__init__.py,sha256=lZtgs0vT-2gub5UZo8BUnREZl3K_-_xYqUP8mjf8vhM,436
-datasourcelib/datasources/azure_devops_source.py,sha256=g-IOCq5vGwwteU21jZPWW_GggMu1_myVJkP0_BmSdGY,7282
-datasourcelib/datasources/blob_source.py,sha256=Qk61_ulqUSPYDaiMzqgvJAu43c4AjTlDRdfFg4VwgDU,3574
-datasourcelib/datasources/datasource_base.py,sha256=N8fOGvTl8oWWAiydLI0Joz66luq73a5yovO0XA9Q3jk,1068
-datasourcelib/datasources/datasource_types.py,sha256=eEiWymYS05X_TxwuB7P3MpphPG1En67h3kRiSGeHjQ0,176
-datasourcelib/datasources/sharepoint_source - Copy.py,sha256=7V1c-zyvTo4IuPN_YMrKwLZFgbtipbP-mtunmXjOLJQ,17664
-datasourcelib/datasources/sharepoint_source.py,sha256=Pv9735Gu2FylVeeT9e_cZlCvgGUwxn-pVRRZQe2PHU8,20196
-datasourcelib/datasources/sql_source.py,sha256=sCYHrmeD82fQVcdQjL9Y2TTTjaqlv2v8B5noAng3Bl4,5450
-datasourcelib/indexes/__init__.py,sha256=S8dz-lyxy1BTuDuLGRJNLrZD_1ku_FIUnDEm6HhMyT0,94
-datasourcelib/indexes/azure_search_index.py,sha256=o3BoSxURBk5jCC3AlNz-v9_igg-dXYS4yUxXZwSfqFg,17265
-datasourcelib/indexes/azure_search_index_only.py,sha256=SulrYPehWGaf3Wi_Dw8UvFneSY-UwEK9viVYXwIlQuI,7120
-datasourcelib/indexes/azure_search_index_vector.py,sha256=4By1vJHv1ORiWOpTqO5wR0sTrq1TaEHP6t8MoOINhok,13410
-datasourcelib-0.1.2.dist-info/licenses/LICENSE,sha256=9S0AcKETmp9XOcC73jEjN7WSkuSWGFGreiBat6ONClo,1087
-datasourcelib-0.1.2.dist-info/METADATA,sha256=DOKGwf3XspFhCQRLYLod8Oqc2sUjhDaFFe15xiKqQhQ,1185
-datasourcelib-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datasourcelib-0.1.2.dist-info/top_level.txt,sha256=wIwiwdIj8T9pAvE2TkGLUvT2oIi43C2vkkTKibUlv3U,14
-datasourcelib-0.1.2.dist-info/RECORD,,