vd-dlt 0.1.0__tar.gz

vd_dlt-0.1.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: vd-dlt
+Version: 0.1.0
+Summary: Core DLT ingestion framework for VibeData pipelines
+Project-URL: Homepage, https://github.com/accelerate-data/vd-dlt-connectors
+Project-URL: Repository, https://github.com/accelerate-data/vd-dlt-connectors
+Author-email: VibeData <info@vibedata.dev>
+License-Expression: MIT
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.9
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: notion
+Requires-Dist: vd-dlt-notion; extra == 'notion'
+Provides-Extra: notion-schema
+Requires-Dist: vd-dlt-notion-schema; extra == 'notion-schema'
+Provides-Extra: pipeline
+Requires-Dist: dlt[deltalake,filesystem]; extra == 'pipeline'
+Requires-Dist: pyarrow>=17.0.0; extra == 'pipeline'
+Description-Content-Type: text/markdown
+
+# vd-dlt
+
+Core DLT ingestion framework for VibeData pipelines. Provides config resolution, credential management (Azure Key Vault), pipeline execution, and observability.
+
+## Installation
+
+```bash
+# Core only
+pip install vd-dlt
+
+# With pipeline dependencies (dlt, pyarrow)
+pip install vd-dlt[pipeline]
+
+# With Notion connector
+pip install vd-dlt[notion]
+
+# With Notion schema (defaults, docs)
+pip install vd-dlt[notion-schema]
+
+# Everything for Notion
+pip install vd-dlt[pipeline,notion,notion-schema]
+```
+
+## Quick Start
+
+```python
+from vd_dlt import PipelineRunner
+
+runner = PipelineRunner(
+    vault_url="https://my-vault.vault.azure.net/",
+)
+result = runner.run("my_source_name")
+print(f"Loaded {result.total_rows_loaded} rows")
+```
+
+## Architecture
+
+The framework uses a 4-level config hierarchy (most specific wins):
+
+1. **Connector Defaults** - from connector schema package
+2. **Source Config** - from source YAML (including `default_sync`)
+3. **Group Config** - optional grouping within source
+4. **Resource Config** - per-table overrides

vd_dlt-0.1.0/README.md

@@ -0,0 +1,43 @@
+# vd-dlt
+
+Core DLT ingestion framework for VibeData pipelines. Provides config resolution, credential management (Azure Key Vault), pipeline execution, and observability.
+
+## Installation
+
+```bash
+# Core only
+pip install vd-dlt
+
+# With pipeline dependencies (dlt, pyarrow)
+pip install vd-dlt[pipeline]
+
+# With Notion connector
+pip install vd-dlt[notion]
+
+# With Notion schema (defaults, docs)
+pip install vd-dlt[notion-schema]
+
+# Everything for Notion
+pip install vd-dlt[pipeline,notion,notion-schema]
+```
+
+## Quick Start
+
+```python
+from vd_dlt import PipelineRunner
+
+runner = PipelineRunner(
+    vault_url="https://my-vault.vault.azure.net/",
+)
+result = runner.run("my_source_name")
+print(f"Loaded {result.total_rows_loaded} rows")
+```
+
+## Architecture
+
+The framework uses a 4-level config hierarchy (most specific wins):
+
+1. **Connector Defaults** - from connector schema package
+2. **Source Config** - from source YAML (including `default_sync`)
+3. **Group Config** - optional grouping within source
+4. **Resource Config** - per-table overrides

vd_dlt-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vd-dlt"
+version = "0.1.0"
+description = "Core DLT ingestion framework for VibeData pipelines"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+authors = [
+    { name = "VibeData", email = "info@vibedata.dev" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "pyyaml>=6.0",
+]
+
+[project.optional-dependencies]
+pipeline = [
+    "dlt[filesystem,deltalake]",
+    "pyarrow>=17.0.0",
+]
+notion = [
+    "vd-dlt-notion",
+]
+notion-schema = [
+    "vd-dlt-notion-schema",
+]
+
+[project.urls]
+Homepage = "https://github.com/accelerate-data/vd-dlt-connectors"
+Repository = "https://github.com/accelerate-data/vd-dlt-connectors"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vd_dlt"]

vd_dlt-0.1.0/src/vd_dlt/__init__.py

@@ -0,0 +1,33 @@
+"""
+vd-dlt: Core DLT Ingestion Framework
+
+Provides:
+- Config resolution (4-level hierarchy)
+- Credential resolution (Key Vault integration)
+- Pipeline execution
+- Observability (logging, sync history)
+- Schema validation
+"""
+
+__version__ = "0.1.0"
+
+from .config_resolver import ConfigResolver, resolve_resource_config
+from .credential_resolver import CredentialResolver, resolve_credentials
+from .pipeline_runner import PipelineRunner, run_source
+from .observability import setup_logging, SyncHistoryTracker, SyncRecord, ResourceResult
+from .schema_validator import ValidationError, validate_all
+
+__all__ = [
+    "ConfigResolver",
+    "resolve_resource_config",
+    "CredentialResolver",
+    "resolve_credentials",
+    "PipelineRunner",
+    "run_source",
+    "setup_logging",
+    "SyncHistoryTracker",
+    "SyncRecord",
+    "ResourceResult",
+    "ValidationError",
+    "validate_all",
+]

vd_dlt-0.1.0/src/vd_dlt/config_resolver.py

@@ -0,0 +1,238 @@
+"""
+Config Resolution Engine - 4-Level Hierarchy
+
+Resolution order (most specific wins):
+1. Connector Defaults - connectors/{name}/defaults.yml
+2. Source - sources/{source_name}.yml (including default_sync)
+3. Group - Optional grouping within source config
+4. Resource - Per-table overrides in source config
+
+Priority: Resource > Group > Source > Connector Defaults
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+from .utils import (
+    deep_merge,
+    load_yaml,
+    get_connectors_path,
+    get_configs_path,
+    file_exists,
+)
+
+
+class ConfigResolver:
+    """
+    Resolves configuration from the 4-level hierarchy.
+
+    Usage:
+        resolver = ConfigResolver()
+        source_config = resolver.load_source_config("notion_wiki")
+        resource_config = resolver.resolve_resource_config(source_config, "database_rows")
+    """
+
+    def __init__(self, base_path: Optional[str] = None):
+        """
+        Initialize config resolver.
+
+        Args:
+            base_path: Base path for configs (default: from environment)
+        """
+        if base_path:
+            self.connectors_path = f"{base_path}/connectors"
+            self.configs_path = f"{base_path}/configs"
+        else:
+            self.connectors_path = get_connectors_path()
+            self.configs_path = get_configs_path()
+
+        self._connector_cache: Dict[str, Dict] = {}
+
+    def load_connector_defaults(self, connector_name: str) -> Dict[str, Any]:
+        """
+        Load connector defaults (Level 1).
+
+        Args:
+            connector_name: Name of the connector (e.g., "notion")
+
+        Returns:
+            Connector defaults dictionary
+        """
+        if connector_name in self._connector_cache:
+            return self._connector_cache[connector_name]
+
+        defaults_path = f"{self.connectors_path}/{connector_name}/defaults.yml"
+        if file_exists(defaults_path):
+            defaults = load_yaml(defaults_path)
+        else:
+            defaults = {}
+
+        self._connector_cache[connector_name] = defaults
+        return defaults
+
+    def load_connector_manifest(self, connector_name: str) -> Dict[str, Any]:
+        """
+        Load connector manifest (metadata).
+
+        Args:
+            connector_name: Name of the connector
+
+        Returns:
+            Connector manifest dictionary
+        """
+        manifest_path = f"{self.connectors_path}/{connector_name}/manifest.yml"
+        if file_exists(manifest_path):
+            return load_yaml(manifest_path)
+        return {}
+
+    def load_source_config(self, source_name: str) -> Dict[str, Any]:
+        """
+        Load source config (Level 2).
+
+        Args:
+            source_name: Name of the source (e.g., "notion_wiki")
+
+        Returns:
+            Source config dictionary
+
+        Raises:
+            FileNotFoundError: If source config doesn't exist
+        """
+        source_path = f"{self.configs_path}/sources/{source_name}.yml"
+        if not file_exists(source_path):
+            raise FileNotFoundError(f"Source config not found: {source_path}")
+
+        return load_yaml(source_path)
+
+    def get_group_for_resource(
+        self, source_config: Dict[str, Any], resource_name: str
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Find the group config for a resource (Level 3).
+
+        Args:
+            source_config: Source configuration
+            resource_name: Name of the resource
+
+        Returns:
+            Group config if resource belongs to a group, None otherwise
+        """
+        groups = source_config.get("groups", {})
+        for group_name, group_config in groups.items():
+            if resource_name in group_config.get("resources", []):
+                # Return group settings without the 'resources' list
+                return {k: v for k, v in group_config.items() if k != "resources"}
+        return None
+
+    def resolve_resource_config(
+        self, source_config: Dict[str, Any], resource_name: str
+    ) -> Dict[str, Any]:
+        """
+        Resolve effective config for a resource.
+
+        Applies the 4-level hierarchy:
+        Priority: Resource > Group > Source > Connector Defaults
+
+        Args:
+            source_config: Source configuration (from load_source_config)
+            resource_name: Name of the resource to resolve
+
+        Returns:
+            Fully resolved configuration for the resource
+        """
+        connector_name = source_config.get("connector", "rest_api_generic")
+
+        # Level 1: Start with connector defaults
+        connector_defaults = self.load_connector_defaults(connector_name)
+        effective = deep_merge({}, connector_defaults.get("defaults", {}))
+
+        # Apply resource-specific connector defaults if exists
+        resource_connector_defaults = connector_defaults.get("resource_defaults", {}).get(resource_name, {})
+        effective = deep_merge(effective, resource_connector_defaults)
+
+        # Level 2: Apply source-level config (including default_sync)
+        source_level = {
+            k: v for k, v in source_config.items()
+            if k not in ["groups", "resources", "source_name", "connector",
+                        "connector_version", "secrets"]
+        }
+        effective = deep_merge(effective, source_level)
+
+        # Level 3: Apply group config (if resource belongs to a group)
+        group_config = self.get_group_for_resource(source_config, resource_name)
+        if group_config:
+            effective = deep_merge(effective, group_config)
+
+        # Level 4: Apply resource-specific overrides
+        # Handle both list format (new) and dict format (legacy)
+        resources = source_config.get("resources", [])
+        resource_config = {}
+        if isinstance(resources, list):
+            for res in resources:
+                if res.get("name") == resource_name:
+                    resource_config = {k: v for k, v in res.items() if k != "name"}
+                    break
+        elif isinstance(resources, dict):
+            resource_config = resources.get(resource_name, {})
+        effective = deep_merge(effective, resource_config)
+
+        # Add metadata
+        effective["_resource_name"] = resource_name
+        effective["_source_name"] = source_config.get("source_name", "unknown")
+        effective["_connector"] = connector_name
+
+        return effective
+
+    def get_enabled_resources(self, source_config: Dict[str, Any]) -> List[str]:
+        """
+        Get list of enabled resources from source config.
+
+        Supports both list format (new) and dict format (legacy):
+        - List: [{"name": "projects", "enabled": true}, ...]
+        - Dict: {"projects": {"enabled": true}, ...}
+
+        Args:
+            source_config: Source configuration
+
+        Returns:
+            List of enabled resource names
+        """
+        resources = source_config.get("resources", [])
+        enabled = []
+
+        if isinstance(resources, list):
+            # New list format
+            for resource in resources:
+                if resource.get("enabled", True):
+                    name = resource.get("name")
+                    if name:
+                        enabled.append(name)
+        elif isinstance(resources, dict):
+            # Legacy dict format
+            for name, config in resources.items():
+                if isinstance(config, dict) and config.get("enabled", True):
+                    enabled.append(name)
+                elif config is None or config == {}:
+                    enabled.append(name)
+
+        return enabled
+
+
+def resolve_resource_config(
+    source_name: str,
+    resource_name: str,
+    base_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Convenience function to resolve resource config.
+
+    Args:
+        source_name: Name of the source
+        resource_name: Name of the resource
+        base_path: Optional base path override
+
+    Returns:
+        Resolved resource configuration
+    """
+    resolver = ConfigResolver(base_path)
+    source_config = resolver.load_source_config(source_name)
+    return resolver.resolve_resource_config(source_config, resource_name)