data-contract-validator 1.0.4a0__tar.gz → 1.0.5__tar.gz

data_contract_validator-1.0.5/CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.0.5] - 2025-01-24
+
+### Fixed
+- **CRITICAL**: Fixed missing return statement in `DBTExtractor.extract_schemas()` that could return `None` instead of dictionary
+  - Added fallback to SQL file parsing when manifest.json is unavailable
+  - Now works reliably with or without DBT CLI installed
+- **HIGH**: Fixed function signature mismatch in `_test_configuration()` causing TypeError on `--dry-run` command
+  - Added missing `disable_manifest` parameter
+  - Enhanced to display manifest parsing status
+- **MEDIUM**: Replaced bare exception handler in `_try_compile_dbt()` with specific exception types
+  - Now properly handles TimeoutExpired, FileNotFoundError
+  - Provides helpful error messages instead of silent failures
+  - Respects keyboard interrupts
+- **MEDIUM**: Removed unused `fastapi_directory` parameter from CLI
+  - Simplified API - use `--fastapi-local` for both files and directories
+- **MEDIUM**: Added comprehensive YAML error handling with user-friendly messages
+  - Catches malformed YAML files with helpful suggestions
+  - Validates required configuration sections
+  - Provides clear error messages instead of Python tracebacks
+- **LOW**: Added GitHub API rate limiting detection and handling
+  - Monitors rate limit headers and warns when limits are low
+  - Provides helpful guidance to use GITHUB_TOKEN for higher limits
+  - Better error messages for 403 and 404 responses
+
+### Improved
+- Enhanced error messages throughout the application
+- Better support for different use-cases:
+  - DBT projects with or without manifest.json
+  - Local files and directories for FastAPI models
+  - GitHub repositories with rate limit awareness
+  - Configuration validation with clear error reporting
+
+## [1.0.0] - 2025-01-XX
+
+### Added
+- Initial release of Data Contract Validator
+- DBT schema extraction from SQL files and manifest.json
+- FastAPI/Pydantic model extraction from local files and GitHub repos
+- Command-line interface with multiple output formats
+- GitHub Actions integration
+- Contract validation with critical/warning/info severity levels
+- Support for multiple repositories and complex validation scenarios
+
+### Features
+- ✅ DBT model schema extraction
+- ✅ FastAPI/Pydantic schema extraction
+- ✅ Cross-repository validation
+- ✅ GitHub Actions workflows
+- ✅ Multiple output formats (terminal, JSON, GitHub Actions)
+- ✅ Comprehensive error reporting with suggested fixes
+- ✅ Type compatibility checking
+- ✅ Missing table/column detection
+
+### Known Limitations
+- Only supports DBT and FastAPI currently
+- Requires manual installation of DBT CLI
+- Limited type inference from SQL
+- No support for complex nested types
+
+[Unreleased]: https://github.com/OGsiji/data-contract-validator/compare/v1.0.5...HEAD
+[1.0.5]: https://github.com/OGsiji/data-contract-validator/releases/tag/v1.0.5
+[1.0.0]: https://github.com/OGsiji/data-contract-validator/releases/tag/v1.0.0

{data_contract_validator-1.0.4a0/data_contract_validator.egg-info → data_contract_validator-1.0.5}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: data-contract-validator
-Version: 1.0.4a0
-Summary: Adding pre-commit-fixes
+Version: 1.0.5
+Summary: Critical bug fixes: DBT extractor return paths, function signatures, error handling, and GitHub API rate limiting
 Author-email: Ogunniran Siji <ogunniransiji@gmail.com>
 Maintainer-email: Ogunniran Siji <ogunniransiji@gmail.com>
 License: MIT

{data_contract_validator-1.0.4a0 → data_contract_validator-1.0.5}/data_contract_validator/cli.py

@@ -494,7 +494,6 @@ def _test_setup(config_file: Path) -> bool:
 @click.option(
     "--fastapi-local", help="Override FastAPI models path (file or directory)"
 )
-@click.option("--fastapi-directory", help="Override FastAPI models directory path")
 @click.option("--fastapi-repo", help="Override FastAPI repo (org/repo)")
 @click.option(
     "--fastapi-path",
@@ -508,10 +507,9 @@ def validate(
     output: str,
     dbt_project: str,
     fastapi_local: str,
-    fastapi_directory: str,
     fastapi_repo: str,
     fastapi_path: str,
-    disable_manifest: bool, 
+    disable_manifest: bool,
 ):
     """🔍 Validate data contracts (prevents production breaks)."""
 
@@ -519,9 +517,34 @@ def validate(
     config_data = {}
     config_file = Path(config)
     if config_file.exists():
-        with open(config_file) as f:
-            config_data = yaml.safe_load(f)
-        click.echo(f"📋 Using config: {config}")
+        try:
+            with open(config_file) as f:
+                config_data = yaml.safe_load(f)
+
+            # Validate that config is a dictionary
+            if not isinstance(config_data, dict):
+                click.echo(f"❌ Configuration file {config} is invalid: expected YAML dictionary")
+                click.echo("💡 Check the file format - it should contain key-value pairs")
+                sys.exit(1)
+
+            # Check for required sections
+            if "source" not in config_data:
+                click.echo(f"⚠️  Warning: 'source' section missing in {config}")
+            if "target" not in config_data:
+                click.echo(f"⚠️  Warning: 'target' section missing in {config}")
+
+            click.echo(f"📋 Using config: {config}")
+        except yaml.YAMLError as e:
+            click.echo(f"❌ Configuration file {config} contains invalid YAML:")
+            click.echo(f"   {e}")
+            click.echo("💡 Check for:")
+            click.echo("   - Incorrect indentation")
+            click.echo("   - Missing colons after keys")
+            click.echo("   - Unmatched quotes or brackets")
+            sys.exit(1)
+        except Exception as e:
+            click.echo(f"❌ Error reading configuration file {config}: {e}")
+            sys.exit(1)
     elif not any([dbt_project, fastapi_local, fastapi_repo]):
         click.echo("❌ No config file found and no command line options provided")
         click.echo("💡 Run 'contract-validator init' to create a config file")
@@ -534,7 +557,7 @@ def validate(
     if dry_run:
         click.echo("🧪 Dry run - testing configuration only")
         _test_configuration(
-            config_data, dbt_project, fastapi_local, fastapi_directory, fastapi_repo, disable_manifest 
+            config_data, dbt_project, fastapi_local, fastapi_repo, disable_manifest
         )
         return
 
@@ -544,14 +567,18 @@ def validate(
         output,
         dbt_project,
         fastapi_local,
-        fastapi_directory,
         fastapi_repo,
         fastapi_path,
+        disable_manifest,
     )
 
 
 def _test_configuration(
-    config_data: Dict[str, Any], dbt_project: str, fastapi_local: str, fastapi_repo: str
+    config_data: Dict[str, Any],
+    dbt_project: str,
+    fastapi_local: str,
+    fastapi_repo: str,
+    disable_manifest: bool = False
 ):
     """Test configuration without running full validation."""
 
@@ -565,12 +592,21 @@ def _test_configuration(
     else:
         click.echo("      ❌ Path not found")
 
+    if disable_manifest or config_data.get("source", {}).get("dbt", {}).get("disable_manifest", False):
+        click.echo("   📄 Manifest parsing: disabled")
+    else:
+        click.echo("   📋 Manifest parsing: enabled")
+
     if fastapi_local:
         click.echo(f"   🎯 FastAPI models: {fastapi_local}")
-        if Path(fastapi_local).exists():
-            click.echo("      ✅ File exists")
+        local_path = Path(fastapi_local)
+        if local_path.exists():
+            if local_path.is_file():
+                click.echo("      ✅ File exists")
+            elif local_path.is_dir():
+                click.echo("      ✅ Directory exists")
         else:
-            click.echo("      ❌ File not found")
+            click.echo("      ❌ Path not found")
 
     if fastapi_repo:
         click.echo(f"   🐙 FastAPI repo: {fastapi_repo}")

{data_contract_validator-1.0.4a0 → data_contract_validator-1.0.5}/data_contract_validator/extractors/dbt.py

@@ -27,13 +27,19 @@ class DBTExtractor(BaseExtractor):
         """Extract schemas from DBT project."""
         print(f"🔍 Extracting DBT schemas from {self.project_path}")
 
-        # Try manifest first, fallback to SQL parsing
+        # Check if manifest should be disabled
+        if self.disable_manifest:
+            print("   📄 Manifest disabled, using SQL file parsing")
+            return self._extract_from_sql_files()
+
+        # Try to use manifest.json if available
         if self._try_compile_dbt() and self.manifest_path.exists():
             print("   📋 Using manifest.json")
             return self._extract_from_manifest()
-        else:
-            print("   📄 Using SQL file parsing")
-            return self._extract_from_sql_files()
+
+        # Fallback to SQL file parsing if manifest is not available
+        print("   📄 Manifest not available, using SQL file parsing")
+        return self._extract_from_sql_files()
 
     def _try_compile_dbt(self) -> bool:
         """Try to compile DBT project."""
@@ -45,7 +51,14 @@ class DBTExtractor(BaseExtractor):
                 timeout=60,
             )
             return result.returncode == 0
-        except:
+        except subprocess.TimeoutExpired:
+            print("   ⚠️  DBT compilation timeout (>60s)")
+            return False
+        except FileNotFoundError:
+            print("   ⚠️  DBT CLI not found (install with: pip install dbt-core)")
+            return False
+        except Exception as e:
+            print(f"   ⚠️  DBT compilation error: {e}")
             return False
 
     def _extract_from_manifest(self) -> Dict[str, Schema]:

{data_contract_validator-1.0.4a0 → data_contract_validator-1.0.5}/data_contract_validator/extractors/fastapi.py

@@ -223,7 +223,7 @@ class FastAPIExtractor(BaseExtractor):
 
     @staticmethod
     def _fetch_github_file(repo: str, path: str, token: str = None) -> Optional[str]:
-        """Fetch file content from GitHub API."""
+        """Fetch file content from GitHub API with rate limit handling."""
         url = f"https://api.github.com/repos/{repo}/contents/{path}"
         headers = {}
 
@@ -232,11 +232,35 @@ class FastAPIExtractor(BaseExtractor):
 
         try:
             response = requests.get(url, headers=headers)
+
+            # Check rate limit headers
+            if "X-RateLimit-Remaining" in response.headers:
+                remaining = int(response.headers["X-RateLimit-Remaining"])
+                if remaining < 10:
+                    print(f"   ⚠️  GitHub API rate limit low: {remaining} requests remaining")
+                    if remaining == 0:
+                        reset_time = int(response.headers.get("X-RateLimit-Reset", 0))
+                        import time
+                        wait_time = max(0, reset_time - int(time.time()))
+                        print(f"   ⏳ Rate limit exceeded. Resets in {wait_time // 60} minutes")
+
             if response.status_code == 200:
                 import base64
 
                 content = base64.b64decode(response.json()["content"]).decode("utf-8")
                 return content
+            elif response.status_code == 403:
+                # Check if it's a rate limit error
+                error_message = response.json().get("message", "")
+                if "rate limit" in error_message.lower():
+                    print(f"   ❌ GitHub API rate limit exceeded")
+                    print(f"   💡 Try setting GITHUB_TOKEN environment variable for higher limits")
+                else:
+                    print(f"   ❌ GitHub API access forbidden: {error_message}")
+                return None
+            elif response.status_code == 404:
+                print(f"   ❌ File not found: {path}")
+                return None
             else:
                 print(f"   ❌ GitHub API error for {path}: {response.status_code}")
                 return None

{data_contract_validator-1.0.4a0 → data_contract_validator-1.0.5/data_contract_validator.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: data-contract-validator
-Version: 1.0.4a0
-Summary: Adding pre-commit-fixes
+Version: 1.0.5
+Summary: Critical bug fixes: DBT extractor return paths, function signatures, error handling, and GitHub API rate limiting
 Author-email: Ogunniran Siji <ogunniransiji@gmail.com>
 Maintainer-email: Ogunniran Siji <ogunniransiji@gmail.com>
 License: MIT

{data_contract_validator-1.0.4a0 → data_contract_validator-1.0.5}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-contract-validator"
-version = "1.0.4a"
-description = "Adding pre-commit-fixes"
+version = "1.0.5"
+description = "Critical bug fixes: DBT extractor return paths, function signatures, error handling, and GitHub API rate limiting"
 readme = "README.md"
 license = {text = "MIT"}
 authors = [

data_contract_validator-1.0.4a0/CHANGELOG.md

@@ -1,38 +0,0 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [Unreleased]
-
-## [1.0.0] - 2025-01-XX
-
-### Added
-- Initial release of Data Contract Validator
-- DBT schema extraction from SQL files and manifest.json
-- FastAPI/Pydantic model extraction from local files and GitHub repos
-- Command-line interface with multiple output formats
-- GitHub Actions integration
-- Contract validation with critical/warning/info severity levels
-- Support for multiple repositories and complex validation scenarios
-
-### Features
-- ✅ DBT model schema extraction
-- ✅ FastAPI/Pydantic schema extraction
-- ✅ Cross-repository validation
-- ✅ GitHub Actions workflows
-- ✅ Multiple output formats (terminal, JSON, GitHub Actions)
-- ✅ Comprehensive error reporting with suggested fixes
-- ✅ Type compatibility checking
-- ✅ Missing table/column detection
-
-### Known Limitations
-- Only supports DBT and FastAPI currently
-- Requires manual installation of DBT CLI
-- Limited type inference from SQL
-- No support for complex nested types
-
-[Unreleased]: https://github.com/OGsiji/retl_validator/compare/v1.0.0...HEAD
-[1.0.0]: https://github.com/OGsiji/retl_validator/releases/tag/v1.0.0