tablesleuth 0.4.2__tar.gz

tablesleuth-0.4.2/.gitattributes

@@ -0,0 +1,24 @@
+# Auto detect text files and normalize line endings to LF in the repository
+* text=auto eol=lf
+
+# Explicitly declare text files you want to always be normalized and converted
+# to native line endings on checkout
+*.py text eol=lf
+*.md text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf
+*.toml text eol=lf
+*.json text eol=lf
+*.txt text eol=lf
+*.sh text eol=lf
+Makefile text eol=lf
+
+# Denote all files that are truly binary and should not be modified
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+*.whl binary
+*.gz binary
+*.zip binary

tablesleuth-0.4.2/.github/workflows/ci.yml

@@ -0,0 +1,80 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.13', '3.14']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      run: pip install uv
+
+    - name: Install dependencies
+      run: uv sync --extra dev
+
+    - name: Run linter
+      run: uv run ruff check .
+
+    - name: Run formatter check
+      run: uv run ruff format --check .
+
+    - name: Run type checker
+      run: uv run mypy src
+
+    - name: Run security scan
+      run: uv run bandit -c pyproject.toml -r src/
+
+    - name: Run tests
+      run: uv run pytest --cov=tablesleuth --cov-report=xml --cov-report=term
+
+    - name: Upload coverage to Codecov
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
+      uses: codecov/codecov-action@v4
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      run: pip install uv
+
+    - name: Build package
+      run: uv build
+
+    - name: Check package
+      run: |
+        pip install twine
+        twine check dist/*
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: dist
+        path: dist/

tablesleuth-0.4.2/.github/workflows/publish.yml

@@ -0,0 +1,43 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/TableSleuth
+    permissions:
+      id-token: write  # Required for trusted publishing
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      run: pip install uv
+
+    - name: Build package
+      run: uv build
+
+    - name: Verify version matches tag
+      run: |
+        TAG_VERSION=${GITHUB_REF#refs/tags/v}
+        PKG_VERSION=$(grep '^version = ' pyproject.toml | cut -d'"' -f2)
+        if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
+          echo "Tag version ($TAG_VERSION) does not match package version ($PKG_VERSION)"
+          exit 1
+        fi
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

tablesleuth-0.4.2/.gitignore

@@ -0,0 +1,88 @@
+# Environment variables
+.env
+.env.local
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+env/
+ENV/
+.venv
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Logs
+*.log
+logs/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+coverage.xml
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Ruff
+.ruff_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+resources/config.json
+*.pem
+input/
+.kiro/
+.chroma/
+.ruff_cache/
+.mypy_cache/
+*.zip
+data/
+scripts/
+docs/archive/
+resources/debug/
+resources/docs/
+debug/
+.pyiceberg.yaml
+
+# Gradio
+gradio_cached_examples/
+flagged/
+
+# Docker
+docker-compose.override.yml
+*.tar.gz
+backups/
+ssl/

tablesleuth-0.4.2/.pre-commit-config.yaml

@@ -0,0 +1,58 @@
+# Pre-commit hooks for table-sleuth
+# Install: uv run pre-commit install
+# Run manually: uv run pre-commit run --all-files
+
+repos:
+  # Ruff - replaces black, isort, flake8
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.4
+    hooks:
+      # Run the linter
+      - id: ruff
+        args: [--fix]
+      # Run the formatter
+      - id: ruff-format
+
+  # mypy - type checking
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.13.0
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - types-toml
+          - types-pyyaml
+          - pydantic>=2.0
+          - textual>=0.47.0
+          - rich>=13.0.0
+          - pyarrow>=14.0.0
+          - pyiceberg>=0.6.0
+          - adbc-driver-flightsql>=0.8.0
+          - boto3>=1.34.0
+          - botocore>=1.34.0
+          - fsspec>=2023.0.0
+          - s3fs>=2023.0.0
+        args: [--config-file=pyproject.toml, src/]
+        pass_filenames: false
+
+  # Standard pre-commit hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+        args: [--maxkb=1000]
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: mixed-line-ending
+        args: [--fix=lf]
+
+  # Security scanning with bandit
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.10
+    hooks:
+      - id: bandit
+        args: [-c, pyproject.toml, -r, src/]
+        additional_dependencies: ["bandit[toml]"]

tablesleuth-0.4.2/CHANGELOG.md

@@ -0,0 +1,179 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.4.2] - 2026-01-17
+
+### Added
+- **Configuration Management Commands**
+  - `tablesleuth init` - Interactive configuration file initialization
+    - Creates `tablesleuth.toml` and `.pyiceberg.yaml` with comprehensive templates
+    - Prompts for home directory (~/) or current directory (./) placement
+    - Includes `--force` flag to overwrite existing files
+    - Generates well-commented templates with multiple catalog examples
+  - `tablesleuth config-check` - Configuration validation and testing
+    - Validates all configuration files and syntax
+    - Tests GizmoSQL connection
+    - Checks PyIceberg catalog configuration
+    - Shows configuration precedence and active values
+    - Supports `-v/--verbose` flag for detailed output
+
+### Changed
+- **Configuration File Locations** - Simplified configuration paths
+  - Removed `~/.config/tablesleuth/` directory approach
+  - Now supports: `./tablesleuth.toml` (local) and `~/tablesleuth.toml` (home)
+  - PyIceberg config: `./.pyiceberg.yaml` (local) and `~/.pyiceberg.yaml` (home)
+  - Respects `PYICEBERG_HOME` environment variable for PyIceberg config location
+
+- **Configuration Priority** - Clear precedence order
+  1. Environment variables (`TABLESLEUTH_*`, `PYICEBERG_*`)
+  2. Local config files (current directory)
+  3. Home config files (home directory)
+  4. Built-in defaults
+
+- **Environment Variable Support**
+  - `TABLESLEUTH_CONFIG` - Override config file path
+  - Existing: `TABLESLEUTH_CATALOG_NAME`, `TABLESLEUTH_GIZMO_*`
+  - PyIceberg native: `PYICEBERG_HOME`
+
+- **Configuration File Renamed** - Consistency with package name
+  - `table_sleuth.toml` → `tablesleuth.toml`
+  - Updated all documentation and code references
+
+### Fixed
+- **Configuration Error Handling** - Improved error messages and handling
+  - Fixed unhandled `FileNotFoundError` in `inspect` and `iceberg` commands when `TABLESLEUTH_CONFIG` points to non-existent file
+  - Fixed unhandled exception in `config-check` command with invalid `TABLESLEUTH_CONFIG` environment variable
+  - Both commands now show helpful error messages suggesting `tablesleuth init` instead of tracebacks
+  - Added proper try-except blocks around `load_config()` calls in main CLI commands
+
+- **Configuration Template TOML Syntax** - Fixed invalid TOML in generated config
+  - Changed `default = null` to commented `# default = ""` (TOML doesn't support null type)
+  - Generated config files now parse correctly without `TOMLDecodeError`
+  - Affects `tablesleuth init` command output
+
+- **S3 Tables Catalog Configuration** - Fixed incorrect catalog type and improved flexibility
+  - Changed S3 Tables catalog from `type: glue` to `type: rest` with proper REST API settings
+  - Added required REST API configuration: `uri`, `rest.sigv4-enabled`, `rest.signing-name`, `rest.signing-region`
+  - Fixed hardcoded catalog name - now supports multiple S3 Tables catalogs
+  - Users can specify which S3 Tables catalog to use with `--catalog` flag when using ARNs
+  - Default catalog name "s3tables" is used when ARN is provided without `--catalog` flag
+  - Added clear documentation and usage examples in template showing multiple S3 Tables catalogs
+  - Clarified difference between Glue catalog and S3 Tables catalog
+
+- **GizmoSQL Optional Component Handling** - Made GizmoSQL truly optional
+  - `config-check` command no longer fails when GizmoSQL connection fails
+  - Added `--with-gizmosql` flag to explicitly test GizmoSQL connection
+  - GizmoSQL test is now skipped by default (shown as "⊘ Skipped")
+  - Exit code 0 (success) when only optional components fail
+  - Consistent with other optional checks like missing PyIceberg config
+
+### Dependencies
+- Added `pyyaml>=6.0.0` for PyIceberg config validation
+
+## [0.4.1] - 2026-01-17
+
+### Changed
+- **Python Module Renamed to `tablesleuth`** - Complete consistency across package
+  - Module directory renamed from `table_sleuth` to `tablesleuth`
+  - All imports now use `from tablesleuth import ...`
+  - Eliminates confusion between package name and import name
+  - **Breaking Change:** Update all imports from `table_sleuth` to `tablesleuth`
+
+### Migration
+If upgrading from v0.4.0 (unreleased), update your imports:
+```python
+# Old
+from table_sleuth import __version__
+from table_sleuth.services import ParquetInspector
+
+# New
+from tablesleuth import __version__
+from tablesleuth.services import ParquetInspector
+```
+
+## [0.4.0] - 2026-01-16 (Unreleased)
+
+### Changed
+- **Package Renamed to `tablesleuth`** - Unified package name for PyPI distribution
+  - CLI command changed from `table-sleuth` to `tablesleuth`
+  - Package name now matches tablesleuth.com domain
+  - Improved discoverability on PyPI
+- **Version Management** - Consolidated version to single source of truth in `__init__.py`
+  - Removed hardcoded version from CLI
+  - Version now imported from package
+- **Enhanced PyPI Metadata**
+  - Upgraded development status from Alpha to Beta
+  - Added comprehensive classifiers for better discoverability
+  - Added project URLs including homepage, documentation, and changelog
+  - Added publishing tools (twine, build) to dev dependencies
+
+### Added
+- **GitHub Actions CI/CD** - Automated testing and publishing workflows
+  - Multi-platform testing (Ubuntu, macOS, Windows)
+  - Multi-version Python testing (3.13, 3.14)
+  - Automated quality checks (ruff, mypy, bandit)
+  - Automated PyPI publishing on release
+  - Support for PyPI Trusted Publishing
+- **PyPI Publishing Guide** - Comprehensive documentation for package publishing
+  - Step-by-step publishing instructions
+  - TestPyPI testing workflow
+  - Automated release process documentation
+  - Troubleshooting guide
+
+## [0.3.0] - 2025-11-29
+
+### Added
+- **Strict MyPy Type Checking** - Comprehensive type annotations across the codebase
+  - Enabled strict mypy configuration with `disallow_untyped_defs`, `disallow_incomplete_defs`, and `warn_return_any`
+  - Added proper type annotations to all service classes and methods
+  - Configured per-module overrides for third-party libraries without type stubs
+  - Integrated mypy into pre-commit hooks with all required dependencies
+  - Zero type errors in production code (only expected import-untyped warnings for PyArrow)
+
+- **Enhanced Documentation**
+  - Streamlined README.md with high-level feature overview and screenshot galleries
+  - Organized documentation with clear navigation to detailed guides
+  - Added visual comparison tables for Parquet and Iceberg interfaces
+  - Improved quick start examples and configuration guidance
+
+- **UI Improvements**
+  - Removed subtitle from TUI header for cleaner interface
+  - Updated application title to "Table Sleuth - Parquet Analysis"
+
+### Changed
+- **Code Quality Improvements**
+  - Fixed import paths for IcebergAdapter (moved to `formats.iceberg`)
+  - Removed unreachable backwards compatibility code in gizmo_duckdb.py
+  - Added explicit type casts where needed for type safety
+  - Improved error handling with proper type annotations
+
+- **Pre-commit Configuration**
+  - Added all required dependencies to mypy pre-commit hook
+  - Configured proper module overrides for untyped libraries (pyarrow, fsspec, s3fs, etc.)
+  - All pre-commit hooks now pass cleanly
+
+### Fixed
+- Type annotation issues in FileDiscoveryService, ParquetInspector, and GizmoDuckDbProfiler
+- Missing return type annotations across multiple service classes
+- Unused type ignore comments after fixing import paths
+- Event handler type annotations in TUI views
+
+## [Unreleased]
+
+### Added
+
+#### Performance Profiling for Merge-on-Read
+- **Added performance profiling models** (`QueryPerformanceProfile`, `MergeOnReadPerformance`)
+  - Measures query execution time with and without delete file application
+  - Calculates merge-on-read overhead in milliseconds and percentage
+  - Tracks rows scanned, rows returned, and rows deleted
+  - Provides timing breakdown for data file scan, delete file scan, and merge operations
+- **Extended ProfilingBackend protocol** with `profile_query_performance()` method
+  - Allows backends to implement performance profiling
+  - Optional method - backends can raise `NotImplementedError` if not supported
+- **Comprehensive test suite** for performance profiling models
+  - Tests overhead calculation, edge cases, and zero-division handling
+- **Updated product specification** with performance profiling user story
+  - Story 6: Performance profiling for merge-on-read queries
+  - Helps engineers make data-driven decisions about table compaction