cognite-data-quality 0.1.0__tar.gz

cognite_data_quality-0.1.0/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @cognitedata/partner-enablement @cognitedata/atlas-ai-team @cognitedata/ai-team @cognitedata/velocity

cognite_data_quality-0.1.0/.github/workflows/build.yml

@@ -0,0 +1,26 @@
+name: Build and Test
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v4
+      - name: Install dependencies
+        run: uv sync --all-extras
+      - name: Lint
+        run: uv run pre-commit run --all-files
+      - name: Tests
+        run: uv run pytest tests/ -v

cognite_data_quality-0.1.0/.github/workflows/deploy.yml

@@ -0,0 +1,343 @@
+name: Deploy Data Quality Validation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'function/**'
+      - 'generate_demo_data/**'
+      - 'config/**'
+      - 'scripts/**'
+      - 'cognite_data_quality/**'
+      - '.github/workflows/deploy.yml'
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'function/**'
+      - 'generate_demo_data/**'
+      - 'config/**'
+      - 'scripts/**'
+      - 'cognite_data_quality/**'
+  
+  workflow_dispatch:
+    inputs:
+      force_function:
+        description: 'Force function redeployment'
+        required: false
+        default: 'false'
+        type: boolean
+      force_views:
+        description: 'Force workflow/trigger redeployment'
+        required: false
+        default: 'false'
+        type: boolean
+      dry_run:
+        description: 'Dry run (show what would be deployed)'
+        required: false
+        default: 'false'
+        type: boolean
+      views:
+        description: 'Specific views to deploy (comma-separated, leave empty for all)'
+        required: false
+        default: ''
+        type: string
+      timeseries_configs:
+        description: 'Specific timeseries configs to deploy (comma-separated, leave empty for all)'
+        required: false
+        default: ''
+        type: string
+      skip_timeseries:
+        description: 'Skip time series validation deployment'
+        required: false
+        default: 'false'
+        type: boolean
+      skip_demo_data:
+        description: 'Skip demo data generator deployment'
+        required: false
+        default: 'false'
+        type: boolean
+      environment:
+        description: 'Deployment environment (controls GitHub env + CONFIG_ENV)'
+        required: false
+        default: 'cog-ai-arn'
+        type: choice
+        options:
+          - cog-ai-arn
+          - abp-dev
+
+env:
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  validate:
+    name: Validate Configuration
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - environment: cog-ai-arn
+            config_env: "cog-ai"
+            enabled: false
+          - environment: abp-dev
+            config_env: "abp-dev"
+            enabled: true
+    env:
+      CONFIG_ENV: ${{ matrix.config_env }}
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        if: matrix.enabled == true
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      
+      - name: Install dependencies
+        if: matrix.enabled == true
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyyaml rdflib cognite-sdk
+      
+      - name: Validate configuration
+        if: matrix.enabled == true
+        run: python scripts/validate_config.py
+
+  deploy:
+    name: Deploy to CDF
+    runs-on: ubuntu-latest
+    needs: validate
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - environment: cog-ai-arn
+            config_env: "cog-ai"
+            enabled: false
+          - environment: abp-dev
+            config_env: "abp-dev"
+            enabled: true
+    environment: ${{ matrix.environment }}
+    
+    # Only deploy on push to main, or on manual trigger
+    # PRs will only run validation
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    
+    env:
+      COGNITE_CLUSTER: ${{ vars.COGNITE_CLUSTER }}
+      COGNITE_PROJECT: ${{ vars.COGNITE_PROJECT }}
+      COGNITE_CLIENT_ID: ${{ secrets.COGNITE_CLIENT_ID }}
+      COGNITE_CLIENT_SECRET: ${{ secrets.COGNITE_CLIENT_SECRET }}
+      COGNITE_BASE_URL: ${{ vars.COGNITE_BASE_URL }}
+      AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
+      CONFIG_ENV: ${{ matrix.config_env }}
+    
+    steps:
+      - name: Skip disabled environment
+        if: matrix.enabled != true
+        run: |
+          echo "Environment '${{ matrix.environment }}' is disabled in matrix (no deployment)."
+
+      - name: Checkout repository
+        if: matrix.enabled == true
+        uses: actions/checkout@v4
+      
+      - name: Set up Python
+        if: matrix.enabled == true
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      
+      - name: Install dependencies
+        if: matrix.enabled == true
+        run: |
+          python -m pip install --upgrade pip
+          pip install cognite-sdk pyyaml
+          pip install -e .
+      
+      - name: Determine deployment flags
+        if: matrix.enabled == true
+        id: flags
+        run: |
+          # Dry run flag
+          if [[ "${{ github.event.inputs.dry_run }}" == "true" ]]; then
+            echo "dry_run=--dry-run" >> $GITHUB_OUTPUT
+            echo "Running in DRY RUN mode"
+          else
+            echo "dry_run=" >> $GITHUB_OUTPUT
+          fi
+          
+          # Force function flag
+          if [[ "${{ github.event.inputs.force_function }}" == "true" ]]; then
+            echo "force_function=--force" >> $GITHUB_OUTPUT
+          else
+            echo "force_function=" >> $GITHUB_OUTPUT
+          fi
+          
+          # Force views flag
+          if [[ "${{ github.event.inputs.force_views }}" == "true" ]]; then
+            echo "force_views=--force" >> $GITHUB_OUTPUT
+          else
+            echo "force_views=" >> $GITHUB_OUTPUT
+          fi
+          
+          # Views filter
+          if [[ -n "${{ github.event.inputs.views }}" ]]; then
+            views=$(echo "${{ github.event.inputs.views }}" | tr ',' ' ')
+            echo "views=--views $views" >> $GITHUB_OUTPUT
+          else
+            echo "views=" >> $GITHUB_OUTPUT
+          fi
+          
+          # Time series configs filter
+          if [[ -n "${{ github.event.inputs.timeseries_configs }}" ]]; then
+            configs=$(echo "${{ github.event.inputs.timeseries_configs }}" | tr ',' ' ')
+            echo "timeseries_configs=--configs $configs" >> $GITHUB_OUTPUT
+          else
+            echo "timeseries_configs=" >> $GITHUB_OUTPUT
+          fi
+          
+          # Skip time series flag
+          if [[ "${{ github.event.inputs.skip_timeseries }}" == "true" ]]; then
+            echo "skip_timeseries=true" >> $GITHUB_OUTPUT
+          else
+            echo "skip_timeseries=false" >> $GITHUB_OUTPUT
+          fi
+          
+          # Skip demo data flag
+          if [[ "${{ github.event.inputs.skip_demo_data }}" == "true" ]]; then
+            echo "skip_demo_data=true" >> $GITHUB_OUTPUT
+          else
+            echo "skip_demo_data=false" >> $GITHUB_OUTPUT
+          fi
+
+          echo "CONFIG_ENV from matrix: $CONFIG_ENV"
+      
+      # Deploy Complete Validation Infrastructure (unified)
+      - name: Deploy Validation Infrastructure
+        if: matrix.enabled == true
+        id: infrastructure
+        working-directory: scripts
+        run: |
+          echo "::group::Deploy Validation Infrastructure"
+          python deploy_infrastructure.py ${{ steps.flags.outputs.dry_run }} ${{ steps.flags.outputs.force_function }} --output ../infrastructure_result.json
+          echo "::endgroup::"
+      
+      # Deploy Demo Data Generator (if enabled in settings.yaml)
+      - name: Deploy Demo Data Generator
+        id: demo_data
+        if: matrix.enabled == true && steps.flags.outputs.skip_demo_data != 'true'
+        working-directory: scripts
+        run: |
+          echo "::group::Deploy Demo Data Generator"
+          python deploy_demo_data.py ${{ steps.flags.outputs.dry_run }} ${{ steps.flags.outputs.force_function }} --output ../demo_data_result.json
+          echo "::endgroup::"
+      
+      - name: Generate Summary
+        if: matrix.enabled == true
+        run: |
+          python3 << 'EOF'
+          import json
+          import os
+
+          summary = []
+          summary.append("## Deployment Summary\n")
+
+          # Check if dry run
+          dry_run = "${{ steps.flags.outputs.dry_run }}" == "--dry-run"
+          if dry_run:
+              summary.append("**Mode:** Dry Run (no changes made)\n")
+          else:
+              summary.append("**Mode:** Production Deployment\n")
+
+          summary.append("")
+
+          # Read unified infrastructure result
+          try:
+              with open("infrastructure_result.json") as f:
+                  result = json.load(f)
+
+              # Function
+              summary.append("### 1. Validation Function\n")
+              func_result = result.get("functions", {}).get("function", {})
+              func_status = func_result.get("status", "unknown")
+              func_id = func_result.get("function", "N/A")
+              if func_status == "deployed":
+                  summary.append(f"- **{func_id}**: ✅ Deployed\n")
+              elif func_status == "skipped":
+                  summary.append(f"- **{func_id}**: ⏭️ Skipped (up to date)\n")
+              else:
+                  summary.append(f"- **{func_id}**: {func_status}\n")
+
+              summary.append("")
+
+              # Instance Validation Workflows
+              summary.append("### 2. Instance Validation Workflows\n")
+              workflows = result.get("workflows", [])
+              if workflows:
+                  deployed = sum(1 for w in workflows if w.get("status") == "deployed")
+                  skipped = sum(1 for w in workflows if w.get("status") == "skipped")
+
+                  summary.append(f"**Total:** {len(workflows)} | **Deployed:** {deployed} | **Skipped:** {skipped}\n")
+                  summary.append("")
+                  summary.append("| View | Workflow | Status |")
+                  summary.append("|------|----------|--------|")
+
+                  for w in workflows:
+                      view = w.get("view", "N/A")
+                      wf_id = w.get("workflow_external_id", "N/A")
+                      status = w.get("status", "unknown")
+                      emoji = "✅" if status == "deployed" else "⏭️"
+                      summary.append(f"| {view} | `{wf_id}` | {emoji} {status} |")
+
+                  summary.append("")
+              else:
+                  summary.append("No instance validation workflows configured.\n")
+
+              # Time Series Validation Workflows
+              summary.append("### 3. Time Series Validation Workflows\n")
+              ts_workflows = result.get("timeseries_workflows", [])
+              if ts_workflows:
+                  deployed = sum(1 for w in ts_workflows if w.get("status") == "deployed")
+                  skipped = sum(1 for w in ts_workflows if w.get("status") == "skipped")
+
+                  summary.append(f"**Total:** {len(ts_workflows)} | **Deployed:** {deployed} | **Skipped:** {skipped}\n")
+                  summary.append("")
+                  summary.append("| Config | Workflow | Trigger | Status |")
+                  summary.append("|--------|----------|---------|--------|")
+
+                  for w in ts_workflows:
+                      name = w.get("name", "N/A")
+                      wf_id = w.get("workflow_external_id", "N/A")
+                      tr_id = w.get("trigger_external_id", "N/A")
+                      status = w.get("status", "unknown")
+                      emoji = "✅" if status == "deployed" else "⏭️"
+                      summary.append(f"| {name} | `{wf_id}` | `{tr_id}` | {emoji} {status} |")
+
+                  summary.append("")
+              else:
+                  summary.append("No time series validation workflows configured.\n")
+
+          except Exception as e:
+              summary.append(f"\n⚠️ Error reading deployment results: {e}\n")
+
+          summary.append("")
+          summary.append("### Configuration\n")
+          summary.append(f"- **Trigger:** ${{ github.event_name }}")
+          summary.append(f"- **Branch:** ${{ github.ref_name }}")
+          summary.append(f"- **Environment:** ${{ matrix.environment }}")
+          summary.append(f"- **CONFIG_ENV:** ${{ env.CONFIG_ENV || 'default' }}")
+          summary.append(f"- **Commit:** `${{ github.sha }}`")
+
+          # Write to summary
+          with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+              f.write("\n".join(summary))
+
+          print("Summary generated successfully")
+          EOF
+

cognite_data_quality-0.1.0/.github/workflows/release.yaml

@@ -0,0 +1,73 @@
+---
+name: release
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install required dependencies
+        run: |
+          uv sync --all-extras
+      - name: Linting and static code checks
+        run: uv run pre-commit run --all-files
+
+  release:
+    runs-on: ubuntu-latest
+    environment: CD
+    needs: lint
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install uv and set the python version
+        uses: astral-sh/setup-uv@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install required dependencies
+        run: |
+          uv sync --all-extras
+      - name: Dump last commit message to file
+        run: git log -1 --pretty=%B > last_git_message.txt
+      - name: Dump last tag to file
+        run: git describe --tags --abbrev=0 > last_version.txt
+      - name: Bump Version
+        run: uv run python dev.py bump --verbose
+      - id: version
+        name: Read the new version
+        run: echo "version=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' cognite_data_quality/_version.py)" >> $GITHUB_ENV
+      - name: Create CHANGELOG entry
+        if: env.version != '0.0.0'
+        run: uv run python dev.py changelog
+      - name: Build package
+        if: env.version != '0.0.0'
+        run: uv build
+
+      - name: Release to PyPI
+        if: env.version != '0.0.0'
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: uv run twine upload --skip-existing --verbose dist/*
+
+      - name: Add entry to CHANGELOG
+        if: env.version != '0.0.0'
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          prerelease: false
+          draft: false
+          tag: ${{ env.version }}
+          bodyFile: last_changelog_entry.md

cognite_data_quality-0.1.0/.gitignore

@@ -0,0 +1,104 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+.DS_Store
+
+# Environment files
+.env
+.env.local
+get.env
+get.env-*
+config.toml
+
+# Jupyter
+.ipynb_checkpoints/
+*.ipynb_checkpoints
+
+# Tooling caches
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+htmlcov/
+.coverage
+
+# MkDocs
+site/
+
+# Test data and samples (keep for reference but don't track)
+# Uncomment these if you want to ignore sample data files
+# *_instances.json
+# *_shacl_rules.ttl
+
+# Wheel files - we need whl files for the function 
+# *.whl
+
+function_deploy.zip
+abp-dev-get.env
+config/environments/cog-sail/cog-sail.toml
+config/environments/abp-dev/abp-dev.toml
+test_and_deploy/cog-sail.toml
+
+# Test/verification scripts (not for deployment)
+manual_verification.py
+verify_validation_results.py
+inspect_single_instance.py
+check_property_coverage.py
+scripts/test*.py
+scripts/check_partition_logs.py
+scripts/check_retry_history.py
+scripts/check_state_instances.py
+scripts/cleanup_duplicate_triggers.py
+scripts/create_test_stream.py
+scripts/delete_state_instances.py
+scripts/reset_state_containers.py
+scripts/list_writable_datasets.py
+scripts/monitor_partition_call.py
+scripts/show_all_partition_logs.py
+scripts/show_call_logs.py
+scripts/show_partition_logs.py
+scripts/show_partition_state.py
+scripts/show_partition_state_raw.py
+scripts/verify_sync_properties.py
+
+# Test/exploration notebooks (not for deployment)
+explore_validation_records.ipynb
+test_orchestrator_monitor.ipynb
+.claude/settings.json
+
+rule_input/*
+
+# Documentation DOCX files (generated from markdown)
+*.docx
+
+# Release PR description drafts (never commit - local use only with gh pr create)
+pr_description*.md

cognite_data_quality-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.2
+    hooks:
+      - id: ruff
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.1
+    hooks:
+      - id: mypy
+        args: ["cognite_data_quality", "tests"]
+        additional_dependencies: ["pydantic>=2.0.0", "types-PyYAML", "types-toml"]
+        pass_filenames: false

cognite_data_quality-0.1.0/.readthedocs.yaml

@@ -0,0 +1,26 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+python:
+  install:
+    - requirements: docs/requirements.txt