datasluice 0.2.0__tar.gz

datasluice-0.2.0/.editorconfig

@@ -0,0 +1,37 @@
+# https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.{html,css,js,json,sh,yml,yaml}]
+indent_size = 2
+
+[*.bat]
+indent_style = tab
+end_of_line = crlf
+
+[LICENSE]
+insert_final_newline = false
+
+[justfile]
+indent_style = space
+indent_size = 4
+
+# Ignore binary or generated files
+[*.{png,jpg,gif,ico,woff,woff2,ttf,eot,svg,pdf}]
+charset = unset
+end_of_line = unset
+indent_style = unset
+indent_size = unset
+trim_trailing_whitespace = unset
+insert_final_newline = unset
+max_line_length = unset
+
+[*.{diff,patch}]
+trim_trailing_whitespace = false

datasluice-0.2.0/.env.example

@@ -0,0 +1,21 @@
+# DataSluice environment variables
+# Copy this file to .env and adjust as needed.
+
+# HTTP client
+DATASLUICE_HTTP_TIMEOUT=30
+DATASLUICE_HTTP_RETRIES=3
+DATASLUICE_HTTP_RATE_LIMIT=10
+
+# Authentication (optional, for portals requiring API keys)
+DATASLUICE_API_KEY=
+DATASLUICE_BEARER_TOKEN=
+
+# Cache
+DATASLUICE_CACHE_DIR=.datasluice/cache
+DATASLUICE_CACHE_TTL=3600
+
+# Logging
+DATASLUICE_LOG_LEVEL=INFO
+
+# User agent
+DATASLUICE_USER_AGENT=datasluice/0.1.0

datasluice-0.2.0/.gitattributes

@@ -0,0 +1 @@
+graphify-out/graph.json merge=graphify

datasluice-0.2.0/.github/FUNDING.yml

@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: nitishraj
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

datasluice-0.2.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,61 @@
+name: Bug Report
+description: Something isn't working as expected.
+title: "BUG: "
+labels: [bug]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: What happened? What did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce
+      description: Minimal steps to trigger the bug.
+      placeholder: |
+        1. Install with `uv pip install datasluice`
+        2. Run `...`
+        3. See error
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: "DataSluice version"
+      placeholder: "0.1.0"
+    validations:
+      required: true
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      placeholder: "3.13"
+    validations:
+      required: true
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating system
+      options:
+        - macOS
+        - Linux
+        - Windows
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+      description: Logs, tracebacks, screenshots, or anything else that helps.
+    validations:
+      required: false

datasluice-0.2.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: true

datasluice-0.2.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,30 @@
+name: Feature Request
+description: Suggest something new.
+title: "FEAT: "
+labels: [enhancement]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the feature
+    validations:
+      required: true
+  - type: textarea
+    id: motivation
+    attributes:
+      label: Motivation
+      description: What problem does this solve? What's the use case?
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+    validations:
+      required: false
+  - type: textarea
+    id: context
+    attributes:
+      label: Additional context
+    validations:
+      required: false

datasluice-0.2.0/.github/codeql/codeql-config.yml

@@ -0,0 +1,5 @@
+name: "CodeQL config"
+
+paths-ignore:
+  - tests
+  - providers/apache-airflow/tests

datasluice-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,304 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint & format
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Ruff format check
+        run: uv run ruff format --check .
+
+      - name: Ruff lint
+        run: uv run ruff check .
+
+  type-check:
+    name: Type check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Type check with ty
+        run: uv run --all-extras ty check .
+
+  provider-matrix:
+    name: Resolve provider compatibility matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.emit.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Emit matrix from compatibility manifest
+        id: emit
+        run: |
+          python - <<'PY'
+          import json
+          import itertools
+          with open("providers/apache-airflow/compatibility.json", encoding="utf-8") as fh:
+              manifest = json.load(fh)
+          include = [
+              {"core_version": core, "airflow": airflow, "python": minor}
+              for core, airflow, minor in itertools.product(
+                  manifest["core"], manifest["airflow"], manifest["python"]
+              )
+          ]
+          with open("/tmp/provider-matrix.json", "w", encoding="utf-8") as fh:
+              json.dump({"include": include}, fh)
+          print(json.dumps({"include": include}))
+          PY
+          echo "matrix=$(cat /tmp/provider-matrix.json)" >> "$GITHUB_OUTPUT"
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Run core tests
+        run: uv run --all-extras --python=${{ matrix.python-version }} coverage run --source=src/datasluice --branch -m pytest tests
+
+      - name: Upload core coverage data
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: core-coverage-${{ matrix.python-version }}
+          path: .coverage.core.*
+          include-hidden-files: true
+          if-no-files-found: error
+
+  coverage-core:
+    name: Core branch coverage (>= 80%)
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Download core coverage data
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: core-coverage-*
+          merge-multiple: true
+          path: .coverage/
+          include-hidden-files: true
+
+      - name: Combine core coverage
+        run: uv run coverage combine --data-file=.coverage.core .coverage/
+        env:
+          COVERAGE_FILE: .coverage.core
+
+      - name: Core coverage report
+        run: |
+          echo '## Core Branch Coverage Report' >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          uv run coverage report --data-file=.coverage.core | tee -a $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+  provider-test:
+    name: Provider ${{ matrix.airflow }} / py${{ matrix.python }}
+    needs: provider-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.provider-matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Run provider tests (local candidate wheels)
+        run: |
+          uv run --python=${{ matrix.python }} python providers/apache-airflow/tests/run_candidate.py \
+            --core-version ${{ matrix.core_version }} \
+            --airflow-version ${{ matrix.airflow }} \
+            -- pytest providers/apache-airflow/tests -x
+
+  coverage-provider:
+    name: Provider branch coverage (>= 80%)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Run provider suite under coverage
+        run: |
+          uv run python providers/apache-airflow/tests/run_candidate.py \
+            --core-version 1.0.0 \
+            --airflow-version 3.2.2 \
+            -- --cover-provider providers/apache-airflow/tests -q
+
+  build:
+    name: Build & validate package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: false
+
+      - name: Build distribution
+        run: uv build
+
+      - name: Validate distribution
+        run: uvx twine check dist/*
+
+      - name: Upload distribution
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: dist
+          path: dist/
+
+  installed-wheel-e2e:
+    name: Installed-wheel CLI E2E (py${{ matrix.python-version }})
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.14"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Installed-wheel bare CLI E2E
+        run: uv run --python=${{ matrix.python-version }} pytest tests/e2e/test_cli_bare.py -x
+
+      - name: Installed-wheel all-extras CLI E2E
+        run: uv run --python=${{ matrix.python-version }} pytest tests/e2e/test_cli_existing.py -x
+        env:
+          DATASLUICE_E2E_EXTRAS: all
+
+  provider-e2e:
+    name: Provider DAG import E2E (py${{ matrix.python-version }})
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.14"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Import sample DAG from installed wheels
+        run: |
+          uv run --python=${{ matrix.python-version }} python providers/apache-airflow/tests/run_candidate.py \
+            --core-version 1.0.0 \
+            --airflow-version 3.2.2 \
+            -- python providers/apache-airflow/tests/dags/example_datasluice.py
+
+  smoke-test:
+    name: Smoke test (install wheel)
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download distribution
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: dist
+          path: dist/
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: false
+
+      - name: Install built wheel & smoke import
+        run: |
+          uv venv --python 3.12 /tmp/smoke
+          uv pip install --python /tmp/smoke dist/*.whl
+          /tmp/smoke/bin/python -c "import datasluice; print(datasluice.__version__)"
+
+  all-checks-pass:
+    name: All checks pass
+    if: always()
+    needs:
+      - lint
+      - type-check
+      - test
+      - coverage-core
+      - provider-test
+      - coverage-provider
+      - build
+      - installed-wheel-e2e
+      - provider-e2e
+      - smoke-test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check job results
+        uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
+        with:
+          jobs: ${{ toJSON(needs) }}

datasluice-0.2.0/.github/workflows/codeql.yml

@@ -0,0 +1,44 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+  schedule:
+    - cron: '0 6 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ${{ 'ubuntu-latest' }}
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: python
+          build-mode: none
+          queries: security-extended
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7.0.0
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:${{matrix.language}}"

datasluice-0.2.0/.github/workflows/docs.yml

@@ -0,0 +1,72 @@
+name: Documentation
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+permissions: {}
+
+concurrency:
+  group: docs-deploy
+  # Let in-flight deployments finish to avoid a half-deployed site.
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+
+      - name: Guard current docs against obsolete Airflow references
+        run: |
+          set -e
+          if grep -rEn -e 'datasluice\[airflow\]' -e 'datasluice\.integrations\.airflow' -e 'DataSluiceOperator' -e 'DataSluiceSession' \
+              README.md docs --include '*.md' --exclude-dir='.planning' --exclude='CHANGELOG.md'; then
+            echo "Obsolete Airflow/core reference found in current docs" >&2
+            exit 1
+          fi
+          grep -q 'apache-airflow-providers-datasluice' docs/install.md
+
+      - name: Run executable public documentation examples
+        run: uv run --group docs --extra pandas --extra streaming --extra storage --extra http pytest tests/e2e/test_docs_examples.py tests/e2e/test_cli_bare.py -x
+
+      - name: Import provider sample DAG from installed wheels
+        run: |
+          uv run python providers/apache-airflow/tests/run_candidate.py \
+            --core-version 1.0.0 \
+            --airflow-version 3.2.2 \
+            -- python providers/apache-airflow/tests/dags/example_datasluice.py
+
+      - name: Build docs (warnings fatal)
+        env:
+          ZENSICAL_WARNINGS_AS_ERRORS: "1"
+        run: uv run --group docs zensical build --clean
+
+      - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
+
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
+        with:
+          path: site
+
+  deploy:
+    name: Deploy
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0
+        id: deployment

datasluice-0.2.0/.github/workflows/ocr-review.yml

@@ -0,0 +1,59 @@
+name: OpenCodeReview PR Review
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  code-review:
+    name: AI code review
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    if: github.event_name == 'pull_request_target' && github.actor != 'renovate[bot]' && github.actor != 'dependabot[bot]'
+    steps:
+      - name: Run OpenCodeReview
+        uses: alibaba/open-code-review@4bcc95acecd629e64ea984c962cfd08dd892a38e # v1.8.10
+        with:
+          llm_url: ${{ secrets.OCR_LLM_URL }}
+          llm_auth_token: ${{ secrets.OCR_LLM_AUTH_TOKEN }}
+          llm_model: ${{ secrets.OCR_LLM_MODEL }}
+          llm_use_anthropic: ${{ secrets.OCR_LLM_USE_ANTHROPIC }}
+
+          llm_extra_body: '{"reasoning_effort": "max"}'
+
+          review_concurrency: '4'
+          language: 'English'
+
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          sticky_summary: 'true'
+          incremental: 'true'
+          incremental_overlap_threshold: '0.8'
+          upload_artifacts: 'true'
+
+          background: >-
+            DataSluice is a Python 3.12+ open-data SDK and CLI for discovery, extraction,
+            format normalization, and pipeline integration against CKAN, data.gouv.fr,
+            Socrata, and custom portals. Preserve public API compatibility and consistent
+            behavior across all portal connectors. Connectors are discovered via the
+            datasluice.connectors entry-points group by the injected runtime
+            PluginManager (no module-level registry, no import side effects; broken
+            third-party plugins are recorded as failures, never fatal). Optional format,
+            storage, and integration dependencies (httpx, fsspec, pyarrow, pandas,
+            polars, openpyxl, zstandard, dlt, duckdb, airflow) must remain lazily
+            imported so the base package works without extras. Prioritize HTTP
+            correctness, pagination, retries, rate limiting, redirect handling, resource
+            and file-descriptor cleanup (streams, readers, clients), credential and
+            secret safety, portal mapping correctness, cache/checksum integrity, sync
+            state correctness, typing, and tests for changed behavior. Avoid style-only
+            findings already covered by Ruff and ty.

datasluice-0.2.0/.github/workflows/pr-agent.yml

@@ -0,0 +1,30 @@
+name: PR Agent
+
+concurrency:
+  group: pr-agent-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize, ready_for_review]
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  pr_description:
+    name: Auto-generate PR title & description (PR-Agent /describe)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    if: ${{ github.actor != 'renovate[bot]' && github.actor != 'dependabot[bot]' }}
+    steps:
+      - name: PR Agent /describe
+        uses: the-pr-agent/pr-agent@743f52d80f3513193dfc7626f778c73478df6d0a # v0.41.1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          config.model: ${{ secrets.OCR_LLM_MODEL }}
+          OPENAI_KEY: ${{ secrets.OCR_LLM_AUTH_TOKEN }}
+          OPENAI.API_BASE: ${{ secrets.OCR_LLM_URL }}
+          config.fallback_models: '${{ vars.PR_AGENT_FALLBACK_MODEL }}'