bnetza-bk6-scraper 0.0.1__tar.gz

bnetza_bk6_scraper-0.0.1/.gitattributes

@@ -0,0 +1 @@
+unittests/fixtures/** -text

bnetza_bk6_scraper-0.0.1/.github/dependabot.yml

@@ -0,0 +1,18 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    reviewers:
+      - "@Hochfrequenz/python-developers-review-team"
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

bnetza_bk6_scraper-0.0.1/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+  schedule:
+    - cron: "29 14 * * 6"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["python"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v4
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4

bnetza_bk6_scraper-0.0.1/.github/workflows/coverage.yml

@@ -0,0 +1,22 @@
+name: "Coverage"
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run Tests and Record Coverage
+        run: |
+          tox -e coverage

bnetza_bk6_scraper-0.0.1/.github/workflows/dependabot_automerge.yml

@@ -0,0 +1,18 @@
+name: Dependabot auto-approve / -merge
+on: pull_request
+
+jobs:
+  dependabot:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    env:
+      PR_URL: ${{github.event.pull_request.html_url}}
+      GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+      - name: Enable auto-merge for Dependabot PRs
+        run: gh pr merge --auto --squash "$PR_URL"

bnetza_bk6_scraper-0.0.1/.github/workflows/dev_test.yml

@@ -0,0 +1,28 @@
+name: "Test Dev Environment"
+# Checks that the dev environment (tox -e dev) can be set up.
+# This might not work, if different linting/testing envs refer to different versions of the same lib (e.g. typing-extensions).
+# Different versions of the same package might work for isolated specific envs (only linting, only testing...) but the dev environment inherits from all of them.
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  check:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Create a Dev Environment
+        run: |
+          tox -e dev

bnetza_bk6_scraper-0.0.1/.github/workflows/formatting.yml

@@ -0,0 +1,25 @@
+name: "Formatting"
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        tool: ["black", "isort"]
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[formatting]
+      - name: ${{ matrix.tool }} Code Formatter
+        run: |
+          ${{ matrix.tool }} . --check

bnetza_bk6_scraper-0.0.1/.github/workflows/no_byte_order_mark.yml

@@ -0,0 +1,15 @@
+name: Prevent ByteOrderMarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request: {}
+
+jobs:
+  bom-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: arma-actions/bom-check@v1
+        name: Check for BOM

bnetza_bk6_scraper-0.0.1/.github/workflows/packaging_test.yml

@@ -0,0 +1,22 @@
+name: "Packaging Test"
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  check_packaging:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run Packaging Test
+        run: |
+          tox -e test_packaging

bnetza_bk6_scraper-0.0.1/.github/workflows/python-publish.yml

@@ -0,0 +1,65 @@
+# This GitHub workflow is only needed for python package releases which are supposed to be published on pypi.
+# It requires the Github "environments" feature (see instructions below) it might not be available for private free accounts (but works for public or organization repos).
+# After creating the "release" environment in the Github repo settings, you need to enter your Github organization/user name + repo name + "python-publish.yml" workflow file name in the PyPI UI to make this work.
+
+# This workflow uploads a Python Package using Twine when a release is created.
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created, edited]
+
+jobs:
+  tests:
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.14"]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run tox
+        run: |
+          tox
+
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+        os: [ ubuntu-latest ]
+    # Specifying a GitHub environment, # Specifying a GitHub environment, which is strongly recommended by PyPI: https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+    # you have to create an environment in your repository settings and add the environment name here
+    environment: release
+    permissions:
+        # IMPORTANT: this permission is mandatory for trusted publishing
+        id-token: write
+    needs: tests
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[packaging]
+      - name: Build wheel and source distributions
+        run: |
+          python -m build
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1

bnetza_bk6_scraper-0.0.1/.github/workflows/pythonlint.yml

@@ -0,0 +1,26 @@
+name: "Linting"
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  pylint:
+    name: Python Code Quality and Lint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        linter-env: ["linting", "type_check", "spell_check"]
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: 3.14
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run ${{ matrix.linter-env }} via Tox
+        run: |
+          tox -e ${{ matrix.linter-env }}

bnetza_bk6_scraper-0.0.1/.github/workflows/unittests.yml

@@ -0,0 +1,26 @@
+name: "Unittests"
+
+on:
+  push:
+    branches: [main]
+  pull_request: {}
+jobs:
+  pytest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run the Unit Tests via Tox
+        run: |
+          tox -e tests

bnetza_bk6_scraper-0.0.1/.gitignore

@@ -0,0 +1,140 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea/
+
+# vscode settings
+.vscode/
+
+src/bnetza_bk6_scraper/_version.py
+
+# local smoke-test output
+_smoke/
+_smoke*/

bnetza_bk6_scraper-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+# to update all repo revisions just run: pre-commit autoupdate
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 23.9.1
+    hooks:
+      - id: black
+        language_version: python3
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+      - id: isort
+        name: isort (cython)
+        types: [cython]
+      - id: isort
+        name: isort (pyi)
+        types: [pyi]

bnetza_bk6_scraper-0.0.1/PKG-INFO

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: bnetza_bk6_scraper
+Version: 0.0.1
+Summary: Scrapes documents of Bundesnetzagentur Beschlusskammer 6 into a structured, git-diffable mirror
+Project-URL: Changelog, https://github.com/Hochfrequenz/bnetza_bk6_scraper/releases
+Project-URL: Homepage, https://github.com/Hochfrequenz/bnetza_bk6_scraper
+Author-email: Hochfrequenz Unternehmensberatung GmbH <info@hochfrequenz.de>
+License: MIT
+Keywords: beschlusskammer,bnetza,bundesnetzagentur,energy,scraper
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: >=3.11
+Requires-Dist: aiohttp<3.14
+Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
+Requires-Dist: pydantic>=2
+Requires-Dist: typer
+Provides-Extra: coverage
+Requires-Dist: coverage==7.15.0; extra == 'coverage'
+Provides-Extra: dev
+Requires-Dist: pip-tools; extra == 'dev'
+Provides-Extra: formatting
+Requires-Dist: black==26.5.1; extra == 'formatting'
+Requires-Dist: isort==8.0.1; extra == 'formatting'
+Provides-Extra: linting
+Requires-Dist: pylint==4.0.6; extra == 'linting'
+Provides-Extra: packaging
+Requires-Dist: build==1.5.0; extra == 'packaging'
+Requires-Dist: twine==6.2.0; extra == 'packaging'
+Provides-Extra: spell-check
+Requires-Dist: codespell==2.4.2; extra == 'spell-check'
+Provides-Extra: tests
+Requires-Dist: aioresponses==0.7.8; extra == 'tests'
+Requires-Dist: pytest-asyncio==1.3.0; extra == 'tests'
+Requires-Dist: pytest==9.1.1; extra == 'tests'
+Provides-Extra: type-check
+Requires-Dist: mypy==2.1.0; extra == 'type-check'
+Description-Content-Type: text/markdown
+
+# bnetza_bk6_scraper
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+![Python Versions (officially) supported](https://img.shields.io/pypi/pyversions/bnetza_bk6_scraper.svg)
+![PyPI Status Badge](https://img.shields.io/pypi/v/bnetza_bk6_scraper)
+![Unittests status badge](https://github.com/Hochfrequenz/bnetza_bk6_scraper/workflows/Unittests/badge.svg)
+![Coverage status badge](https://github.com/Hochfrequenz/bnetza_bk6_scraper/workflows/Coverage/badge.svg)
+![Linting status badge](https://github.com/Hochfrequenz/bnetza_bk6_scraper/workflows/Linting/badge.svg)
+![Formatting status badge](https://github.com/Hochfrequenz/bnetza_bk6_scraper/workflows/Formatting/badge.svg)
+
+`bnetza_bk6_scraper` mirrors the documents published by the German
+Bundesnetzagentur (BNetzA) **Beschlusskammer 6** (BK6) into a structured,
+git-diffable directory tree. BK6 regulates electricity network access and is a
+constant source of consultations, rulings (Festlegungen) and their attachments.
+Because the agency publishes these as loose PDFs on HTML pages with no changelog,
+tracking *what* changed and *when* is painful. This tool discovers every BK6
+proceeding, downloads its PDFs and a normalized HTML snapshot of each phase page,
+and records structured metadata. Committing the output to git turns every
+regulatory update into a reviewable diff.
+
+## Installation
+
+```bash
+pip install bnetza_bk6_scraper
+```
+
+## Usage
+
+The package installs a single console command, `bnetza-bk6-scraper`, with a
+`mirror` subcommand:
+
+```bash
+bnetza-bk6-scraper mirror --target <dir> [--concurrency N] [--year YYYY] [-v]
+```
+
+| Option          | Default | Description                                        |
+| --------------- | ------- | -------------------------------------------------- |
+| `--target`      | *(required)* | Output directory (the mirror repository root). |
+| `--concurrency` | `4`     | Number of parallel HTTP fetches.                   |
+| `--year`        | *(all)* | Restrict the run to a single year, e.g. `2023`.    |
+| `-v`, `--verbose` | off   | Enable debug logging.                              |
+
+Example — mirror only the 2023 proceedings into `./mirror`:
+
+```bash
+bnetza-bk6-scraper mirror --target ./mirror --year 2023 -v
+```
+
+Each run logs a summary such as
+`run summary: 7 proceedings, 16 documents written, 0 failures`.
+
+## Output layout
+
+Proceedings are written under `/{year}/{aktenzeichen}/`, with a top-level
+`index.json` listing every mirrored proceeding:
+
+```text
+<target>/
+├── index.json                          # summary of all proceedings
+└── 2023/
+    └── BK6-23-241/
+        ├── metadata.json               # structured proceeding metadata
+        ├── BK6-23-241_beschluss.html   # normalized HTML snapshot of a phase page
+        ├── BK6-23-241_beschluss_vom_07.05.26.pdf
+        ├── BK6-23-241_bilarem.pdf
+        └── BK6-23-241_anlage_bilarem.pdf
+```
+
+- `metadata.json` captures the Aktenzeichen, year, title, status, `Stand`
+  (last-modified date), any submission deadline (Frist), the phase pages, and
+  one entry per document (title, type, source URL, filename).
+- The normalized `*.html` files are trimmed, stable snapshots of the source
+  phase pages so that content changes surface as small diffs.
+- The PDFs are the proceeding's documents, downloaded verbatim.
+
+Change detection is intentionally "dumb": the tool always writes the current
+state, and `git diff` in the mirror repository reveals what changed.
+
+## Mirror repository
+
+The scraper is designed to feed a separate mirror repository,
+`Hochfrequenz/bnetza_bk6_mirror`. A scheduled GitHub Action there will
+periodically:
+
+```bash
+pip install bnetza_bk6_scraper
+bnetza-bk6-scraper mirror --target .
+git add -A && git commit -m "update BK6 mirror"
+```
+
+so that regulatory changes at BK6 become visible as reviewable git diffs and
+commit history. That Action is future work and does not live in this repository.
+
+## WAF / browser User-Agent
+
+The BNetzA website sits behind a Web Application Firewall that rejects
+non-browser clients by serving a `200 OK` "The requested URL was rejected" page
+instead of the real content. To get through, the scraper sends browser-like
+`User-Agent` and `Accept` headers and treats the rejection page as a retryable
+error. No credentials or API keys are required.
+
+## Contribute
+
+This project uses [tox](https://tox.wiki) for all quality gates. Create a
+one-shot development environment with everything installed:
+
+```bash
+tox -e dev
+```
+
+Individual gates: `tox -e tests`, `tox -e linting`, `tox -e type_check`,
+`tox -e coverage`, and `tox -e spell_check`. Run the full suite with `tox`.