technologydata 0.2.1__tar.gz → 0.3.0__tar.gz

technologydata-0.3.0/.claude/settings.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(uv sync:*)",
+      "Bash(uv run pytest:*)",
+      "Bash(uv run ruff:*)",
+      "Bash(uv run mypy:*)",
+      "Bash(uv run pre-commit:*)",
+      "Bash(uv run reuse:*)",
+      "Bash(uv run mkdocs build:*)"
+    ]
+  }
+}

technologydata-0.3.0/.claude/settings.json.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: technologydata contributors
+
+SPDX-License-Identifier: MIT

technologydata-0.3.0/.claude/skills/dataset-factsheet/SKILL.md

@@ -0,0 +1,110 @@
+---
+name: dataset-factsheet
+description: Create or update the fact sheet in docs/datasets/<key>.md for a dataset shipped with technologydata, including its Parser API section, once its parser and parsed JSON exist. Use when a data source or a new version of one is added, when a parser changes, or when a fact sheet's doctests fail after the data changed.
+---
+
+# Dataset fact sheet
+
+<!--
+SPDX-FileCopyrightText: technologydata contributors
+
+SPDX-License-Identifier: MIT
+
+-->
+
+The template, section order and writing guidelines are in `docs/contributing/adding_a_dataset.md`; read it first and follow it exactly.
+`docs/datasets/dea_energy_storage.md` and `docs/datasets/legacy_input_data.md` are finished examples.
+How a parser is structured is described in `docs/contributing/writing_a_parser.md`.
+This skill covers how to find each fact, how to check the parser docstrings and how to check the page.
+
+Readers are experienced modellers and AI agents: tables and short bullets, no description of the parser code.
+Only state what you verified in the code or the data; if something cannot be verified (e.g. the upstream license), write what is known and tell the user.
+
+## Inputs
+
+Determine, or ask the user if unclear:
+
+- `<key>`: the `DataSourceName` value in `src/technologydata/data_accessor.py`.
+- `<version>`: the directory in `src/technologydata/parsers/<key>/`.
+- The parser module `src/technologydata/parsers/<key>/`, the raw files in `src/technologydata/parsers/raw/<key>/`, and `sources.json` of the version.
+
+For a new version of an existing dataset, update the existing pages instead (see the end of the checklist in `adding_a_dataset.md`), including a `:::` block for the new version parser in `Parser API`.
+
+## Where each fact comes from
+
+| Section | Source |
+|---|---|
+| Front matter, At a glance | `sources.json` (title, authors, `url`, `url_archive`, `url_date`), the license in `REUSE.toml` and upstream, the parser (region, currency) |
+| Years, cases, currency | the loaded data, see the snippet below |
+| Available versions | `get_supported_versions()` of the parser, the parse options that reproduce the shipped JSON |
+| Field mapping | the raw file's columns and the parser; trace one real row from raw value to parsed value |
+| Naming conventions | the parser's cleaning functions (regexes, renames, case maps, unit replacements); one real example each |
+| Assumptions and deviations | hard-coded values in the parser: fixed region, parameter filters, filled-in units, dropped rows, merged cases, row-level sources not kept |
+| Known limitations | checks below |
+| Parser API | the dispatcher class in `src/technologydata/parsers/<key>/__init__.py` and the version parser class of each entry in its `get_supported_versions()` |
+
+```python
+from technologydata import DataAccessor
+
+df = DataAccessor(data_source="<key>", version="<version>").load().technologies.to_dataframe()
+print(df[["region", "case"]].value_counts(), df["year"].min(), df["year"].max())
+```
+
+Run all Python from the repository root: `DataAccessor.load()` resolves the data relative to the working directory.
+
+## Generate the code block outputs
+
+Copy the `Accessing the data` and `Contents` blocks from the template, change only `data_source` and `version`, and pick a representative technology and parameter for `Accessing the data`.
+Put a placeholder line as the expected output, then run:
+
+```bash
+uv run pytest test/test_docs.py --test-docs -k <key>
+```
+
+The failure report shows `Got:` for each block; paste that output into the page and run the test again until it passes.
+Do not edit the output by hand; the tests compare it with whitespace normalised only.
+
+## Checks for known limitations
+
+Look for these and quantify them where cheap ("7 of about 76 parameters"):
+
+- **Overwritten values**: parameters are stored in a dict per technology, so raw rows mapping to the same technology, case, year and parameter key overwrite each other.
+  Count duplicates in the raw data after applying the parser's name cleaning.
+- **Dropped data**: raw columns not mapped, rows removed by filters or failed parsing (compare raw row count with parsed parameter count).
+- **Heterogeneous units**: visible in the `Parameters` table when one parameter has several reference units.
+- **Order of operations** that loses precision, such as rounding before unit scaling.
+- **Provenance**: whether row-level sources are kept, whether an archived copy of the source exists.
+
+## Verify Reproduce
+
+The documented `parse()` call must reproduce the shipped files.
+It overwrites them, so check the working tree is clean for the output directory first:
+
+```bash
+git status --short src/technologydata/parsers/<key>/<version>/
+uv run python -c 'from technologydata import DataAccessor; DataAccessor(data_source="<key>", version="<version>").parse(input_file_names=["<file>"], num_digits=3)'
+git diff --stat src/technologydata/parsers/<key>/<version>/
+git checkout -- src/technologydata/parsers/<key>/<version>/
+```
+
+Restore only the output directory `<key>/<version>/`: the parser code next to it may have uncommitted changes.
+
+`input_file_names` lists every raw file of the version, as file names inside `src/technologydata/parsers/raw/<key>/`.
+Only a trailing-newline difference is acceptable; otherwise adjust the documented options until the output matches.
+Never use `archive_source=True` here: it calls the Wayback Machine and rewrites `sources.json`.
+The test suite also re-parses the shipped files in place; restore them with `git checkout` after running it.
+
+## Parser API section
+
+The `Parser API` section at the end of the fact sheet renders the docstrings of the parser classes with mkdocstrings, one `:::` block per class with `heading_level: 3`, as in the template: first the dispatcher, then one block per supported version.
+It is the only per-parser documentation, so the docstrings must be complete and match the code: parameters and their types, what the parser writes where, raised errors, which options it ignores, and examples.
+Fix wrong docstrings in the parser, or tell the user if the fix is not obvious.
+
+## Wire up and check
+
+1. Add the fact sheet to the `Datasets` nav in `mkdocs.yaml` and a row to the table in `docs/datasets/index.md`.
+2. Run `uv run pytest test/test_docs.py --test-docs` and `uv run pre-commit run --all-files`.
+   Codespell may flag domain abbreviations; add real terms to `.codespell.ignore`.
+3. Run `READTHEDOCS_CANONICAL_URL=http://localhost/ uv run mkdocs build --strict -d /tmp/site` to check links, anchors and the `:::` blocks.
+   Warnings `has no git logs` for new, uncommitted files can be ignored.
+4. Report to the user: anything not verified, and parser problems found (such as overwritten values or wrong docstrings) as candidates for issues.

{technologydata-0.2.1 → technologydata-0.3.0}/.codespell.ignore

@@ -2,3 +2,5 @@
 #
 # SPDX-License-Identifier: MIT
 CAF
+FOM
+ore

technologydata-0.3.0/.github/CODEOWNERS

@@ -0,0 +1,23 @@
+# CODEOWNERS file for technology-data
+# These owners will be the default owners for everything in the repo
+# and will be requested for review when someone opens a pull request.
+
+# Default owners for the entire repository
+* @euronion @finozzifa
+
+# Documentation
+/docs/ @euronion @finozzifa
+
+# Source code
+/src/ @euronion @finozzifa
+
+# Tests
+/tests/ @euronion @finozzifa
+
+# CI/CD and workflows
+/.github/ @euronion @finozzifa
+
+# Configuration files
+*.toml @euronion @finozzifa
+*.yaml @euronion @finozzifa
+*.yml @euronion @finozzifa

technologydata-0.3.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,18 @@
+<!--
+SPDX-FileCopyrightText: technologydata contributors
+
+SPDX-License-Identifier: MIT
+
+-->
+
+# Summary
+
+<!-- What does this PR change, and why? Link related issues with "Closes #ISSUE_NUMBER". -->
+
+## Checklist
+
+- [ ] Tests added/updated for the change and `uv run pytest` passes
+- [ ] New files carry SPDX license info (checked by REUSE)
+- [ ] `uv run pre-commit run --all-files` passes (lint, format, mypy, REUSE)
+- [ ] Documentation/docstrings has been updated
+- [ ] (If AI-assisted) Changes were reviewed by a human before opening this PR

technologydata-0.3.0/.github/dependabot.yml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+
+# dependabot
+# Ref: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+# ------------------------------------------------------------------------------
+version: 2
+updates:
+- package-ecosystem: github-actions
+  directory: /
+  schedule:
+    interval: weekly
+  groups:
+    # open a single pull-request for all GitHub actions updates
+    github-actions:
+      patterns:
+      - '*'

technologydata-0.3.0/.github/release.yml

@@ -0,0 +1,20 @@
+changelog:
+  categories:
+  - title: Breaking Changes
+    labels:
+    - breaking change
+  - title: Features
+    labels:
+    - feature
+  - title: Bug Fixes
+    labels:
+    - bug
+  - title: Data
+    labels:
+    - data
+  - title: Documentation
+    labels:
+    - documentation
+  - title: Other Changes
+    labels:
+    - "*"

{technologydata-0.2.1 → technologydata-0.3.0}/.github/workflows/ci.yaml

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: MIT
 name: Technologydata prototype CI
 
+permissions: read-all
+
 on:
   push:
     branches: [prototype-2]
@@ -15,16 +17,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
 
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: "3.13"
 
     - name: Cache Python dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v 6.1.0
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('**/uv.lock') }}

technologydata-0.3.0/.github/workflows/codeql.yml

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+
+name: "CodeQL Advanced"
+
+permissions: read-all
+
+on:
+  push:
+    branches: ["prototype-2"]
+  pull_request:
+    branches: ["prototype-2"]
+  schedule:
+  - cron: '0 8 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: actions
+          build-mode: none
+        - language: python
+          build-mode: none
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd  # v4.38.1
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd  # v4.38.1
+      with:
+        category: "/language:${{matrix.language}}"

technologydata-0.3.0/.github/workflows/docs.yml

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+name: Docs
+
+permissions: read-all
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  check-docs:
+    name: Check docs
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
+
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+      with:
+        python-version: "3.13"
+
+    - name: Cache uv dependencies
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v 6.1.0
+      with:
+        path: ~/.cache/uv
+        key: ${{ runner.os }}-uv-${{ hashFiles('**/uv.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-uv-
+
+    # The tutorial's currency-conversion example downloads World Bank
+    # deflator data via pydeflate on first use. Caching it keeps reruns
+    # fast and avoids hitting the network every time.
+    - name: Cache pydeflate data
+      uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v 6.1.0
+      with:
+        path: pydeflate_data
+        key: pydeflate-worldbank-data
+
+    - name: Install package and dependencies
+      run: |
+        python -m pip install uv
+        uv sync --frozen --group dev
+
+    - name: Run documentation doctests
+      run: |
+        uv run pytest test/test_docs.py --test-docs

technologydata-0.3.0/.github/workflows/link-check.yml

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+
+name: Check documentation links
+
+permissions: read-all
+
+on:
+  schedule:
+  - cron: "0 4 * * 1" # every Monday at 04:00 UTC
+  workflow_dispatch:
+
+jobs:
+  link-check:
+    name: Check external links
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
+
+    - name: Check external links
+      uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8  # v2.9.0
+      with:
+        args: "--config .lychee.toml --no-progress './docs/**/*.md' './README.md'"
+        fail: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{technologydata-0.2.1 → technologydata-0.3.0}/.github/workflows/release.yml

@@ -4,6 +4,8 @@
 
 name: Release
 
+permissions: read-all
+
 on:
   push:
     tags:
@@ -19,7 +21,7 @@ jobs:
     outputs:
       prepared: ${{ steps.validate.outputs.prepared }}
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
 
     - name: Validate commit message
       id: validate
@@ -48,7 +50,7 @@ jobs:
         echo "Branch found: $branch"
         echo "BRANCH_NAME=$branch" >> $GITHUB_ENV
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   #v7.0.1
       with:
         fetch-depth: 0
         ref: ${{ env.BRANCH_NAME }}
@@ -56,7 +58,7 @@ jobs:
 
     # Start of preparation script
 
-    - uses: actions/setup-python@v6
+    - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: 3.12
 
@@ -92,7 +94,7 @@ jobs:
         git push origin --delete ${{ github.ref_name }}
 
     - name: Commit changes
-      uses: stefanzweifel/git-auto-commit-action@v7
+      uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d  #v7.2.0
       with:
         branch: ${{ env.BRANCH_NAME }}
         commit_message: '${{ env.PREPARATION_COMMIT }}'
@@ -106,8 +108,8 @@ jobs:
     if: ${{ needs.check-preparation.outputs.prepared == 'true' }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
-    - uses: hynek/build-and-inspect-python-package@v2
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   #v7.0.1
+    - uses: hynek/build-and-inspect-python-package@2abe76da66d0a6a4a227101f9348ee855797cfa5  #v3.0.1
 
   release:
     name: Create GitHub release
@@ -116,12 +118,9 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v6
-    - uses: softprops/action-gh-release@v2
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   #v7.0.1
+    - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64  #v3.0.3
       with:
-        body: |
-          Revised release notes are available in the documentation.
-        append_body: true
         generate_release_notes: true
 
   publish:
@@ -134,8 +133,8 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/download-artifact@v6
+    - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
       with:
         name: Packages
         path: dist
-    - uses: pypa/gh-action-pypi-publish@release/v1
+    - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2

technologydata-0.3.0/.github/workflows/scorecard.yml

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+  - cron: '0 8 * * 1'
+  push:
+    branches: ["prototype-2"]
+  workflow_dispatch:
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.
+    if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request'
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+    - name: "Checkout code"
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1   # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: "Run analysis"
+      uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc   # v2.4.4
+      with:
+        results_file: results.sarif
+        results_format: sarif
+        publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+    - name: "Upload artifact"
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a   # v7.0.1
+      with:
+        name: SARIF file
+        path: results.sarif
+        retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
+    - name: "Upload to code-scanning"
+      uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd  # v4.38.1
+      with:
+        sarif_file: results.sarif

{technologydata-0.2.1 → technologydata-0.3.0}/.gitignore

@@ -78,3 +78,9 @@ dev/
 
 # pydeflate cache
 pydeflate_data/
+
+# site
+site/
+
+# Claude Code agent worktrees
+.claude/worktrees/

technologydata-0.3.0/.lychee.toml

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: technologydata contributors
+#
+# SPDX-License-Identifier: MIT
+
+# lychee link checker config (https://lychee.cli.rs).
+# Checks external links referenced from the documentation and README.
+scheme = ["https", "http"]
+
+# Accept every reachable response except definitive "not found" codes, so only
+# 404 and 410 fail. Publishers routinely answer bots with 400/403/429, which is
+# not link rot.
+accept = ["100..=403", "405..=409", "411..=599"]
+
+# Slow or rate-limiting hosts time out on a different random subset each run.
+# Timeouts are not link rot, so tolerate them; genuine 404/410 and connection
+# failures still fail the check.
+accept_timeouts = true
+
+# Placeholder URL used in code examples throughout the docs, not a real link.
+exclude = [
+  "example.com",
+]
+
+timeout = 30
+max_retries = 5
+max_redirects = 10
+host_concurrency = 4

{technologydata-0.2.1 → technologydata-0.3.0}/.pre-commit-config.yaml

@@ -8,7 +8,7 @@ ci:
 repos:
   # Basic pre-commit hooks for whitespace, merge conflicts, etc.
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
   - id: check-merge-conflict
   - id: check-added-large-files
@@ -18,7 +18,7 @@ repos:
 
   # Ruff: Python linter and formatter
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.10
+  rev: v0.15.19
   hooks:
   - id: ruff
     args: [--fix]
@@ -26,7 +26,7 @@ repos:
 
   # Codespell: Checks for common misspellings
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.4.1
+  rev: v2.4.2
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words=.codespell.ignore']
@@ -41,14 +41,14 @@ repos:
 
   # Pretty-format-yaml: YAML formatter
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.14.0
+  rev: v2.16.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2', --preserve-quotes]
 
   # Mypy: Static type checker for Python
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: 'v1.15.0'
+  rev: 'v2.1.0'
   hooks:
   - id: mypy
     args: [--strict, --ignore-missing-imports, --config-file=pyproject.toml]
@@ -57,19 +57,19 @@ repos:
 
   # Pyupgrade: Automatically upgrades Python syntax
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.16.0
+  rev: v3.21.2
   hooks:
   - id: pyupgrade
     args: ["--py311-plus"]
 
   # REUSE: Checks for REUSE license compliance
 - repo: https://github.com/fsfe/reuse-tool
-  rev: v5.0.2
+  rev: v6.2.0
   hooks:
   - id: reuse
 
 - repo: https://github.com/rvben/rumdl-pre-commit
-  rev: v0.0.99  # Use the latest release tag
+  rev: v0.2.22  # Use the latest release tag
   hooks:
   - id: rumdl
-    args: ["--fix", "--disable=MD006, MD013, MD024, MD030, MD033, MD041"]
+    args: ["--fix", "--disable=MD013, MD024, MD030, MD033, MD041"]