DEMNE 1.3.0__tar.gz

demne-1.3.0/.gitattributes

@@ -0,0 +1 @@
+*.py text working-tree-encoding=UTF-8 eol=lf

demne-1.3.0/.github/workflows/ci.yml

@@ -0,0 +1,141 @@
+name: CI
+
+on:
+  push:
+    # Triggered on development branches AND on vX.Y.Z tags
+    # (tags act as the trigger for the publish job)
+    branches: [main, develop]
+    tags: ["v*.*.*"]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint-and-test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # -----------------------------------------------------------------------
+      # Cache pip: GitHub caches ~/.cache/pip between runs.
+      # Key includes OS, Python version, and a hash of pyproject.toml.
+      # If pyproject.toml hasn't changed, the cache is reused → ~30-60 s
+      # of installation saved per runner.
+      # -----------------------------------------------------------------------
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            pip-${{ runner.os }}-${{ matrix.python-version }}-
+
+      - name: Install dependencies
+        run: |
+          pip install -e ".[dev,ner]"
+
+      - name: Cache pre-commit environments
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Lint and format check (ruff + black via pre-commit)
+        run: pre-commit run --all-files
+
+      # -----------------------------------------------------------------------
+      # Type-check with mypy.
+      # Installed separately (not in the dev group) to keep the prod env light.
+      # --ignore-missing-imports suppresses errors for libs without stubs
+      # (eco2ai, seqeval…).
+      # -----------------------------------------------------------------------
+      - name: Type-check with mypy
+        run: |
+          pip install mypy
+          mypy src/demne --ignore-missing-imports --no-error-summary
+
+      # -----------------------------------------------------------------------
+      # CPU-only tests: GPU tests are explicitly excluded
+      # (-m "not cuda and not rocm") so the job never tries to load a model
+      # on a non-existent GPU (which would raise a CUDA error instead of a
+      # clean skip).
+      # Coverage gate: --cov-fail-under=75 fails the job if overall coverage
+      # drops below 75 %.
+      # -----------------------------------------------------------------------
+      - name: Run unit and integration tests (CPU only)
+        run: >
+          pytest
+          -m "not cuda and not rocm"
+          --cov=demne
+          --cov-report=xml
+          --cov-fail-under=75
+          --tb=short
+          -q
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          flags: unittests
+
+  # ---------------------------------------------------------------------------
+  # Publish job: runs ONLY on a vX.Y.Z tag pushed to main.
+  # Depends on lint-and-test (needs:) → the package is published only if all
+  # tests pass across the 9 OS×Python combinations.
+  # Required secrets in the repo's GitHub Settings:
+  #   PYPI_API_TOKEN  → token generated on pypi.org (scope: DEMNE project)
+  # ---------------------------------------------------------------------------
+  publish:
+    name: Publish to PyPI
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    # Condition: this job runs only when the event is a vX.Y.Z tag push
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+
+    permissions:
+      # Required for OIDC attestation (optional but recommended by PyPI)
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip (publish job)
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-ubuntu-3.12-${{ hashFiles('pyproject.toml') }}
+
+      # build generates dist/*.whl and dist/*.tar.gz
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      # twine checks package consistency before upload
+      - name: Check package with twine
+        run: |
+          pip install twine
+          twine check dist/*
+
+      # Upload to PyPI using the token stored in repo secrets
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload dist/*

demne-1.3.0/.github/workflows/gpu.yml

@@ -0,0 +1,163 @@
+name: GPU Tests
+
+# This workflow is triggered manually (workflow_dispatch) or on push/PR to main.
+# It runs ONLY on self-hosted runners labelled [self-hosted, gpu] — GitHub Actions
+# will leave this job pending (not fail) if no matching runner is registered,
+# which is the intended behaviour while waiting for a runner.
+#
+# How to register a GPU runner:
+#   1. GitHub repo → Settings → Actions → Runners → New self-hosted runner
+#   2. Follow the installation instructions (Linux recommended for CUDA/ROCm)
+#   3. Add the labels "gpu" and "cuda" or "rocm" during configuration
+#   4. Make sure CUDA Toolkit (≥11.8) or ROCm (≥5.4) is installed on the machine
+
+on:
+  # Only manual dispatch: no push trigger, so no job is ever queued automatically.
+  # GPU runners are not registered — automatic triggers would leave jobs pending
+  # indefinitely. Trigger manually from Actions → Run workflow when a runner is
+  # available.
+  workflow_dispatch:
+    inputs:
+      gpu_type:
+        description: "GPU type to target"
+        required: false
+        default: "any"
+        type: choice
+        options: [any, cuda, rocm]
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # CUDA job — self-hosted runner with NVIDIA GPU
+  # labels: [self-hosted, gpu, cuda]
+  # ---------------------------------------------------------------------------
+  test-cuda:
+    name: "Tests CUDA (NVIDIA self-hosted)"
+    # runs-on accepts a list of labels: the runner must have ALL of them
+    runs-on: [self-hosted, gpu, cuda]
+    # If no runner matches, the job waits — timeout added to avoid
+    # blocking a PR indefinitely.
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # The self-hosted runner may already have a warm pip cache.
+      # Sharing it anyway benefits successive runs.
+      - name: Cache pip (CUDA runner)
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-cuda-3.12-${{ hashFiles('pyproject.toml') }}
+          restore-keys: pip-cuda-3.12-
+
+      # Install [gpu] + [dev] + [ner] extras.
+      # [gpu] installs torch with the right CUDA index via the
+      # PIP_EXTRA_INDEX_URL env var set on the self-hosted runner
+      # (e.g. https://download.pytorch.org/whl/cu121).
+      # If the variable is not set, pip falls back to PyPI (CPU wheel).
+      - name: Install dependencies (CUDA)
+        run: pip install -e ".[dev,ner,gpu]"
+
+      # Quick check that torch sees the GPU before running the tests
+      - name: Smoke check — torch sees CUDA
+        run: |
+          python -c "
+          import torch
+          assert torch.cuda.is_available(), 'CUDA not available on this runner'
+          print(f'GPU: {torch.cuda.get_device_name(0)}, CUDA {torch.version.cuda}')
+          "
+
+      # -m "cuda": runs only tests marked @pytest.mark.cuda
+      # --no-header: more compact output for CI logs
+      - name: Run CUDA tests
+        run: >
+          pytest tests/test_gpu.py
+          -m "cuda"
+          -v
+          --tb=short
+          --no-header
+
+      # Unit and integration tests also run on the GPU runner to detect
+      # CPU-side regressions in the same environment.
+      - name: Run unit + integration tests (on GPU runner)
+        run: >
+          pytest
+          -m "not rocm"
+          --cov=demne
+          --cov-report=xml
+          --cov-fail-under=70
+          --tb=short
+          -q
+
+      - name: Upload coverage (CUDA)
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          flags: cuda
+
+  # ---------------------------------------------------------------------------
+  # ROCm job — self-hosted runner with AMD GPU
+  # labels: [self-hosted, gpu, rocm]
+  # ---------------------------------------------------------------------------
+  test-rocm:
+    name: "Tests ROCm (AMD self-hosted)"
+    runs-on: [self-hosted, gpu, rocm]
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip (ROCm runner)
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-rocm-3.12-${{ hashFiles('pyproject.toml') }}
+          restore-keys: pip-rocm-3.12-
+
+      # For ROCm, the self-hosted runner must have PIP_EXTRA_INDEX_URL pointing
+      # to https://download.pytorch.org/whl/rocm5.7 (or the installed version).
+      - name: Install dependencies (ROCm)
+        run: pip install -e ".[dev,ner,gpu]"
+
+      - name: Smoke check — torch sees ROCm
+        run: |
+          python -c "
+          import torch
+          assert torch.cuda.is_available(), 'ROCm/CUDA API not available'
+          assert torch.version.hip is not None, 'This torch build is not ROCm'
+          print(f'GPU: {torch.cuda.get_device_name(0)}, ROCm {torch.version.hip}')
+          "
+
+      - name: Run ROCm tests
+        run: >
+          pytest tests/test_gpu.py
+          -m "rocm"
+          -v
+          --tb=short
+          --no-header
+
+      - name: Run unit + integration tests (on ROCm runner)
+        run: >
+          pytest
+          -m "not cuda"
+          --cov=demne
+          --cov-report=xml
+          --cov-fail-under=70
+          --tb=short
+          -q
+
+      - name: Upload coverage (ROCm)
+        uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          flags: rocm

demne-1.3.0/.gitignore

@@ -0,0 +1,76 @@
+# === Python ===
+__pycache__/
+*.py[cod]
+*.so
+*.egg-info/
+*.egg
+dist/
+build/
+
+# === Environnements ===
+.venv/
+venv/
+env/
+.env
+
+# === IDE / Éditeurs ===
+.vscode/
+.idea/
+.claude/
+*.swp
+*.swo
+*~
+
+# === Jupyter ===
+.ipynb_checkpoints/
+
+# === OS ===
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# === Données et modèles ===
+*.pdf
+*.pptx
+*.rar
+*.zip
+*.tar.gz
+data/*
+!data/README.md
+# Recalibration layout (python main.py optimize_params): folders versioned, corpora not
+!data/*_CORPORA/
+data/*_CORPORA/*
+!data/*_CORPORA/.gitkeep
+models/
+NER/models/
+
+# === Résultats générés ===
+Results/*
+!Results/demne_params_published.json
+!Results/figures/
+Results/figures/*
+!Results/figures/front_pareto_exemple.png
+!Results/figures/Graph_decision.png
+src/demne/Rules/Results/
+src/demne/Rules/src/Results/
+NER/sweep_results*.csv
+
+# === Artefacts runtime ===
+*.log
+*.tmp
+out_errs.json
+jupyter_server_config.json
+energy_consumption_DEMNE.csv
+decision_config.json
+output_decision.txt
+modifications.ps1
+src/demne/REST_interface/Results/
+
+# === Tests / Coverage ===
+.pytest_cache/
+htmlcov/
+.coverage
+coverage.xml
+
+# === Annotations BRAT : jamais diffusées (local-only, Zenodo/PyPI) ===
+*.ann

demne-1.3.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: local
+    hooks:
+      - id: autofix
+        name: black + ruff (format, fix, auto-stage)
+        language: system
+        entry: bash -c 'black --line-length=100 . && ruff check --fix . && git add -u'
+        pass_filenames: false
+        always_run: true
+
+      - id: pytest-fast
+        name: pytest (CPU-only, no GPU)
+        language: system
+        entry: python -m pytest -m "not cuda and not rocm" --tb=short -q --no-header
+        pass_filenames: false
+        always_run: true

demne-1.3.0/CHANGELOG.md

@@ -0,0 +1,74 @@
+# Changelog
+
+## [Unreleased]
+
+## [1.3.0] - 2026-10-21
+### Added
+
+- `demne` console command (source checkout only: delegates to `main.py`) and `py.typed` marker
+
+### Changed
+
+- Core dependencies reduced to numpy, pandas, scikit-learn, scipy, Unidecode; matplotlib, plotly and seaborn moved to the `[viz]` and `[rest]` extras
+- numpy 2.x allowed (removed the `<2.0` cap); development status Beta; author contact email
+
+### Added (since 0.4.0)
+
+- `python main.py optimize_params --in_domain | --all_corpora`: Optuna recalibration of `demne_params.json` on user corpora (`data/*_CORPORA/`, extra `[optimize]`), see `docs/OPTIMIZE_PARAMS.md`
+- Presets `CALIBRATED` / `FRUGAL` / `QUALITY` derived from the calibrated thresholds (`preset_margin`) instead of stored absolute values
+- Extra `[energy]` for eco2ai
+- Tests for the `main.py` sub-commands, the dashboard metrics/parser and the scorers' output files; coverage 70 % → 83 %, floor 75 %
+- mypy `disallow_untyped_defs` on the package (all functions annotated) and public docstrings
+
+### Changed
+
+- Removed commands `tree`, `export-csv`, `rest` and `rest-config --import`
+- `eco2ai` is now an optional dependency (`demne._tracking` is a no-op without it)
+- One BRAT parser (`demne._brat`) for the scorers, frequency, TF-IDF coverage, optimizer and the dashboard parser
+- One copy of the risk lexicons (`demne.params`) for the CLI scorer, the dashboard and the optimizer
+- Dashboard R now includes the contradiction term, as the CLI scorer does (it was fixed to 0)
+- Dashboard entity preselection comes from `data/decision_config.json` instead of a hardcoded list
+- `main.py info` reads `demne.__version__`
+
+### Fixed
+
+- Annotation files (`*.ann`) could ship in the wheel/sdist: removed from `package-data`, excluded in `MANIFEST.in`
+
+## [0.4.0] - 2026-05-29
+### Changed
+
+- Full HDR audit: removed 49 files (artefacts, duplicates, PDFs, caches)
+- Encoding fix UTF-16 → UTF-8 on 5 files (ci.yml, Makefile, CONTRIBUTING.md, DATA_GOVERNANCE.md, reproducibility.yaml)
+- Removed UTF-8 BOM from 4 files
+- Fixed broken Python imports (3 tests + sensitivity_analysis + REST __init__)
+- Replaced 6 absolute Windows paths with DEMNE_ESMO_DIR environment variable
+- Rewrote .gitignore (90 duplicated lines → 35 clean lines)
+- Rewrote Makefile (UTF-16 + broken target → UTF-8 + valid targets)
+
+### Added
+
+- pyproject.toml: 14 missing dependencies in 6 optional groups (rest, viz, notebook, reports, gui, ner[pycrysuite])
+- config/example_env.sh: environment variable template
+- Graph_decision.png in Results/figures/ (fixes README image)
+
+### Fixed
+
+- Broken README image (Graph_decision_biss.png → Graph_decision.png)
+- Renamed lunch.py → launch.py (typo)
+- README/CITATION.cff URLs
+
+## [0.3.0] - 2026-04-19
+### Changed
+
+- Purged 182 ghost files from the Git index
+- pyproject.toml: fixed build-backend (setuptools.build_meta)
+- Removed requirements.txt (pyproject.toml is now the single source)
+- README: CI/License/Python badges, corrected references
+
+### Added
+
+- CITATION.cff, CHANGELOG.md, data/README.md
+
+### Fixed
+
+- Regression from commit c0d5795 (4 files re-added)

demne-1.3.0/CITATION.cff

@@ -0,0 +1,13 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "DEMNE: Adaptive Selection of Named-Entity Extraction Method for Frugal Clinical NLP in Oncology"
+version: 1.3.0
+date-released: 2026-10-21
+authors:
+  - family-names: Longeac
+    given-names: Clément
+    affiliation: ESIEE Paris
+    email: clement.longeac@gmail.com
+repository-code: "https://github.com/longeacc/DEMNE-Determination-of-Extraction-Methode-for-Named-Entity"
+license: MIT
+keywords: [NLP, NER, oncology, frugal-AI, biomarkers]

demne-1.3.0/CONTRIBUTING.md

@@ -0,0 +1,25 @@
+# Contributing to DEMNE
+
+We welcome contributions to DEMNE! 
+
+## Getting Started
+
+1. Fork the repository and create your branch from main.
+2. Install the development dependencies:
+   ```bash
+   pip install -e .[dev,ner]
+   ```
+3. Create a branch for your feature or bugfix.
+4. Ensure you write tests for any new code. Run tests with:
+   ```bash
+   pytest --cov
+   ```
+5. Follow code style guidelines: We use black, isort, and 
+ruff for formatting and linting.
+
+## Pull Requests
+- Format your commit messages using Conventional Commits (e.g., feat: ..., fix: ..., docs: ...).
+- Reference issues you are resolving (e.g. Resolves #123).
+
+## Data Governance
+If you're contributing validation code, ensure it uses synthetic or fully anonymized datatypes matching docs/DATA_GOVERNANCE.md.

demne-1.3.0/LICENSE

@@ -0,0 +1,18 @@
+MIT License
+
+Copyright (c) 2024-2026 Clément LONGEAC — ESIEE Paris, Université Gustave Eiffel 
+Copyright (c) 2024-2026 Akram REDJDAL — Laboratoire de Biomécanique Appliquée (LBA), Aix-Marseille Université
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

demne-1.3.0/MANIFEST.in

@@ -0,0 +1,18 @@
+include LICENSE README.md CHANGELOG.md CONTRIBUTING.md CITATION.cff Makefile
+include main.py pyproject.toml
+
+recursive-include dashboard *
+recursive-include .github *
+include .gitattributes .gitignore .pre-commit-config.yaml
+include Results/demne_params_published.json
+recursive-include data *
+recursive-include docs *
+recursive-include scripts *
+recursive-include src/demne *
+recursive-include tests *
+
+# Clinical annotation data must never ship in an sdist (PyPI / Zenodo).
+global-exclude *.ann
+global-exclude __pycache__
+global-exclude *.py[cod]
+global-exclude *.egg-info

demne-1.3.0/Makefile

@@ -0,0 +1,22 @@
+.PHONY: install test lint evaluate dashboard clean
+
+install:
+	pip install -e .[dev,ner]
+
+test:
+	pytest --cov=demne --tb=short -q
+
+lint:
+	ruff check .
+	black --check .
+
+evaluate:
+	python main.py evaluate --no-visualize
+
+dashboard:
+	streamlit run dashboard/app.py
+
+clean:
+	find . -type d -name __pycache__ -exec rm -rf {} +
+	find . -name '*.pyc' -delete
+	rm -rf .pytest_cache htmlcov .coverage coverage.xml