ecgdatakit 2.0.2__tar.gz → 2.0.4__tar.gz

{ecgdatakit-2.0.2/src/ecgdatakit.egg-info → ecgdatakit-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ecgdatakit
-Version: 2.0.2
+Version: 2.0.4
 Summary: Multi-format ECG file parsing and signal processing library
 Author-email: Ahmad Fall <ahmad.fall@ird.fr>
 Maintainer-email: Ahmad Fall <ahmad.fall@ird.fr>
@@ -20,12 +20,10 @@ Classifier: Intended Audience :: Healthcare Industry
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy>=1.24
@@ -50,6 +48,8 @@ Requires-Dist: scipy>=1.10; extra == "plotting-interactive"
 Provides-Extra: anonymize
 Requires-Dist: shortuuid>=1.0; extra == "anonymize"
 Requires-Dist: watchdog>=3.0; extra == "anonymize"
+Provides-Extra: export
+Requires-Dist: lxml>=4.9; extra == "export"
 Provides-Extra: all
 Requires-Dist: pydicom>=2.4; extra == "all"
 Requires-Dist: scipy>=1.10; extra == "all"
@@ -60,10 +60,12 @@ Requires-Dist: peakutils; extra == "all"
 Requires-Dist: neurokit2; extra == "all"
 Requires-Dist: shortuuid>=1.0; extra == "all"
 Requires-Dist: watchdog>=3.0; extra == "all"
+Requires-Dist: lxml>=4.9; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=9.0; extra == "dev"
 Requires-Dist: pytest-cov>=7.0; extra == "dev"
 Requires-Dist: ruff>=0.15; extra == "dev"
+Requires-Dist: lxml>=4.9; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: sphinx>=8.0; extra == "docs"
 Requires-Dist: furo>=2024.8; extra == "docs"
@@ -74,13 +76,13 @@ Requires-Dist: sphinx-sitemap>=2.6; extra == "docs"
 Requires-Dist: sphinxext-opengraph>=0.9; extra == "docs"
 Dynamic: license-file
 
-# ECGDataKit
+# ECGDataKit 2.0.4
 
 [![PyPI](https://img.shields.io/pypi/v/ecgdatakit)](https://pypi.org/project/ecgdatakit/)
 [![Release](https://img.shields.io/github/v/release/UMMISCO/ECGDataKit)](https://github.com/UMMISCO/ECGDataKit/releases)
 [![Tests](https://github.com/UMMISCO/ECGDataKit/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/UMMISCO/ECGDataKit/actions/workflows/tests.yml)
 [![Docs](https://github.com/UMMISCO/ECGDataKit/actions/workflows/docs.yml/badge.svg?branch=main)](https://ecgdatakit.ummisco.fr)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/)
 [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/UMMISCO/ECGDataKit/blob/main/LICENSE)
 [![GitLab](https://img.shields.io/badge/GitLab-mirror-orange?logo=gitlab)](https://git.ummisco.fr/open/ecgdatakit)
 
@@ -94,7 +96,7 @@ Developed at [UMMISCO](https://www.ummisco.fr) / [IRD](https://www.ird.fr) by Ah
 
 ## Features
 
-### Parsing - 13 ECG formats, one unified data model
+### Parsing - 14 ECG formats, one unified data model
 
 Formats are detected from the file content, not the extension.
 
@@ -113,6 +115,7 @@ Formats are detected from the file content, not the extension.
 | EDAN ARC Holter (experimental) | `patient.hea` + `ecgraw.dat`, `.arc` | `patient.hea` with sibling `ecgraw.dat`, or an `.arc` with a known signature |
 | AliveCor Kardia JSON | `.json` | JSON object with Kardia recording keys (`recordedAt`, `algorithmDetermination`, ...) |
 | XLE ECG (experimental) | `.xle` | `.xle` file with a 256 KiB header holding the patient field labels |
+| Cardionics Holter chain | `.chn` + `.ecg` | `.chn` file whose first record holds a valid date and an `.ecg` file name or `End` |
 
 Parsers report what the file stores. They never compute clinical values (intervals, heart rate, QTc, axes, pacemaker status) or invent dates. The only derived values are those a format requires (duration from the sample count, age from the birth date when no age is stored), and a default taken from a format specification is flagged with a warning and in `raw_metadata`.
 
@@ -150,6 +153,17 @@ Parsers report what the file stores. They never compute clinical values (interva
 ecgdatakit anonymize run /path/to/source
 ```
 
+### Export
+
+`HL7aECGExporter` writes a record read from any supported format as an HL7 Annotated ECG XML file that conforms to the official HL7 schema. EDAN ARC Holter files are the exception, since they do not state a voltage scale. See [Export](https://ecgdatakit.ummisco.fr/api/export.html) in the docs.
+
+```python
+from ecgdatakit import FileParser
+from ecgdatakit.exporter import HL7aECGExporter
+
+HL7aECGExporter().export(FileParser.parse("recording.dcm"), "recording.xml")
+```
+
 ## Installation
 
 ```bash
@@ -174,6 +188,9 @@ pip install "ecgdatakit[dicom]"
 # With anonymization (ecgdatakit anonymize)
 pip install "ecgdatakit[anonymize]"
 
+# With HL7 aECG schema checking of exported files (lxml)
+pip install "ecgdatakit[export]"
+
 # With the experimental DeepFADE denoising autoencoder (requires torch)
 pip install "ecgdatakit[denoising]"
 
@@ -241,21 +258,22 @@ Every parser returns the same `ECGRecord`, so downstream code stays identical no
 
 ```
 ECGRecord
-  patient          PatientInfo             ID, name, birth date, sex, race, age, weight, height, medications, history, pacemaker
-  recording        RecordingInfo           date, end date, duration, technician, physician, room, location
-    ├─ device      DeviceInfo              manufacturer, model, serial number, software version, institution
-    └─ acquisition AcquisitionSetup
-         ├─ signal SignalCharacteristics   sampling rate, resolution, bits/sample, encoding, compression, channels
-         └─ filters FilterSettings         highpass, lowpass, notch frequencies
-  leads            list[Lead]              label, samples (float64), sampling rate, resolution, units, is_raw
-  measurements     GlobalMeasurements      HR, PR, QRS, QT, QTc (Bazett/Fridericia), P/QRS/T axes, RR interval
-  interpretation   Interpretation          statements, severity, source, interpreter
-  median_beats     list[Lead]              median/template beats, when available
-  leads_enhanced   list[Lead]              the same leads filtered by the device, when the file stores them (AliveCor)
-  annotations      dict[str, str]          additional key-value annotations
-  source_format    str                     parser identifier (e.g. "sierra_xml")
-  file_format      FileFormatInfo          file format version and creation date
-  raw_metadata     dict                    original format-specific metadata
+  patient               PatientInfo             ID, name, birth date, sex, race, age, weight, height, medications, history, pacemaker
+  recording             RecordingInfo           date, end date, duration, technician, physician, room, location
+    ├─ device           DeviceInfo              manufacturer, model, serial number, software version, institution
+    └─ acquisition      AcquisitionSetup
+         ├─ signal      SignalCharacteristics   sampling rate, resolution, bits/sample, encoding, compression, channels
+         └─ filters     FilterSettings          highpass, lowpass, notch frequencies
+  leads                 list[Lead]              label, samples (float64), sampling rate, resolution, units, is_raw
+  measurements          GlobalMeasurements      HR, PR, QRS, QT, QTc (Bazett/Fridericia), P/QRS/T axes, RR interval
+  interpretation        Interpretation          statements, severity, source, interpreter
+  representative_beats  list[Lead]              representative beats (median, average or template), when available
+  leads_enhanced        list[Lead]              the same leads filtered by the device, when the file stores them (AliveCor)
+  ecg_files             list[ECGFile]           name, start, end, samples and read error of each file, when the record spans several files
+  annotations           dict[str, str]          additional key-value annotations
+  source_format         str                     parser identifier (e.g. "sierra_xml")
+  file_format           FileFormatInfo          file format version and creation date
+  raw_metadata          dict                    original format-specific metadata
 ```
 
 ## Author

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/README.md

@@ -1,10 +1,10 @@
-# ECGDataKit
+# ECGDataKit 2.0.4
 
 [![PyPI](https://img.shields.io/pypi/v/ecgdatakit)](https://pypi.org/project/ecgdatakit/)
 [![Release](https://img.shields.io/github/v/release/UMMISCO/ECGDataKit)](https://github.com/UMMISCO/ECGDataKit/releases)
 [![Tests](https://github.com/UMMISCO/ECGDataKit/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/UMMISCO/ECGDataKit/actions/workflows/tests.yml)
 [![Docs](https://github.com/UMMISCO/ECGDataKit/actions/workflows/docs.yml/badge.svg?branch=main)](https://ecgdatakit.ummisco.fr)
-[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue.svg)](https://www.python.org/downloads/)
 [![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](https://github.com/UMMISCO/ECGDataKit/blob/main/LICENSE)
 [![GitLab](https://img.shields.io/badge/GitLab-mirror-orange?logo=gitlab)](https://git.ummisco.fr/open/ecgdatakit)
 
@@ -18,7 +18,7 @@ Developed at [UMMISCO](https://www.ummisco.fr) / [IRD](https://www.ird.fr) by Ah
 
 ## Features
 
-### Parsing - 13 ECG formats, one unified data model
+### Parsing - 14 ECG formats, one unified data model
 
 Formats are detected from the file content, not the extension.
 
@@ -37,6 +37,7 @@ Formats are detected from the file content, not the extension.
 | EDAN ARC Holter (experimental) | `patient.hea` + `ecgraw.dat`, `.arc` | `patient.hea` with sibling `ecgraw.dat`, or an `.arc` with a known signature |
 | AliveCor Kardia JSON | `.json` | JSON object with Kardia recording keys (`recordedAt`, `algorithmDetermination`, ...) |
 | XLE ECG (experimental) | `.xle` | `.xle` file with a 256 KiB header holding the patient field labels |
+| Cardionics Holter chain | `.chn` + `.ecg` | `.chn` file whose first record holds a valid date and an `.ecg` file name or `End` |
 
 Parsers report what the file stores. They never compute clinical values (intervals, heart rate, QTc, axes, pacemaker status) or invent dates. The only derived values are those a format requires (duration from the sample count, age from the birth date when no age is stored), and a default taken from a format specification is flagged with a warning and in `raw_metadata`.
 
@@ -74,6 +75,17 @@ Parsers report what the file stores. They never compute clinical values (interva
 ecgdatakit anonymize run /path/to/source
 ```
 
+### Export
+
+`HL7aECGExporter` writes a record read from any supported format as an HL7 Annotated ECG XML file that conforms to the official HL7 schema. EDAN ARC Holter files are the exception, since they do not state a voltage scale. See [Export](https://ecgdatakit.ummisco.fr/api/export.html) in the docs.
+
+```python
+from ecgdatakit import FileParser
+from ecgdatakit.exporter import HL7aECGExporter
+
+HL7aECGExporter().export(FileParser.parse("recording.dcm"), "recording.xml")
+```
+
 ## Installation
 
 ```bash
@@ -98,6 +110,9 @@ pip install "ecgdatakit[dicom]"
 # With anonymization (ecgdatakit anonymize)
 pip install "ecgdatakit[anonymize]"
 
+# With HL7 aECG schema checking of exported files (lxml)
+pip install "ecgdatakit[export]"
+
 # With the experimental DeepFADE denoising autoencoder (requires torch)
 pip install "ecgdatakit[denoising]"
 
@@ -165,21 +180,22 @@ Every parser returns the same `ECGRecord`, so downstream code stays identical no
 
 ```
 ECGRecord
-  patient          PatientInfo             ID, name, birth date, sex, race, age, weight, height, medications, history, pacemaker
-  recording        RecordingInfo           date, end date, duration, technician, physician, room, location
-    ├─ device      DeviceInfo              manufacturer, model, serial number, software version, institution
-    └─ acquisition AcquisitionSetup
-         ├─ signal SignalCharacteristics   sampling rate, resolution, bits/sample, encoding, compression, channels
-         └─ filters FilterSettings         highpass, lowpass, notch frequencies
-  leads            list[Lead]              label, samples (float64), sampling rate, resolution, units, is_raw
-  measurements     GlobalMeasurements      HR, PR, QRS, QT, QTc (Bazett/Fridericia), P/QRS/T axes, RR interval
-  interpretation   Interpretation          statements, severity, source, interpreter
-  median_beats     list[Lead]              median/template beats, when available
-  leads_enhanced   list[Lead]              the same leads filtered by the device, when the file stores them (AliveCor)
-  annotations      dict[str, str]          additional key-value annotations
-  source_format    str                     parser identifier (e.g. "sierra_xml")
-  file_format      FileFormatInfo          file format version and creation date
-  raw_metadata     dict                    original format-specific metadata
+  patient               PatientInfo             ID, name, birth date, sex, race, age, weight, height, medications, history, pacemaker
+  recording             RecordingInfo           date, end date, duration, technician, physician, room, location
+    ├─ device           DeviceInfo              manufacturer, model, serial number, software version, institution
+    └─ acquisition      AcquisitionSetup
+         ├─ signal      SignalCharacteristics   sampling rate, resolution, bits/sample, encoding, compression, channels
+         └─ filters     FilterSettings          highpass, lowpass, notch frequencies
+  leads                 list[Lead]              label, samples (float64), sampling rate, resolution, units, is_raw
+  measurements          GlobalMeasurements      HR, PR, QRS, QT, QTc (Bazett/Fridericia), P/QRS/T axes, RR interval
+  interpretation        Interpretation          statements, severity, source, interpreter
+  representative_beats  list[Lead]              representative beats (median, average or template), when available
+  leads_enhanced        list[Lead]              the same leads filtered by the device, when the file stores them (AliveCor)
+  ecg_files             list[ECGFile]           name, start, end, samples and read error of each file, when the record spans several files
+  annotations           dict[str, str]          additional key-value annotations
+  source_format         str                     parser identifier (e.g. "sierra_xml")
+  file_format           FileFormatInfo          file format version and creation date
+  raw_metadata          dict                    original format-specific metadata
 ```
 
 ## Author

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/pyproject.toml

@@ -11,7 +11,7 @@ authors = [{name = "Ahmad Fall", email = "ahmad.fall@ird.fr"}]
 maintainers = [{name = "Ahmad Fall", email = "ahmad.fall@ird.fr"}]
 license = "Apache-2.0"
 license-files = ["LICENSE"]
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 dependencies = [
     "numpy>=1.24",
     "defusedxml>=0.7.1",
@@ -26,8 +26,6 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Medical Science Apps.",
     "Topic :: Scientific/Engineering :: Bio-Informatics",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Operating System :: OS Independent",
@@ -54,6 +52,8 @@ plotting = ["matplotlib>=3.7", "scipy>=1.10"]
 plotting-interactive = ["plotly>=5.15", "scipy>=1.10"]
 # watchdog: file-system events for the anonymization daemon
 anonymize = ["shortuuid>=1.0", "watchdog>=3.0"]
+# lxml: checking exported HL7 aECG files against the HL7 schema
+export = ["lxml>=4.9"]
 all = [
     "pydicom>=2.4",
     "scipy>=1.10",
@@ -64,8 +64,10 @@ all = [
     "neurokit2",
     "shortuuid>=1.0",
     "watchdog>=3.0",
+    "lxml>=4.9",
 ]
-dev = ["pytest>=9.0", "pytest-cov>=7.0", "ruff>=0.15"]
+# lxml: the tests check every exported HL7 aECG document against the schema
+dev = ["pytest>=9.0", "pytest-cov>=7.0", "ruff>=0.15", "lxml>=4.9"]
 docs = [
     "sphinx>=8.0",
     "furo>=2024.8",
@@ -89,8 +91,12 @@ where = ["src"]
 [tool.setuptools.dynamic]
 version = {attr = "ecgdatakit.__version__"}
 
+[tool.uv]
+# neurokit2 caps setuptools below 82 but does not import it
+override-dependencies = ["setuptools>=84.1.0"]
+
 [tool.ruff]
-target-version = "py310"
+target-version = "py312"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/__init__.py

@@ -1,10 +1,12 @@
 """ECGDataKit - Multi-format ECG file parsing and processing library."""
 
-__version__ = "2.0.2"
+__version__ = "2.0.4"
 
 from ecgdatakit.models import (
     AcquisitionSetup,
+    BeatAnnotations,
     DeviceInfo,
+    ECGFile,
     ECGRecord,
     FileFormatInfo,
     FilterSettings,
@@ -26,12 +28,16 @@ from ecgdatakit.exceptions import (
     ChecksumError,
     ChecksumWarning,
     RawSamplesError,
+    ExportError,
 )
 from ecgdatakit.parsing.batch import BatchParseWarning, parse_batch
+from ecgdatakit.parsing.parsers.ishne_holter import read_ishne_annotations
 
 __all__ = [
     "AcquisitionSetup",
+    "BeatAnnotations",
     "DeviceInfo",
+    "ECGFile",
     "ECGRecord",
     "FileFormatInfo",
     "FilterSettings",
@@ -52,6 +58,8 @@ __all__ = [
     "ChecksumError",
     "ChecksumWarning",
     "RawSamplesError",
+    "ExportError",
     "BatchParseWarning",
     "parse_batch",
+    "read_ishne_annotations",
 ]

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/_catalog.py

@@ -29,6 +29,7 @@ COLUMNS = [
     "original_file_name",
     "anonymized_file_name",
     "folder_anonymized",
+    "file_name_anonymized",
     "companions",
     "duplicate",
     "raw_size",

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/_engine.py

@@ -22,7 +22,7 @@ import time
 import uuid
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
 
 from ecgdatakit import __version__
@@ -40,6 +40,7 @@ from ecgdatakit.anonymize._stats import (
     FileRecord,
     Stat,
     compare,
+    name_pseudonym,
     write_stats,
 )
 from ecgdatakit.anonymize._verify import VerificationError, verify
@@ -58,7 +59,7 @@ _SAVE_EVERY = 200
 
 
 def _now() -> str:
-    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
+    return datetime.now(UTC).astimezone().isoformat(timespec="seconds")
 
 
 def _hidden(name: str) -> bool:
@@ -179,6 +180,12 @@ class Anonymizer:
         after the patient pseudonym in the output, and its name is replaced
         wherever it appears in the files. Every other folder, sub-folders of
         the patient folder included, keeps its name.
+    anonymize_file_names : bool
+        ``False`` (default): each anonymized file keeps its raw name. A name
+        holding a known identity value (name, ID, anonymized patient folder
+        name) is reported as risk. ``True``: each anonymized file is named
+        ``<patient pseudonym>_<ECG pseudonym>.<extension>``, nothing of the
+        raw name is kept.
     out_dir : str
         Output folder created in each dataset, with the same sub-folders
         as the dataset.
@@ -197,7 +204,7 @@ class Anonymizer:
 
     def __init__(self, source: str | Path, datasets: bool = False,
                  patients_dir_name: str | None = None, group_by_patient_folders: bool = False,
-                 anonymize_patient_folders: bool = False,
+                 anonymize_patient_folders: bool = False, anonymize_file_names: bool = False,
                  out_dir: str = "ANONYMIZED", catalog_name: str = "anonymization_catalog.csv",
                  threads: int = 8, settle_seconds: float = 0.0,
                  lock_stale_seconds: float = 12 * 3600, dry_run: bool = False) -> None:
@@ -214,6 +221,7 @@ class Anonymizer:
         self.patients_dir_name = patients_dir_name or None
         self.by_folder = bool(group_by_patient_folders)
         self.folders = bool(anonymize_patient_folders)
+        self.file_names = bool(anonymize_file_names)
         if self.folders and not self.by_folder:
             raise ValueError("Renaming patient folders needs group_by_patient_folders "
                              "(--group-by-patient-folders): without it there are no "
@@ -424,6 +432,9 @@ class Anonymizer:
                         continue
                     handler = self._detect(path)
                     if handler is None:
+                        if path.suffix.lower() == ".ann" and any(
+                                n.lower() == path.stem.lower() + ".ecg" for n in names):
+                            continue  # ISHNE beat annotations, handled with their recording
                         report.skipped_unsupported += 1
                         continue
                     if isinstance(handler, WFDBHandler) and path.suffix.lower() != ".hea":
@@ -470,7 +481,7 @@ class Anonymizer:
             if anon_root.is_dir() and all(r != anon_root for _, _, r in anon_roots):
                 anon_roots.append((anon_root, True, anon_root))
         raw = self._records(dataset, raw_roots, skip=out)
-        anon = self._records(out, anon_roots, skip=None)
+        anon = self._records(out, anon_roots, skip=None, pseudonyms=self._ecg_pseudonyms(dataset))
         raw_dirs = self._patient_dirs(dataset, raw_roots, out)
         anon_dirs = self._patient_dirs(out, anon_roots, None)
         stats = compare(raw, anon, grouped=self.by_folder, renamed=self.folders,
@@ -478,6 +489,22 @@ class Anonymizer:
         empty = sorted(raw_dirs - {r.patient_folder for r in raw})
         return stats, empty
 
+    def _ecg_pseudonyms(self, dataset: Path) -> dict[str, str] | None:
+        """ECG pseudonym of each anonymized file, by its path relative to
+        the output folder, from the catalog. ``None`` when the files are
+        named after their pseudonyms: the names are read instead."""
+        if self.file_names:
+            return None
+        catalog = Catalog(self.catalog_path(dataset))
+        out = Path(self.out_dir)
+        result: dict[str, str] = {}
+        for r in catalog.active():
+            for p in _outputs(r):
+                path = Path(p)
+                if path.parts[:len(out.parts)] == out.parts:
+                    result[path.relative_to(out).as_posix()] = r["anonymized_ecg_id"]
+        return result
+
     def _patient_dirs(self, base: Path, roots, skip: Path | None) -> set[str]:
         """Every patient folder on disk (the folders directly under the
         roots), with or without ECG files, relative to *base*."""
@@ -493,9 +520,12 @@ class Anonymizer:
                     dirs.add(Path(e.path).relative_to(base).as_posix())
         return dirs
 
-    def _records(self, base: Path, roots, skip: Path | None) -> list[FileRecord]:
+    def _records(self, base: Path, roots, skip: Path | None,
+                 pseudonyms: dict[str, str] | None = None) -> list[FileRecord]:
         """ECG files under *roots*, with their patient folder and the
-        identity values read in them."""
+        identity values read in them. For anonymized files, *pseudonyms*
+        gives the ECG pseudonym of each file (``None``: read from the file
+        names)."""
         found: list[tuple[Path, str]] = []
         for folder, rec, root in roots:
             for _, files in self._list(folder, rec, skip=skip):
@@ -517,9 +547,15 @@ class Anonymizer:
                 identity = handler.identity(path)
             except Exception:  # noqa: BLE001 - an unreadable file is counted without values
                 identity = Identity()
-            return FileRecord(path.relative_to(base).as_posix(), patient_folder,
+            rel = path.relative_to(base).as_posix()
+            if pseudonyms is None:
+                pseudonym = name_pseudonym(rel)
+            else:
+                # A file missing from the catalog counts as its own ECG
+                pseudonym = pseudonyms.get(rel, "") or rel
+            return FileRecord(rel, patient_folder,
                               tuple(identity.patient_ids), tuple(identity.ecg_ids),
-                              _sha256([path]))
+                              _sha256([path]), pseudonym.replace("-", ""))
 
         with ThreadPoolExecutor(self.threads) as pool:
             return [r for r in pool.map(read, found) if r is not None]
@@ -539,7 +575,8 @@ class Anonymizer:
                 # Unchanged on disk; re-done only when its output disappeared
                 # or the folder mode changed
                 if (all((dataset / p).exists() for p in _outputs(old))
-                        and old["folder_anonymized"] == self._folder_flag()):
+                        and old["folder_anonymized"] == self._folder_flag()
+                        and _names_flag(old) == self._names_flag()):
                     report.unchanged += 1
                     continue
                 entry.handler = self._detect(entry.path)
@@ -600,6 +637,7 @@ class Anonymizer:
         now = _now()
         if (old and old["status"] in ACTIVE and old["raw_sha256"] == sha
                 and old["folder_anonymized"] == self._folder_flag()
+                and _names_flag(old) == self._names_flag()
                 and all((dataset / p).exists() for p in _outputs(old))):
             row = dict(old, raw_size=str(entry.size), raw_mtime_ns=str(entry.mtime_ns))
             row["status"] = "unchanged"
@@ -652,9 +690,17 @@ class Anonymizer:
             for v in unmatched:
                 risks.append(f"patient ID field ({v}) does not match the patient folder "
                              f"name, replaced")
-        # The anonymized file gets a new name: nothing of the raw name is
-        # kept, since a name may hold identity in any form
-        if isinstance(handler, WFDBHandler) and handler.is_multi_segment(entry.path):
+        replacer = Replacer(identity, patient_code, ecg_code, extra_ids=folder_ids)
+        # With anonymize_file_names the file gets a new name: nothing of the
+        # raw name is kept, since a name may hold identity in any form.
+        # Otherwise the raw name is kept, and reported when it holds a known
+        # identity value.
+        if not self.file_names:
+            new_name = entry.path.name
+            if replacer.replace(new_name) != new_name:
+                risks.append("file name kept as is and holds identity text "
+                             "(see --anonymize-file-names)")
+        elif isinstance(handler, WFDBHandler) and handler.is_multi_segment(entry.path):
             new_name = entry.path.name
             risks.append("multi-segment WFDB record name kept as is (it may hold identity)")
         else:
@@ -668,7 +714,6 @@ class Anonymizer:
                                      new_name, ecg_code)
         out_path = out_dir / new_name
 
-        replacer = Replacer(identity, patient_code, ecg_code, extra_ids=folder_ids)
         codes = Codes(patient_code, ecg_code, frozenset(kept_ids))
         rewrite = handler.rewrite(entry.path, out_path, identity, codes, replacer)
         risks += rewrite.notes
@@ -690,6 +735,7 @@ class Anonymizer:
             "original_file_name": entry.path.name,
             "anonymized_file_name": new_name,
             "folder_anonymized": self._folder_flag(),
+            "file_name_anonymized": self._names_flag(),
             "companions": SEPARATOR.join(
                 (Path(self.out_dir) / rel_dir / p.name).as_posix()
                 for p in rewrite.outputs if p != out_path),
@@ -711,6 +757,10 @@ class Anonymizer:
             staged = self._stage(rewrite.outputs, staging)
             verify(entry.path, staged[out_path], replacer, patient_code, kept=kept_ids)
             values = handler.free_text(_head(rewrite.outputs[out_path]))
+            # Companions with a rewritten header of their own (ISHNE .ann) are checked too
+            for p, output in rewrite.outputs.items():
+                if p != out_path and isinstance(output, Spliced) and output.head:
+                    values += handler.free_text(output.head)
             left = sum(1 for v in values if replacer.replace_value(v, is_data(v)) != v)
             if left:
                 risks.append(f"{left} field value(s) of the anonymized file still hold identity text")
@@ -746,6 +796,9 @@ class Anonymizer:
     def _folder_flag(self) -> str:
         return "TRUE" if self.folders else "FALSE"
 
+    def _names_flag(self) -> str:
+        return "TRUE" if self.file_names else "FALSE"
+
     def _output_dir(self, entry: Entry, patient_code: str) -> Path:
         """Output sub-folder of *entry*: the raw one, with only the patient
         folder renamed to the pseudonym when patient folders are anonymized.
@@ -939,6 +992,12 @@ def _outputs(row: dict) -> list[str]:
     return [p for p in paths if p]
 
 
+def _names_flag(row: dict) -> str:
+    """``file_name_anonymized`` of a catalog row. Catalogs written before
+    the option existed always anonymized file names."""
+    return row.get("file_name_anonymized") or "TRUE"
+
+
 def _in_scope(path: Path, scanned: list[tuple[Path, bool, bool]]) -> bool:
     """True when *path* was in a folder (or is the file) listed by this run."""
     for target, recursive, is_file in scanned:

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/_identity.py

@@ -57,12 +57,19 @@ def name_tokens(*names: str, min_len: int = MIN_NAME_TOKEN) -> list[str]:
     return sorted(tokens, key=lambda t: (-len(t), t))
 
 
+#: Values a writer puts in a field it has nothing for (THEW Holters say "none"
+#: for both names). They are no one's identity: replacing them would also
+#: change every other field that holds the same word (a compression "none")
+_PLACEHOLDERS = {"none", "null", "nil", "unknown", "n/a", "-", "?", "anonymous"}
+
+
 @dataclass
 class Identity:
     """Identity values stored in one file.
 
     Every value is kept as written in the file, so the CSV catalog holds the
-    exact original text.
+    exact original text. Placeholders such as ``none`` or ``unknown`` are not
+    identity values and are left out.
     """
 
     patient_ids: list[str] = field(default_factory=list)
@@ -73,7 +80,7 @@ class Identity:
     def add(self, kind: str, value: str | None) -> None:
         value = (value or "").strip()
         values = getattr(self, kind)
-        if value and value not in values:
+        if value and value.lower() not in _PLACEHOLDERS and value not in values:
             values.append(value)
 
     def merge(self, other: Identity) -> None:

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/_stats.py

@@ -30,6 +30,9 @@ class FileRecord(NamedTuple):
     patient_ids: tuple[str, ...]
     ecg_ids: tuple[str, ...]
     sha256: str
+    ecg_pseudonym: str = ""
+    """ECG pseudonym of an anonymized file, without hyphens: read from its
+    name when file names are anonymized, otherwise from the catalog."""
 
 
 class Stat(NamedTuple):
@@ -50,10 +53,10 @@ def _duplicates(records: list[FileRecord]) -> int:
     return sum(n - 1 for n in Counter(r.sha256 for r in records).values())
 
 
-def _name_pseudonym(r: FileRecord) -> str:
+def name_pseudonym(path: str) -> str:
     """ECG pseudonym written in an anonymized file name
     (``<patient>_<ecg>[_<n>].<ext>``), without hyphens (WFDB form)."""
-    parts = PurePosixPath(r.path).name.split(".")[0].split("_")
+    parts = PurePosixPath(path).name.split(".")[0].split("_")
     return parts[1].replace("-", "") if len(parts) >= 2 else ""
 
 
@@ -110,9 +113,9 @@ def compare(raw: list[FileRecord], anon: list[FileRecord], grouped: bool,
     raw_not = sum(1 for r in raw_in if _not_folder_raw(r)) if grouped else 0
     anon_not = _not_folder_anonymized(anon, renamed) if grouped else 0
     raw_ecgs = len({r.sha256 for r in raw})
-    anon_ecgs = len({_name_pseudonym(r) for r in anon})
+    anon_ecgs = len({r.ecg_pseudonym for r in anon})
     anon_ecg_ids = len({v for r in anon for v in r.ecg_ids})
-    anon_with_ecg_id = len({_name_pseudonym(r) for r in anon if r.ecg_ids})
+    anon_with_ecg_id = len({r.ecg_pseudonym for r in anon if r.ecg_ids})
     if grouped:
         # One ID per patient folder (folders whose files only hold dates
         # have none), and no ID shared by two folders

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/_verify.py

@@ -1,7 +1,7 @@
 """Check of each anonymized file before it is published.
 
 Both files are read with ecgdatakit. The anonymized file must be read by
-the same parser, hold the same samples bit for bit (leads, median beats,
+the same parser, hold the same samples bit for bit (leads, representative beats,
 enhanced leads), have only the pseudonym in the patient name and ID fields,
 and give every other field the raw value with the identity text replaced.
 """
@@ -76,7 +76,7 @@ def verify(raw: Path, anonymized: Path, replacer: Replacer, patient_code: str,
             f"read by {after.file_format.parser} after anonymization, "
             f"{before.file_format.parser} before")
     _same_signals(before.leads, after.leads, "leads")
-    _same_signals(before.median_beats, after.median_beats, "median beats")
+    _same_signals(before.representative_beats, after.representative_beats, "representative beats")
     _same_signals(before.leads_enhanced, after.leads_enhanced, "enhanced leads")
 
     old = before.to_dict(include_samples=False)

{ecgdatakit-2.0.2 → ecgdatakit-2.0.4}/src/ecgdatakit/anonymize/formats/__init__.py

@@ -1,7 +1,8 @@
 """Format handlers, one per ecgdatakit parser.
 
 EDAN Holter is not handled: the format is undocumented. XLE, also
-undocumented, is handled from its reverse-engineered layout.
+undocumented, is handled from its reverse-engineered layout. Cardionics
+chains are not handled, the ISHNE files they list are.
 """
 
 from __future__ import annotations