dicom-dre 1.0.0.dev6__tar.gz → 1.0.0.dev7__tar.gz

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/.devcontainer/post-create.sh

@@ -32,3 +32,7 @@ uv sync
 echo -e "${BLUE}Compiling JPEG-DCT acceleration extension...${NC}"
 (cd src && uv run python -m dicom_dre._jpeg_dct_accel_build)
 rm -f src/dicom_dre/_jpeg_dct_accel.c src/dicom_dre/_jpeg_dct_accel.o
+
+# Install git hooks so PHI-protection checks run on commit, push, and commit-msg.
+echo -e "${BLUE}Installing git hooks...${NC}"
+uv run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg

dicom_dre-1.0.0.dev7/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

dicom_dre-1.0.0.dev7/.github/workflows/docs.yml

@@ -0,0 +1,124 @@
+name: Deploy Documentation
+
+# Publishes the Sphinx documentation to GitHub Pages, versioned by branch.
+#
+# Pushes to `development` publish under /development/ (pre-release docs).
+# Pushes to `main` publish under /stable/ (released docs).
+#
+# Each version lives in its own subdirectory on the `gh-pages` branch. A run
+# reconstructs the full tree (previously published versions plus the version it
+# just built), then force-publishes it, so removed pages do not linger and other
+# versions are preserved. The site root redirects to /stable/ when it exists and
+# otherwise to /development/.
+#
+# GitHub Pages must be configured to serve from the `gh-pages` branch (root).
+
+on:
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - "docs/**"
+      - "src/**"
+      - "pyproject.toml"
+      - ".github/workflows/docs.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+# Serialize deployments so concurrent pushes to main and development do not race
+# when reconstructing and publishing the shared gh-pages tree.
+concurrency:
+  group: docs-pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Determine version slug
+        id: slug
+        run: |
+          case "${GITHUB_REF_NAME}" in
+            main) echo "slug=stable" >> "$GITHUB_OUTPUT" ;;
+            development) echo "slug=development" >> "$GITHUB_OUTPUT" ;;
+            *) echo "slug=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" ;;
+          esac
+
+      - name: Build HTML documentation
+        run: uv run --group docs sphinx-build -b html docs build/html
+
+      - name: Checkout existing published docs
+        uses: actions/checkout@v4
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+
+      - name: Assemble versioned site
+        run: |
+          set -euo pipefail
+          # A non-matching glob expands to nothing rather than the literal
+          # pattern; the gh-pages checkout leaves an empty directory on the first
+          # run before the branch exists.
+          shopt -s nullglob
+          slug="${{ steps.slug.outputs.slug }}"
+          mkdir -p site
+
+          # Carry forward every previously published version except the one being
+          # rebuilt in this run.
+          for path in gh-pages/*/; do
+            name="$(basename "$path")"
+            [ "$name" = "${slug}" ] && continue
+            cp -r "$path" "site/${name}"
+          done
+
+          # Place the freshly built version.
+          rm -rf "site/${slug}"
+          cp -r build/html "site/${slug}"
+
+          # Serve underscore-prefixed asset directories (Sphinx _static, etc.).
+          touch site/.nojekyll
+
+          # Root redirect: prefer stable, fall back to development, then the slug
+          # just built.
+          if [ -d site/stable ]; then
+            target="stable"
+          elif [ -d site/development ]; then
+            target="development"
+          else
+            target="${slug}"
+          fi
+          cat > site/index.html <<EOF
+          <!doctype html>
+          <meta charset="utf-8">
+          <meta http-equiv="refresh" content="0; url=./${target}/">
+          <link rel="canonical" href="./${target}/">
+          <title>dicom-dre documentation</title>
+          <a href="./${target}/">Continue to the documentation</a>
+          EOF
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_branch: gh-pages
+          publish_dir: ./site
+          force_orphan: true
+          user_name: "github-actions[bot]"
+          user_email: "github-actions[bot]@users.noreply.github.com"

dicom_dre-1.0.0.dev7/.gitignore

@@ -0,0 +1,70 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.o
+
+# Packaging / build
+build/
+dist/
+*.egg-info/
+.eggs/
+src/dicom_dre/_version.py
+
+# Sphinx documentation
+docs/_build/
+
+# CFFI build scratch (generated by the JPEG-DCT build hook)
+src/dicom_dre/dicom_dre/
+src/dicom_dre/_jpeg_dct_accel.c
+
+# Virtual environments
+.venv/
+venv/
+
+# Tooling caches
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
+# uv
+.uv/
+
+# OS / editor
+.DS_Store
+
+# Local working files
+temp/
+
+# --- PHI leak prevention -------------------------------------------------
+# Block data-file types at the repo root where ad-hoc tooling drops output.
+# Scoped to root with a leading slash so tracked fixtures and resources deeper
+# in the tree (allow lists, catalog fixtures) are unaffected.
+/*.json
+/*.csv
+/*.sql
+/regression*
+/~*
+
+# Scratch / local working directories
+/reference/**
+/scripts/**
+local_workspace/
+temp-test/*
+**/ignore/*
+**/DO_NOT_COMMIT.*
+
+# Databases and logs
+db.sqlite3
+*.log
+
+# DICOM and derived image binaries (may carry burned-in PHI)
+*.dcm
+*.dicom
+*.ima
+*.DCM

dicom_dre-1.0.0.dev7/.pre-commit-config.yaml

@@ -0,0 +1,73 @@
+default_language_version:
+  python: python3.12
+fail_fast: false
+default_stages: [pre-commit, pre-push, manual]
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-added-large-files
+        name: check for added large files
+        description: prevents giant files (for example DICOM binaries) from being committed.
+        entry: check-added-large-files
+        language: python
+        exclude: '^(uv\.lock|src/dicom_dre/resources/allow_lists/default\.csv)$'
+      - id: check-toml
+        name: check toml
+        description: checks toml files for parseable syntax.
+        entry: check-toml
+        language: python
+        types: [toml]
+      - id: check-yaml
+        name: check yaml
+        description: checks yaml files for parseable syntax.
+        entry: check-yaml
+        language: python
+        types: [yaml]
+      - id: end-of-file-fixer
+        name: fix end of files
+        description: ensures that a file is either empty, or ends with one newline.
+        entry: end-of-file-fixer
+        language: python
+        types: [text]
+        exclude: '^(docs/|.*\.py$)' # Python files are handled by ruff
+      - id: detect-private-key
+        name: detect private key
+        description: detects the presence of private keys.
+        entry: detect-private-key
+        language: python
+        types: [text]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.14.14
+    hooks:
+      - id: ruff
+        name: ruff linter
+        args: [--fix]
+        files: '^(src|tests)/'
+        types: [python]
+        stages: [manual] # Only run when explicitly called, not during commit
+      - id: ruff-format
+        name: ruff formatter
+        files: '^(src|tests)/'
+        types: [python]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.21.2
+    hooks:
+      - id: pyupgrade
+        types: [python]
+        args: [--py312-plus]
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v4.3.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+        args: [--strict, build, chore, ci, docs, feat, fix, perf, refactor, style, test]
+  - repo: local
+    hooks:
+      - id: block-dicom-content
+        name: block DICOM content
+        description: refuses to commit any file carrying the DICOM magic (DICM at byte offset 128), even if misnamed.
+        entry: bash
+        language: system
+        args: [-c, 'for f in "$@"; do if [ "$(dd if="$f" bs=1 skip=128 count=4 2>/dev/null)" = "DICM" ]; then echo "Refusing to commit DICOM file $f"; exit 1; fi; done', --]
+        stages: [pre-commit]

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 <!-- version list -->
 
+## v1.0.0-dev.6 (2026-07-29)
+
+### Bug Fixes
+
+- Public api change, bump version
+  ([`a2ec45c`](https://github.com/susom/dicom-dre/commit/a2ec45c6d9e00fa30136bead4eada1c4d1e30d31))
+
+
 ## v1.0.0-dev.5 (2026-07-28)
 
 ### Bug Fixes

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dicom-dre
-Version: 1.0.0.dev6
+Version: 1.0.0.dev7
 Summary: DICOM De-identification & Redaction Engine
 Project-URL: repository, https://github.com/susom/dicom-dre
 Author-email: Stanford <people@stanford.edu>

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/docs/conf.py

@@ -15,8 +15,8 @@ sys.path.insert(0, os.path.abspath("../src"))
 # -- Project information ------------------------------------------------------
 
 project = "dicom-dre"
-author = "Stanford"
-copyright = "2026, Stanford"
+author = "DICOM-DRE contributors"
+copyright = "2026"
 
 try:
     release = importlib.metadata.version("dicom-dre")

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/docs/index.md

@@ -67,6 +67,13 @@ guides/local-validation
 reference/api
 ```
 
+```{toctree}
+:maxdepth: 2
+:caption: Notes
+
+notes/iso-2022-specific-character-set
+```
+
 ```{toctree}
 :maxdepth: 2
 :caption: About

dicom_dre-1.0.0.dev7/docs/notes/iso-2022-specific-character-set.md

@@ -0,0 +1,173 @@
+# Non-conformant SpecificCharacterSet and ISO 2022 escape sequences
+
+:::{note}
+This is a recorded finding. The behavior is understood and benign for run
+outcomes. No change to the engine has been made for it yet.
+:::
+
+## Symptom
+
+When de-identifying certain instances with a date-preserving profile, pydicom
+emits:
+
+```
+WARNING | pydicom - Found unknown escape sequence in encoded string value - using encoding latin_1
+.../pydicom/charset.py:481: UserWarning: Found unknown escape sequence in encoded string value - using encoding latin_1
+```
+
+Profiles that do not preserve dates do not emit it, even though every profile
+reads the same source files.
+
+## Root cause
+
+A source instance has an internally inconsistent character-set declaration:
+
+- `SpecificCharacterSet` (0008,0005) = `ISO_IR 192` (UTF-8), which has no ISO 2022
+  code-extension mechanism.
+- A text element, `Image Comments` (0020,4000), VR `LT`, contains raw ISO 2022-JP
+  (JIS X 0208) escape sequences.
+
+The offending value bytes are:
+
+```
+\x1b$B  !z!z  \x1b(B
+```
+
+- `\x1b$B` designates JIS X 0208 (Japanese) to G0.
+- `!z!z` are the two encoded characters.
+- `\x1b(B` returns to ASCII.
+
+Decoded as ISO 2022-JP this is the two-character string of full-width black stars
+(U+2605 U+2605). Because the declared charset (`ISO_IR 192`) cannot handle the
+`\x1b$B` escape, pydicom reports "unknown escape sequence" and falls back to
+`latin_1`, which decodes the bytes one-for-one and yields the literal 10-character
+string `\x1b$B!z!z\x1b(B` (escape codes preserved as control/ASCII characters)
+instead of the intended two stars.
+
+## Why only date-preserving profiles surface it
+
+The warning fires when a text element is decoded. Whether an element is decoded
+depends on which tags a profile touches:
+
+- A profile that preserves dates (`preserve_dates=True`) calls
+  `_should_skip_for_date_preservation`, which reads `ds[tag].VR` for elements as
+  it decides whether to keep them. Accessing `ds[tag]` forces pydicom to lazily
+  decode the raw element value, hitting the bad escape sequence.
+- A profile that does not preserve dates never runs that check, so it never
+  decodes that element and never emits the warning.
+
+Same source files; different tag handling.
+
+## Impact
+
+The warning is benign for run outcome: pydicom recovers via `latin_1` and
+continues, so it does not change filter/scrub/quarantine results. The practical
+downside is that a profile which *preserves* the element writes the garbled
+`latin_1` text (raw escape bytes) into the output instead of the intended
+characters.
+
+## Two conformant ways to keep the characters
+
+Both were verified to decode with no warning:
+
+- Option A (content-preserving): keep the ISO 2022-JP bytes and declare the
+  matching code extension. `SpecificCharacterSet` = `["", "ISO 2022 IR 87"]`
+  (empty first value defaults to ISO 2022 IR 6). No re-encoding; byte-length
+  safe.
+- Option B (canonicalizing): re-encode the text as real UTF-8
+  (`b'\xe2\x98\x85\xe2\x98\x85'`) and leave `SpecificCharacterSet` =
+  `ISO_IR 192`. Changes value bytes/length.
+
+## Generalized fix for the DRE engine
+
+A pre-decode normalization pass, run immediately after each `dcmread` and before
+any text element is decoded. It scans undecoded `RawDataElement` bytes for
+ISO 2022 escape sequences, maps each escape to its DICOM defined term using
+pydicom's own tables, and — when the declared `SpecificCharacterSet` does not
+already cover those terms — widens (0008,0005) to the multi-valued ISO 2022 form
+and resets the dataset's original encoding so pydicom re-decodes the existing
+bytes correctly. This is content-preserving (bytes unchanged), conformant, and
+handles any ISO 2022 code-extension charset (not just Japanese).
+
+Verified end-to-end: an input declaring `ISO_IR 192` with `\x1b$B!z!z\x1b(B` in
+`Image Comments` normalized to `SpecificCharacterSet = ['', 'ISO 2022 IR 87']`
+and decoded to the two stars with no warning.
+
+```python
+from pydicom.dataelem import RawDataElement
+from pydicom import charset as _charset
+
+# Reverse pydicom's own tables: ISO 2022 escape -> DICOM defined term.
+_ENC_TO_TERM = {v: k for k, v in _charset.python_encoding.items()}
+_ESC_TO_TERM = {
+    esc: _ENC_TO_TERM[enc]
+    for esc, enc in _charset.CODES_TO_ENCODINGS.items()
+    if enc in _ENC_TO_TERM
+}
+
+
+def normalize_character_set(ds) -> bool:
+    """Widen SpecificCharacterSet to match ISO 2022 escapes present in raw text.
+
+    Operates on undecoded RawDataElement bytes so no element is decoded (and no
+    charset warning is emitted) before the declaration is corrected. Recurses
+    into sequence items. Returns True if the dataset was modified.
+    """
+    needed: list[str] = []
+
+    def _scan(dataset) -> None:
+        for elem in dataset._dict.values():
+            raw = elem.value if isinstance(elem, RawDataElement) else None
+            if isinstance(raw, (bytes, bytearray)) and _charset.ESC in raw:
+                i = raw.find(_charset.ESC)
+                while i != -1:
+                    for esc, term in _ESC_TO_TERM.items():
+                        if (
+                            raw[i:i + len(esc)] == esc
+                            and term != "ISO 2022 IR 6"
+                            and term not in needed
+                        ):
+                            needed.append(term)
+                    i = raw.find(_charset.ESC, i + 1)
+            # Recurse into already-parsed sequence items.
+            if getattr(elem, "VR", None) == "SQ" and not isinstance(elem, RawDataElement):
+                for item in (elem.value or []):
+                    _scan(item)
+
+    _scan(ds)
+    if not needed:
+        return False
+
+    ds.SpecificCharacterSet = [""] + needed  # empty first value = default ISO 2022 IR 6
+    ds.set_original_encoding(
+        ds._read_implicit,
+        ds._read_little,
+        _charset.convert_encodings(ds.SpecificCharacterSet),
+    )
+    return True
+```
+
+### Where to hook it
+
+In the DRE `deidentify_file` pipeline, call `normalize_character_set(ds)` after
+each read and before catalog evaluation, element rules, and write:
+
+- after the initial `pydicom.dcmread(...)` of the input.
+- after the post-pixel-scrub re-read of the working file.
+
+It also supersedes the profile's default `_ensure_specific_character_set`
+behavior for the escape case: when escapes are present, normalization sets the
+correct code-extension charset instead of defaulting to `ISO_IR 100`.
+
+### Notes and alternatives
+
+- The two key pydicom details that make this work: decode uses
+  `original_character_set` (the read-time `_read_charset`), so rewriting only
+  (0008,0005) is not enough — `set_original_encoding` must update the original
+  encoding too; and inspecting `RawDataElement.value` returns raw bytes without
+  triggering a decode.
+- A warning filter alone would only silence the message; it would not fix the
+  garbled preserved value, so it is not a substitute for normalization.
+- Option B (normalize all text to UTF-8, `ISO_IR 192`) is an alternative if a
+  single canonical charset is preferred downstream, at the cost of rewriting
+  values.

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/justfile

@@ -9,6 +9,14 @@ default:
 sync:
     uv sync
 
+# Install git hooks (pre-commit, pre-push, commit-msg)
+git-hooks:
+    uv run pre-commit install --hook-type pre-commit --hook-type pre-push --hook-type commit-msg
+
+# Run pre-commit checks on staged files (or all files with FILES=--all-files)
+precommit *FILES:
+    uv run pre-commit run {{ FILES }}
+
 # Compile the CFFI JPEG-DCT acceleration extension
 build-ext:
     cd src && uv run python -m dicom_dre._jpeg_dct_accel_build

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/pyproject.toml

@@ -66,6 +66,7 @@ dev = [
     "coverage[toml]",
     "hatch-vcs",
     "hatchling",
+    "pre-commit",
     "pyrefly",
     "pillow",
     "ruff",

{dicom_dre-1.0.0.dev6 → dicom_dre-1.0.0.dev7}/src/dicom_dre/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.0.dev6'
-__version_tuple__ = version_tuple = (1, 0, 0, 'dev6')
+__version__ = version = '1.0.0.dev7'
+__version_tuple__ = version_tuple = (1, 0, 0, 'dev7')
 
 __commit_id__ = commit_id = None