regcraft 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- regcraft/__init__.py +0 -0
- regcraft/cli/__init__.py +15 -0
- regcraft/cli/combine.py +103 -0
- regcraft/cli/dataset.py +71 -0
- regcraft/cli/fetch.py +58 -0
- regcraft/code_schemes/__init__.py +10 -0
- regcraft/code_schemes/chapters.py +96 -0
- regcraft/code_schemes/parse.py +25 -0
- regcraft/code_schemes/types.py +18 -0
- regcraft/datasets/__init__.py +0 -0
- regcraft/datasets/delfi.py +386 -0
- regcraft/datasets/utils.py +119 -0
- regcraft/io.py +33 -0
- regcraft/polars.py +10 -0
- regcraft/registries/__init__.py +0 -0
- regcraft/registries/columns.py +43 -0
- regcraft/registries/combine.py +44 -0
- regcraft/registries/fetchers/__init__.py +7 -0
- regcraft/registries/fetchers/cpr.py +21 -0
- regcraft/registries/fetchers/factory.py +115 -0
- regcraft/registries/fetchers/lpr.py +95 -0
- regcraft/registries/fetchers/lsr.py +68 -0
- regcraft-0.1.0.dist-info/METADATA +19 -0
- regcraft-0.1.0.dist-info/RECORD +28 -0
- regcraft-0.1.0.dist-info/WHEEL +5 -0
- regcraft-0.1.0.dist-info/entry_points.txt +2 -0
- regcraft-0.1.0.dist-info/licenses/LICENSE +21 -0
- regcraft-0.1.0.dist-info/top_level.txt +1 -0
regcraft/__init__.py
ADDED
|
File without changes
|
regcraft/cli/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
import regcraft.cli.combine as c
|
|
4
|
+
import regcraft.cli.dataset as d
|
|
5
|
+
import regcraft.cli.fetch as f
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@click.group()
|
|
9
|
+
def cli():
|
|
10
|
+
"""Registry toolkit."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
cli.add_command(f.fetch)
|
|
14
|
+
cli.add_command(d.dataset)
|
|
15
|
+
cli.add_command(c.combine)
|
regcraft/cli/combine.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import click
|
|
4
|
+
import polars as pl
|
|
5
|
+
|
|
6
|
+
from regcraft.io import write_parquet
|
|
7
|
+
from regcraft.registries.columns import Lpr2AdmCols
|
|
8
|
+
from regcraft.registries.combine import join_diag_adm, join_lsr
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@click.group()
|
|
12
|
+
def combine() -> None:
|
|
13
|
+
"""Create long versions of LSR and Diag Adms"""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_input_file = click.Path(
|
|
17
|
+
exists=True,
|
|
18
|
+
file_okay=True,
|
|
19
|
+
dir_okay=False,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
_output_file = click.Path(
|
|
23
|
+
exists=False,
|
|
24
|
+
file_okay=True,
|
|
25
|
+
dir_okay=False,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@combine.command("lsr", help="Create combined version of Epikur and Laegemiddelinfo")
|
|
30
|
+
@click.argument("epikur_path", type=_input_file)
|
|
31
|
+
@click.argument("laegemiddel_path", type=_input_file)
|
|
32
|
+
@click.argument("output_path", type=_output_file)
|
|
33
|
+
@click.option(
|
|
34
|
+
"--collect",
|
|
35
|
+
is_flag=True,
|
|
36
|
+
default=False,
|
|
37
|
+
help="Collect result before writing output",
|
|
38
|
+
)
|
|
39
|
+
def lsr(epikur_path: str, laegemiddel_path: str, output_path: str, collect: bool):
|
|
40
|
+
lsr = _load_join(epikur_path, laegemiddel_path, join_lsr)
|
|
41
|
+
|
|
42
|
+
write_parquet(lsr, output_path, collect=collect)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@combine.command(
|
|
46
|
+
"diag-adm", help="Create combined version of Diag and Adms for LPR2 and LPR3"
|
|
47
|
+
)
|
|
48
|
+
@click.argument("output_path", type=_output_file)
|
|
49
|
+
@click.option("--lpr2-diag-path", type=_input_file)
|
|
50
|
+
@click.option("--lpr2-adm-path", type=_input_file)
|
|
51
|
+
@click.option("--lpr3-diag-path", type=_input_file)
|
|
52
|
+
@click.option("--lpr3-adm-path", type=_input_file)
|
|
53
|
+
def diag_adm(
|
|
54
|
+
output_path: str,
|
|
55
|
+
lpr2_diag_path: str | None = None,
|
|
56
|
+
lpr2_adm_path: str | None = None,
|
|
57
|
+
lpr3_adm_path: str | None = None,
|
|
58
|
+
lpr3_diag_path: str | None = None,
|
|
59
|
+
):
|
|
60
|
+
|
|
61
|
+
def _raise_specify_both_usage_error(arg1: str, arg2: str):
|
|
62
|
+
raise click.UsageError(f"{arg1} and {arg2} must be specified together.")
|
|
63
|
+
|
|
64
|
+
if (lpr2_diag_path is None) != (lpr2_adm_path is None):
|
|
65
|
+
_raise_specify_both_usage_error("--lpr2-diag-path", "--lpr2-adm-path")
|
|
66
|
+
elif (lpr3_diag_path is None) != (lpr3_adm_path is None):
|
|
67
|
+
_raise_specify_both_usage_error("--lpr3-diag-path", "--lpr3-adm-path")
|
|
68
|
+
elif lpr2_diag_path is None and lpr3_diag_path is None:
|
|
69
|
+
raise click.UsageError("Either lpr3 and or lpr2 paths must be specified.")
|
|
70
|
+
|
|
71
|
+
diag_adm_parts = []
|
|
72
|
+
if lpr2_diag_path is not None and lpr2_adm_path is not None:
|
|
73
|
+
diag_adm_parts.append(
|
|
74
|
+
_load_join(
|
|
75
|
+
lpr2_diag_path,
|
|
76
|
+
lpr2_adm_path,
|
|
77
|
+
join_diag_adm,
|
|
78
|
+
on=Lpr2AdmCols.record_number,
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
if lpr3_diag_path is not None and lpr3_adm_path is not None:
|
|
83
|
+
diag_adm_parts.append(
|
|
84
|
+
_load_join(
|
|
85
|
+
lpr3_diag_path,
|
|
86
|
+
lpr3_adm_path,
|
|
87
|
+
join_diag_adm,
|
|
88
|
+
on=Lpr2AdmCols.record_number,
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
if len(diag_adm_parts) == 2:
|
|
93
|
+
diag_adm = pl.concat(diag_adm_parts)
|
|
94
|
+
else:
|
|
95
|
+
diag_adm = diag_adm_parts[0]
|
|
96
|
+
|
|
97
|
+
write_parquet(diag_adm, output_path)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _load_join(lf1_path: str, lf2_path: str, join_fn: Any, **kwargs) -> pl.LazyFrame:
|
|
101
|
+
lf1 = pl.scan_parquet(lf1_path)
|
|
102
|
+
lf2 = pl.scan_parquet(lf2_path)
|
|
103
|
+
return join_fn(lf1, lf2, **kwargs)
|
regcraft/cli/dataset.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import click
|
|
2
|
+
import polars as pl
|
|
3
|
+
|
|
4
|
+
from regcraft.datasets.delfi import build_delfi_dataset
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@click.group()
|
|
8
|
+
def dataset() -> None:
|
|
9
|
+
"""Build commonly used dataset formats for EHR machine learning models"""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataset.command("delfi", help="Build Delfi dataset")
|
|
13
|
+
@click.argument(
|
|
14
|
+
"cpr_path",
|
|
15
|
+
type=click.Path(
|
|
16
|
+
exists=True,
|
|
17
|
+
file_okay=True,
|
|
18
|
+
dir_okay=False,
|
|
19
|
+
),
|
|
20
|
+
)
|
|
21
|
+
@click.argument(
|
|
22
|
+
"output_dir",
|
|
23
|
+
type=click.Path(
|
|
24
|
+
exists=False,
|
|
25
|
+
file_okay=True,
|
|
26
|
+
dir_okay=False,
|
|
27
|
+
),
|
|
28
|
+
)
|
|
29
|
+
@click.option(
|
|
30
|
+
"--lsr-path",
|
|
31
|
+
type=click.Path(exists=True, dir_okay=False),
|
|
32
|
+
)
|
|
33
|
+
@click.option(
|
|
34
|
+
"--diag-adm-path",
|
|
35
|
+
type=click.Path(exists=True, dir_okay=False),
|
|
36
|
+
)
|
|
37
|
+
@click.option("--atc-token-level", type=int, default=4, show_default=True)
|
|
38
|
+
@click.option("--icd10-token-level", type=int, default=3, show_default=True)
|
|
39
|
+
@click.option("--min-atc-tokens", type=int, default=5, show_default=True)
|
|
40
|
+
@click.option("--min-icd10-tokens", type=int, default=5, show_default=True)
|
|
41
|
+
@click.option("--val-percent", type=float, default=0.1, show_default=True)
|
|
42
|
+
@click.option("--test-percent", type=float, default=0.2, show_default=True)
|
|
43
|
+
def build_delfi(
|
|
44
|
+
cpr_path: str,
|
|
45
|
+
output_dir: str,
|
|
46
|
+
lsr_path: str | None,
|
|
47
|
+
diag_adm_path: str | None,
|
|
48
|
+
atc_token_level: int,
|
|
49
|
+
icd10_token_level: int,
|
|
50
|
+
min_atc_tokens: int,
|
|
51
|
+
min_icd10_tokens: int,
|
|
52
|
+
val_percent: float,
|
|
53
|
+
test_percent: float,
|
|
54
|
+
):
|
|
55
|
+
cpr = pl.scan_parquet(cpr_path)
|
|
56
|
+
if lsr_path is None and diag_adm_path is None:
|
|
57
|
+
raise click.UsageError("Either lsr_path or diag_adms_path must be specified")
|
|
58
|
+
lsr = pl.scan_parquet(lsr_path) if lsr_path else None
|
|
59
|
+
diag_adms = pl.scan_parquet(diag_adm_path) if diag_adm_path else None
|
|
60
|
+
dataset = build_delfi_dataset(
|
|
61
|
+
cpr=cpr,
|
|
62
|
+
lsr=lsr,
|
|
63
|
+
diag_adms=diag_adms,
|
|
64
|
+
atc_token_level=atc_token_level,
|
|
65
|
+
icd10_token_level=icd10_token_level,
|
|
66
|
+
min_atc_tokens=min_atc_tokens,
|
|
67
|
+
min_icd10_tokens=min_icd10_tokens,
|
|
68
|
+
val_percent=val_percent,
|
|
69
|
+
test_percent=test_percent,
|
|
70
|
+
)
|
|
71
|
+
dataset.save(output_dir)
|
regcraft/cli/fetch.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from regcraft.io import write_parquet
|
|
4
|
+
from regcraft.registries.fetchers.factory import (
|
|
5
|
+
FetcherType,
|
|
6
|
+
RegistryVersion,
|
|
7
|
+
get_registryfetcher,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@click.command(help="Fetch and normalize Danish health registry")
|
|
12
|
+
@click.argument(
|
|
13
|
+
"input_path",
|
|
14
|
+
type=click.Path(
|
|
15
|
+
exists=True,
|
|
16
|
+
file_okay=True,
|
|
17
|
+
dir_okay=False,
|
|
18
|
+
),
|
|
19
|
+
)
|
|
20
|
+
@click.argument(
|
|
21
|
+
"output_path",
|
|
22
|
+
type=click.Path(
|
|
23
|
+
exists=False,
|
|
24
|
+
file_okay=True,
|
|
25
|
+
dir_okay=False,
|
|
26
|
+
),
|
|
27
|
+
)
|
|
28
|
+
@click.option(
|
|
29
|
+
"--version",
|
|
30
|
+
"version",
|
|
31
|
+
type=click.Choice(
|
|
32
|
+
[version.value for version in RegistryVersion],
|
|
33
|
+
case_sensitive=False,
|
|
34
|
+
),
|
|
35
|
+
help="Registry version. (default: Inferred from path)",
|
|
36
|
+
)
|
|
37
|
+
@click.option(
|
|
38
|
+
"--type",
|
|
39
|
+
"fetcher_type",
|
|
40
|
+
type=click.Choice(
|
|
41
|
+
[version.value for version in FetcherType],
|
|
42
|
+
case_sensitive=False,
|
|
43
|
+
),
|
|
44
|
+
help="Registry type. (default: Inferred from path)",
|
|
45
|
+
)
|
|
46
|
+
def fetch(
|
|
47
|
+
input_path: str,
|
|
48
|
+
output_path: str,
|
|
49
|
+
version: RegistryVersion | None,
|
|
50
|
+
fetcher_type: FetcherType | None,
|
|
51
|
+
):
|
|
52
|
+
version = version if version else RegistryVersion.from_path(input_path)
|
|
53
|
+
fetcher_type = fetcher_type if fetcher_type else FetcherType.from_path(input_path)
|
|
54
|
+
fetcher = get_registryfetcher(fetcher_type)(version=version)
|
|
55
|
+
|
|
56
|
+
lf = fetcher.parse(input_path)
|
|
57
|
+
|
|
58
|
+
write_parquet(lf, output_path)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
|
|
3
|
+
from .types import CodeMetadata, CodeType
|
|
4
|
+
|
|
5
|
+
ICD10_CHAPTERS = {
|
|
6
|
+
1: ("A00", "B99", "Certain infectious and parasitic diseases"),
|
|
7
|
+
2: ("C00", "D48", "Neoplasms"),
|
|
8
|
+
3: ("D50", "D89", "Diseases of the blood and blood-forming organs"),
|
|
9
|
+
4: ("E00", "E90", "Endocrine, nutritional and metabolic diseases"),
|
|
10
|
+
5: ("F00", "F99", "Mental and behavioural disorders"),
|
|
11
|
+
6: ("G00", "G99", "Diseases of the nervous system"),
|
|
12
|
+
7: ("H00", "H59", "Diseases of the eye and adnexa"),
|
|
13
|
+
8: ("H60", "H95", "Diseases of the ear and mastoid process"),
|
|
14
|
+
9: ("I00", "I99", "Diseases of the circulatory system"),
|
|
15
|
+
10: ("J00", "J99", "Diseases of the respiratory system"),
|
|
16
|
+
11: ("K00", "K93", "Diseases of the digestive system"),
|
|
17
|
+
12: ("L00", "L99", "Diseases of the skin and subcutaneous tissue"),
|
|
18
|
+
13: ("M00", "M99", "Diseases of the musculoskeletal system"),
|
|
19
|
+
14: ("N00", "N99", "Diseases of the genitourinary system"),
|
|
20
|
+
15: ("O00", "O99", "Pregnancy, childbirth and the puerperium"),
|
|
21
|
+
16: ("P00", "P96", "Certain conditions originating in the perinatal period"),
|
|
22
|
+
17: ("Q00", "Q99", "Congenital malformations and chromosomal abnormalities"),
|
|
23
|
+
18: ("R00", "R99", "Symptoms, signs and abnormal clinical findings"),
|
|
24
|
+
19: (
|
|
25
|
+
"S00",
|
|
26
|
+
"T98",
|
|
27
|
+
"Injury, poisoning and certain other consequences of external causes",
|
|
28
|
+
),
|
|
29
|
+
20: ("V01", "Y98", "External causes of morbidity and mortality"),
|
|
30
|
+
21: (
|
|
31
|
+
"Z00",
|
|
32
|
+
"Z99",
|
|
33
|
+
"Factors influencing health status and contact with health services",
|
|
34
|
+
),
|
|
35
|
+
22: ("U00", "U99", "Codes for special purposes"),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def icd10_chapter(code: str) -> CodeMetadata:
|
|
40
|
+
code = code.strip().upper().replace(".", "")
|
|
41
|
+
category = code[:3]
|
|
42
|
+
|
|
43
|
+
chapter = next(
|
|
44
|
+
(
|
|
45
|
+
(index, description)
|
|
46
|
+
for index, (start, end, description) in ICD10_CHAPTERS.items()
|
|
47
|
+
if start <= category <= end
|
|
48
|
+
),
|
|
49
|
+
(None, None),
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
return CodeMetadata(
|
|
53
|
+
code,
|
|
54
|
+
CodeType.ICD10,
|
|
55
|
+
chapter[0],
|
|
56
|
+
chapter[1],
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
ATC_CHAPTERS: Mapping[str, tuple[int, str]] = {
|
|
61
|
+
"A": (1, "Alimentary tract and metabolism"),
|
|
62
|
+
"B": (2, "Blood and blood forming organs"),
|
|
63
|
+
"C": (3, "Cardiovascular system"),
|
|
64
|
+
"D": (4, "Dermatologicals"),
|
|
65
|
+
"G": (5, "Genito-urinary system and sex hormones"),
|
|
66
|
+
"H": (6, "Systemic hormonal preparations"),
|
|
67
|
+
"J": (7, "Antiinfectives for systemic use"),
|
|
68
|
+
"L": (8, "Antineoplastic and immunomodulating agents"),
|
|
69
|
+
"M": (9, "Musculo-skeletal system"),
|
|
70
|
+
"N": (10, "Nervous system"),
|
|
71
|
+
"P": (11, "Antiparasitic products, insecticides and repellents"),
|
|
72
|
+
"R": (12, "Respiratory system"),
|
|
73
|
+
"S": (13, "Sensory organs"),
|
|
74
|
+
"V": (14, "Various"),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def atc_chapter(code: str) -> CodeMetadata:
|
|
79
|
+
"""Map an ATC code to its chapter index and description."""
|
|
80
|
+
code = code.strip().upper()
|
|
81
|
+
chapter = ATC_CHAPTERS.get(code[0])
|
|
82
|
+
|
|
83
|
+
return CodeMetadata(
|
|
84
|
+
code,
|
|
85
|
+
CodeType.ATC,
|
|
86
|
+
chapter[0] if chapter is not None else None,
|
|
87
|
+
chapter[1] if chapter is not None else None,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def code_to_chapter(code: str, code_type: CodeType) -> CodeMetadata:
|
|
92
|
+
match code_type:
|
|
93
|
+
case CodeType.ATC:
|
|
94
|
+
return atc_chapter(code)
|
|
95
|
+
case CodeType.ICD10:
|
|
96
|
+
return icd10_chapter(code)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import polars as pl
|
|
2
|
+
|
|
3
|
+
from regcraft.registries.combine import DiagAdmCols, LsrCols
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def format_icd10(icd_level: int):
|
|
7
|
+
return _format_code(length=icd_level, col=DiagAdmCols.diag, prefix="ICD10", start=1)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def format_atc(atc_level: int):
|
|
11
|
+
"""Length is level + 1 (e.g. level 4: A10AB)"""
|
|
12
|
+
return _format_code(length=atc_level + 1, col=LsrCols.atc, prefix="ATC")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _format_code(
|
|
16
|
+
length: int,
|
|
17
|
+
col: str,
|
|
18
|
+
prefix: str | None = None,
|
|
19
|
+
start: int = 0,
|
|
20
|
+
separator: str = " ",
|
|
21
|
+
):
|
|
22
|
+
expr = pl.col(col).str.slice(start, length)
|
|
23
|
+
if prefix is not None:
|
|
24
|
+
expr = pl.concat_str([prefix, expr], separator=separator)
|
|
25
|
+
return expr
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
type CodeQuery = Callable[[str], CodeMetadata]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CodeType(Enum):
|
|
9
|
+
ICD10 = 0
|
|
10
|
+
ATC = 1
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class CodeMetadata:
|
|
15
|
+
code: str
|
|
16
|
+
code_type: CodeType
|
|
17
|
+
category: int | None = None
|
|
18
|
+
description: str | None = None
|
|
File without changes
|