mrfkit 0.1.0__py3-none-any.whl

mrfkit/records.py

@@ -0,0 +1,260 @@
+"""The rows mrfkit produces.
+
+Every record carries its charge item's key (code, code_type, billing_class,
+setting, modifiers), so standard charges, payer rates and unmapped cells join
+back to their item without a database. Values are what the file says after
+normalization: mrfkit never truncates text or clamps numbers.
+
+Insurer Transparency in Coverage records (``Tic*``) come last and use the TiC
+schema's own field names.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import ClassVar, Optional
+
+
+@dataclass(slots=True)
+class ChargeItem:
+    """One billable item. Emitted once per distinct item key in a file."""
+
+    TABLE: ClassVar[str] = "charge_items"
+
+    code: Optional[str]
+    code_type: Optional[str]
+    description: Optional[str]
+    billing_class: str = ""
+    setting: str = ""
+    modifiers: Optional[str] = None
+    drug_unit_of_measurement: Optional[str] = None
+    drug_type_of_measurement: Optional[str] = None
+    additional_generic_notes: Optional[str] = None
+
+
+@dataclass(slots=True)
+class StandardCharge:
+    """The payer-independent prices for an item."""
+
+    TABLE: ClassVar[str] = "standard_charges"
+
+    code: Optional[str]
+    code_type: Optional[str]
+    description: Optional[str]
+    billing_class: str = ""
+    setting: str = ""
+    modifiers: Optional[str] = None
+    gross_charge: Optional[float] = None
+    discounted_cash_price: Optional[float] = None
+    min_negotiated_rate: Optional[float] = None
+    max_negotiated_rate: Optional[float] = None
+
+
+@dataclass(slots=True)
+class PayerRate:
+    """One negotiated rate for an item under one payer and plan.
+
+    ``billing_class``, ``setting`` and ``modifiers`` are the item's key.
+    ``rate_billing_class`` and ``rate_setting`` are what the rate itself says,
+    which can differ from the item's.
+    """
+
+    TABLE: ClassVar[str] = "payer_rates"
+
+    code: Optional[str]
+    code_type: Optional[str]
+    description: Optional[str]
+    billing_class: str = ""
+    setting: str = ""
+    modifiers: Optional[str] = None
+    payer_name: Optional[str] = None       # normalized
+    raw_payer_name: Optional[str] = None   # as written in the file
+    plan_name: Optional[str] = None
+    plan_category: str = "Other"
+    plan_network: Optional[str] = None
+    negotiated_rate: Optional[float] = None
+    negotiated_percentage: Optional[float] = None
+    negotiated_algorithm: Optional[str] = None
+    methodology: Optional[str] = None       # as written, minus audit notes and URLs
+    methodology_type: Optional[str] = None  # normalized category
+    estimated_amount: Optional[float] = None
+    rate_billing_class: str = ""
+    rate_setting: str = ""
+    additional_notes: Optional[str] = None
+    footnote: Optional[str] = None
+    median_amount: Optional[float] = None
+    pct_10: Optional[float] = None
+    pct_90: Optional[float] = None
+    claim_count: Optional[int] = None
+
+
+@dataclass(slots=True)
+class UnmappedCell:
+    """A value from a column mrfkit could not map to a field.
+
+    Kept so nothing in the file is silently lost, and so new header synonyms
+    can be found. Capped per column by the readers.
+    """
+
+    TABLE: ClassVar[str] = "unmapped_cells"
+
+    code: Optional[str]
+    code_type: Optional[str]
+    description: Optional[str]
+    billing_class: str = ""
+    setting: str = ""
+    modifiers: Optional[str] = None
+    source_column: str = ""
+    value: Optional[str] = None
+
+
+@dataclass(slots=True)
+class FileMetadata:
+    """What the file says about itself: hospital, license, dates, attestation.
+
+    Several locations or addresses are joined with ``|``, as the CMS CSV
+    template writes them.
+    """
+
+    TABLE: ClassVar[str] = "file_metadata"
+
+    hospital_name: Optional[str] = None
+    hospital_location: Optional[str] = None
+    hospital_address: Optional[str] = None
+    cms_certification_number: Optional[str] = None
+    license_number: Optional[str] = None
+    license_state: Optional[str] = None
+    type_2_npi: Optional[str] = None
+    ein: Optional[str] = None
+    last_updated_on: Optional[str] = None
+    version: Optional[str] = None
+    attestation: Optional[str] = None
+    attester_name: Optional[str] = None
+    confirm_attestation: Optional[bool] = None
+
+
+@dataclass(slots=True)
+class HeaderMapping:
+    """How one source column (or JSON key) was read.
+
+    ``mapped_to`` is a canonical field name, a layout tag such as
+    ``wide_payer:negotiated_rate``, or None for an unmapped column.
+    """
+
+    TABLE: ClassVar[str] = "header_mappings"
+
+    source_header: str
+    normalized: str
+    mapped_to: Optional[str] = None
+
+
+@dataclass(slots=True)
+class ModifierInfo:
+    """A modifier the file defines (CMS 3.0 ``modifier_information``).
+
+    One general record per modifier code, then one per payer with its own
+    description. ``payer_name`` is normalized; ``raw_payer_name`` is as written.
+    """
+
+    TABLE: ClassVar[str] = "modifiers"
+
+    code: str
+    description: Optional[str] = None
+    setting: Optional[str] = None
+    payer_name: Optional[str] = None
+    raw_payer_name: Optional[str] = None
+    plan_name: Optional[str] = None
+    payer_description: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Insurer Transparency in Coverage (TiC) files
+# ---------------------------------------------------------------------------
+
+@dataclass(slots=True)
+class TicFileMetadata:
+    """What a TiC in-network rates file says about itself.
+
+    ``network_name`` is the first provider reference's network label, which
+    payers repeat on every reference.
+    """
+
+    TABLE: ClassVar[str] = "tic_file_metadata"
+
+    reporting_entity_name: Optional[str] = None
+    reporting_entity_type: Optional[str] = None
+    last_updated_on: Optional[str] = None
+    version: Optional[str] = None
+    network_name: Optional[str] = None
+
+
+@dataclass(slots=True)
+class TicProviderGroup:
+    """One provider group, a TIN and its NPIs, behind a provider reference.
+
+    ``group_id`` is the file's own ``provider_group_id``, so it is unique
+    only within one file, and a reference holding several groups has several
+    rows. A group listed inline on a rate gets a synthetic ``inline-N`` id.
+    ``tin`` is digits only. Several NPIs are joined with ``|``.
+    """
+
+    TABLE: ClassVar[str] = "tic_provider_groups"
+
+    group_id: str
+    tin_type: Optional[str]
+    tin: str
+    npis: str
+    business_name: Optional[str] = None
+
+
+@dataclass(slots=True)
+class TicRate:
+    """One negotiated price for a billing code, shared by provider groups.
+
+    Identical prices for one code are merged into a single record whose
+    ``provider_group_ids`` (``|``-joined, sorted) lists every
+    :class:`TicProviderGroup` that has it, instead of one row per provider
+    or NPI. A ``percentage`` price fills ``negotiated_percentage``, every
+    other type ``negotiated_rate``. ``expiration_date`` is ``9999-12-31``
+    when the price does not expire, and ``setting`` is empty when the file
+    gives none. Several service codes or modifiers are joined with ``|``.
+    """
+
+    TABLE: ClassVar[str] = "tic_rates"
+
+    billing_code: Optional[str]
+    billing_code_type: Optional[str]
+    billing_code_type_version: Optional[str] = None
+    negotiation_arrangement: Optional[str] = None
+    negotiated_type: str = ""
+    negotiated_rate: Optional[float] = None
+    negotiated_percentage: Optional[float] = None
+    expiration_date: str = "9999-12-31"
+    billing_class: str = ""
+    setting: str = ""
+    service_codes: Optional[str] = None
+    modifiers: Optional[str] = None
+    provider_group_ids: str = ""
+
+
+@dataclass(slots=True)
+class TicIndexEntry:
+    """One in-network file a plan uses, from a TiC table of contents.
+
+    One record per (reporting plan, in-network file) pair of a
+    ``reporting_structure``: a structure with two plans and three files
+    gives six records. ``allowed_amount_location`` is the structure's
+    allowed-amount file, which mrfkit does not read.
+    """
+
+    TABLE: ClassVar[str] = "tic_index"
+
+    reporting_entity_name: Optional[str] = None
+    reporting_entity_type: Optional[str] = None
+    plan_name: Optional[str] = None
+    plan_id: Optional[str] = None
+    plan_id_type: Optional[str] = None
+    plan_market_type: Optional[str] = None
+    in_network_location: str = ""
+    in_network_description: Optional[str] = None
+    allowed_amount_location: Optional[str] = None

mrfkit/reference.py

@@ -0,0 +1,136 @@
+"""Optional reference data and parse counters for the normalizers.
+
+Every normalizer works with no reference data at all. When you have lookup
+tables (published code lists, payer aliases you curate, payers you already
+know), put them in a ``ReferenceData`` and pass it as ``ref=``. When you want
+to know how many rows were rejected or repaired, pass a ``ParseStats`` as
+``stats=``.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, FrozenSet, List, Mapping, Optional
+
+from mrfkit.payers import _clean_payer_key
+
+
+@dataclass(frozen=True)
+class ReferenceData:
+    """Lookup tables that sharpen normalization. Every field is optional.
+
+    code_prefixes
+        ``{'CPT': frozenset(codes), 'HCPCS': frozenset(codes)}`` of published
+        codes. When set, a modifier baked into a code (``73721TC``) is split
+        only if the remaining code is in this list. ``None`` skips the check;
+        an empty set for a type rejects every split of that type.
+    modifier_validity
+        CPT code -> frozenset of digit modifiers CMS marks valid for it
+        (``{'73721': frozenset({'50'})}``). Digit modifiers baked into a code
+        split only when this confirms the pair. ``None`` means they never
+        split.
+    code_descriptions
+        CPT code -> a short description you hold the rights to. When set, a
+        digit modifier split also requires the row description to resemble
+        it.
+    payer_aliases
+        Raw payer name -> canonical payer name, checked before the built-in
+        map. Keys are cleaned (quotes, brackets, underscores, spacing) and
+        uppercased for you.
+    payer_match_index
+        ``payer_match_key(name)`` -> canonical payer name of payers you
+        already know. Spelling variants of an otherwise unknown payer fold
+        onto the name here. When the index is not empty, new payers are added
+        to it as they are seen, so later variants in the same run fold onto
+        the first spelling.
+    methodology_aliases
+        Raw methodology text -> methodology type, checked before the built-in
+        map. Keys are uppercased for you.
+    """
+
+    code_prefixes: Optional[Mapping[str, FrozenSet[str]]] = None
+    modifier_validity: Optional[Mapping[str, FrozenSet[str]]] = None
+    code_descriptions: Optional[Mapping[str, str]] = None
+    payer_aliases: Dict[str, str] = field(default_factory=dict)
+    payer_match_index: Dict[str, str] = field(default_factory=dict)
+    methodology_aliases: Dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        aliases = {}
+        for alias, canonical in self.payer_aliases.items():
+            key = _clean_payer_key(alias)
+            if key and canonical:
+                aliases[key] = canonical
+        object.__setattr__(self, 'payer_aliases', aliases)
+        object.__setattr__(self, 'payer_match_index', dict(self.payer_match_index))
+        object.__setattr__(
+            self, 'methodology_aliases',
+            {k.upper(): v for k, v in self.methodology_aliases.items()},
+        )
+
+
+@dataclass
+class ParseStats:
+    """Counts of rows the normalizers rejected or repaired.
+
+    Pass one instance per file as ``stats=`` to ``is_rejected_code`` and
+    ``apply_baked_modifier_split``. Sample lists keep at most
+    ``sample_limit`` entries, with each value cut short, so a file with
+    millions of bad rows cannot blow up memory. The TiC readers count each
+    JSON path they could not map in ``unmapped_paths``.
+    """
+
+    sample_limit: int = 20
+    rows_read: int = 0
+    rows_skipped: int = 0
+    warnings: List[str] = field(default_factory=list)
+    rejected_codes: int = 0
+    rejected_code_samples: List[Dict[str, str]] = field(default_factory=list)
+    baked_modifier_splits: int = 0
+    baked_modifiers: Dict[str, int] = field(default_factory=dict)
+    baked_modifier_samples: List[Dict[str, str]] = field(default_factory=list)
+    unmapped_path_limit: int = 500
+    unmapped_paths: Dict[str, int] = field(default_factory=dict)
+
+    def warn(self, message: str) -> None:
+        """Keep *message* for the caller and log it."""
+        self.warnings.append(message)
+        logging.getLogger("mrfkit").warning(message)
+
+    def record_unmapped_path(self, path: str) -> None:
+        """Count one occurrence of *path*: a JSON path a reader could not map.
+
+        Only the path and a count are kept, never the value found there, so
+        free text in an unknown field cannot end up in the output. At most
+        ``unmapped_path_limit`` distinct paths are kept.
+        """
+        if path in self.unmapped_paths:
+            self.unmapped_paths[path] += 1
+        elif len(self.unmapped_paths) < self.unmapped_path_limit:
+            self.unmapped_paths[path] = 1
+
+    def record_rejected_code(
+        self, code: Optional[str], code_type: Optional[str],
+    ) -> None:
+        self.rejected_codes += 1
+        if len(self.rejected_code_samples) < self.sample_limit:
+            self.rejected_code_samples.append({
+                'code': (code or '')[:32],
+                'code_type': (code_type or '')[:32],
+            })
+
+    def record_baked_modifier_split(
+        self, baked_modifier: Optional[str], source_code_type: Optional[str],
+    ) -> None:
+        if not baked_modifier:
+            return
+        self.baked_modifier_splits += 1
+        self.baked_modifiers[baked_modifier] = (
+            self.baked_modifiers.get(baked_modifier, 0) + 1
+        )
+        if len(self.baked_modifier_samples) < self.sample_limit:
+            self.baked_modifier_samples.append({
+                'baked_modifier': baked_modifier[:8],
+                'source_code_type': (source_code_type or '')[:16],
+            })

mrfkit/sinks.py

@@ -0,0 +1,126 @@
+"""Write records to disk: one file per record type, in CSV or Parquet."""
+
+from __future__ import annotations
+
+import csv
+import dataclasses
+import operator
+import typing
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Tuple, Type
+
+
+def _columns(record_type: type) -> Tuple[List[str], Callable[[Any], tuple]]:
+    names = [f.name for f in dataclasses.fields(record_type)]
+    getter = operator.attrgetter(*names)
+    if len(names) == 1:
+        return names, lambda r: (getter(r),)
+    return names, getter
+
+
+class CsvSink:
+    """Write each record type to ``<out_dir>/<TABLE>.csv``. None becomes an empty cell."""
+
+    def __init__(self, out_dir: Path):
+        self.out_dir = Path(out_dir)
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self._open: Dict[type, Tuple[Any, Any, Callable]] = {}
+
+    def write(self, record: Any) -> None:
+        entry = self._open.get(type(record))
+        if entry is None:
+            names, getter = _columns(type(record))
+            fh = open(self.out_dir / f"{record.TABLE}.csv", "w", newline="", encoding="utf-8")
+            writer = csv.writer(fh)
+            writer.writerow(names)
+            entry = self._open[type(record)] = (fh, writer, getter)
+        entry[1].writerow(entry[2](record))
+
+    def close(self) -> None:
+        for fh, _, _ in self._open.values():
+            fh.close()
+        self._open.clear()
+
+    def __enter__(self) -> "CsvSink":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+
+class ParquetSink:
+    """Write each record type to ``<out_dir>/<TABLE>.parquet``, in row groups of *batch_size*."""
+
+    def __init__(self, out_dir: Path, batch_size: int = 50_000):
+        try:
+            import pyarrow
+            import pyarrow.parquet
+        except ImportError as exc:
+            raise ImportError('Parquet output needs pyarrow: pip install "mrfkit[parquet]"') from exc
+        self._pa = pyarrow
+        self._pq = pyarrow.parquet
+        self.out_dir = Path(out_dir)
+        self.out_dir.mkdir(parents=True, exist_ok=True)
+        self.batch_size = batch_size
+        self._rows: Dict[type, list] = {}
+        self._writers: Dict[type, Any] = {}
+        self._layout: Dict[type, Tuple[List[str], Callable, Any]] = {}
+
+    def write(self, record: Any) -> None:
+        record_type = type(record)
+        if record_type not in self._layout:
+            names, getter = _columns(record_type)
+            self._layout[record_type] = (names, getter, self._schema(record_type, names))
+            self._rows[record_type] = []
+        rows = self._rows[record_type]
+        rows.append(self._layout[record_type][1](record))
+        if len(rows) >= self.batch_size:
+            self._flush(record_type)
+
+    def _schema(self, record_type: type, names: List[str]):
+        pa = self._pa
+        hints = typing.get_type_hints(record_type)
+        by_type = {str: pa.string(), float: pa.float64(), int: pa.int64(), bool: pa.bool_()}
+        fields = []
+        for name in names:
+            args = [a for a in typing.get_args(hints[name]) if a is not type(None)] or [hints[name]]
+            fields.append(pa.field(name, by_type[args[0]]))
+        return pa.schema(fields)
+
+    def _flush(self, record_type: Type) -> None:
+        rows = self._rows[record_type]
+        if not rows:
+            return
+        names, _, schema = self._layout[record_type]
+        table = self._pa.Table.from_arrays(
+            [self._pa.array(col, type=schema.field(i).type) for i, col in enumerate(zip(*rows, strict=True))],
+            schema=schema,
+        )
+        writer = self._writers.get(record_type)
+        if writer is None:
+            path = self.out_dir / f"{record_type.TABLE}.parquet"
+            writer = self._writers[record_type] = self._pq.ParquetWriter(path, schema)
+        writer.write_table(table)
+        rows.clear()
+
+    def close(self) -> None:
+        for record_type in list(self._rows):
+            self._flush(record_type)
+        for writer in self._writers.values():
+            writer.close()
+        self._writers.clear()
+
+    def __enter__(self) -> "ParquetSink":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+
+def open_sink(out_dir: Path, fmt: str = "csv"):
+    """Return a CSV or Parquet sink for *out_dir*."""
+    if fmt == "csv":
+        return CsvSink(out_dir)
+    if fmt == "parquet":
+        return ParquetSink(out_dir)
+    raise ValueError(f"Unknown output format {fmt!r}: use 'csv' or 'parquet'")