mrfkit 0.1.0__py3-none-any.whl

mrfkit/__init__.py

@@ -0,0 +1,68 @@
+"""Parse US price transparency files (MRFs) into clean, normalized rows.
+
+    import mrfkit
+
+    for record in mrfkit.iter_records("hospital_standardcharges.csv"):
+        ...
+
+Hospital files (CSV or JSON) and insurer Transparency in Coverage files
+(in-network rates and tables of contents) are both read. See
+:mod:`mrfkit.records` for what comes out.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterator, Optional, Union
+
+from .csv_reader import iter_csv
+from .files import detect_file_format
+from .json_reader import iter_json
+from .records import (
+    ChargeItem,
+    FileMetadata,
+    HeaderMapping,
+    ModifierInfo,
+    PayerRate,
+    StandardCharge,
+    TicFileMetadata,
+    TicIndexEntry,
+    TicProviderGroup,
+    TicRate,
+    UnmappedCell,
+)
+from .reference import ParseStats, ReferenceData
+from .sinks import CsvSink, ParquetSink, open_sink
+from .tic import detect_tic_file, group_tic_networks, iter_tic_in_network, iter_tic_index, tic_network_key
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ChargeItem", "CsvSink", "FileMetadata", "HeaderMapping", "ModifierInfo", "ParquetSink",
+    "ParseStats", "PayerRate", "ReferenceData", "StandardCharge", "TicFileMetadata",
+    "TicIndexEntry", "TicProviderGroup", "TicRate", "UnmappedCell", "detect_file_format",
+    "group_tic_networks", "iter_csv", "iter_json", "iter_records", "iter_tic_in_network",
+    "iter_tic_index", "open_sink", "tic_network_key",
+]
+
+
+def iter_records(path: Union[str, Path], compression: Optional[str] = None, **options: Any) -> Iterator[Any]:
+    """Yield every record in an MRF, CSV or JSON, plain or compressed.
+
+    A JSON file whose root has ``in_network`` or ``provider_references`` is
+    read by :func:`iter_tic_in_network`, one with ``reporting_structure`` by
+    :func:`iter_tic_index`, and every other file as a hospital MRF.
+
+    *options* go to :func:`iter_csv` / :func:`iter_json`: ``stats``,
+    ``extra_synonyms``, ``header_overrides``, ``code_extraction``, ``ref``.
+    The TiC readers take only ``stats``.
+    """
+    fmt, detected = detect_file_format(Path(path))
+    compression = compression or detected
+    if fmt == "json":
+        kind = detect_tic_file(path, compression)
+        if kind is not None:
+            reader = iter_tic_in_network if kind == "in_network" else iter_tic_index
+            return reader(path, compression, stats=options.get("stats"))
+    reader = iter_csv if fmt == "csv" else iter_json
+    return reader(path, compression, **options)

mrfkit/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

mrfkit/cli.py

@@ -0,0 +1,79 @@
+"""``mrfkit FILE...``: turn hospital and insurer MRFs into CSV or Parquet tables."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import List, Optional
+
+from . import __version__, iter_records
+from .reference import ParseStats
+from .sinks import open_sink
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="mrfkit",
+        description="Turn US price transparency files (hospital CSV or JSON, insurer "
+                    "Transparency in Coverage JSON; plain, gzip or zip) into clean tables: "
+                    "one file per record type.")
+    p.add_argument("files", nargs="+", type=Path, help="MRF files to read")
+    p.add_argument("-o", "--out", type=Path, default=Path("mrfkit-out"),
+                   help="output directory (default: mrfkit-out). With several input "
+                        "files, each gets its own subdirectory")
+    p.add_argument("-f", "--format", choices=["csv", "parquet"], default="csv",
+                   help="output format (parquet needs: pip install \"mrfkit[parquet]\")")
+    p.add_argument("--synonyms", type=Path,
+                   help="JSON file mapping extra header names to fields, "
+                        "e.g. {\"charge_amt\": \"gross_charge\"}")
+    p.add_argument("-v", "--verbose", action="store_true",
+                   help="log layout detection and list unmapped JSON paths")
+    p.add_argument("--version", action="version", version=f"mrfkit {__version__}")
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = _parser()
+    args = parser.parse_args(argv)
+    if args.format == "parquet":
+        try:
+            import pyarrow  # noqa: F401
+        except ImportError:
+            parser.error('Parquet output needs pyarrow: pip install "mrfkit[parquet]"')
+    logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING,
+                        format="%(levelname)s %(message)s")
+    synonyms = json.loads(args.synonyms.read_text()) if args.synonyms else None
+
+    failed = 0
+    for path in args.files:
+        out_dir = args.out if len(args.files) == 1 else args.out / path.name.split(".")[0]
+        stats = ParseStats()
+        counts: Counter = Counter()
+        try:
+            with open_sink(out_dir, args.format) as sink:
+                for record in iter_records(path, stats=stats, extra_synonyms=synonyms):
+                    sink.write(record)
+                    counts[record.TABLE] += 1
+        except (OSError, ValueError) as exc:
+            failed += 1
+            print(f"{path}: error: {exc}", file=sys.stderr)
+            continue
+        summary = ", ".join(f"{n:,} {table}" for table, n in sorted(counts.items()))
+        print(f"{path} -> {out_dir}: {stats.rows_read:,} rows read; {summary or 'no records'}",
+              file=sys.stderr)
+        if stats.rejected_codes:
+            print(f"  {stats.rejected_codes:,} rows dropped for corrupt codes", file=sys.stderr)
+        for warning in stats.warnings:
+            print(f"  warning: {warning}", file=sys.stderr)
+        if stats.unmapped_paths:
+            n_paths = len(stats.unmapped_paths)
+            print(f"  {n_paths:,} unmapped JSON path{'' if n_paths == 1 else 's'}"
+                  + ("" if args.verbose else " (-v lists them)"), file=sys.stderr)
+            if args.verbose:
+                for json_path, n in sorted(stats.unmapped_paths.items()):
+                    print(f"    {json_path}: {n:,}", file=sys.stderr)
+    return 1 if failed else 0