specatwrap83 0.2.0__py3-none-any.whl

specatwrap83/__init__.py

@@ -0,0 +1,39 @@
+"""
+Specatwrap - A wrapper for processing healthcare data.
+
+This module provides a CLI for converting and processing healthcare data files.
+"""
+
+import click
+
+from .sas_converter import sas2parquet
+from .prep import prep
+from .sas_preview import preview
+from .printable import print_file
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli():
+    """
+    Specatwrap - A wrapper for processing healthcare data.
+
+    A command-line tool for processing and converting healthcare data files.
+    """
+    pass
+
+
+# Register command groups
+cli.add_command(sas2parquet)
+cli.add_command(prep)
+cli.add_command(preview)
+cli.add_command(print_file)
+
+
+def main():
+    """Entry point for the CLI application."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

specatwrap83/prep/__init__.py

@@ -0,0 +1,124 @@
+"""
+Prep command group for preprocessing healthcare data files.
+
+This module provides commands for filtering and preprocessing parquet files
+before converting them to XES format.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from .io_handler import process_parquet_files
+
+
+@click.group()
+def prep():
+    """
+    Preprocess and filter healthcare data files.
+
+    Commands in this group help prepare raw data files by filtering,
+    cleaning, and transforming them before further processing.
+    """
+    pass
+
+
+@prep.command()
+@click.option(
+    "-i",
+    "--input",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="Path to directory containing parquet files to process.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_file",
+    type=click.Path(dir_okay=False, file_okay=True, path_type=Path),
+    required=True,
+    help="Path to output parquet file.",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
+def diagnosis(input_dir, output_file, verbose):
+    """
+    Filter and preprocess diagnosis parquet files.
+
+    This command lazily reads all parquet files from a directory, applies
+    filtering and preprocessing transformations, and writes the results to
+    a single output parquet file.
+
+    INPUT: Directory path containing parquet files to process.
+
+    OUTPUT: Path to output parquet file for processed data.
+
+    Example usage:
+
+        specatwrap prep diagnosis -i ./data/parquet_files/ -o ./processed/diagnosis.parquet
+
+        specatwrap prep diagnosis --input ./raw_data/ --output ./clean_data.parquet -v
+    """
+    try:
+        # Display processing information
+        click.echo(f"Input directory: {input_dir}")
+        click.echo(f"Output file: {output_file}")
+        click.echo()
+
+        # Find all parquet files in input directory (for verbose output)
+        if verbose:
+            parquet_pattern = str(input_dir / "*.parquet")
+            click.echo(f"Searching for parquet files: {parquet_pattern}")
+            parquet_files = list(input_dir.glob("*.parquet"))
+            if parquet_files:
+                click.echo(f"Found {len(parquet_files)} parquet file(s):")
+                for f in parquet_files:
+                    click.echo(f"  - {f.name}")
+                click.echo()
+
+        # Process the parquet files using the io_handler
+        click.echo("Loading and applying filters...")
+        with click.progressbar(
+            length=100, label="Processing", show_eta=False, show_percent=True
+        ) as bar:
+            # Call the generic I/O handler with "diagnosis" filter type
+            process_parquet_files(
+                input_dir=input_dir,
+                output_file=output_file,
+                filter_type="diagnosis",
+                verbose=verbose,
+            )
+            bar.update(100)
+
+        # Display success message
+        click.secho("✓ Processing completed successfully!", fg="green", bold=True)
+        click.echo(f"Output file: {output_file}")
+
+        # Display file statistics
+        if output_file.exists():
+            size_mb = output_file.stat().st_size / (1024 * 1024)
+            click.echo(f"File size: {size_mb:.2f} MB")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File or directory not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            "✗ Error: Out of memory. Try processing smaller batches.",
+            fg="red",
+            err=True,
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)

specatwrap83/prep/diagnosis.py

@@ -0,0 +1,426 @@
+"""
+Diagnosis filter classes.
+
+This module provides filter classes for preprocessing diagnosis parquet files.
+"""
+
+from abc import ABC, abstractmethod
+import polars as pl
+
+
+class BaseFilter(ABC):
+    """
+    Abstract base class for parquet file filters.
+
+    Each filter type (diagnosis, procedure, medication, etc.) should inherit
+    from this class and implement the apply() method with their specific
+    filtering and transformation logic.
+    """
+
+    @abstractmethod
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply filtering and preprocessing logic to a LazyFrame.
+
+        Args:
+            lazy_frame: Input Polars LazyFrame to filter/transform
+
+        Returns:
+            Transformed Polars LazyFrame
+        """
+        pass
+
+    def get_name(self) -> str:
+        """
+        Get the name of this filter type.
+
+        Returns:
+            Filter type name (defaults to class name without 'Filter' suffix)
+        """
+        class_name = self.__class__.__name__
+        if class_name.endswith("Filter"):
+            return class_name[:-6].lower()
+        return class_name.lower()
+
+
+class DiagnosisFilter(BaseFilter):
+    """
+    Filter for diagnosis parquet files.
+
+    Applies diagnosis-specific filtering and column transformations:
+    - Filters by birthdate (year > 1980)
+    - Filters by region (Region Sjælland)
+    - Renames columns to standardized event log format
+    """
+
+    CASE_ATTR = {
+        k: f"case:{v}"
+        for k, v in {
+            "BORGER_FOEDSELSDATO": "BDay",
+            "PNR": "PNR",
+            "BORGER_KOEN": "gender"
+        }.items()
+    }
+
+    EVENT_ATTR = {
+        "TORRA_DIAG": "TDiag",
+        "ADIAG": "ADiag",
+        "ADIAG_TEKST": "diagText",
+        "KONT_ANS_GEO_REG_TEKST": "region",
+        "KONT_LPR_ENTITY_ID": "org:id",
+        "KONT_INST_EJERTYPE": "org:type",
+        "KONT_STARTTIDSPUNKT": "startTime",
+        "KONT_SLUTTIDSPUNKT": "endTime",
+        # "BORGER_ALDER_AAR_IND": "patient:age",
+        "PRIORITET_TEKST": "priority",
+        "KONT_TYPE_TEKST": "contact_type",
+    }
+
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply diagnosis-specific filtering and transformations.
+
+        Args:
+            lazy_frame: Input LazyFrame containing diagnosis data
+
+        Returns:
+            Filtered and transformed LazyFrame
+        """
+
+        lf = self._add_torra_diag(lf)
+
+        return (
+            lf.filter(pl.col("TORRA_DIAG").list.len() > 0)
+            .with_columns(pl.col("TORRA_DIAG").list.join(", "))
+            .select(list((self.CASE_ATTR | self.EVENT_ATTR).keys()))
+            .rename(self.CASE_ATTR | self.EVENT_ATTR)
+        )
+
+    def _add_torra_diag(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        col = pl.col("ADIAG")
+
+        def starts_with(prefixes: list[str], label: str) -> pl.Expr:
+            pattern = f"^(?:{'|'.join(prefixes)})"
+            return (
+                pl.when(col.str.contains(pattern))
+                .then(pl.lit(label))
+                .otherwise(pl.lit(None))
+            )
+
+        return lazy_frame.with_columns(
+            TORRA_DIAG=pl.concat_list(
+                [
+                    starts_with(
+                        ["DJ41", "DJ42", "DJ43", "DJ44", "DJ45", "DJ46"], "LUNG"
+                    ),
+                    starts_with(
+                        [
+                            "DL405",
+                            "DM05",
+                            "DM06",
+                            "DM07",
+                            "DM15",
+                            "DM16",
+                            "DM17",
+                            "DM45",
+                            "DM47",
+                            "DM50",
+                            "DM51",
+                            "DM53",
+                            "DM54",
+                            "DM80",
+                            "DM81",
+                            "DM82",
+                        ],
+                        "MUSCULOSKELETAL",
+                    ),
+                    starts_with(
+                        ["DE03", "DE05", "DE10", "DE11", "DE12", "DE13", "DE14"],
+                        "ENDOCRINE",
+                    ),
+                    starts_with(["DE03", "DE05"], "THYROIDEA"),
+                    starts_with(["DE10", "DE11", "DE12", "DE13", "DE14"], "DIABETES"),
+                    starts_with(["DG30", "DG318", "DG319", "DF"], "MENTAL"),
+                    # CANCER: Matches DC but explicitly excludes DC44
+                    pl.when(col.str.contains("^DC") & ~col.str.contains("^DC44"))
+                    .then(pl.lit("CANCER"))
+                    .otherwise(pl.lit(None)),
+                    starts_with(
+                        [
+                            "DI60",
+                            "DI61",
+                            "DI62",
+                            "DI63",
+                            "DI64",
+                            "DI69",
+                            "DG20",
+                            "DG35",
+                            "DG40",
+                            "DG43",
+                        ],
+                        "NEUROLOGICAL",
+                    ),
+                    starts_with(
+                        [
+                            "DK30",
+                            "DK50",
+                            "DK51",
+                            "DK58",
+                            "DK70",
+                            "DK71",
+                            "DK72",
+                            "DK73",
+                            "DK74",
+                            "DK75",
+                            "DK76",
+                            "DK860",
+                            "DK861",
+                        ],
+                        "GASTROINTESTINAL",
+                    ),
+                    starts_with(
+                        [
+                            "DI20",
+                            "DI21",
+                            "DI22",
+                            "DI23",
+                            "DI24",
+                            "DI25",
+                            "DI47",
+                            "DI48",
+                            "DI49",
+                            "DI50",
+                            "DI05",
+                            "DI06",
+                            "DI07",
+                            "DI08",
+                            "DI34",
+                            "DI35",
+                            "DI36",
+                            "DI37",
+                            "DI441",
+                            "DI442",
+                            "DI443",
+                            "DI444",
+                            "DI445",
+                            "DI446",
+                            "DI447",
+                            "DI452",
+                            "DI453",
+                            "DI454",
+                            "DI455",
+                            "DI456",
+                            "DI457",
+                            "DI458",
+                            "DI459",
+                        ],
+                        "CARDIOVASCULAR",
+                    ),
+                    starts_with(
+                        [
+                            "DN03",
+                            "DN04",
+                            "DN05",
+                            "DN11",
+                            "DN12",
+                            "DN18",
+                            "DN19",
+                            "DZ49",
+                            "DN80",
+                            "DZ992",
+                            "DN393",
+                            "DN394",
+                        ],
+                        "GENITURINARY",
+                    ),
+                    starts_with(
+                        [
+                            "DH40",
+                            "DH91",
+                            "DL40",
+                            "DH540",
+                            "DH541",
+                            "DH542",
+                            "DH543",
+                            "DH547",
+                            "DH900",
+                            "DH902",
+                            "DH903",
+                            "DH905",
+                            "DH906",
+                            "DH908",
+                        ],
+                        "SENSORY ORGANS",
+                    ),
+                    starts_with(
+                        ["DL23", "DL24", "DL25", "DJ30", "DL500", "DJ450"], "ALLERGY"
+                    ),
+                ]
+            ).list.drop_nulls()
+        )
+
+    def print_torra_diag_method(self) -> None:
+        """Print the _add_torra_diag method to stdout."""
+        method_code = """    def _add_torra_diag(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        col = pl.col("ADIAG")
+
+        def starts_with(prefixes: list[str], label: str) -> pl.Expr:
+            pattern = f"^(?:{'|'.join(prefixes)})"
+            return (
+                pl.when(col.str.contains(pattern))
+                .then(pl.lit(label))
+                .otherwise(pl.lit(None))
+            )
+
+        return lazy_frame.with_columns(
+            TORRA_DIAG=pl.concat_list(
+                [
+                    starts_with(
+                        ["DJ41", "DJ42", "DJ43", "DJ44", "DJ45", "DJ46"], "LUNG"
+                    ),
+                    starts_with(
+                        [
+                            "DL405",
+                            "DM05",
+                            "DM06",
+                            "DM07",
+                            "DM15",
+                            "DM16",
+                            "DM17",
+                            "DM45",
+                            "DM47",
+                            "DM50",
+                            "DM51",
+                            "DM53",
+                            "DM54",
+                            "DM80",
+                            "DM81",
+                            "DM82",
+                        ],
+                        "MUSCULOSKELETAL",
+                    ),
+                    starts_with(
+                        ["DE03", "DE05", "DE10", "DE11", "DE12", "DE13", "DE14"],
+                        "ENDOCRINE",
+                    ),
+                    starts_with(["DE03", "DE05"], "THYROIDEA"),
+                    starts_with(["DE10", "DE11", "DE12", "DE13", "DE14"], "DIABETES"),
+                    starts_with(["DG30", "DG318", "DG319", "DF"], "MENTAL"),
+                    # CANCER: Matches DC but explicitly excludes DC44
+                    pl.when(col.str.contains("^DC") & ~col.str.contains("^DC44"))
+                    .then(pl.lit("CANCER"))
+                    .otherwise(pl.lit(None)),
+                    starts_with(
+                        [
+                            "DI60",
+                            "DI61",
+                            "DI62",
+                            "DI63",
+                            "DI64",
+                            "DI69",
+                            "DG20",
+                            "DG35",
+                            "DG40",
+                            "DG43",
+                        ],
+                        "NEUROLOGICAL",
+                    ),
+                    starts_with(
+                        [
+                            "DK30",
+                            "DK50",
+                            "DK51",
+                            "DK58",
+                            "DK70",
+                            "DK71",
+                            "DK72",
+                            "DK73",
+                            "DK74",
+                            "DK75",
+                            "DK76",
+                            "DK860",
+                            "DK861",
+                        ],
+                        "GASTROINTESTINAL",
+                    ),
+                    starts_with(
+                        [
+                            "DI20",
+                            "DI21",
+                            "DI22",
+                            "DI23",
+                            "DI24",
+                            "DI25",
+                            "DI47",
+                            "DI48",
+                            "DI49",
+                            "DI50",
+                            "DI05",
+                            "DI06",
+                            "DI07",
+                            "DI08",
+                            "DI34",
+                            "DI35",
+                            "DI36",
+                            "DI37",
+                            "DI441",
+                            "DI442",
+                            "DI443",
+                            "DI444",
+                            "DI445",
+                            "DI446",
+                            "DI447",
+                            "DI452",
+                            "DI453",
+                            "DI454",
+                            "DI455",
+                            "DI456",
+                            "DI457",
+                            "DI458",
+                            "DI459",
+                        ],
+                        "CARDIOVASCULAR",
+                    ),
+                    starts_with(
+                        [
+                            "DN03",
+                            "DN04",
+                            "DN05",
+                            "DN11",
+                            "DN12",
+                            "DN18",
+                            "DN19",
+                            "DZ49",
+                            "DN80",
+                            "DZ992",
+                            "DN393",
+                            "DN394",
+                        ],
+                        "GENITURINARY",
+                    ),
+                    starts_with(
+                        [
+                            "DH40",
+                            "DH91",
+                            "DL40",
+                            "DH540",
+                            "DH541",
+                            "DH542",
+                            "DH543",
+                            "DH547",
+                            "DH900",
+                            "DH902",
+                            "DH903",
+                            "DH905",
+                            "DH906",
+                            "DH908",
+                        ],
+                        "SENSORY ORGANS",
+                    ),
+                    starts_with(
+                        ["DL23", "DL24", "DL25", "DJ30", "DL500", "DJ450"], "ALLERGY"
+                    ),
+                ]
+            ).list.drop_nulls()
+        )"""
+        print(method_code)

specatwrap83/prep/io_handler.py

@@ -0,0 +1,86 @@
+"""
+I/O handler for parquet file processing.
+
+This module provides generic I/O functionality for reading, filtering,
+and writing parquet files. It uses a filter registry pattern to support
+different filter types (diagnosis, procedure, medication, etc.).
+"""
+
+from pathlib import Path
+from typing import Type
+import polars as pl
+
+from .diagnosis import BaseFilter, DiagnosisFilter
+
+
+# Registry mapping filter type strings to filter classes
+FILTER_REGISTRY: dict[str, Type[BaseFilter]] = {
+    "diagnosis": DiagnosisFilter,
+    # Future additions:
+    # "procedure": ProcedureFilter,
+    # "medication": MedicationFilter,
+}
+
+
+def process_parquet_files(
+    input_dir: Path,
+    output_file: Path,
+    filter_type: str,
+    verbose: bool = False,
+):
+    """
+    Process parquet files with a specified filter type.
+
+    This function handles all I/O operations:
+    - Discovers parquet files in the input directory
+    - Lazily loads them using Polars
+    - Applies the specified filter transformation
+    - Collects the results and writes to output file
+
+    Args:
+        input_dir: Directory containing input parquet files
+        output_file: Path to output parquet file
+        filter_type: Type of filter to apply (e.g., "diagnosis", "procedure")
+        verbose: Whether to include verbose processing (currently unused in I/O layer)
+
+    Returns:
+        Tuple of (row_count, column_count, column_names)
+
+    Raises:
+        FileNotFoundError: If no parquet files found in input directory
+        ValueError: If filter_type is not registered
+        PermissionError: If unable to read/write files
+        MemoryError: If insufficient memory to process data
+    """
+    # Validate filter type
+    if filter_type not in FILTER_REGISTRY:
+        available_filters = ", ".join(FILTER_REGISTRY.keys())
+        raise ValueError(
+            f"Unknown filter type '{filter_type}'. "
+            f"Available filters: {available_filters}"
+        )
+
+    # Check if input directory contains parquet files
+    parquet_files = list(input_dir.glob("*.parquet"))
+    if not parquet_files:
+        raise FileNotFoundError(f"No parquet files found in {input_dir}")
+
+    # Create parquet file pattern for lazy loading
+    parquet_pattern = str(input_dir / "*.parquet")
+
+    # Lazy load all parquet files
+    lazy_frame = pl.scan_parquet(parquet_pattern)
+
+    # Instantiate the appropriate filter
+    filter_class = FILTER_REGISTRY[filter_type]
+    filter_instance = filter_class()
+
+    # Apply the filter transformation
+    filtered_lazy_frame = filter_instance.apply(lazy_frame)
+
+    # Collect the lazy frame (materialize the data)
+
+    # Ensure output directory exists
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    filtered_lazy_frame.sink_parquet(output_file)