specatwrap25 0.2.0__tar.gz

specatwrap25-0.2.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.3
+Name: specatwrap25
+Version: 0.2.0
+Summary: A simple wrapper
+Author: Casper Lauge Nørup Koch
+Author-email: Casper Lauge Nørup Koch <kochcasper@gmail.com>
+Requires-Dist: click>=8.3.1
+Requires-Dist: pm4py>=2.7.19.8
+Requires-Dist: polars>=1.38.1
+Requires-Dist: pyreadstat>=1.3.3
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+

specatwrap25-0.2.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "specatwrap25"
+version = "0.2.0"
+description = "A simple wrapper"
+readme = "README.md"
+authors = [
+    { name = "Casper Lauge Nørup Koch", email = "kochcasper@gmail.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.3.1",
+    "pm4py>=2.7.19.8",
+    "polars>=1.38.1",
+    "pyreadstat>=1.3.3",
+]
+
+[project.scripts]
+specatwrap = "specatwrap25:main"
+
+[build-system]
+requires = ["uv_build>=0.10.3,<0.11.0"]
+build-backend = "uv_build"
+
+
+[tool.pyright]
+reportUnannotatedClassAttribute = "none"
+reportImplicitOverride = "none"
+reportUnknownParameterType = "none"
+reportMissingParameterType = "none"
+reportUnknownVariableType = "none"
+reportUnknownMemberType = "none"
+reportUnknownArgumentType = "none"
+reportAny = "none"

specatwrap25-0.2.0/src/specatwrap25/__init__.py

@@ -0,0 +1,39 @@
+"""
+Specatwrap - A wrapper for processing healthcare data.
+
+This module provides a CLI for converting and processing healthcare data files.
+"""
+
+import click
+
+from .sas_converter import sas2parquet
+from .prep import prep
+from .sas_preview import preview
+from .printable import print_file
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli():
+    """
+    Specatwrap - A wrapper for processing healthcare data.
+
+    A command-line tool for processing and converting healthcare data files.
+    """
+    pass
+
+
+# Register command groups
+cli.add_command(sas2parquet)
+cli.add_command(prep)
+cli.add_command(preview)
+cli.add_command(print_file)
+
+
+def main():
+    """Entry point for the CLI application."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

specatwrap25-0.2.0/src/specatwrap25/prep/__init__.py

@@ -0,0 +1,124 @@
+"""
+Prep command group for preprocessing healthcare data files.
+
+This module provides commands for filtering and preprocessing parquet files
+before converting them to XES format.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from .io_handler import process_parquet_files
+
+
+@click.group()
+def prep():
+    """
+    Preprocess and filter healthcare data files.
+
+    Commands in this group help prepare raw data files by filtering,
+    cleaning, and transforming them before further processing.
+    """
+    pass
+
+
+@prep.command()
+@click.option(
+    "-i",
+    "--input",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="Path to directory containing parquet files to process.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_file",
+    type=click.Path(dir_okay=False, file_okay=True, path_type=Path),
+    required=True,
+    help="Path to output parquet file.",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
+def diagnosis(input_dir, output_file, verbose):
+    """
+    Filter and preprocess diagnosis parquet files.
+
+    This command lazily reads all parquet files from a directory, applies
+    filtering and preprocessing transformations, and writes the results to
+    a single output parquet file.
+
+    INPUT: Directory path containing parquet files to process.
+
+    OUTPUT: Path to output parquet file for processed data.
+
+    Example usage:
+
+        specatwrap prep diagnosis -i ./data/parquet_files/ -o ./processed/diagnosis.parquet
+
+        specatwrap prep diagnosis --input ./raw_data/ --output ./clean_data.parquet -v
+    """
+    try:
+        # Display processing information
+        click.echo(f"Input directory: {input_dir}")
+        click.echo(f"Output file: {output_file}")
+        click.echo()
+
+        # Find all parquet files in input directory (for verbose output)
+        if verbose:
+            parquet_pattern = str(input_dir / "*.parquet")
+            click.echo(f"Searching for parquet files: {parquet_pattern}")
+            parquet_files = list(input_dir.glob("*.parquet"))
+            if parquet_files:
+                click.echo(f"Found {len(parquet_files)} parquet file(s):")
+                for f in parquet_files:
+                    click.echo(f"  - {f.name}")
+                click.echo()
+
+        # Process the parquet files using the io_handler
+        click.echo("Loading and applying filters...")
+        with click.progressbar(
+            length=100, label="Processing", show_eta=False, show_percent=True
+        ) as bar:
+            # Call the generic I/O handler with "diagnosis" filter type
+            process_parquet_files(
+                input_dir=input_dir,
+                output_file=output_file,
+                filter_type="diagnosis",
+                verbose=verbose,
+            )
+            bar.update(100)
+
+        # Display success message
+        click.secho("✓ Processing completed successfully!", fg="green", bold=True)
+        click.echo(f"Output file: {output_file}")
+
+        # Display file statistics
+        if output_file.exists():
+            size_mb = output_file.stat().st_size / (1024 * 1024)
+            click.echo(f"File size: {size_mb:.2f} MB")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File or directory not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            "✗ Error: Out of memory. Try processing smaller batches.",
+            fg="red",
+            err=True,
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)

specatwrap25-0.2.0/src/specatwrap25/prep/diagnosis.py

@@ -0,0 +1,424 @@
+"""
+Diagnosis filter classes.
+
+This module provides filter classes for preprocessing diagnosis parquet files.
+"""
+
+from abc import ABC, abstractmethod
+import polars as pl
+
+
+class BaseFilter(ABC):
+    """
+    Abstract base class for parquet file filters.
+
+    Each filter type (diagnosis, procedure, medication, etc.) should inherit
+    from this class and implement the apply() method with their specific
+    filtering and transformation logic.
+    """
+
+    @abstractmethod
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply filtering and preprocessing logic to a LazyFrame.
+
+        Args:
+            lazy_frame: Input Polars LazyFrame to filter/transform
+
+        Returns:
+            Transformed Polars LazyFrame
+        """
+        pass
+
+    def get_name(self) -> str:
+        """
+        Get the name of this filter type.
+
+        Returns:
+            Filter type name (defaults to class name without 'Filter' suffix)
+        """
+        class_name = self.__class__.__name__
+        if class_name.endswith("Filter"):
+            return class_name[:-6].lower()
+        return class_name.lower()
+
+
+class DiagnosisFilter(BaseFilter):
+    """
+    Filter for diagnosis parquet files.
+
+    Applies diagnosis-specific filtering and column transformations:
+    - Filters by birthdate (year > 1980)
+    - Filters by region (Region Sjælland)
+    - Renames columns to standardized event log format
+    """
+
+    CASE_ATTR = {
+        k: f"case:{v}"
+        for k, v in {
+            "BORGER_FOEDSELSDATO": "birthdate",
+            "PNR": "concept:name",
+        }.items()
+    }
+
+    EVENT_ATTR = {
+        "TORRA_DIAG": "concept:name",
+        "ADIAG_TEKST": "diag",
+        "KONT_ANS_GEO_REG_TEKST": "region",
+        "KONT_LPR_ENTITY_ID": "org:id",
+        "KONT_INST_EJERTYPE": "org:type",
+        "KONT_STARTTIDSPUNKT": "time:timestamp",
+        "KONT_SLUTTIDSPUNKT": "end_time",
+        # "BORGER_ALDER_AAR_IND": "patient:age",
+        "PRIORITET_TEKST": "priority",
+        "KONT_TYPE_TEKST": "contact_type",
+    }
+
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply diagnosis-specific filtering and transformations.
+
+        Args:
+            lazy_frame: Input LazyFrame containing diagnosis data
+
+        Returns:
+            Filtered and transformed LazyFrame
+        """
+
+        lf = self._add_torra_diag(lf)
+
+        return (
+            lf.filter(pl.col("TORRA_DIAG").list.len() > 0)
+            .with_columns(pl.col("TORRA_DIAG").list.join(", "))
+            .select(list((self.CASE_ATTR | self.EVENT_ATTR).keys()))
+            .rename(self.CASE_ATTR | self.EVENT_ATTR)
+        )
+
+    def _add_torra_diag(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        col = pl.col("ADIAG")
+
+        def starts_with(prefixes: list[str], label: str) -> pl.Expr:
+            pattern = f"^(?:{'|'.join(prefixes)})"
+            return (
+                pl.when(col.str.contains(pattern))
+                .then(pl.lit(label))
+                .otherwise(pl.lit(None))
+            )
+
+        return lazy_frame.with_columns(
+            TORRA_DIAG=pl.concat_list(
+                [
+                    starts_with(
+                        ["DJ41", "DJ42", "DJ43", "DJ44", "DJ45", "DJ46"], "LUNG"
+                    ),
+                    starts_with(
+                        [
+                            "DL405",
+                            "DM05",
+                            "DM06",
+                            "DM07",
+                            "DM15",
+                            "DM16",
+                            "DM17",
+                            "DM45",
+                            "DM47",
+                            "DM50",
+                            "DM51",
+                            "DM53",
+                            "DM54",
+                            "DM80",
+                            "DM81",
+                            "DM82",
+                        ],
+                        "MUSCULOSKELETAL",
+                    ),
+                    starts_with(
+                        ["DE03", "DE05", "DE10", "DE11", "DE12", "DE13", "DE14"],
+                        "ENDOCRINE",
+                    ),
+                    starts_with(["DE03", "DE05"], "THYROIDEA"),
+                    starts_with(["DE10", "DE11", "DE12", "DE13", "DE14"], "DIABETES"),
+                    starts_with(["DG30", "DG318", "DG319", "DF"], "MENTAL"),
+                    # CANCER: Matches DC but explicitly excludes DC44
+                    pl.when(col.str.contains("^DC") & ~col.str.contains("^DC44"))
+                    .then(pl.lit("CANCER"))
+                    .otherwise(pl.lit(None)),
+                    starts_with(
+                        [
+                            "DI60",
+                            "DI61",
+                            "DI62",
+                            "DI63",
+                            "DI64",
+                            "DI69",
+                            "DG20",
+                            "DG35",
+                            "DG40",
+                            "DG43",
+                        ],
+                        "NEUROLOGICAL",
+                    ),
+                    starts_with(
+                        [
+                            "DK30",
+                            "DK50",
+                            "DK51",
+                            "DK58",
+                            "DK70",
+                            "DK71",
+                            "DK72",
+                            "DK73",
+                            "DK74",
+                            "DK75",
+                            "DK76",
+                            "DK860",
+                            "DK861",
+                        ],
+                        "GASTROINTESTINAL",
+                    ),
+                    starts_with(
+                        [
+                            "DI20",
+                            "DI21",
+                            "DI22",
+                            "DI23",
+                            "DI24",
+                            "DI25",
+                            "DI47",
+                            "DI48",
+                            "DI49",
+                            "DI50",
+                            "DI05",
+                            "DI06",
+                            "DI07",
+                            "DI08",
+                            "DI34",
+                            "DI35",
+                            "DI36",
+                            "DI37",
+                            "DI441",
+                            "DI442",
+                            "DI443",
+                            "DI444",
+                            "DI445",
+                            "DI446",
+                            "DI447",
+                            "DI452",
+                            "DI453",
+                            "DI454",
+                            "DI455",
+                            "DI456",
+                            "DI457",
+                            "DI458",
+                            "DI459",
+                        ],
+                        "CARDIOVASCULAR",
+                    ),
+                    starts_with(
+                        [
+                            "DN03",
+                            "DN04",
+                            "DN05",
+                            "DN11",
+                            "DN12",
+                            "DN18",
+                            "DN19",
+                            "DZ49",
+                            "DN80",
+                            "DZ992",
+                            "DN393",
+                            "DN394",
+                        ],
+                        "GENITURINARY",
+                    ),
+                    starts_with(
+                        [
+                            "DH40",
+                            "DH91",
+                            "DL40",
+                            "DH540",
+                            "DH541",
+                            "DH542",
+                            "DH543",
+                            "DH547",
+                            "DH900",
+                            "DH902",
+                            "DH903",
+                            "DH905",
+                            "DH906",
+                            "DH908",
+                        ],
+                        "SENSORY ORGANS",
+                    ),
+                    starts_with(
+                        ["DL23", "DL24", "DL25", "DJ30", "DL500", "DJ450"], "ALLERGY"
+                    ),
+                ]
+            ).list.drop_nulls()
+        )
+
+    def print_torra_diag_method(self) -> None:
+        """Print the _add_torra_diag method to stdout."""
+        method_code = """    def _add_torra_diag(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        col = pl.col("ADIAG")
+
+        def starts_with(prefixes: list[str], label: str) -> pl.Expr:
+            pattern = f"^(?:{'|'.join(prefixes)})"
+            return (
+                pl.when(col.str.contains(pattern))
+                .then(pl.lit(label))
+                .otherwise(pl.lit(None))
+            )
+
+        return lazy_frame.with_columns(
+            TORRA_DIAG=pl.concat_list(
+                [
+                    starts_with(
+                        ["DJ41", "DJ42", "DJ43", "DJ44", "DJ45", "DJ46"], "LUNG"
+                    ),
+                    starts_with(
+                        [
+                            "DL405",
+                            "DM05",
+                            "DM06",
+                            "DM07",
+                            "DM15",
+                            "DM16",
+                            "DM17",
+                            "DM45",
+                            "DM47",
+                            "DM50",
+                            "DM51",
+                            "DM53",
+                            "DM54",
+                            "DM80",
+                            "DM81",
+                            "DM82",
+                        ],
+                        "MUSCULOSKELETAL",
+                    ),
+                    starts_with(
+                        ["DE03", "DE05", "DE10", "DE11", "DE12", "DE13", "DE14"],
+                        "ENDOCRINE",
+                    ),
+                    starts_with(["DE03", "DE05"], "THYROIDEA"),
+                    starts_with(["DE10", "DE11", "DE12", "DE13", "DE14"], "DIABETES"),
+                    starts_with(["DG30", "DG318", "DG319", "DF"], "MENTAL"),
+                    # CANCER: Matches DC but explicitly excludes DC44
+                    pl.when(col.str.contains("^DC") & ~col.str.contains("^DC44"))
+                    .then(pl.lit("CANCER"))
+                    .otherwise(pl.lit(None)),
+                    starts_with(
+                        [
+                            "DI60",
+                            "DI61",
+                            "DI62",
+                            "DI63",
+                            "DI64",
+                            "DI69",
+                            "DG20",
+                            "DG35",
+                            "DG40",
+                            "DG43",
+                        ],
+                        "NEUROLOGICAL",
+                    ),
+                    starts_with(
+                        [
+                            "DK30",
+                            "DK50",
+                            "DK51",
+                            "DK58",
+                            "DK70",
+                            "DK71",
+                            "DK72",
+                            "DK73",
+                            "DK74",
+                            "DK75",
+                            "DK76",
+                            "DK860",
+                            "DK861",
+                        ],
+                        "GASTROINTESTINAL",
+                    ),
+                    starts_with(
+                        [
+                            "DI20",
+                            "DI21",
+                            "DI22",
+                            "DI23",
+                            "DI24",
+                            "DI25",
+                            "DI47",
+                            "DI48",
+                            "DI49",
+                            "DI50",
+                            "DI05",
+                            "DI06",
+                            "DI07",
+                            "DI08",
+                            "DI34",
+                            "DI35",
+                            "DI36",
+                            "DI37",
+                            "DI441",
+                            "DI442",
+                            "DI443",
+                            "DI444",
+                            "DI445",
+                            "DI446",
+                            "DI447",
+                            "DI452",
+                            "DI453",
+                            "DI454",
+                            "DI455",
+                            "DI456",
+                            "DI457",
+                            "DI458",
+                            "DI459",
+                        ],
+                        "CARDIOVASCULAR",
+                    ),
+                    starts_with(
+                        [
+                            "DN03",
+                            "DN04",
+                            "DN05",
+                            "DN11",
+                            "DN12",
+                            "DN18",
+                            "DN19",
+                            "DZ49",
+                            "DN80",
+                            "DZ992",
+                            "DN393",
+                            "DN394",
+                        ],
+                        "GENITURINARY",
+                    ),
+                    starts_with(
+                        [
+                            "DH40",
+                            "DH91",
+                            "DL40",
+                            "DH540",
+                            "DH541",
+                            "DH542",
+                            "DH543",
+                            "DH547",
+                            "DH900",
+                            "DH902",
+                            "DH903",
+                            "DH905",
+                            "DH906",
+                            "DH908",
+                        ],
+                        "SENSORY ORGANS",
+                    ),
+                    starts_with(
+                        ["DL23", "DL24", "DL25", "DJ30", "DL500", "DJ450"], "ALLERGY"
+                    ),
+                ]
+            ).list.drop_nulls()
+        )"""
+        print(method_code)

specatwrap25-0.2.0/src/specatwrap25/prep/io_handler.py

@@ -0,0 +1,86 @@
+"""
+I/O handler for parquet file processing.
+
+This module provides generic I/O functionality for reading, filtering,
+and writing parquet files. It uses a filter registry pattern to support
+different filter types (diagnosis, procedure, medication, etc.).
+"""
+
+from pathlib import Path
+from typing import Type
+import polars as pl
+
+from .diagnosis import BaseFilter, DiagnosisFilter
+
+
+# Registry mapping filter type strings to filter classes
+FILTER_REGISTRY: dict[str, Type[BaseFilter]] = {
+    "diagnosis": DiagnosisFilter,
+    # Future additions:
+    # "procedure": ProcedureFilter,
+    # "medication": MedicationFilter,
+}
+
+
+def process_parquet_files(
+    input_dir: Path,
+    output_file: Path,
+    filter_type: str,
+    verbose: bool = False,
+):
+    """
+    Process parquet files with a specified filter type.
+
+    This function handles all I/O operations:
+    - Discovers parquet files in the input directory
+    - Lazily loads them using Polars
+    - Applies the specified filter transformation
+    - Collects the results and writes to output file
+
+    Args:
+        input_dir: Directory containing input parquet files
+        output_file: Path to output parquet file
+        filter_type: Type of filter to apply (e.g., "diagnosis", "procedure")
+        verbose: Whether to include verbose processing (currently unused in I/O layer)
+
+    Returns:
+        Tuple of (row_count, column_count, column_names)
+
+    Raises:
+        FileNotFoundError: If no parquet files found in input directory
+        ValueError: If filter_type is not registered
+        PermissionError: If unable to read/write files
+        MemoryError: If insufficient memory to process data
+    """
+    # Validate filter type
+    if filter_type not in FILTER_REGISTRY:
+        available_filters = ", ".join(FILTER_REGISTRY.keys())
+        raise ValueError(
+            f"Unknown filter type '{filter_type}'. "
+            f"Available filters: {available_filters}"
+        )
+
+    # Check if input directory contains parquet files
+    parquet_files = list(input_dir.glob("*.parquet"))
+    if not parquet_files:
+        raise FileNotFoundError(f"No parquet files found in {input_dir}")
+
+    # Create parquet file pattern for lazy loading
+    parquet_pattern = str(input_dir / "*.parquet")
+
+    # Lazy load all parquet files
+    lazy_frame = pl.scan_parquet(parquet_pattern)
+
+    # Instantiate the appropriate filter
+    filter_class = FILTER_REGISTRY[filter_type]
+    filter_instance = filter_class()
+
+    # Apply the filter transformation
+    filtered_lazy_frame = filter_instance.apply(lazy_frame)
+
+    # Collect the lazy frame (materialize the data)
+
+    # Ensure output directory exists
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    filtered_lazy_frame.sink_parquet(output_file)

specatwrap25-0.2.0/src/specatwrap25/printable/__init__.py

@@ -0,0 +1,76 @@
+"""
+CLI command to print Python files from the printable folder.
+
+This allows easy copying of scripts on the remote protected machine.
+"""
+
+from pathlib import Path
+
+import click
+
+
+@click.command("print")
+@click.argument("filename", required=False)
+@click.option("--list", "-l", is_flag=True, help="List all available printable files")
+def print_file(filename: str | None, list: bool) -> None:
+    """
+    Print Python files from the printable folder to stdout.
+
+    This command helps transfer code to the restricted environment by printing
+    file contents that can be easily copied.
+
+    Usage:
+
+        specatwrap print diagnosis_kpi > diagnosis_kpi.py
+
+        specatwrap print --list
+
+    \b
+    Arguments:
+        FILENAME: Name of the file to print (without .py extension)
+    """
+    printable_dir = Path(__file__).parent
+
+    # List available files
+    if list:
+        python_files = sorted(printable_dir.glob("*.py"))
+        python_files = [f for f in python_files if f.name != "__init__.py"]
+
+        if not python_files:
+            click.echo("No printable files available.")
+            return
+
+        click.echo("Available printable files:")
+        for file in python_files:
+            click.echo(f"  - {file.stem}")
+        return
+
+    # Print specific file
+    if not filename:
+        click.echo(
+            "Error: Please provide a filename or use --list to see available files."
+        )
+        click.echo("Usage: specatwrap print <filename>")
+        click.echo("       specatwrap print --list")
+        raise click.Abort()
+
+    # Add .py extension if not present
+    if not filename.endswith(".py"):
+        filename = f"{filename}.py"
+
+    file_path = printable_dir / filename
+
+    if not file_path.exists():
+        click.echo(f"Error: File '{filename}' not found in printable folder.")
+        click.echo("\nUse 'specatwrap print --list' to see available files.")
+        raise click.Abort()
+
+    # Read and write to tmp.txt file
+    try:
+        content = file_path.read_text()
+        output_path = Path("tmp.txt")
+        output_path.write_text(content)
+        click.echo(f"Content written to {output_path.absolute()}")
+    except Exception as e:
+        click.echo(f"Error reading/writing file: {e}")
+        raise click.Abort()

specatwrap25-0.2.0/src/specatwrap25/printable/diagnosis_kpi.py

@@ -0,0 +1,78 @@
+import polars as pl
+
+lf = pl.scan_parquet("diag.parquet")
+lf_death = pl.scan_parquet("death.parquet")
+
+lf_death = lf_death.rename(
+    {"PNR": "case:concept:name", "DODDATO": "start_timestamp"}
+).with_columns(
+    pl.lit("DØD").alias("concept:name"),
+    pl.col("start_timestamp").cast(pl.Datetime("us")),
+)
+
+# 1. Deduplication & Sorting
+lf = (
+    pl.concat([lf, lf_death], how="diagonal")
+    .sort("start_timestamp")  # Ensures keep='first' grabs the earliest
+    .unique(subset=["case:concept:name", "concept:name"], keep="first")
+    .sort(
+        ["case:concept:name", "start_timestamp"]
+    )  # Groups data physically for cache-friendly window functions
+)
+
+# 2. Combined Window Functions (No Temp Columns)
+
+lf = lf.with_columns(
+    pl.col("start_timestamp")
+    .diff()
+    .over("case:concept:name")
+    .filter(pl.col("concept:name") != "DØD")
+    .mean()
+    .over("case:concept:name")
+    .alias("case:avg_inter_diagnosis_time"),
+    pl.when(
+        (pl.col("concept:name") == "DØD") & (pl.len().over("case:concept:name") > 1)
+    )
+    .then(
+        pl.col("start_timestamp")
+        - pl.col("start_timestamp").shift(1).over("case:concept:name")
+    )
+    .otherwise(None)
+    .alias("case:time_to_death_from_last_diag"),
+)
+
+# 3. Aggregation without maintain_order
+lf_variants = lf.group_by("case:concept:name").agg(
+    pl.col("concept:name").sort_by("start_timestamp").str.join(">").alias("variants"),
+    pl.col("case:avg_inter_diagnosis_time").first(),
+    pl.col(
+        "case:time_to_death_from_last_diag"
+    ).last(),  # .last() is safer here since DØD is the last event
+)
+
+# 4. Inline KPI Calculation
+lf_variants = lf_variants.with_columns(
+    pl.col("variants")
+    .str.contains("DØD")
+    .mean()  # Boolean mean implicitly calculates the ratio
+    .over(
+        pl.col("variants").str.replace(r">DØD$", "")
+    )  # Group by the expression directly
+    .alias("mortality_kpi")
+)
+
+# 5. Summary
+lf_summary = lf_variants.group_by("variants").agg(
+    pl.col("mortality_kpi").first().alias("mortality_kpi"),
+    pl.len().alias("count"),
+    pl.col("case:avg_inter_diagnosis_time").mean().alias("avg_inter_diagnosis_time"),
+    pl.col("case:time_to_death_from_last_diag")
+    .mean()
+    .alias("avg_inter_diagnosis_time_to_death"),
+)
+
+# 6. Broadcast
+patient_kpi_map = lf_variants.select(
+    "case:concept:name", pl.col("mortality_kpi").alias("case:mortality_kpi")
+)
+lf = lf.join(patient_kpi_map, on="case:concept:name", how="left")

specatwrap25-0.2.0/src/specatwrap25/sas_converter.py

@@ -0,0 +1,299 @@
+"""
+SAS to Parquet converter module.
+
+This module provides functionality to convert large SAS7BDAT files to Parquet format
+in a memory-efficient way by processing the data in chunks.
+"""
+
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+import shutil
+import click
+import pyreadstat
+import polars as pl
+
+
+# Table-specific column mappings
+TABLE_TYPE_COLUMNS = {
+    "diagnosis": [
+        "BORGER_FOEDSELSDATO",
+        "PNR",
+        "ADIAG",
+        "ADIAG_TEKST",
+        "KONT_ANS_GEO_REG_TEKST",
+        "KONT_LPR_ENTITY_ID",
+        "KONT_INST_EJERTYPE",
+        "KONT_STARTTIDSPUNKT",
+        "KONT_SLUTTIDSPUNKT",
+        "PRIORITET_TEKST",
+        "KONT_TYPE_TEKST",
+    ],
+}
+
+
+def convert_sas_to_parquet(
+    input_file: Path,
+    output_path: Path,
+    chunk_size: int = 100000,
+    compression: str = "zstd",
+    encoding: Optional[str] = None,
+    overwrite: bool = False,
+    verbose: bool = False,
+    cols: Optional[list[str]] = None,
+):
+    """
+    Convert a SAS7BDAT file to Parquet format in chunks.
+
+    Parameters
+    ----------
+    input_file : Path
+        Path to the SAS file (.sas7bdat)
+    output_path : Path
+        Path to output directory for Parquet files
+    chunk_size : int, optional
+        Number of rows to read per chunk (default: 100000)
+    compression : str, optional
+        Parquet compression algorithm (default: 'zstd')
+    encoding : str, optional
+        File encoding (default: auto-detect)
+    overwrite : bool, optional
+        Whether to overwrite existing output directory (default: False)
+    verbose : bool, optional
+        Enable verbose output (default: False)
+    cols : list[str], optional
+        List of columns to read from the SAS file (default: None, reads all columns)
+
+    Raises
+    ------
+    FileNotFoundError
+        If the input file does not exist
+    ValueError
+        If the input file is not a .sas7bdat file or output path is invalid
+    PermissionError
+        If there are permission issues with files/directories
+    MemoryError
+        If there's insufficient memory (suggest smaller chunk_size)
+    """
+    # Validate input file
+    if not input_file.exists():
+        click.secho(f"✗ Error: Input file not found: {input_file}", fg="red", err=True)
+        sys.exit(1)
+
+    if input_file.suffix.lower() != ".sas7bdat":
+        click.secho("✗ Error: Input file must be a .sas7bdat file", fg="red", err=True)
+        sys.exit(1)
+
+    # Handle output directory
+    output_path = Path(output_path)
+
+    if output_path.exists():
+        if not output_path.is_dir():
+            click.secho(
+                f"✗ Error: Output path exists and is not a directory: {output_path}",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if overwrite:
+            if verbose:
+                click.echo(f"Removing existing directory: {output_path}")
+            shutil.rmtree(output_path)
+            output_path.mkdir(parents=True, exist_ok=True)
+        else:
+            # Check if directory has parquet files
+            existing_parquet = list(output_path.glob("*.parquet"))
+            if existing_parquet:
+                click.secho(
+                    f"⚠ Warning: Output directory already contains {len(existing_parquet)} .parquet file(s)",
+                    fg="yellow",
+                )
+                if not click.confirm(
+                    "Do you want to continue and potentially mix files?"
+                ):
+                    click.echo("Aborted.")
+                    sys.exit(0)
+    else:
+        output_path.mkdir(parents=True, exist_ok=True)
+
+    # Display processing information
+    click.echo(f"Processing: {input_file}")
+    click.echo(f"Output directory: {output_path}")
+    if verbose:
+        click.echo(f"Chunk size: {chunk_size:,} rows")
+        click.echo(f"Compression: {compression}")
+        click.echo(f"Encoding: {encoding if encoding else 'auto-detect'}")
+    click.echo()
+
+    # Process file in chunks
+    click.echo("Converting SAS to Parquet...")
+
+    reader = pyreadstat.read_file_in_chunks(
+        pyreadstat.read_sas7bdat,
+        str(input_file),
+        chunksize=chunk_size,
+        encoding=encoding,
+        output_format="polars",
+        usecols=cols,
+    )
+
+    chunk_num = 0
+    total_rows = 0
+    num_columns = 0
+
+    for df_polars, meta in reader:
+        # Start timing for this chunk
+        chunk_start_time = time.time()
+
+        # Generate chunk filename with zero-padded numbering
+        chunk_filename = f"chunk_{chunk_num:04d}.parquet"
+        chunk_path = output_path / chunk_filename
+
+        # Write chunk to parquet
+        df_polars.write_parquet(chunk_path, compression=compression)
+
+        # Calculate elapsed time
+        chunk_elapsed_time = time.time() - chunk_start_time
+
+        # Update statistics
+        chunk_rows = len(df_polars)
+        total_rows += chunk_rows
+        num_columns = len(meta.column_names)
+        chunk_num += 1
+
+        if verbose:
+            click.echo(
+                f"  Wrote chunk {chunk_num}: {chunk_rows:,} rows → {chunk_filename} (took {chunk_elapsed_time:.2f}s)"
+            )
+        else:
+            click.echo(
+                f"  Wrote chunk {chunk_num}: {chunk_rows:,} rows → {chunk_filename} (took {chunk_elapsed_time:.2f}s)"
+            )
+
+    # Calculate file sizes
+    input_size = input_file.stat().st_size / (1024 * 1024)  # MB
+    output_files = list(output_path.glob("*.parquet"))
+    output_size = sum(f.stat().st_size for f in output_files) / (1024 * 1024)  # MB
+    compression_ratio = input_size / output_size if output_size > 0 else 0
+
+    # Display success message
+    click.echo()
+    click.secho("✓ Conversion completed successfully!", fg="green", bold=True)
+    click.echo(f"Input file:      {input_size:.2f} MB ({input_file.name})")
+    click.echo(f"Output files:    {output_size:.2f} MB ({len(output_files)} chunks)")
+    click.echo(f"Compression:     {compression_ratio:.2f}x")
+    click.echo(f"Total rows:      {total_rows:,}")
+    click.echo(f"Columns:         {num_columns}")
+    click.echo(f"Chunk size:      {chunk_size:,} rows/chunk")
+    click.echo()
+    click.echo("To read the data:")
+    click.echo(f"  import polars as pl")
+    click.echo(f"  df = pl.read_parquet('{output_path}/*.parquet')")
+
+
+@click.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_path", type=click.Path(path_type=Path))
+@click.option(
+    "--table-type",
+    type=click.Choice(["diagnosis"], case_sensitive=False),
+    default=None,
+    help="Type of table to process (determines which columns to extract). If not specified, all columns are read.",
+)
+@click.option(
+    "-c",
+    "--chunk-size",
+    type=int,
+    default=100000,
+    help="Number of rows to read per chunk (default: 100000)",
+)
+@click.option(
+    "--compression",
+    type=click.Choice(["zstd", "lz4", "snappy", "gzip"], case_sensitive=False),
+    default="zstd",
+    help="Parquet compression algorithm (default: zstd)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+@click.option(
+    "--overwrite",
+    is_flag=True,
+    default=False,
+    help="Overwrite existing output directory if it exists",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output")
+def sas2parquet(
+    input_file,
+    output_path,
+    table_type,
+    chunk_size,
+    compression,
+    encoding,
+    overwrite,
+    verbose,
+):
+    """
+    Convert a SAS file to Parquet format in chunks.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    OUTPUT_PATH: Path to output directory for Parquet files
+
+    This command processes large SAS files in chunks to minimize memory usage.
+    Each chunk is written to a separate Parquet file in the output directory.
+
+    Example usage:
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --table-type diagnosis
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --chunk-size 50000 -v
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --table-type diagnosis --compression lz4
+
+    Reading the data later with Polars:
+
+        import polars as pl
+
+        df = pl.read_parquet("output_dir/*.parquet")
+    """
+    # Get columns for the specified table type (None means all columns)
+    cols = TABLE_TYPE_COLUMNS.get(table_type.lower()) if table_type else None
+
+    try:
+        convert_sas_to_parquet(
+            input_file=input_file,
+            output_path=output_path,
+            chunk_size=chunk_size,
+            compression=compression,
+            encoding=encoding,
+            overwrite=overwrite,
+            verbose=verbose,
+            cols=cols,
+        )
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            f"✗ Error: Out of memory. Try a smaller --chunk-size", fg="red", err=True
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)

specatwrap25-0.2.0/src/specatwrap25/sas_preview.py

@@ -0,0 +1,93 @@
+"""
+SAS file preview module.
+
+This module provides functionality to quickly preview the first N rows
+of a SAS7BDAT file for inspection purposes.
+"""
+
+import click
+from pathlib import Path
+import sys
+import pyreadstat
+
+
+@click.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "-n",
+    "--num-rows",
+    type=int,
+    default=10,
+    help="Number of rows to display (default: 10)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+def preview(input_file, num_rows, encoding):
+    """
+    Preview the first N rows of a SAS file.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    This command quickly reads and displays the first few rows of a SAS file
+    to help you inspect the data structure and content without converting
+    the entire file.
+
+    Example usage:
+
+        specatwrap preview input.sas7bdat
+
+        specatwrap preview input.sas7bdat --num-rows 20
+
+        specatwrap preview input.sas7bdat --encoding latin1
+    """
+    try:
+        # Validate file exists
+        if not input_file.exists():
+            click.secho(f"✗ Error: File not found: {input_file}", fg="red", err=True)
+            sys.exit(1)
+
+        if input_file.suffix.lower() != ".sas7bdat":
+            click.secho(
+                "✗ Error: Input file must be a .sas7bdat file", fg="red", err=True
+            )
+            sys.exit(1)
+
+        click.echo(f"Reading first {num_rows} rows from: {input_file.name}")
+        click.echo()
+
+        # Read first N rows using pyreadstat
+        df, meta = pyreadstat.read_sas7bdat(
+            str(input_file),
+            row_limit=num_rows,
+            encoding=encoding,
+            output_format="polars",
+        )
+
+        # Display metadata
+        click.echo(f"Total columns: {len(meta.column_names)}")
+        click.echo(f"Column names: {', '.join(meta.column_names)}")
+        if encoding:
+            click.echo(f"Encoding: {encoding}")
+        click.echo()
+
+        # Display the data using Polars' pretty print
+        click.echo(str(df))
+        click.echo()
+        click.secho(f"✓ Showing {len(df)} row(s)", fg="green")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        sys.exit(1)