specatwrap8 0.2.0__tar.gz

specatwrap8-0.2.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.3
+Name: specatwrap8
+Version: 0.2.0
+Summary: A simple wrapper
+Author: Casper Lauge Nørup Koch
+Author-email: Casper Lauge Nørup Koch <kochcasper@gmail.com>
+Requires-Dist: click>=8.3.1
+Requires-Dist: pm4py>=2.7.19.8
+Requires-Dist: polars>=1.38.1
+Requires-Dist: pyreadstat>=1.3.3
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+

specatwrap8-0.2.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "specatwrap8"
+version = "0.2.0"
+description = "A simple wrapper"
+readme = "README.md"
+authors = [
+    { name = "Casper Lauge Nørup Koch", email = "kochcasper@gmail.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.3.1",
+    "pm4py>=2.7.19.8",
+    "polars>=1.38.1",
+    "pyreadstat>=1.3.3",
+]
+
+[project.scripts]
+specatwrap = "specatwrap8:main"
+
+[build-system]
+requires = ["uv_build>=0.10.3,<0.11.0"]
+build-backend = "uv_build"
+
+
+[tool.pyright]
+reportUnannotatedClassAttribute = "none"
+reportImplicitOverride = "none"
+reportUnknownParameterType = "none"
+reportMissingParameterType = "none"
+reportUnknownVariableType = "none"
+reportUnknownMemberType = "none"
+reportUnknownArgumentType = "none"
+reportAny = "none"

specatwrap8-0.2.0/src/specatwrap8/__init__.py

@@ -0,0 +1,208 @@
+"""
+Specatwrap - A wrapper for processing healthcare data.
+
+This module provides a CLI for converting and processing healthcare data files.
+"""
+
+import click
+from pathlib import Path
+import sys
+import pyreadstat
+
+from .sas_converter import convert_sas_to_parquet
+from .prep import prep
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli():
+    """
+    Specatwrap - A wrapper for processing healthcare data.
+
+    A command-line tool for processing and converting healthcare data files.
+    """
+    pass
+
+
+# Register command groups
+cli.add_command(prep)
+
+
+@cli.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_path", type=click.Path(path_type=Path))
+@click.option(
+    "-c",
+    "--chunk-size",
+    type=int,
+    default=100000,
+    help="Number of rows to read per chunk (default: 100000)",
+)
+@click.option(
+    "--compression",
+    type=click.Choice(["zstd", "lz4", "snappy", "gzip"], case_sensitive=False),
+    default="zstd",
+    help="Parquet compression algorithm (default: zstd)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+@click.option(
+    "--overwrite",
+    is_flag=True,
+    default=False,
+    help="Overwrite existing output directory if it exists",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output")
+def sas2parquet(
+    input_file, output_path, chunk_size, compression, encoding, overwrite, verbose
+):
+    """
+    Convert a SAS file to Parquet format in chunks.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    OUTPUT_PATH: Path to output directory for Parquet files
+
+    This command processes large SAS files in chunks to minimize memory usage.
+    Each chunk is written to a separate Parquet file in the output directory.
+
+    Example usage:
+
+        specatwrap sas2parquet input.sas7bdat output_dir/
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --chunk-size 50000 -v
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --compression lz4
+
+    Reading the data later with Polars:
+
+        import polars as pl
+
+        df = pl.read_parquet("output_dir/*.parquet")
+    """
+    try:
+        convert_sas_to_parquet(
+            input_file=input_file,
+            output_path=output_path,
+            chunk_size=chunk_size,
+            compression=compression,
+            encoding=encoding,
+            overwrite=overwrite,
+            verbose=verbose,
+        )
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            f"✗ Error: Out of memory. Try a smaller --chunk-size", fg="red", err=True
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)
+
+
+@cli.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "-n",
+    "--num-rows",
+    type=int,
+    default=10,
+    help="Number of rows to display (default: 10)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+def preview(input_file, num_rows, encoding):
+    """
+    Preview the first N rows of a SAS file.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    This command quickly reads and displays the first few rows of a SAS file
+    to help you inspect the data structure and content without converting
+    the entire file.
+
+    Example usage:
+
+        specatwrap preview input.sas7bdat
+
+        specatwrap preview input.sas7bdat --num-rows 20
+
+        specatwrap preview input.sas7bdat --encoding latin1
+    """
+    try:
+        # Validate file exists
+        if not input_file.exists():
+            click.secho(f"✗ Error: File not found: {input_file}", fg="red", err=True)
+            sys.exit(1)
+
+        if input_file.suffix.lower() != ".sas7bdat":
+            click.secho(
+                "✗ Error: Input file must be a .sas7bdat file", fg="red", err=True
+            )
+            sys.exit(1)
+
+        click.echo(f"Reading first {num_rows} rows from: {input_file.name}")
+        click.echo()
+
+        # Read first N rows using pyreadstat
+        df, meta = pyreadstat.read_sas7bdat(
+            str(input_file),
+            row_limit=num_rows,
+            encoding=encoding,
+            output_format="polars",
+        )
+
+        # Display metadata
+        click.echo(f"Total columns: {len(meta.column_names)}")
+        click.echo(f"Column names: {', '.join(meta.column_names)}")
+        if encoding:
+            click.echo(f"Encoding: {encoding}")
+        click.echo()
+
+        # Display the data using Polars' pretty print
+        click.echo(str(df))
+        click.echo()
+        click.secho(f"✓ Showing {len(df)} row(s)", fg="green")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        sys.exit(1)
+
+
+def main():
+    """Entry point for the CLI application."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

specatwrap8-0.2.0/src/specatwrap8/prep/__init__.py

@@ -0,0 +1,124 @@
+"""
+Prep command group for preprocessing healthcare data files.
+
+This module provides commands for filtering and preprocessing parquet files
+before converting them to XES format.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from .io_handler import process_parquet_files
+
+
+@click.group()
+def prep():
+    """
+    Preprocess and filter healthcare data files.
+
+    Commands in this group help prepare raw data files by filtering,
+    cleaning, and transforming them before further processing.
+    """
+    pass
+
+
+@prep.command()
+@click.option(
+    "-i",
+    "--input",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="Path to directory containing parquet files to process.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_file",
+    type=click.Path(dir_okay=False, file_okay=True, path_type=Path),
+    required=True,
+    help="Path to output parquet file.",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
+def diagnosis(input_dir, output_file, verbose):
+    """
+    Filter and preprocess diagnosis parquet files.
+
+    This command lazily reads all parquet files from a directory, applies
+    filtering and preprocessing transformations, and writes the results to
+    a single output parquet file.
+
+    INPUT: Directory path containing parquet files to process.
+
+    OUTPUT: Path to output parquet file for processed data.
+
+    Example usage:
+
+        specatwrap prep diagnosis -i ./data/parquet_files/ -o ./processed/diagnosis.parquet
+
+        specatwrap prep diagnosis --input ./raw_data/ --output ./clean_data.parquet -v
+    """
+    try:
+        # Display processing information
+        click.echo(f"Input directory: {input_dir}")
+        click.echo(f"Output file: {output_file}")
+        click.echo()
+
+        # Find all parquet files in input directory (for verbose output)
+        if verbose:
+            parquet_pattern = str(input_dir / "*.parquet")
+            click.echo(f"Searching for parquet files: {parquet_pattern}")
+            parquet_files = list(input_dir.glob("*.parquet"))
+            if parquet_files:
+                click.echo(f"Found {len(parquet_files)} parquet file(s):")
+                for f in parquet_files:
+                    click.echo(f"  - {f.name}")
+                click.echo()
+
+        # Process the parquet files using the io_handler
+        click.echo("Loading and applying filters...")
+        with click.progressbar(
+            length=100, label="Processing", show_eta=False, show_percent=True
+        ) as bar:
+            # Call the generic I/O handler with "diagnosis" filter type
+            process_parquet_files(
+                input_dir=input_dir,
+                output_file=output_file,
+                filter_type="diagnosis",
+                verbose=verbose,
+            )
+            bar.update(100)
+
+        # Display success message
+        click.secho("✓ Processing completed successfully!", fg="green", bold=True)
+        click.echo(f"Output file: {output_file}")
+
+        # Display file statistics
+        if output_file.exists():
+            size_mb = output_file.stat().st_size / (1024 * 1024)
+            click.echo(f"File size: {size_mb:.2f} MB")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File or directory not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            "✗ Error: Out of memory. Try processing smaller batches.",
+            fg="red",
+            err=True,
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)

specatwrap8-0.2.0/src/specatwrap8/prep/diagnosis.py

@@ -0,0 +1,94 @@
+"""
+Diagnosis filter classes.
+
+This module provides filter classes for preprocessing diagnosis parquet files.
+"""
+
+from abc import ABC, abstractmethod
+import polars as pl
+
+
+class BaseFilter(ABC):
+    """
+    Abstract base class for parquet file filters.
+
+    Each filter type (diagnosis, procedure, medication, etc.) should inherit
+    from this class and implement the apply() method with their specific
+    filtering and transformation logic.
+    """
+
+    @abstractmethod
+    def apply(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply filtering and preprocessing logic to a LazyFrame.
+
+        Args:
+            lazy_frame: Input Polars LazyFrame to filter/transform
+
+        Returns:
+            Transformed Polars LazyFrame
+        """
+        pass
+
+    def get_name(self) -> str:
+        """
+        Get the name of this filter type.
+
+        Returns:
+            Filter type name (defaults to class name without 'Filter' suffix)
+        """
+        class_name = self.__class__.__name__
+        if class_name.endswith("Filter"):
+            return class_name[:-6].lower()
+        return class_name.lower()
+
+
+class DiagnosisFilter(BaseFilter):
+    """
+    Filter for diagnosis parquet files.
+
+    Applies diagnosis-specific filtering and column transformations:
+    - Filters by birthdate (year > 1980)
+    - Filters by region (Region Sjælland)
+    - Renames columns to standardized event log format
+    """
+
+    CASE_ATTR = {
+        k: f"case:{v}"
+        for k, v in {
+            "BORGER_FOEDSELSDATO": "patient:birthdate",
+            "Personnummer": "concept:name",
+        }.items()
+    }
+
+    EVENT_ATTR = {
+        "ADIAG": "concept:name",
+        "KONT_ANS_GEO_REG_TEKST": "org:group",
+        "KONT_LPR_ENTITY_ID": "org:resource",
+        "KONT_INST_EJERTYPE": "org:role",
+        "KONT_STARTTIDSPUNKT": "start_timestamp",
+        "KONT_SLUTTIDSPUNKT": "time:timestamp",
+        # "BORGER_ALDER_AAR_IND": "patient:age",
+        "PRIORITET_TEKST": "medical:priority",
+        "KONT_TYPE_TEKST": "medical:contact_type",
+    }
+
+    def apply(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply diagnosis-specific filtering and transformations.
+
+        Args:
+            lazy_frame: Input LazyFrame containing diagnosis data
+
+        Returns:
+            Filtered and transformed LazyFrame
+        """
+        return (
+            lazy_frame
+            # Filter by Birthdate (assuming it's a date/datetime type)
+            .filter(pl.col("BORGER_FOEDSELSDATO").dt.year() > 1980)
+            # Filter by Region
+            .filter(pl.col("KONT_ANS_GEO_REG_TEKST") == "Region Sjælland")
+            # Apply column name mappings
+            .rename(self.CASE_ATTR | self.EVENT_ATTR)
+        )

specatwrap8-0.2.0/src/specatwrap8/prep/io_handler.py

@@ -0,0 +1,86 @@
+"""
+I/O handler for parquet file processing.
+
+This module provides generic I/O functionality for reading, filtering,
+and writing parquet files. It uses a filter registry pattern to support
+different filter types (diagnosis, procedure, medication, etc.).
+"""
+
+from pathlib import Path
+from typing import Type
+import polars as pl
+
+from .diagnosis import BaseFilter, DiagnosisFilter
+
+
+# Registry mapping filter type strings to filter classes
+FILTER_REGISTRY: dict[str, Type[BaseFilter]] = {
+    "diagnosis": DiagnosisFilter,
+    # Future additions:
+    # "procedure": ProcedureFilter,
+    # "medication": MedicationFilter,
+}
+
+
+def process_parquet_files(
+    input_dir: Path,
+    output_file: Path,
+    filter_type: str,
+    verbose: bool = False,
+):
+    """
+    Process parquet files with a specified filter type.
+
+    This function handles all I/O operations:
+    - Discovers parquet files in the input directory
+    - Lazily loads them using Polars
+    - Applies the specified filter transformation
+    - Collects the results and writes to output file
+
+    Args:
+        input_dir: Directory containing input parquet files
+        output_file: Path to output parquet file
+        filter_type: Type of filter to apply (e.g., "diagnosis", "procedure")
+        verbose: Whether to include verbose processing (currently unused in I/O layer)
+
+    Returns:
+        Tuple of (row_count, column_count, column_names)
+
+    Raises:
+        FileNotFoundError: If no parquet files found in input directory
+        ValueError: If filter_type is not registered
+        PermissionError: If unable to read/write files
+        MemoryError: If insufficient memory to process data
+    """
+    # Validate filter type
+    if filter_type not in FILTER_REGISTRY:
+        available_filters = ", ".join(FILTER_REGISTRY.keys())
+        raise ValueError(
+            f"Unknown filter type '{filter_type}'. "
+            f"Available filters: {available_filters}"
+        )
+
+    # Check if input directory contains parquet files
+    parquet_files = list(input_dir.glob("*.parquet"))
+    if not parquet_files:
+        raise FileNotFoundError(f"No parquet files found in {input_dir}")
+
+    # Create parquet file pattern for lazy loading
+    parquet_pattern = str(input_dir / "*.parquet")
+
+    # Lazy load all parquet files
+    lazy_frame = pl.scan_parquet(parquet_pattern)
+
+    # Instantiate the appropriate filter
+    filter_class = FILTER_REGISTRY[filter_type]
+    filter_instance = filter_class()
+
+    # Apply the filter transformation
+    filtered_lazy_frame = filter_instance.apply(lazy_frame)
+
+    # Collect the lazy frame (materialize the data)
+
+    # Ensure output directory exists
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    filtered_lazy_frame.sink_parquet(output_file)

specatwrap8-0.2.0/src/specatwrap8/sas_converter.py

@@ -0,0 +1,163 @@
+"""
+SAS to Parquet converter module.
+
+This module provides functionality to convert large SAS7BDAT files to Parquet format
+in a memory-efficient way by processing the data in chunks.
+"""
+
+import sys
+from pathlib import Path
+from typing import Optional
+import shutil
+import click
+import pyreadstat
+import polars as pl
+
+
+def convert_sas_to_parquet(
+    input_file: Path,
+    output_path: Path,
+    chunk_size: int = 100000,
+    compression: str = "zstd",
+    encoding: Optional[str] = None,
+    overwrite: bool = False,
+    verbose: bool = False,
+):
+    """
+    Convert a SAS7BDAT file to Parquet format in chunks.
+
+    Parameters
+    ----------
+    input_file : Path
+        Path to the SAS file (.sas7bdat)
+    output_path : Path
+        Path to output directory for Parquet files
+    chunk_size : int, optional
+        Number of rows to read per chunk (default: 100000)
+    compression : str, optional
+        Parquet compression algorithm (default: 'zstd')
+    encoding : str, optional
+        File encoding (default: auto-detect)
+    overwrite : bool, optional
+        Whether to overwrite existing output directory (default: False)
+    verbose : bool, optional
+        Enable verbose output (default: False)
+
+    Raises
+    ------
+    FileNotFoundError
+        If the input file does not exist
+    ValueError
+        If the input file is not a .sas7bdat file or output path is invalid
+    PermissionError
+        If there are permission issues with files/directories
+    MemoryError
+        If there's insufficient memory (suggest smaller chunk_size)
+    """
+    # Validate input file
+    if not input_file.exists():
+        click.secho(f"✗ Error: Input file not found: {input_file}", fg="red", err=True)
+        sys.exit(1)
+
+    if input_file.suffix.lower() != ".sas7bdat":
+        click.secho("✗ Error: Input file must be a .sas7bdat file", fg="red", err=True)
+        sys.exit(1)
+
+    # Handle output directory
+    output_path = Path(output_path)
+
+    if output_path.exists():
+        if not output_path.is_dir():
+            click.secho(
+                f"✗ Error: Output path exists and is not a directory: {output_path}",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if overwrite:
+            if verbose:
+                click.echo(f"Removing existing directory: {output_path}")
+            shutil.rmtree(output_path)
+            output_path.mkdir(parents=True, exist_ok=True)
+        else:
+            # Check if directory has parquet files
+            existing_parquet = list(output_path.glob("*.parquet"))
+            if existing_parquet:
+                click.secho(
+                    f"⚠ Warning: Output directory already contains {len(existing_parquet)} .parquet file(s)",
+                    fg="yellow",
+                )
+                if not click.confirm(
+                    "Do you want to continue and potentially mix files?"
+                ):
+                    click.echo("Aborted.")
+                    sys.exit(0)
+    else:
+        output_path.mkdir(parents=True, exist_ok=True)
+
+    # Display processing information
+    click.echo(f"Processing: {input_file}")
+    click.echo(f"Output directory: {output_path}")
+    if verbose:
+        click.echo(f"Chunk size: {chunk_size:,} rows")
+        click.echo(f"Compression: {compression}")
+        click.echo(f"Encoding: {encoding if encoding else 'auto-detect'}")
+    click.echo()
+
+    # Process file in chunks
+    click.echo("Converting SAS to Parquet...")
+
+    reader = pyreadstat.read_file_in_chunks(
+        pyreadstat.read_sas7bdat,
+        str(input_file),
+        chunksize=chunk_size,
+        encoding=encoding,
+        output_format="polars",
+    )
+
+    chunk_num = 0
+    total_rows = 0
+    num_columns = 0
+
+    for df_polars, meta in reader:
+        # Generate chunk filename with zero-padded numbering
+        chunk_filename = f"chunk_{chunk_num:04d}.parquet"
+        chunk_path = output_path / chunk_filename
+
+        # Write chunk to parquet
+        df_polars.write_parquet(chunk_path, compression=compression)
+
+        # Update statistics
+        chunk_rows = len(df_polars)
+        total_rows += chunk_rows
+        num_columns = len(meta.column_names)
+        chunk_num += 1
+
+        if verbose:
+            click.echo(
+                f"  Wrote chunk {chunk_num}: {chunk_rows:,} rows → {chunk_filename}"
+            )
+        elif chunk_num % 10 == 0:
+            # Show progress every 10 chunks in non-verbose mode
+            click.echo(f"  Processed {chunk_num} chunks ({total_rows:,} rows)...")
+
+    # Calculate file sizes
+    input_size = input_file.stat().st_size / (1024 * 1024)  # MB
+    output_files = list(output_path.glob("*.parquet"))
+    output_size = sum(f.stat().st_size for f in output_files) / (1024 * 1024)  # MB
+    compression_ratio = input_size / output_size if output_size > 0 else 0
+
+    # Display success message
+    click.echo()
+    click.secho("✓ Conversion completed successfully!", fg="green", bold=True)
+    click.echo(f"Input file:      {input_size:.2f} MB ({input_file.name})")
+    click.echo(f"Output files:    {output_size:.2f} MB ({len(output_files)} chunks)")
+    click.echo(f"Compression:     {compression_ratio:.2f}x")
+    click.echo(f"Total rows:      {total_rows:,}")
+    click.echo(f"Columns:         {num_columns}")
+    click.echo(f"Chunk size:      {chunk_size:,} rows/chunk")
+    click.echo()
+    click.echo("To read the data:")
+    click.echo(f"  import polars as pl")
+    click.echo(f"  df = pl.read_parquet('{output_path}/*.parquet')")