specatwrap14 0.2.0__tar.gz

specatwrap14-0.2.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.3
+Name: specatwrap14
+Version: 0.2.0
+Summary: A simple wrapper
+Author: Casper Lauge Nørup Koch
+Author-email: Casper Lauge Nørup Koch <kochcasper@gmail.com>
+Requires-Dist: click>=8.3.1
+Requires-Dist: pm4py>=2.7.19.8
+Requires-Dist: polars>=1.38.1
+Requires-Dist: pyreadstat>=1.3.3
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+

specatwrap14-0.2.0/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "specatwrap14"
+version = "0.2.0"
+description = "A simple wrapper"
+readme = "README.md"
+authors = [
+    { name = "Casper Lauge Nørup Koch", email = "kochcasper@gmail.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.3.1",
+    "pm4py>=2.7.19.8",
+    "polars>=1.38.1",
+    "pyreadstat>=1.3.3",
+]
+
+[project.scripts]
+specatwrap = "specatwrap14:main"
+
+[build-system]
+requires = ["uv_build>=0.10.3,<0.11.0"]
+build-backend = "uv_build"
+
+
+[tool.pyright]
+reportUnannotatedClassAttribute = "none"
+reportImplicitOverride = "none"
+reportUnknownParameterType = "none"
+reportMissingParameterType = "none"
+reportUnknownVariableType = "none"
+reportUnknownMemberType = "none"
+reportUnknownArgumentType = "none"
+reportAny = "none"

specatwrap14-0.2.0/src/specatwrap14/__init__.py

@@ -0,0 +1,129 @@
+"""
+Specatwrap - A wrapper for processing healthcare data.
+
+This module provides a CLI for converting and processing healthcare data files.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from .sas_converter import convert_sas_to_parquet
+from .prep import prep
+from .sas_preview import preview
+from .print_update import print_update_script
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli():
+    """
+    Specatwrap - A wrapper for processing healthcare data.
+
+    A command-line tool for processing and converting healthcare data files.
+    """
+    pass
+
+
+# Register command groups
+cli.add_command(prep)
+cli.add_command(preview)
+cli.add_command(print_update_script)
+
+
+@cli.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.argument("output_path", type=click.Path(path_type=Path))
+@click.option(
+    "-c",
+    "--chunk-size",
+    type=int,
+    default=100000,
+    help="Number of rows to read per chunk (default: 100000)",
+)
+@click.option(
+    "--compression",
+    type=click.Choice(["zstd", "lz4", "snappy", "gzip"], case_sensitive=False),
+    default="zstd",
+    help="Parquet compression algorithm (default: zstd)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+@click.option(
+    "--overwrite",
+    is_flag=True,
+    default=False,
+    help="Overwrite existing output directory if it exists",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output")
+def sas2parquet(
+    input_file, output_path, chunk_size, compression, encoding, overwrite, verbose
+):
+    """
+    Convert a SAS file to Parquet format in chunks.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    OUTPUT_PATH: Path to output directory for Parquet files
+
+    This command processes large SAS files in chunks to minimize memory usage.
+    Each chunk is written to a separate Parquet file in the output directory.
+
+    Example usage:
+
+        specatwrap sas2parquet input.sas7bdat output_dir/
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --chunk-size 50000 -v
+
+        specatwrap sas2parquet input.sas7bdat output_dir/ --compression lz4
+
+    Reading the data later with Polars:
+
+        import polars as pl
+
+        df = pl.read_parquet("output_dir/*.parquet")
+    """
+    try:
+        convert_sas_to_parquet(
+            input_file=input_file,
+            output_path=output_path,
+            chunk_size=chunk_size,
+            compression=compression,
+            encoding=encoding,
+            overwrite=overwrite,
+            verbose=verbose,
+        )
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            f"✗ Error: Out of memory. Try a smaller --chunk-size", fg="red", err=True
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)
+
+
+def main():
+    """Entry point for the CLI application."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

specatwrap14-0.2.0/src/specatwrap14/prep/__init__.py

@@ -0,0 +1,124 @@
+"""
+Prep command group for preprocessing healthcare data files.
+
+This module provides commands for filtering and preprocessing parquet files
+before converting them to XES format.
+"""
+
+import click
+from pathlib import Path
+import sys
+
+from .io_handler import process_parquet_files
+
+
+@click.group()
+def prep():
+    """
+    Preprocess and filter healthcare data files.
+
+    Commands in this group help prepare raw data files by filtering,
+    cleaning, and transforming them before further processing.
+    """
+    pass
+
+
+@prep.command()
+@click.option(
+    "-i",
+    "--input",
+    "input_dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="Path to directory containing parquet files to process.",
+)
+@click.option(
+    "-o",
+    "--output",
+    "output_file",
+    type=click.Path(dir_okay=False, file_okay=True, path_type=Path),
+    required=True,
+    help="Path to output parquet file.",
+)
+@click.option("-v", "--verbose", is_flag=True, help="Enable verbose output.")
+def diagnosis(input_dir, output_file, verbose):
+    """
+    Filter and preprocess diagnosis parquet files.
+
+    This command lazily reads all parquet files from a directory, applies
+    filtering and preprocessing transformations, and writes the results to
+    a single output parquet file.
+
+    INPUT: Directory path containing parquet files to process.
+
+    OUTPUT: Path to output parquet file for processed data.
+
+    Example usage:
+
+        specatwrap prep diagnosis -i ./data/parquet_files/ -o ./processed/diagnosis.parquet
+
+        specatwrap prep diagnosis --input ./raw_data/ --output ./clean_data.parquet -v
+    """
+    try:
+        # Display processing information
+        click.echo(f"Input directory: {input_dir}")
+        click.echo(f"Output file: {output_file}")
+        click.echo()
+
+        # Find all parquet files in input directory (for verbose output)
+        if verbose:
+            parquet_pattern = str(input_dir / "*.parquet")
+            click.echo(f"Searching for parquet files: {parquet_pattern}")
+            parquet_files = list(input_dir.glob("*.parquet"))
+            if parquet_files:
+                click.echo(f"Found {len(parquet_files)} parquet file(s):")
+                for f in parquet_files:
+                    click.echo(f"  - {f.name}")
+                click.echo()
+
+        # Process the parquet files using the io_handler
+        click.echo("Loading and applying filters...")
+        with click.progressbar(
+            length=100, label="Processing", show_eta=False, show_percent=True
+        ) as bar:
+            # Call the generic I/O handler with "diagnosis" filter type
+            process_parquet_files(
+                input_dir=input_dir,
+                output_file=output_file,
+                filter_type="diagnosis",
+                verbose=verbose,
+            )
+            bar.update(100)
+
+        # Display success message
+        click.secho("✓ Processing completed successfully!", fg="green", bold=True)
+        click.echo(f"Output file: {output_file}")
+
+        # Display file statistics
+        if output_file.exists():
+            size_mb = output_file.stat().st_size / (1024 * 1024)
+            click.echo(f"File size: {size_mb:.2f} MB")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File or directory not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except MemoryError:
+        click.secho(
+            "✗ Error: Out of memory. Try processing smaller batches.",
+            fg="red",
+            err=True,
+        )
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        if verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)

specatwrap14-0.2.0/src/specatwrap14/prep/diagnosis.py

@@ -0,0 +1,265 @@
+"""
+Diagnosis filter classes.
+
+This module provides filter classes for preprocessing diagnosis parquet files.
+"""
+
+from abc import ABC, abstractmethod
+import polars as pl
+
+
+class BaseFilter(ABC):
+    """
+    Abstract base class for parquet file filters.
+
+    Each filter type (diagnosis, procedure, medication, etc.) should inherit
+    from this class and implement the apply() method with their specific
+    filtering and transformation logic.
+    """
+
+    @abstractmethod
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply filtering and preprocessing logic to a LazyFrame.
+
+        Args:
+            lazy_frame: Input Polars LazyFrame to filter/transform
+
+        Returns:
+            Transformed Polars LazyFrame
+        """
+        pass
+
+    def get_name(self) -> str:
+        """
+        Get the name of this filter type.
+
+        Returns:
+            Filter type name (defaults to class name without 'Filter' suffix)
+        """
+        class_name = self.__class__.__name__
+        if class_name.endswith("Filter"):
+            return class_name[:-6].lower()
+        return class_name.lower()
+
+
+class DiagnosisFilter(BaseFilter):
+    """
+    Filter for diagnosis parquet files.
+
+    Applies diagnosis-specific filtering and column transformations:
+    - Filters by birthdate (year > 1980)
+    - Filters by region (Region Sjælland)
+    - Renames columns to standardized event log format
+    """
+
+    CASE_ATTR = {
+        k: f"case:{v}"
+        for k, v in {
+            "BORGER_FOEDSELSDATO": "patient:birthdate",
+            "PNR": "concept:name",
+        }.items()
+    }
+
+    EVENT_ATTR = {
+        "TORRA_DIAG": "concept:name",
+        "ADIAG_TEKST": "medical:diag",
+        "KONT_ANS_GEO_REG_TEKST": "org:group",
+        "KONT_LPR_ENTITY_ID": "org:resource",
+        "KONT_INST_EJERTYPE": "org:role",
+        "KONT_STARTTIDSPUNKT": "start_timestamp",
+        "KONT_SLUTTIDSPUNKT": "time:timestamp",
+        # "BORGER_ALDER_AAR_IND": "patient:age",
+        "PRIORITET_TEKST": "medical:priority",
+        "KONT_TYPE_TEKST": "medical:contact_type",
+    }
+
+    def _columns_to_select(self) -> list[str]:
+        a = list((self.CASE_ATTR | self.EVENT_ATTR).keys())
+        a.append("ADIAG")  # Needed for TORRA_DIAG transformation
+        a.remove("TORRA_DIAG")  # We will create this column in the transformation step
+        return a
+
+    def apply(self, lf: pl.LazyFrame) -> pl.LazyFrame:
+        """
+        Apply diagnosis-specific filtering and transformations.
+
+        Args:
+            lazy_frame: Input LazyFrame containing diagnosis data
+
+        Returns:
+            Filtered and transformed LazyFrame
+        """
+
+        lf = lf.select(self._columns_to_select())
+
+        lf = self._add_torra_diag(lf)
+
+        return (
+            lf#.filter(pl.col("TORRA_DIAG").list.len() > 4)
+            .with_columns(pl.col("TORRA_DIAG").list.join(", "))
+            .rename(self.CASE_ATTR | self.EVENT_ATTR)
+        )
+
+    def _add_torra_diag(self, lazy_frame: pl.LazyFrame) -> pl.LazyFrame:
+        col = pl.col("ADIAG")
+
+        def starts_with(prefixes: list[str], label: str) -> pl.Expr:
+            pattern = f"^(?:{'|'.join(prefixes)})"
+            return (
+                pl.when(col.str.contains(pattern))
+                .then(pl.lit(label))
+                .otherwise(pl.lit(None))
+            )
+
+        return lazy_frame.with_columns(
+            TORRA_DIAG=pl.concat_list(
+                [
+                    starts_with(
+                        ["DJ41", "DJ42", "DJ43", "DJ44", "DJ45", "DJ46"], "LUNG"
+                    ),
+                    starts_with(
+                        [
+                            "DL405",
+                            "DM05",
+                            "DM06",
+                            "DM07",
+                            "DM15",
+                            "DM16",
+                            "DM17",
+                            "DM45",
+                            "DM47",
+                            "DM50",
+                            "DM51",
+                            "DM53",
+                            "DM54",
+                            "DM80",
+                            "DM81",
+                            "DM82",
+                        ],
+                        "MUSCULOSKELETAL",
+                    ),
+                    starts_with(
+                        ["DE03", "DE05", "DE10", "DE11", "DE12", "DE13", "DE14"],
+                        "ENDOCRINE",
+                    ),
+                    starts_with(["DE03", "DE05"], "THYROIDEA"),
+                    starts_with(["DE10", "DE11", "DE12", "DE13", "DE14"], "DIABETES"),
+                    starts_with(["DG30", "DG318", "DG319", "DF"], "MENTAL"),
+                    # CANCER: Matches DC but explicitly excludes DC44
+                    pl.when(col.str.contains("^DC") & ~col.str.contains("^DC44"))
+                    .then(pl.lit("CANCER"))
+                    .otherwise(pl.lit(None)),
+                    starts_with(
+                        [
+                            "DI60",
+                            "DI61",
+                            "DI62",
+                            "DI63",
+                            "DI64",
+                            "DI69",
+                            "DG20",
+                            "DG35",
+                            "DG40",
+                            "DG43",
+                        ],
+                        "NEUROLOGICAL",
+                    ),
+                    starts_with(
+                        [
+                            "DK30",
+                            "DK50",
+                            "DK51",
+                            "DK58",
+                            "DK70",
+                            "DK71",
+                            "DK72",
+                            "DK73",
+                            "DK74",
+                            "DK75",
+                            "DK76",
+                            "DK860",
+                            "DK861",
+                        ],
+                        "GASTROINTESTINAL",
+                    ),
+                    starts_with(
+                        [
+                            "DI20",
+                            "DI21",
+                            "DI22",
+                            "DI23",
+                            "DI24",
+                            "DI25",
+                            "DI47",
+                            "DI48",
+                            "DI49",
+                            "DI50",
+                            "DI05",
+                            "DI06",
+                            "DI07",
+                            "DI08",
+                            "DI34",
+                            "DI35",
+                            "DI36",
+                            "DI37",
+                            "DI441",
+                            "DI442",
+                            "DI443",
+                            "DI444",
+                            "DI445",
+                            "DI446",
+                            "DI447",
+                            "DI452",
+                            "DI453",
+                            "DI454",
+                            "DI455",
+                            "DI456",
+                            "DI457",
+                            "DI458",
+                            "DI459",
+                        ],
+                        "CARDIOVASCULAR",
+                    ),
+                    starts_with(
+                        [
+                            "DN03",
+                            "DN04",
+                            "DN05",
+                            "DN11",
+                            "DN12",
+                            "DN18",
+                            "DN19",
+                            "DZ49",
+                            "DN80",
+                            "DZ992",
+                            "DN393",
+                            "DN394",
+                        ],
+                        "GENITURINARY",
+                    ),
+                    starts_with(
+                        [
+                            "DH40",
+                            "DH91",
+                            "DL40",
+                            "DH540",
+                            "DH541",
+                            "DH542",
+                            "DH543",
+                            "DH547",
+                            "DH900",
+                            "DH902",
+                            "DH903",
+                            "DH905",
+                            "DH906",
+                            "DH908",
+                        ],
+                        "SENSORY ORGANS",
+                    ),
+                    starts_with(
+                        ["DL23", "DL24", "DL25", "DJ30", "DL500", "DJ450"], "ALLERGY"
+                    ),
+                ]
+            ).list.drop_nulls()
+        )

specatwrap14-0.2.0/src/specatwrap14/prep/io_handler.py

@@ -0,0 +1,86 @@
+"""
+I/O handler for parquet file processing.
+
+This module provides generic I/O functionality for reading, filtering,
+and writing parquet files. It uses a filter registry pattern to support
+different filter types (diagnosis, procedure, medication, etc.).
+"""
+
+from pathlib import Path
+from typing import Type
+import polars as pl
+
+from .diagnosis import BaseFilter, DiagnosisFilter
+
+
+# Registry mapping filter type strings to filter classes
+FILTER_REGISTRY: dict[str, Type[BaseFilter]] = {
+    "diagnosis": DiagnosisFilter,
+    # Future additions:
+    # "procedure": ProcedureFilter,
+    # "medication": MedicationFilter,
+}
+
+
+def process_parquet_files(
+    input_dir: Path,
+    output_file: Path,
+    filter_type: str,
+    verbose: bool = False,
+):
+    """
+    Process parquet files with a specified filter type.
+
+    This function handles all I/O operations:
+    - Discovers parquet files in the input directory
+    - Lazily loads them using Polars
+    - Applies the specified filter transformation
+    - Collects the results and writes to output file
+
+    Args:
+        input_dir: Directory containing input parquet files
+        output_file: Path to output parquet file
+        filter_type: Type of filter to apply (e.g., "diagnosis", "procedure")
+        verbose: Whether to include verbose processing (currently unused in I/O layer)
+
+    Returns:
+        Tuple of (row_count, column_count, column_names)
+
+    Raises:
+        FileNotFoundError: If no parquet files found in input directory
+        ValueError: If filter_type is not registered
+        PermissionError: If unable to read/write files
+        MemoryError: If insufficient memory to process data
+    """
+    # Validate filter type
+    if filter_type not in FILTER_REGISTRY:
+        available_filters = ", ".join(FILTER_REGISTRY.keys())
+        raise ValueError(
+            f"Unknown filter type '{filter_type}'. "
+            f"Available filters: {available_filters}"
+        )
+
+    # Check if input directory contains parquet files
+    parquet_files = list(input_dir.glob("*.parquet"))
+    if not parquet_files:
+        raise FileNotFoundError(f"No parquet files found in {input_dir}")
+
+    # Create parquet file pattern for lazy loading
+    parquet_pattern = str(input_dir / "*.parquet")
+
+    # Lazy load all parquet files
+    lazy_frame = pl.scan_parquet(parquet_pattern)
+
+    # Instantiate the appropriate filter
+    filter_class = FILTER_REGISTRY[filter_type]
+    filter_instance = filter_class()
+
+    # Apply the filter transformation
+    filtered_lazy_frame = filter_instance.apply(lazy_frame)
+
+    # Collect the lazy frame (materialize the data)
+
+    # Ensure output directory exists
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    filtered_lazy_frame.sink_parquet(output_file)

specatwrap14-0.2.0/src/specatwrap14/print_update.py

@@ -0,0 +1,161 @@
+"""
+CLI command to print the standalone update script for the restricted environment.
+"""
+
+import click
+
+
+@click.command("print-update-script")
+def print_update_script():
+    """
+    Print a standalone Python script for updating specatwrap on the restricted environment.
+
+    The script finds the currently installed version (e.g., specatwrap10),
+    uninstalls it, and installs the next version (specatwrap11).
+
+    Usage on development machine:
+
+        specatwrap print-update-script > update_specatwrap.py
+
+    Then transfer update_specatwrap.py to the restricted environment and run:
+
+        python update_specatwrap.py
+    """
+    script = '''#!/usr/bin/env python3
+"""
+Standalone update script for specatwrap package.
+
+This script:
+1. Finds the currently installed specatwrap version (e.g., specatwrap10)
+2. Uninstalls it without confirmation
+3. Installs the next version (e.g., specatwrap11)
+
+Usage:
+    python update_specatwrap.py
+"""
+
+import subprocess
+import sys
+import re
+
+
+def run_command(cmd, capture=True):
+    """Run a shell command and return the result."""
+    try:
+        result = subprocess.run(
+            cmd,
+            shell=True,
+            capture_output=capture,
+            text=True,
+            check=False
+        )
+        return result
+    except Exception as e:
+        print(f"Error running command: {e}")
+        sys.exit(1)
+
+
+def find_installed_version():
+    """Find the currently installed specatwrap package."""
+    result = run_command("pip list")
+    
+    if result.returncode != 0:
+        print("Error: Failed to run 'pip list'")
+        print(result.stderr)
+        sys.exit(1)
+    
+    # Look for specatwrap packages in pip list output
+    lines = result.stdout.split('\\n')
+    specatwrap_packages = []
+    
+    for line in lines:
+        # Match lines starting with specatwrap (with or without version number)
+        match = re.match(r'^(specatwrap\\d*)\\s+', line.strip())
+        if match:
+            specatwrap_packages.append(match.group(1))
+    
+    if not specatwrap_packages:
+        print("Error: No specatwrap package found.")
+        print("Please install specatwrap manually first.")
+        sys.exit(1)
+    
+    if len(specatwrap_packages) > 1:
+        print(f"Error: Multiple specatwrap packages found: {specatwrap_packages}")
+        print("Please uninstall extra versions manually.")
+        sys.exit(1)
+    
+    return specatwrap_packages[0]
+
+
+def parse_version(package_name):
+    """Extract version number from package name."""
+    # Match specatwrap followed by digits
+    match = re.match(r'^specatwrap(\\d+)$', package_name)
+    
+    if not match:
+        # Package is just "specatwrap" without a number
+        print(f"Error: Found '{package_name}' without version number.")
+        print("This should not exist in the mirror.")
+        sys.exit(1)
+    
+    return int(match.group(1))
+
+
+def main():
+    print("=" * 60)
+    print("specatwrap Update Script")
+    print("=" * 60)
+    
+    # Step 1: Find currently installed version
+    print("\\n[1/3] Finding currently installed version...")
+    current_package = find_installed_version()
+    print(f"Found: {current_package}")
+    
+    # Parse version number and calculate next version
+    current_version = parse_version(current_package)
+    next_version = current_version + 1
+    next_package = f"specatwrap{next_version}"
+    
+    print(f"Will update: {current_package} -> {next_package}")
+    
+    # Step 2: Uninstall old version
+    print(f"\\n[2/3] Uninstalling {current_package}...")
+    result = run_command(f"pip uninstall -y {current_package}", capture=False)
+    
+    if result.returncode != 0:
+        print(f"\\nError: Failed to uninstall {current_package}")
+        sys.exit(1)
+    
+    print(f"Successfully uninstalled {current_package}")
+    
+    # Step 3: Install new version
+    print(f"\\n[3/3] Installing {next_package}...")
+    result = run_command(f"pip install {next_package}", capture=False)
+    
+    if result.returncode != 0:
+        print(f"\\nError: Failed to install {next_package}")
+        print("\\nPossible reasons:")
+        print("  - The new version may not be available yet in the PyPI mirror")
+        print("  - The mirror updates once daily")
+        print("  - Please try again later or check if the package was published")
+        sys.exit(1)
+    
+    print(f"\\nSuccessfully installed {next_package}")
+    
+    print("\\n" + "=" * 60)
+    print("Update complete!")
+    print("=" * 60)
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        print("\\nUpdate cancelled.")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\\nUnexpected error: {e}")
+        sys.exit(1)
+'''
+    click.echo(script)

specatwrap14-0.2.0/src/specatwrap14/sas_converter.py

@@ -0,0 +1,171 @@
+"""
+SAS to Parquet converter module.
+
+This module provides functionality to convert large SAS7BDAT files to Parquet format
+in a memory-efficient way by processing the data in chunks.
+"""
+
+import sys
+import time
+from pathlib import Path
+from typing import Optional
+import shutil
+import click
+import pyreadstat
+import polars as pl
+
+
+def convert_sas_to_parquet(
+    input_file: Path,
+    output_path: Path,
+    chunk_size: int = 100000,
+    compression: str = "zstd",
+    encoding: Optional[str] = None,
+    overwrite: bool = False,
+    verbose: bool = False,
+):
+    """
+    Convert a SAS7BDAT file to Parquet format in chunks.
+
+    Parameters
+    ----------
+    input_file : Path
+        Path to the SAS file (.sas7bdat)
+    output_path : Path
+        Path to output directory for Parquet files
+    chunk_size : int, optional
+        Number of rows to read per chunk (default: 100000)
+    compression : str, optional
+        Parquet compression algorithm (default: 'zstd')
+    encoding : str, optional
+        File encoding (default: auto-detect)
+    overwrite : bool, optional
+        Whether to overwrite existing output directory (default: False)
+    verbose : bool, optional
+        Enable verbose output (default: False)
+
+    Raises
+    ------
+    FileNotFoundError
+        If the input file does not exist
+    ValueError
+        If the input file is not a .sas7bdat file or output path is invalid
+    PermissionError
+        If there are permission issues with files/directories
+    MemoryError
+        If there's insufficient memory (suggest smaller chunk_size)
+    """
+    # Validate input file
+    if not input_file.exists():
+        click.secho(f"✗ Error: Input file not found: {input_file}", fg="red", err=True)
+        sys.exit(1)
+
+    if input_file.suffix.lower() != ".sas7bdat":
+        click.secho("✗ Error: Input file must be a .sas7bdat file", fg="red", err=True)
+        sys.exit(1)
+
+    # Handle output directory
+    output_path = Path(output_path)
+
+    if output_path.exists():
+        if not output_path.is_dir():
+            click.secho(
+                f"✗ Error: Output path exists and is not a directory: {output_path}",
+                fg="red",
+                err=True,
+            )
+            sys.exit(1)
+
+        if overwrite:
+            if verbose:
+                click.echo(f"Removing existing directory: {output_path}")
+            shutil.rmtree(output_path)
+            output_path.mkdir(parents=True, exist_ok=True)
+        else:
+            # Check if directory has parquet files
+            existing_parquet = list(output_path.glob("*.parquet"))
+            if existing_parquet:
+                click.secho(
+                    f"⚠ Warning: Output directory already contains {len(existing_parquet)} .parquet file(s)",
+                    fg="yellow",
+                )
+                if not click.confirm(
+                    "Do you want to continue and potentially mix files?"
+                ):
+                    click.echo("Aborted.")
+                    sys.exit(0)
+    else:
+        output_path.mkdir(parents=True, exist_ok=True)
+
+    # Display processing information
+    click.echo(f"Processing: {input_file}")
+    click.echo(f"Output directory: {output_path}")
+    if verbose:
+        click.echo(f"Chunk size: {chunk_size:,} rows")
+        click.echo(f"Compression: {compression}")
+        click.echo(f"Encoding: {encoding if encoding else 'auto-detect'}")
+    click.echo()
+
+    # Process file in chunks
+    click.echo("Converting SAS to Parquet...")
+
+    reader = pyreadstat.read_file_in_chunks(
+        pyreadstat.read_sas7bdat,
+        str(input_file),
+        chunksize=chunk_size,
+        encoding=encoding,
+        output_format="polars",
+    )
+
+    chunk_num = 0
+    total_rows = 0
+    num_columns = 0
+
+    for df_polars, meta in reader:
+        # Start timing for this chunk
+        chunk_start_time = time.time()
+
+        # Generate chunk filename with zero-padded numbering
+        chunk_filename = f"chunk_{chunk_num:04d}.parquet"
+        chunk_path = output_path / chunk_filename
+
+        # Write chunk to parquet
+        df_polars.write_parquet(chunk_path, compression=compression)
+
+        # Calculate elapsed time
+        chunk_elapsed_time = time.time() - chunk_start_time
+
+        # Update statistics
+        chunk_rows = len(df_polars)
+        total_rows += chunk_rows
+        num_columns = len(meta.column_names)
+        chunk_num += 1
+
+        if verbose:
+            click.echo(
+                f"  Wrote chunk {chunk_num}: {chunk_rows:,} rows → {chunk_filename} (took {chunk_elapsed_time:.2f}s)"
+            )
+        else:
+            click.echo(
+                f"  Wrote chunk {chunk_num}: {chunk_rows:,} rows → {chunk_filename} (took {chunk_elapsed_time:.2f}s)"
+            )
+
+    # Calculate file sizes
+    input_size = input_file.stat().st_size / (1024 * 1024)  # MB
+    output_files = list(output_path.glob("*.parquet"))
+    output_size = sum(f.stat().st_size for f in output_files) / (1024 * 1024)  # MB
+    compression_ratio = input_size / output_size if output_size > 0 else 0
+
+    # Display success message
+    click.echo()
+    click.secho("✓ Conversion completed successfully!", fg="green", bold=True)
+    click.echo(f"Input file:      {input_size:.2f} MB ({input_file.name})")
+    click.echo(f"Output files:    {output_size:.2f} MB ({len(output_files)} chunks)")
+    click.echo(f"Compression:     {compression_ratio:.2f}x")
+    click.echo(f"Total rows:      {total_rows:,}")
+    click.echo(f"Columns:         {num_columns}")
+    click.echo(f"Chunk size:      {chunk_size:,} rows/chunk")
+    click.echo()
+    click.echo("To read the data:")
+    click.echo(f"  import polars as pl")
+    click.echo(f"  df = pl.read_parquet('{output_path}/*.parquet')")

specatwrap14-0.2.0/src/specatwrap14/sas_preview.py

@@ -0,0 +1,93 @@
+"""
+SAS file preview module.
+
+This module provides functionality to quickly preview the first N rows
+of a SAS7BDAT file for inspection purposes.
+"""
+
+import click
+from pathlib import Path
+import sys
+import pyreadstat
+
+
+@click.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "-n",
+    "--num-rows",
+    type=int,
+    default=10,
+    help="Number of rows to display (default: 10)",
+)
+@click.option(
+    "--encoding",
+    type=str,
+    default=None,
+    help="File encoding (default: auto-detect)",
+)
+def preview(input_file, num_rows, encoding):
+    """
+    Preview the first N rows of a SAS file.
+
+    INPUT_FILE: Path to the SAS file (.sas7bdat)
+
+    This command quickly reads and displays the first few rows of a SAS file
+    to help you inspect the data structure and content without converting
+    the entire file.
+
+    Example usage:
+
+        specatwrap preview input.sas7bdat
+
+        specatwrap preview input.sas7bdat --num-rows 20
+
+        specatwrap preview input.sas7bdat --encoding latin1
+    """
+    try:
+        # Validate file exists
+        if not input_file.exists():
+            click.secho(f"✗ Error: File not found: {input_file}", fg="red", err=True)
+            sys.exit(1)
+
+        if input_file.suffix.lower() != ".sas7bdat":
+            click.secho(
+                "✗ Error: Input file must be a .sas7bdat file", fg="red", err=True
+            )
+            sys.exit(1)
+
+        click.echo(f"Reading first {num_rows} rows from: {input_file.name}")
+        click.echo()
+
+        # Read first N rows using pyreadstat
+        df, meta = pyreadstat.read_sas7bdat(
+            str(input_file),
+            row_limit=num_rows,
+            encoding=encoding,
+            output_format="polars",
+        )
+
+        # Display metadata
+        click.echo(f"Total columns: {len(meta.column_names)}")
+        click.echo(f"Column names: {', '.join(meta.column_names)}")
+        if encoding:
+            click.echo(f"Encoding: {encoding}")
+        click.echo()
+
+        # Display the data using Polars' pretty print
+        click.echo(str(df))
+        click.echo()
+        click.secho(f"✓ Showing {len(df)} row(s)", fg="green")
+
+    except FileNotFoundError as e:
+        click.secho(f"✗ Error: File not found - {e}", fg="red", err=True)
+        sys.exit(1)
+    except PermissionError as e:
+        click.secho(f"✗ Error: Permission denied - {e}", fg="red", err=True)
+        sys.exit(1)
+    except ValueError as e:
+        click.secho(f"✗ Error: Invalid input - {e}", fg="red", err=True)
+        sys.exit(1)
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red", err=True)
+        sys.exit(1)