ncdb-tools 0.1.0__tar.gz

ncdb_tools-0.1.0/.gitignore

@@ -0,0 +1,58 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+*.egg-info/
+dist/
+build/
+
+# Data - NEVER commit any data files
+data/
+docs/
+scripts/
+*.csv
+*.txt
+*.dat
+*.parquet
+*.feather
+*.json
+*.xml
+*.sas
+*.pdf
+*.rtf
+
+# Documentation/instruction files
+development-guidelines.md
+CLAUDE.md
+claude.md
+ncdb_instructions.md
+
+# Results and outputs
+results/
+outputs/
+figures/
+*.log
+
+# Environment
+.env
+.DS_Store
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Jupyter
+.ipynb_checkpoints
+*.ipynb
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/

ncdb_tools-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

ncdb_tools-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Contributing to NCDB Tools
+
+## Development Setup
+
+1. Clone the repository
+2. Install dependencies with `uv sync --all-extras`
+3. Run tests with `uv run pytest`
+
+## Code Style
+
+- Use `ruff` for formatting and linting
+- Follow Google-style docstrings
+- Add type hints to all functions
+
+## Security
+
+- Never commit data files
+- Never include PHI or patient information
+- Check .gitignore before committing
+
+## Pull Request Process
+
+1. Create a feature branch
+2. Make your changes with tests
+3. Run `uv run ruff check --fix .`
+4. Submit PR with clear description

ncdb_tools-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 The NCDB Tools Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ncdb_tools-0.1.0/MANIFEST.in

@@ -0,0 +1,16 @@
+include README.md
+include LICENSE
+include CONTRIBUTING.md
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]
+recursive-exclude * *.sas
+recursive-exclude * *.dat
+recursive-exclude * *.csv
+recursive-exclude * *.parquet
+recursive-exclude * *.txt
+recursive-exclude * *.pdf
+recursive-exclude * *.rtf
+recursive-exclude docs *
+recursive-exclude scripts *
+recursive-exclude * CLAUDE.md
+recursive-exclude * claude.md

ncdb_tools-0.1.0/PKG-INFO

@@ -0,0 +1,128 @@
+Metadata-Version: 2.4
+Name: ncdb-tools
+Version: 0.1.0
+Summary: Tools for processing and analyzing National Cancer Database (NCDB) data
+Author: NCDB Tools Contributors
+License: MIT
+License-File: LICENSE
+Keywords: analysis,cancer,database,medical-research,ncdb,parquet
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Requires-Python: >=3.9
+Requires-Dist: click>=8.0.0
+Requires-Dist: polars>=0.19.0
+Requires-Dist: pyarrow>=10.0.0
+Requires-Dist: rich>=10.0.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.0.280; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# ncdb-tools
+
+A Python package for efficiently processing and analyzing National Cancer Database (NCDB) data files.
+
+## Installation
+
+```bash
+pip install ncdb-tools
+```
+
+## Important Notice
+
+This package provides tools for processing NCDB data files. **You must obtain NCDB data through official channels** - this package does not include any patient data. The National Cancer Database (NCDB) is a clinical oncology database sourced from hospital registry data that are collected in more than 1,500 Commission on Cancer (CoC)-accredited facilities.
+
+## Quick Start
+
+```python
+import ncdb_tools
+
+# Convert all NCDB data files in a directory to parquet format
+paths = ncdb_tools.build_database("/path/to/NCDB_DATA/")
+
+# The function will:
+# 1. Find all .dat files
+# 2. Find the SAS labels file
+# 3. Create a new subdirectory with today's date
+# 4. Convert all files to parquet format
+# 5. Generate a comprehensive data dictionary
+# 6. Create a summary report
+
+print(f"Database created in: {paths['output_dir']}")
+```
+
+## Working with the Data
+
+After building the database, you can query the parquet files using NCDB-specific filters and standard Polars operations:
+
+```python
+import polars as pl
+
+# Load data with NCDB-specific filters
+query = ncdb_tools.load_data("path/to/parquet_directory/")
+
+# Chain NCDB filters, then use Polars for analysis
+df = (
+    query
+    .filter_by_year(2021)
+    .filter_by_primary_site("C509")  # Breast
+    .filter_by_histology([8140, 8500])  # Adenocarcinoma codes
+    .drop_missing_vital_status()
+    .lazy_frame()  # Get Polars LazyFrame
+)
+
+# Use standard Polars operations
+results = (
+    df
+    .filter(pl.col("AGE") >= 50)
+    .group_by(["SEX", "RACE"])
+    .agg([
+        pl.count().alias("count"),
+        pl.col("AGE").mean().alias("mean_age")
+    ])
+    .collect()
+)
+```
+
+The query interface provides these NCDB-specific filters:
+- `filter_by_year()` - Filter by year of diagnosis
+- `filter_by_primary_site()` - Filter by ICD-O-3 primary site codes
+- `filter_by_histology()` - Filter by histology codes (accepts integers or strings)
+- `drop_missing_vital_status()` - Remove cases with missing vital status
+
+After applying NCDB filters, use `.lazy_frame()` to access the Polars LazyFrame for further analysis.
+
+## Features
+
+- Efficiently converts NCDB fixed-width text files to parquet format
+- Automatically parses SAS labels for meaningful column names
+- Generates comprehensive data dictionaries in CSV, JSON, and HTML formats
+- Memory-efficient processing using Polars
+- Simple, high-level API for common tasks
+- NCDB-specific data filters and transformations
+- Compatible with all Python 3.9+ versions
+
+## Requirements
+
+- Python 3.9 or higher
+- NCDB data files (obtained through official channels)
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file for details.
+
+## Disclaimer
+
+This software is provided for research purposes. Users are responsible for ensuring compliance with all applicable data use agreements and privacy regulations when working with NCDB data.

ncdb_tools-0.1.0/README.md

@@ -0,0 +1,96 @@
+# ncdb-tools
+
+A Python package for efficiently processing and analyzing National Cancer Database (NCDB) data files.
+
+## Installation
+
+```bash
+pip install ncdb-tools
+```
+
+## Important Notice
+
+This package provides tools for processing NCDB data files. **You must obtain NCDB data through official channels** - this package does not include any patient data. The National Cancer Database (NCDB) is a clinical oncology database sourced from hospital registry data that are collected in more than 1,500 Commission on Cancer (CoC)-accredited facilities.
+
+## Quick Start
+
+```python
+import ncdb_tools
+
+# Convert all NCDB data files in a directory to parquet format
+paths = ncdb_tools.build_database("/path/to/NCDB_DATA/")
+
+# The function will:
+# 1. Find all .dat files
+# 2. Find the SAS labels file
+# 3. Create a new subdirectory with today's date
+# 4. Convert all files to parquet format
+# 5. Generate a comprehensive data dictionary
+# 6. Create a summary report
+
+print(f"Database created in: {paths['output_dir']}")
+```
+
+## Working with the Data
+
+After building the database, you can query the parquet files using NCDB-specific filters and standard Polars operations:
+
+```python
+import polars as pl
+
+# Load data with NCDB-specific filters
+query = ncdb_tools.load_data("path/to/parquet_directory/")
+
+# Chain NCDB filters, then use Polars for analysis
+df = (
+    query
+    .filter_by_year(2021)
+    .filter_by_primary_site("C509")  # Breast
+    .filter_by_histology([8140, 8500])  # Adenocarcinoma codes
+    .drop_missing_vital_status()
+    .lazy_frame()  # Get Polars LazyFrame
+)
+
+# Use standard Polars operations
+results = (
+    df
+    .filter(pl.col("AGE") >= 50)
+    .group_by(["SEX", "RACE"])
+    .agg([
+        pl.count().alias("count"),
+        pl.col("AGE").mean().alias("mean_age")
+    ])
+    .collect()
+)
+```
+
+The query interface provides these NCDB-specific filters:
+- `filter_by_year()` - Filter by year of diagnosis
+- `filter_by_primary_site()` - Filter by ICD-O-3 primary site codes
+- `filter_by_histology()` - Filter by histology codes (accepts integers or strings)
+- `drop_missing_vital_status()` - Remove cases with missing vital status
+
+After applying NCDB filters, use `.lazy_frame()` to access the Polars LazyFrame for further analysis.
+
+## Features
+
+- Efficiently converts NCDB fixed-width text files to parquet format
+- Automatically parses SAS labels for meaningful column names
+- Generates comprehensive data dictionaries in CSV, JSON, and HTML formats
+- Memory-efficient processing using Polars
+- Simple, high-level API for common tasks
+- NCDB-specific data filters and transformations
+- Compatible with all Python 3.9+ versions
+
+## Requirements
+
+- Python 3.9 or higher
+- NCDB data files (obtained through official channels)
+
+## License
+
+MIT License - see [LICENSE](LICENSE) file for details.
+
+## Disclaimer
+
+This software is provided for research purposes. Users are responsible for ensuring compliance with all applicable data use agreements and privacy regulations when working with NCDB data.

ncdb_tools-0.1.0/build_database.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+Command-line script to build NCDB database from .dat files.
+
+Usage:
+    uv run build_database.py <data_directory>
+
+This script will:
+1. Find all .dat files in the specified directory
+2. Locate the SAS labels file
+3. Create a timestamped output subdirectory
+4. Convert all .dat files to parquet format
+5. Generate comprehensive data dictionaries (CSV, JSON, HTML)
+"""
+
+import sys
+from pathlib import Path
+from ncdb_tools import build_database
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: uv run build_database.py <data_directory>")
+        print("\nExample:")
+        print("  uv run build_database.py \"R:\\Jason\\NCDB\\NCDB_PUF_DATA_Sep-14-2024\"")
+        return 1
+    
+    data_dir = sys.argv[1]
+    
+    try:
+        result = build_database(data_dir=data_dir)
+        return 0
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+if __name__ == "__main__":
+    exit(main())

ncdb_tools-0.1.0/main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from ncdb-tools!")
+
+
+if __name__ == "__main__":
+    main()

ncdb_tools-0.1.0/pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "ncdb-tools"
+version = "0.1.0"
+description = "Tools for processing and analyzing National Cancer Database (NCDB) data"
+readme = "README.md"
+requires-python = ">=3.9"
+license = {text = "MIT"}
+authors = [
+    {name = "NCDB Tools Contributors"}
+]
+keywords = ["ncdb", "cancer", "database", "analysis", "parquet", "medical-research"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Healthcare Industry",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10", 
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Medical Science Apps.",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "polars>=0.19.0",
+    "click>=8.0.0",
+    "rich>=10.0.0",
+    "pyarrow>=10.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.0.280",
+    "mypy>=1.0.0",
+]
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.ruff]
+line-length = 88
+target-version = "py39"
+select = ["E", "F", "I", "N", "W", "RUF"]
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+
+[dependency-groups]
+dev = [
+    "mypy>=1.16.0",
+    "pytest>=8.4.0",
+    "pytest-cov>=6.1.1",
+    "ruff>=0.11.12",
+]

ncdb_tools-0.1.0/src/ncdb_tools/__init__.py

@@ -0,0 +1,17 @@
+"""NCDB Tools - Tools for managing and analyzing National Cancer Database data."""
+
+__version__ = "0.1.0"
+
+# Core functionality
+from .data_dictionary import generate_data_dictionary
+from .database_builder import build_database
+from .dataset_builder import build_dataset
+from .query import NCDBQuery, load_data
+
+__all__ = [
+    "NCDBQuery",
+    "build_database",  # High-level function for most users
+    "build_dataset",
+    "generate_data_dictionary",
+    "load_data",
+]

ncdb_tools-0.1.0/src/ncdb_tools/_internal/__init__.py

@@ -0,0 +1 @@
+"""Internal utilities for NCDB Tools."""

ncdb_tools-0.1.0/src/ncdb_tools/_internal/sas_parser.py

@@ -0,0 +1,115 @@
+"""Parser for SAS label files."""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+
+def parse_sas_labels(sas_file_path: Path) -> Tuple[Dict[str, str], Dict[str, Dict[str, str]]]:
+    """Parse SAS file to extract variable labels and value formats.
+    
+    Args:
+        sas_file_path: Path to SAS labels file
+        
+    Returns:
+        Tuple of (variable_labels, value_formats)
+        - variable_labels: Dict mapping variable names to descriptions
+        - value_formats: Dict mapping variable names to value->label mappings
+    """
+    with open(sas_file_path, 'r', encoding='latin-1') as f:
+        content = f.read()
+
+    # Extract variable labels (e.g., AGE = 'Age at Diagnosis')
+    variable_labels = {}
+    label_pattern = r"(\w+)\s*=\s*'([^']+)'"
+
+    # Find the label section
+    label_section = re.search(r'label\s+(.*?);', content, re.DOTALL | re.IGNORECASE)
+    if label_section:
+        for match in re.finditer(label_pattern, label_section.group(1)):
+            var_name = match.group(1)
+            var_label = match.group(2)
+            variable_labels[var_name] = var_label
+
+    # Extract value formats (simplified - would need more robust parsing)
+    value_formats = {}
+
+    # Look for proc format value statements
+    format_blocks = re.findall(r'value\s+(\w+)(.*?);\s*(?=value|\s*run|$)', content, re.DOTALL | re.IGNORECASE)
+
+    for format_name, format_content in format_blocks:
+        # Extract value mappings
+        value_map = {}
+
+        # Pattern for numeric or string values
+        value_pattern = r"(['\"]?)([^'\"=]+)\1\s*=\s*['\"]([^'\"]+)['\"]"
+
+        for match in re.finditer(value_pattern, format_content):
+            value = match.group(2).strip()
+            label = match.group(3).strip()
+            value_map[value] = label
+
+        if value_map:
+            # Try to find which variables use this format
+            # Look for format statements
+            format_usage = re.findall(rf'(\w+)\s+{format_name}\.', content)
+            for var in format_usage:
+                value_formats[var] = value_map
+
+    return variable_labels, value_formats
+
+
+def parse_column_positions(sas_file_path: Path) -> List[Dict[str, any]]:
+    """Parse column positions from SAS input statement.
+    
+    Args:
+        sas_file_path: Path to SAS file
+        
+    Returns:
+        List of column definitions with name, start, end positions
+    """
+    with open(sas_file_path, 'r', encoding='latin-1') as f:
+        content = f.read()
+
+    columns = []
+
+    # Find input statement
+    input_section = re.search(r'input\s+(.*?);\s*', content, re.DOTALL | re.IGNORECASE)
+    if not input_section:
+        return columns
+
+    input_text = input_section.group(1)
+
+    # Parse column definitions
+    # Handle both formats: "AGE $ 50-52" and "@50 AGE $3."
+
+    # First try the position-name format: @50 AGE $3.
+    at_pattern = r'@(\d+)\s+(\w+)\s+\$?(\d+)\.'
+    for match in re.finditer(at_pattern, input_text):
+        start_pos = int(match.group(1)) - 1  # Convert to 0-based
+        name = match.group(2)
+        width = int(match.group(3))
+
+        columns.append({
+            'name': name,
+            'start': start_pos,
+            'end': start_pos + width,
+            'width': width
+        })
+
+    # If no columns found, try the range format: AGE $ 50-52
+    if not columns:
+        range_pattern = r'(\w+)\s*\$?\s*(\d+)-(\d+)'
+        for match in re.finditer(range_pattern, input_text):
+            name = match.group(1)
+            start = int(match.group(2)) - 1  # Convert to 0-based
+            end = int(match.group(3))
+
+            columns.append({
+                'name': name,
+                'start': start,
+                'end': end,
+                'width': end - start
+            })
+
+    return columns

ncdb_tools-0.1.0/src/ncdb_tools/constants.py

@@ -0,0 +1,87 @@
+"""Constants for NCDB Tools."""
+
+from typing import List, Set
+
+# File patterns
+DATA_FILE_PATTERN = "NCDBPUF_*.dat"
+PARQUET_EXTENSION = ".parquet"
+
+# Data specifications
+NCDB_RECORD_LENGTH = 1032
+NCDB_COLUMN_COUNT = 338
+
+# Columns that should never be converted to numeric
+NEVER_NUMERIC_COLUMNS: Set[str] = {
+    "PUF_CASE_ID",
+    "PUF_FACILITY_ID",
+    "PRIMARY_SITE",
+    "HISTOLOGY",
+    "HISTOLOGY_ICDO3",
+    "BEHAVIOR",
+    "LATERALITY",
+    "CLASS_OF_CASE",
+    "YEAR_OF_DIAGNOSIS",
+    "SEQUENCE_NUMBER",
+    "FACILITY_TYPE_CD",
+    "FACILITY_LOCATION_CD",
+    "ZIP",
+}
+
+# Standard column groups for convenience
+DEMOGRAPHIC_COLUMNS: List[str] = [
+    "AGE",
+    "SEX",
+    "RACE",
+    "SPANISH_HISPANIC_ORIGIN",
+    "INSURANCE_STATUS",
+    "CDCC_TOTAL_BEST",
+    "MED_INC_QUAR_00",
+    "NO_HSD_QUAR_00",
+    "UR_CD_03",
+]
+
+TREATMENT_COLUMNS: List[str] = [
+    "RX_SUMM_SURG_PRIM_SITE",
+    "RX_SUMM_RADIATION",
+    "RX_SUMM_CHEMO",
+    "RX_SUMM_HORMONE",
+    "RX_SUMM_IMMUNOTHERAPY",
+    "RX_SUMM_SYSTEMIC_SUR_SEQ",
+    "RX_SUMM_TREATMENT_STATUS",
+]
+
+OUTCOME_COLUMNS: List[str] = [
+    "PUF_VITAL_STATUS",
+    "DX_LASTCONTACT_DEATH_MONTHS",
+    "READM_HOSP_30_DAYS",
+    "REASON_FOR_NO_SURGERY",
+]
+
+# Tumor types found in the actual data
+TUMOR_TYPES: List[str] = [
+    "BoneJont",
+    "Brain",
+    "CNS",
+    "EyeOrbit",
+    "GumOtMth",
+    "HodgExtr",
+    "HodgNdal",
+    "Hypophar",
+    "Kaposi",
+    "Langerhans",
+    "Larynx",
+    "Lip",
+    "Melanoma",
+    "MouthFlr",
+    "Nasal",
+    "Nasophar",
+    "NHLExtr",
+    "NHLNdal",
+    "Orophary",
+    "Pharynx",
+    "SalivGld",
+    "SoftTiss",
+    "Thyroid",
+    "Tongue",
+    "Tonsil",
+]