clinical-data-validators 0.4.0__tar.gz

clinical_data_validators-0.4.0/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: clinical-data-validators
+Version: 0.4.0
+Summary: A library for validating clinical data quality
+Author: Navin Kumar
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.3.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Clinical Data Validators
+
+A Python library for validating clinical datasets.
+
+## Installation
+
+pip install clinical-data-validators
+
+## Usage
+
+python
+import pandas as pd
+from clinical_validators import validate_missing_critical_fields
+
+df = pd.read_csv("data.csv")
+result = validate_missing_critical_fields(df)
+print(result)
+
+
+## Validators
+1. Missing Critical Fields
+2. Invalid Data Types
+3. Future Dates
+4. Out of Range Values
+5. Duplicate Records
+6. Invalid Patient IDs
+7. Missing Visit Data
+8. Age Consistency
+9. Gender-Based Validation
+10. Data Completeness

clinical_data_validators-0.4.0/README.md

@@ -0,0 +1,30 @@
+# Clinical Data Validators
+
+A Python library for validating clinical datasets.
+
+## Installation
+
+pip install clinical-data-validators
+
+## Usage
+
+python
+import pandas as pd
+from clinical_validators import validate_missing_critical_fields
+
+df = pd.read_csv("data.csv")
+result = validate_missing_critical_fields(df)
+print(result)
+
+
+## Validators
+1. Missing Critical Fields
+2. Invalid Data Types
+3. Future Dates
+4. Out of Range Values
+5. Duplicate Records
+6. Invalid Patient IDs
+7. Missing Visit Data
+8. Age Consistency
+9. Gender-Based Validation
+10. Data Completeness

clinical_data_validators-0.4.0/clinical_data_validators.egg-info/PKG-INFO

@@ -0,0 +1,49 @@
+Metadata-Version: 2.4
+Name: clinical-data-validators
+Version: 0.4.0
+Summary: A library for validating clinical data quality
+Author: Navin Kumar
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.3.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Clinical Data Validators
+
+A Python library for validating clinical datasets.
+
+## Installation
+
+pip install clinical-data-validators
+
+## Usage
+
+python
+import pandas as pd
+from clinical_validators import validate_missing_critical_fields
+
+df = pd.read_csv("data.csv")
+result = validate_missing_critical_fields(df)
+print(result)
+
+
+## Validators
+1. Missing Critical Fields
+2. Invalid Data Types
+3. Future Dates
+4. Out of Range Values
+5. Duplicate Records
+6. Invalid Patient IDs
+7. Missing Visit Data
+8. Age Consistency
+9. Gender-Based Validation
+10. Data Completeness

clinical_data_validators-0.4.0/clinical_data_validators.egg-info/SOURCES.txt

@@ -0,0 +1,28 @@
+README.md
+setup.py
+clinical_data_validators.egg-info/PKG-INFO
+clinical_data_validators.egg-info/SOURCES.txt
+clinical_data_validators.egg-info/dependency_links.txt
+clinical_data_validators.egg-info/requires.txt
+clinical_data_validators.egg-info/top_level.txt
+clinical_validators/__init__.py
+clinical_validators/age_consistency.py
+clinical_validators/data_completeness.py
+clinical_validators/duplicate_records.py
+clinical_validators/future_dates.py
+clinical_validators/gender_based_validation.py
+clinical_validators/invalid_data_types.py
+clinical_validators/invalid_patient_ids.py
+clinical_validators/missing_fields.py
+clinical_validators/missing_visit_data.py
+clinical_validators/out_of_range_values.py
+tests/test_age_consistency.py
+tests/test_data_completeness.py
+tests/test_duplicate_records.py
+tests/test_future_dates.py
+tests/test_gender_based_validation.py
+tests/test_invalid_data_types.py
+tests/test_invalid_patient_ids.py
+tests/test_missing_fields.py
+tests/test_missing_visit_data.py
+tests/test_out_of_range_values.py

clinical_data_validators-0.4.0/clinical_data_validators.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

clinical_data_validators-0.4.0/clinical_data_validators.egg-info/requires.txt

@@ -0,0 +1 @@
+pandas>=1.3.0

clinical_data_validators-0.4.0/clinical_data_validators.egg-info/top_level.txt

@@ -0,0 +1 @@
+clinical_validators

clinical_data_validators-0.4.0/clinical_validators/__init__.py

@@ -0,0 +1,24 @@
+from .missing_fields import validate_missing_critical_fields
+from .invalid_data_types import validate_invalid_data_types
+from .future_dates import validate_future_dates
+from .out_of_range_values import validate_out_of_range_values
+from .duplicate_records import validate_duplicate_records
+from .invalid_patient_ids import validate_patient_ids
+from .missing_visit_data import validate_missing_visit_data
+from .age_consistency import validate_age_consistency
+from .gender_based_validation import validate_gender_based_tests
+from .data_completeness import validate_data_completeness
+
+__version__ = "0.4.0"
+__all__ = [
+    'validate_missing_critical_fields',
+    'validate_invalid_data_types',
+    'validate_future_dates',
+    'validate_out_of_range_values',
+    'validate_duplicate_records',
+    'validate_patient_ids',
+    'validate_missing_visit_data',
+    'validate_age_consistency',
+    'validate_gender_based_tests',
+    'validate_data_completeness'
+]

clinical_data_validators-0.4.0/clinical_validators/age_consistency.py

@@ -0,0 +1,56 @@
+import pandas as pd
+from datetime import datetime
+
+def validate_age_consistency(dataframe, age_field='age', birth_date_field='birth_date', tolerance_years=1):
+    """
+    Validates that the recorded age is consistent with the birth date.
+    
+    Args:
+        dataframe: pandas DataFrame.
+        age_field: Column name for age.
+        birth_date_field: Column name for birth date.
+        tolerance_years: Allowed difference in years (default 1 for leap year/rounding).
+        
+    Returns:
+        dict with validation results.
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    failures = []
+    failed_row_indices = set()
+    now = datetime.now()
+
+    for idx, row in dataframe.iterrows():
+        age = row.get(age_field)
+        birth_date = row.get(birth_date_field)
+
+        if pd.isna(age) or pd.isna(birth_date):
+            continue
+
+        try:
+            birth_dt = pd.to_datetime(birth_date).to_pydatetime()
+            calculated_age = (now - birth_dt).days / 365.25
+            if abs(calculated_age - float(age)) > tolerance_years:
+                failed_row_indices.add(idx)
+                patient_id = row.get('patient_id', 'UNKNOWN')
+                failures.append({
+                    'row_index': idx,
+                    'patient_id': patient_id,
+                    'recorded_age': age,
+                    'calculated_age': round(calculated_age, 1),
+                    'error': 'Age does not match birth date'
+                })
+        except (ValueError, TypeError):
+            continue
+
+    return {
+        'validator_name': 'age_consistency',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/data_completeness.py

@@ -0,0 +1,57 @@
+import pandas as pd
+
+def validate_data_completeness(dataframe, test_field='test_type', completeness_rules=None):
+    """
+    Validates that required fields are present for specific test types.
+    
+    Args:
+        dataframe: pandas DataFrame.
+        test_field: Column name indicating the test type.
+        completeness_rules: Dict mapping test types to lists of required fields.
+                            Default: {'Blood': ['hemoglobin', 'wbc'], 'Urine': ['ph', 'protein']}
+                            
+    Returns:
+        dict with validation results.
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    if completeness_rules is None:
+        completeness_rules = {
+            'Blood': ['hemoglobin', 'wbc'],
+            'Urine': ['ph', 'protein']
+        }
+
+    failures = []
+    failed_row_indices = set()
+
+    for idx, row in dataframe.iterrows():
+        test_type = row.get(test_field)
+        if pd.isna(test_type):
+            continue
+
+        test_str = str(test_type).strip()
+        if test_str in completeness_rules:
+            required_fields = completeness_rules[test_str]
+            for field in required_fields:
+                if field not in dataframe.columns or pd.isna(row.get(field)) or str(row.get(field)).strip() == '':
+                    failed_row_indices.add(idx)
+                    patient_id = row.get('patient_id', 'UNKNOWN')
+                    failures.append({
+                        'row_index': idx,
+                        'patient_id': patient_id,
+                        'test_type': test_str,
+                        'missing_field': field,
+                        'error': f"Missing required field '{field}' for {test_str} test"
+                    })
+
+    return {
+        'validator_name': 'data_completeness',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/duplicate_records.py

@@ -0,0 +1,62 @@
+import pandas as pd
+
+def validate_duplicate_records(dataframe, duplicate_fields=None):
+    """
+    Validates that there are no duplicate records based on specified fields.
+    Typical use case: same patient + same test + same date
+    
+    Args:
+        dataframe: pandas DataFrame with clinical data
+        duplicate_fields: list of field names to check for duplicates
+                         Default: ['patient_id', 'test_name', 'test_date']
+    
+    Returns:
+        dict with validation results
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+    
+    if duplicate_fields is None:
+        duplicate_fields = ['patient_id', 'test_name', 'test_date']
+    
+    # Check if all required fields exist
+    available_fields = [f for f in duplicate_fields if f in dataframe.columns]
+    if len(available_fields) < 2:
+        raise ValueError(f"Need at least 2 fields to check duplicates. Found: {available_fields}")
+    
+    # Find duplicates
+    duplicates = dataframe[dataframe.duplicated(subset=available_fields, keep=False)]
+    
+    failures = []
+    if not duplicates.empty:
+        # Get unique duplicate groups
+        duplicate_groups = duplicates.groupby(available_fields).size().reset_index(name='count')
+        duplicate_groups = duplicate_groups[duplicate_groups['count'] > 1]
+        
+        for _, row in duplicate_groups.iterrows():
+            # Find all rows in this duplicate group
+            mask = True
+            for field in available_fields:
+                mask = mask & (duplicates[field] == row[field])
+            duplicate_indices = duplicates[mask].index.tolist()
+            
+            for idx in duplicate_indices:
+                failures.append({
+                    'row_index': idx,
+                    'duplicate_fields': available_fields,
+                    'duplicate_values': {field: row[field] for field in available_fields},
+                    'occurrences': int(row['count'])
+                })
+    
+    failed_row_indices = set(f['row_index'] for f in failures)
+    
+    return {
+        'validator_name': 'duplicate_records',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/future_dates.py

@@ -0,0 +1,37 @@
+import pandas as pd
+
+def validate_future_dates(dataframe, date_fields=None):
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    if date_fields is None:
+        date_fields = ['visit_date', 'test_date']
+
+    failures = []
+    failed_row_indices = set()
+    now = pd.Timestamp.now()
+
+    for field in date_fields:
+        if field not in dataframe.columns: continue
+        for idx, val in dataframe[field].items():
+            if pd.isna(val) or str(val).strip() == '': continue
+            try:
+                # QA FIX: Force tz-naive to prevent comparison crashes with mixed timezone data
+                date_val = pd.to_datetime(val).tz_localize(None) 
+                if date_val > now:
+                    failed_row_indices.add(idx)
+                    patient_id = dataframe.at[idx, 'patient_id'] if 'patient_id' in dataframe.columns else 'UNKNOWN'
+                    failures.append({'row_index': idx, 'field': field, 'invalid_value': str(val), 'error': 'Date is in the future', 'patient_id': patient_id})
+            except (ValueError, TypeError):
+                continue
+
+    return {
+        'validator_name': 'future_dates',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/gender_based_validation.py

@@ -0,0 +1,61 @@
+import pandas as pd
+
+def validate_gender_based_tests(dataframe, test_field='test_name', gender_field='gender', rules=None):
+    """
+    Validates that certain medical tests are only performed on appropriate genders.
+    
+    Args:
+        dataframe: pandas DataFrame.
+        test_field: Column name for test name.
+        gender_field: Column name for gender.
+        rules: Dict mapping test names to allowed genders. 
+               Default: {'Prostate_Specific_Antigen': ['M', 'Male'], 'Pap_Smear': ['F', 'Female']}
+               
+    Returns:
+        dict with validation results.
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    if rules is None:
+        rules = {
+            'Prostate_Specific_Antigen': ['M', 'Male'],
+            'Pap_Smear': ['F', 'Female']
+        }
+
+    failures = []
+    failed_row_indices = set()
+
+    for idx, row in dataframe.iterrows():
+        test_name = row.get(test_field)
+        gender = row.get(gender_field)
+
+        if pd.isna(test_name) or pd.isna(gender):
+            continue
+
+        test_str = str(test_name).strip()
+        gender_str = str(gender).strip()
+
+        if test_str in rules:
+            allowed_genders = [g.lower() for g in rules[test_str]]
+            if gender_str.lower() not in allowed_genders:
+                failed_row_indices.add(idx)
+                patient_id = row.get('patient_id', 'UNKNOWN')
+                failures.append({
+                    'row_index': idx,
+                    'patient_id': patient_id,
+                    'test': test_str,
+                    'gender': gender_str,
+                    'error': f"Test {test_str} is invalid for gender {gender_str}"
+                })
+
+    return {
+        'validator_name': 'gender_based_validation',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/invalid_data_types.py

@@ -0,0 +1,46 @@
+import pandas as pd
+
+def validate_invalid_data_types(dataframe, numeric_fields=None, date_fields=None):
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    if numeric_fields is None:
+        numeric_fields = ['lab_value']
+    if date_fields is None:
+        date_fields = ['visit_date', 'test_date']
+
+    failures = []
+    failed_row_indices = set()
+
+    for field in numeric_fields:
+        if field not in dataframe.columns: continue
+        for idx, val in dataframe[field].items():
+            if pd.isna(val) or str(val).strip() == '': continue
+            try:
+                float(val)
+            except (ValueError, TypeError):
+                failed_row_indices.add(idx)
+                patient_id = dataframe.at[idx, 'patient_id'] if 'patient_id' in dataframe.columns else 'UNKNOWN'
+                failures.append({'row_index': idx, 'field': field, 'invalid_value': str(val), 'expected_type': 'numeric', 'patient_id': patient_id})
+
+    for field in date_fields:
+        if field not in dataframe.columns: continue
+        for idx, val in dataframe[field].items():
+            if pd.isna(val) or str(val).strip() == '': continue
+            try:
+                pd.to_datetime(val)
+            except (ValueError, TypeError):
+                failed_row_indices.add(idx)
+                patient_id = dataframe.at[idx, 'patient_id'] if 'patient_id' in dataframe.columns else 'UNKNOWN'
+                failures.append({'row_index': idx, 'field': field, 'invalid_value': str(val), 'expected_type': 'datetime', 'patient_id': patient_id})
+
+    return {
+        'validator_name': 'invalid_data_types',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/invalid_patient_ids.py

@@ -0,0 +1,74 @@
+import pandas as pd
+import re
+
+def validate_patient_ids(dataframe, patient_id_field='patient_id', pattern=None, min_length=None, max_length=None):
+    """
+    Validates patient ID format according to specified rules.
+    
+    Args:
+        dataframe: pandas DataFrame with clinical data
+        patient_id_field: name of the patient ID column (default: 'patient_id')
+        pattern: regex pattern for validation (default: None)
+        min_length: minimum length of patient ID (default: None)
+        max_length: maximum length of patient ID (default: None)
+    
+    Default behavior: Validates that patient IDs are exactly 5 digits
+    
+    Returns:
+        dict with validation results
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+    
+    if patient_id_field not in dataframe.columns:
+        raise ValueError(f"Column '{patient_id_field}' not found in DataFrame")
+    
+    # Default: 5-digit numeric ID
+    if pattern is None and min_length is None and max_length is None:
+        pattern = r'^\d{5}$'  # Exactly 5 digits
+    
+    failures = []
+    failed_row_indices = set()
+    
+    for idx, val in dataframe[patient_id_field].items():
+        if pd.isna(val) or str(val).strip() == '':
+            continue  # Skip missing values
+        
+        patient_id_str = str(val).strip()
+        is_valid = True
+        error_reason = ""
+        
+        # Check pattern
+        if pattern:
+            if not re.match(pattern, patient_id_str):
+                is_valid = False
+                error_reason = f"Does not match pattern: {pattern}"
+        
+        # Check length
+        if is_valid and (min_length is not None or max_length is not None):
+            if min_length is not None and len(patient_id_str) < min_length:
+                is_valid = False
+                error_reason = f"Length {len(patient_id_str)} < minimum {min_length}"
+            elif max_length is not None and len(patient_id_str) > max_length:
+                is_valid = False
+                error_reason = f"Length {len(patient_id_str)} > maximum {max_length}"
+        
+        if not is_valid:
+            failed_row_indices.add(idx)
+            failures.append({
+                'row_index': idx,
+                'field': patient_id_field,
+                'invalid_value': patient_id_str,
+                'error': error_reason
+            })
+    
+    return {
+        'validator_name': 'invalid_patient_ids',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/missing_fields.py

@@ -0,0 +1,113 @@
+import pandas as pd
+from typing import Dict, List, Optional
+
+def validate_missing_critical_fields(
+    dataframe: pd.DataFrame, 
+    critical_fields: Optional[List[str]] = None
+) -> Dict:
+    """
+    Validates that critical clinical fields are not NULL/empty.
+    
+    This validator checks for missing values (NaN, None, empty strings) in 
+    critical clinical data fields. It's designed to catch common data quality 
+    issues in clinical research datasets.
+    
+    Args:
+        dataframe: pandas DataFrame with clinical data
+        critical_fields: list of required field names. If None, uses default 
+                        clinical fields (patient_id, visit_date, lab_test_name, 
+                        lab_value, test_date)
+        
+    Returns:
+        Dictionary with validation results containing:
+        - validator_name: name of the validator
+        - status: 'PASS' or 'FAIL'
+        - total_records: total rows in dataframe
+        - failed_records: number of rows with missing values
+        - failure_count: total number of missing value instances
+        - critical_fields_checked: list of fields that were validated
+        - failures: list of detailed failure information
+        
+    Raises:
+        ValueError: if dataframe is empty or critical_fields is empty list
+        TypeError: if dataframe is not a pandas DataFrame
+        
+    Example:
+        >>> import pandas as pd
+        >>> from clinical_validators import validate_missing_critical_fields
+        >>> df = pd.read_csv('lab_data.csv')
+        >>> result = validate_missing_critical_fields(df)
+        >>> print(result['status'])
+        'FAIL'
+        >>> print(f"Failed records: {result['failed_records']}")
+        'Failed records: 5'
+    """
+    
+    # ===== INPUT VALIDATION =====
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError(f"Expected pandas DataFrame, got {type(dataframe).__name__}")
+    
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty - cannot validate empty dataset")
+    
+    # Default critical fields for clinical data
+    if critical_fields is None:
+        critical_fields = [
+            'patient_id', 
+            'visit_date', 
+            'lab_test_name', 
+            'lab_value', 
+            'test_date'
+        ]
+    
+    if isinstance(critical_fields, list) and len(critical_fields) == 0:
+        raise ValueError("critical_fields list cannot be empty")
+    
+    # ===== VALIDATION LOGIC =====
+    failures = []
+    
+    # Check each critical field
+    for field in critical_fields:
+        # Skip if field doesn't exist in dataframe
+        if field not in dataframe.columns:
+            continue
+        
+        # Find rows where field is NULL, NaN, or empty string
+        # This handles: None, np.nan, pd.NaT, and empty strings
+        missing_mask = (dataframe[field].isna()) | (dataframe[field] == '')
+        missing_indices = dataframe[missing_mask].index.tolist()
+        
+        # Record each failure with context
+        for idx in missing_indices:
+            # Safely retrieve patient_id for context
+            patient_id = 'UNKNOWN'
+            try:
+                if 'patient_id' in dataframe.columns:
+                    pid_value = dataframe.at[idx, 'patient_id']
+                    # Check if patient_id itself is missing
+                    if pd.isna(pid_value) or pid_value == '':
+                        patient_id = 'UNKNOWN'
+                    else:
+                        patient_id = str(pid_value)
+            except (KeyError, IndexError, TypeError):
+                patient_id = 'UNKNOWN'
+            
+            failures.append({
+                'row_index': idx,
+                'missing_field': field,
+                'patient_id': patient_id
+            })
+    
+    # Count unique rows with any failures
+    failed_row_indices = set([f['row_index'] for f in failures])
+    
+    # ===== RETURN RESULTS =====
+    return {
+        'validator_name': 'missing_critical_fields',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'critical_fields_checked': critical_fields,
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/missing_visit_data.py

@@ -0,0 +1,43 @@
+import pandas as pd
+
+def validate_missing_visit_data(dataframe, lab_field='lab_value', visit_field='visit_date'):
+    """
+    Validates that if a patient has a lab result, they must have a corresponding visit record.
+    
+    Args:
+        dataframe: pandas DataFrame with clinical data.
+        lab_field: Column name containing lab results.
+        visit_field: Column name containing visit dates.
+        
+    Returns:
+        dict with validation results.
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+
+    failures = []
+    failed_row_indices = set()
+
+    for idx, row in dataframe.iterrows():
+        has_lab = pd.notna(row.get(lab_field)) and str(row.get(lab_field)).strip() != ''
+        has_visit = pd.notna(row.get(visit_field)) and str(row.get(visit_field)).strip() != ''
+
+        if has_lab and not has_visit:
+            failed_row_indices.add(idx)
+            patient_id = row.get('patient_id', 'UNKNOWN')
+            failures.append({
+                'row_index': idx,
+                'patient_id': patient_id,
+                'error': f"Has lab value '{row[lab_field]}' but missing {visit_field}"
+            })
+
+    return {
+        'validator_name': 'missing_visit_data',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/clinical_validators/out_of_range_values.py

@@ -0,0 +1,61 @@
+import pandas as pd
+
+def validate_out_of_range_values(dataframe, range_rules=None):
+    """
+    Validates that numeric fields fall within acceptable clinical ranges.
+    
+    Args:
+        dataframe: pandas DataFrame with clinical data
+        range_rules: dict with field names as keys and (min, max) tuples as values
+                    Example: {'lab_value': (0, 1000), 'age': (0, 120)}
+    
+    Returns:
+        dict with validation results
+    """
+    if not isinstance(dataframe, pd.DataFrame):
+        raise TypeError("Expected pandas DataFrame")
+    if dataframe.empty:
+        raise ValueError("DataFrame is empty")
+    
+    # Default clinical ranges
+    if range_rules is None:
+        range_rules = {
+            'lab_value': (0, 10000),  # Generic lab value range
+            'age': (0, 120)  # Human age range
+        }
+    
+    failures = []
+    failed_row_indices = set()
+    
+    for field, (min_val, max_val) in range_rules.items():
+        if field not in dataframe.columns:
+            continue
+        
+        for idx, val in dataframe[field].items():
+            if pd.isna(val):
+                continue  # Skip missing values
+            
+            try:
+                numeric_val = float(val)
+                if numeric_val < min_val or numeric_val > max_val:
+                    failed_row_indices.add(idx)
+                    patient_id = dataframe.at[idx, 'patient_id'] if 'patient_id' in dataframe.columns else 'UNKNOWN'
+                    failures.append({
+                        'row_index': idx,
+                        'field': field,
+                        'invalid_value': numeric_val,
+                        'expected_range': f"{min_val}-{max_val}",
+                        'patient_id': patient_id
+                    })
+            except (ValueError, TypeError):
+                # Skip non-numeric values (handled by invalid_data_types validator)
+                continue
+    
+    return {
+        'validator_name': 'out_of_range_values',
+        'status': 'PASS' if len(failed_row_indices) == 0 else 'FAIL',
+        'total_records': len(dataframe),
+        'failed_records': len(failed_row_indices),
+        'failure_count': len(failures),
+        'failures': failures
+    }

clinical_data_validators-0.4.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

clinical_data_validators-0.4.0/setup.py

@@ -0,0 +1,23 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+setup(
+    name="clinical-data-validators",
+    version="0.4.0",
+    author="Navin Kumar",
+    description="A library for validating clinical data quality",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    packages=find_packages(),
+    python_requires=">=3.8",
+    install_requires=[
+        "pandas>=1.3.0",
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)

clinical_data_validators-0.4.0/tests/test_age_consistency.py

@@ -0,0 +1,20 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+from datetime import datetime, timedelta
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.age_consistency import validate_age_consistency
+
+class TestAgeConsistency:
+    def test_valid_age(self):
+        birth = (datetime.now() - timedelta(days=30*365)).strftime('%Y-%m-%d')
+        df = pd.DataFrame({'patient_id': [1], 'age': [30], 'birth_date': [birth]})
+        assert validate_age_consistency(df)['status'] == 'PASS'
+    def test_invalid_age(self):
+        birth = (datetime.now() - timedelta(days=30*365)).strftime('%Y-%m-%d')
+        df = pd.DataFrame({'patient_id': [1], 'age': [50], 'birth_date': [birth]})
+        assert validate_age_consistency(df)['status'] == 'FAIL'
+    def test_missing_data(self):
+        df = pd.DataFrame({'patient_id': [1], 'age': [None], 'birth_date': [None]})
+        assert validate_age_consistency(df)['status'] == 'PASS'

clinical_data_validators-0.4.0/tests/test_data_completeness.py

@@ -0,0 +1,27 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.data_completeness import validate_data_completeness
+
+class TestDataCompleteness:
+    def test_complete_blood(self):
+        df = pd.DataFrame({'patient_id': [1], 'test_type': ['Blood'], 'hemoglobin': [14], 'wbc': [5]})
+        assert validate_data_completeness(df)['status'] == 'PASS'
+        
+    def test_incomplete_blood(self):
+        df = pd.DataFrame({'patient_id': [1], 'test_type': ['Blood'], 'hemoglobin': [14], 'wbc': [None]})
+        assert validate_data_completeness(df)['status'] == 'FAIL'
+        
+    def test_different_test_type(self):
+        # Urine test requires 'ph' and 'protein'. 'hemoglobin' and 'wbc' are for Blood, so they can be None.
+        df = pd.DataFrame({
+            'patient_id': [1], 
+            'test_type': ['Urine'], 
+            'ph': [6.0], 
+            'protein': ['Negative'], 
+            'hemoglobin': [None], 
+            'wbc': [None]
+        })
+        assert validate_data_completeness(df)['status'] == 'PASS'

clinical_data_validators-0.4.0/tests/test_duplicate_records.py

@@ -0,0 +1,37 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.duplicate_records import validate_duplicate_records
+
+class TestDuplicateRecords:
+    def test_no_duplicates(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1002, 1003],
+            'test_name': ['Blood', 'Urine', 'X-Ray'],
+            'test_date': ['2026-01-15', '2026-01-16', '2026-01-17']
+        })
+        result = validate_duplicate_records(df)
+        assert result['status'] == 'PASS'
+        assert result['failed_records'] == 0
+    
+    def test_exact_duplicates(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1001],  # Same patient
+            'test_name': ['Blood', 'Blood'],  # Same test
+            'test_date': ['2026-01-15', '2026-01-15']  # Same date
+        })
+        result = validate_duplicate_records(df)
+        assert result['status'] == 'FAIL'
+        assert result['failed_records'] == 2  # Both rows are duplicates
+    
+    def test_partial_duplicates_not_flagged(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1001],  # Same patient
+            'test_name': ['Blood', 'Urine'],  # Different test
+            'test_date': ['2026-01-15', '2026-01-15']  # Same date
+        })
+        result = validate_duplicate_records(df)
+        assert result['status'] == 'PASS'  # Not exact duplicates

clinical_data_validators-0.4.0/tests/test_future_dates.py

@@ -0,0 +1,23 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+from datetime import datetime, timedelta
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.future_dates import validate_future_dates
+
+class TestFutureDates:
+    def test_valid_past_dates(self):
+        df = pd.DataFrame({'patient_id': [1001], 'visit_date': ['2023-01-15'], 'test_date': ['2023-01-15']})
+        assert validate_future_dates(df)['status'] == 'PASS'
+
+    def test_future_visit_date(self):
+        future_date = (datetime.now() + timedelta(days=10)).strftime('%Y-%m-%d')
+        df = pd.DataFrame({'patient_id': [1001], 'visit_date': [future_date], 'test_date': ['2023-01-15']})
+        result = validate_future_dates(df)
+        assert result['status'] == 'FAIL'
+        assert result['failures'][0]['field'] == 'visit_date'
+
+    def test_empty_dataframe_raises_error(self):
+        with pytest.raises(ValueError): validate_future_dates(pd.DataFrame())

clinical_data_validators-0.4.0/tests/test_gender_based_validation.py

@@ -0,0 +1,17 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.gender_based_validation import validate_gender_based_tests
+
+class TestGenderValidation:
+    def test_valid_gender(self):
+        df = pd.DataFrame({'patient_id': [1], 'test_name': ['Prostate_Specific_Antigen'], 'gender': ['M']})
+        assert validate_gender_based_tests(df)['status'] == 'PASS'
+    def test_invalid_gender(self):
+        df = pd.DataFrame({'patient_id': [1], 'test_name': ['Prostate_Specific_Antigen'], 'gender': ['F']})
+        assert validate_gender_based_tests(df)['status'] == 'FAIL'
+    def test_unrestricted_test(self):
+        df = pd.DataFrame({'patient_id': [1], 'test_name': ['Blood_Test'], 'gender': ['F']})
+        assert validate_gender_based_tests(df)['status'] == 'PASS'

clinical_data_validators-0.4.0/tests/test_invalid_data_types.py

@@ -0,0 +1,27 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.invalid_data_types import validate_invalid_data_types
+
+class TestInvalidDataTypes:
+    def test_valid_data_types(self):
+        df = pd.DataFrame({'patient_id': [1001], 'lab_value': [14.5], 'visit_date': ['2026-01-15'], 'test_date': ['2026-01-15']})
+        assert validate_invalid_data_types(df)['status'] == 'PASS'
+
+    def test_invalid_numeric_value(self):
+        df = pd.DataFrame({'patient_id': [1001], 'lab_value': ['abc'], 'visit_date': ['2026-01-15'], 'test_date': ['2026-01-15']})
+        result = validate_invalid_data_types(df)
+        assert result['status'] == 'FAIL'
+        assert result['failures'][0]['field'] == 'lab_value'
+
+    def test_invalid_date_format(self):
+        df = pd.DataFrame({'patient_id': [1001], 'lab_value': [14.5], 'visit_date': ['not-a-date'], 'test_date': ['2026-01-15']})
+        result = validate_invalid_data_types(df)
+        assert result['status'] == 'FAIL'
+        assert result['failures'][0]['field'] == 'visit_date'
+
+    def test_empty_dataframe_raises_error(self):
+        with pytest.raises(ValueError): validate_invalid_data_types(pd.DataFrame())

clinical_data_validators-0.4.0/tests/test_invalid_patient_ids.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.invalid_patient_ids import validate_patient_ids
+
+class TestInvalidPatientIds:
+    def test_valid_patient_ids(self):
+        df = pd.DataFrame({
+            'patient_id': ['12345', '67890', '11111']
+        })
+        result = validate_patient_ids(df)
+        assert result['status'] == 'PASS'
+        assert result['failed_records'] == 0
+    
+    def test_invalid_format_too_short(self):
+        df = pd.DataFrame({
+            'patient_id': ['12345', '1234']  # 1234 is only 4 digits
+        })
+        result = validate_patient_ids(df)
+        assert result['status'] == 'FAIL'
+        assert result['failed_records'] == 1
+    
+    def test_invalid_format_non_numeric(self):
+        df = pd.DataFrame({
+            'patient_id': ['12345', 'ABC12']  # ABC12 contains letters
+        })
+        result = validate_patient_ids(df)
+        assert result['status'] == 'FAIL'
+        assert result['failed_records'] == 1

clinical_data_validators-0.4.0/tests/test_missing_fields.py

@@ -0,0 +1,182 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from clinical_validators.missing_fields import validate_missing_critical_fields
+
+
+class TestMissingCriticalFields:
+    """Test suite for missing_critical_fields validator"""
+    
+    def test_valid_dataframe_no_missing_values(self):
+        """Test: Clean data should pass validation"""
+        data = {
+            'patient_id': [1001, 1002, 1003],
+            'visit_date': ['2026-01-15', '2026-01-16', '2026-01-17'],
+            'lab_test_name': ['Hemoglobin', 'Glucose', 'Cholesterol'],
+            'lab_value': [14.5, 95, 180],
+            'test_date': ['2026-01-15', '2026-01-16', '2026-01-17']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'PASS', "Clean data should pass"
+        assert result['failed_records'] == 0, "Should have 0 failed records"
+        assert result['total_records'] == 3, "Should have 3 total records"
+        assert len(result['failures']) == 0, "Failures list should be empty"
+        print("✓ Test 1 PASSED: Clean data validated successfully")
+    
+    def test_missing_patient_id(self):
+        """Test: Missing patient_id should be detected"""
+        data = {
+            'patient_id': [1001, None, 1003],
+            'visit_date': ['2026-01-15', '2026-01-16', '2026-01-17'],
+            'lab_test_name': ['Hemoglobin', 'Glucose', 'Cholesterol'],
+            'lab_value': [14.5, 95, 180],
+            'test_date': ['2026-01-15', '2026-01-16', '2026-01-17']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'FAIL', "Should fail with missing patient_id"
+        assert result['failed_records'] == 1, "Should have 1 failed record"
+        assert any(f['missing_field'] == 'patient_id' for f in result['failures']), \
+            "Should identify missing patient_id"
+        print("✓ Test 2 PASSED: Missing patient_id detected")
+    
+    def test_missing_multiple_fields(self):
+        """Test: Multiple missing fields across different rows"""
+        data = {
+            'patient_id': [1001, 1002, None],
+            'visit_date': [None, '2026-01-16', '2026-01-17'],
+            'lab_test_name': ['Hemoglobin', 'Glucose', 'Cholesterol'],
+            'lab_value': [14.5, 95, 180],
+            'test_date': ['2026-01-15', None, '2026-01-17']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'FAIL', "Should fail with missing fields"
+        assert result['failed_records'] == 3, "Should have 3 failed records"
+        assert result['failure_count'] == 3, "Should have 3 total failures"
+        print("✓ Test 3 PASSED: Multiple missing fields detected correctly")
+    
+    def test_missing_lab_test_name(self):
+        """Test: Missing lab_test_name detection (empty string)"""
+        data = {
+            'patient_id': [1001, 1002],
+            'visit_date': ['2026-01-15', '2026-01-16'],
+            'lab_test_name': ['Hemoglobin', ''],  # Empty string
+            'lab_value': [14.5, 95],
+            'test_date': ['2026-01-15', '2026-01-16']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'FAIL', "Should detect empty string as missing"
+        assert result['failed_records'] == 1, "Should flag empty string row"
+        assert any(f['missing_field'] == 'lab_test_name' and f['row_index'] == 1 
+                   for f in result['failures']), "Should identify lab_test_name empty"
+        print("✓ Test 4 PASSED: Empty string detected as missing value")
+    
+    def test_all_fields_missing(self):
+        """Test: Edge case - all critical fields missing in one row"""
+        data = {
+            'patient_id': [None, 1002],
+            'visit_date': [None, '2026-01-16'],
+            'lab_test_name': [None, 'Glucose'],
+            'lab_value': [None, 95],
+            'test_date': [None, '2026-01-16']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['failed_records'] == 1, "Should detect row with all missing fields"
+        assert result['failure_count'] == 5, "Should count 5 failures (1 per field)"
+        print("✓ Test 5 PASSED: All missing fields in single row detected")
+    
+    def test_empty_dataframe_raises_error(self):
+        """Test: Empty DataFrame should raise ValueError"""
+        df = pd.DataFrame()
+        
+        with pytest.raises(ValueError):
+            validate_missing_critical_fields(df)
+        
+        print("✓ Test 6 PASSED: Empty DataFrame raises ValueError")
+    
+    def test_invalid_input_type_raises_error(self):
+        """Test: Non-DataFrame input should raise TypeError"""
+        # Test with string input
+        with pytest.raises(TypeError):
+            validate_missing_critical_fields("not a dataframe")
+        
+        # Test with list input
+        with pytest.raises(TypeError):
+            validate_missing_critical_fields([1, 2, 3])
+        
+        print("✓ Test 7 PASSED: Invalid input types raise TypeError")
+    
+    def test_custom_critical_fields(self):
+        """Test: Validator should work with custom field list"""
+        data = {
+            'patient_id': [1001, None],
+            'visit_date': ['2026-01-15', '2026-01-16'],
+            'other_field': ['A', 'B']
+        }
+        df = pd.DataFrame(data)
+        custom_fields = ['patient_id', 'other_field']
+        result = validate_missing_critical_fields(df, critical_fields=custom_fields)
+        
+        assert 'visit_date' not in result['critical_fields_checked'], \
+            "Should only check specified fields"
+        assert result['failed_records'] == 1, "Should detect missing patient_id"
+        print("✓ Test 8 PASSED: Custom field list works correctly")
+    
+    def test_extra_columns_ignored(self):
+        """Test: Extra columns should not affect validation"""
+        data = {
+            'patient_id': [1001, 1002],
+            'visit_date': ['2026-01-15', '2026-01-16'],
+            'lab_test_name': ['Hemoglobin', 'Glucose'],
+            'lab_value': [14.5, 95],
+            'test_date': ['2026-01-15', '2026-01-16'],
+            'extra_column_1': ['X', 'Y'],
+            'extra_column_2': [100, 200]
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'PASS', "Extra columns should not cause failures"
+        assert result['failed_records'] == 0, "Should have no failures"
+        print("✓ Test 9 PASSED: Extra columns ignored correctly")
+    
+    def test_nan_vs_none_consistency(self):
+        """Test: Both NaN and None are treated as missing"""
+        data = {
+            'patient_id': [1001, None, 1003],
+            'visit_date': ['2026-01-15', '2026-01-16', pd.NaT],
+            'lab_test_name': ['Hemoglobin', 'Glucose', 'Cholesterol'],
+            'lab_value': [14.5, 95, 180],
+            'test_date': ['2026-01-15', '2026-01-16', '2026-01-17']
+        }
+        df = pd.DataFrame(data)
+        result = validate_missing_critical_fields(df)
+        
+        assert result['status'] == 'FAIL', "Should detect both None and NaT"
+        assert result['failed_records'] == 2, "Should have 2 failed records"
+        print("✓ Test 10 PASSED: NaN and None handled consistently")
+
+
+if __name__ == '__main__':
+    print("=" * 80)
+    print("CLINICAL DATA VALIDATORS - TEST SUITE")
+    print("Validator: missing_critical_fields")
+    print("=" * 80)
+    print()
+    
+    # Run with pytest
+    pytest.main([__file__, '-v', '--tb=short'])

clinical_data_validators-0.4.0/tests/test_missing_visit_data.py

@@ -0,0 +1,17 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.missing_visit_data import validate_missing_visit_data
+
+class TestMissingVisitData:
+    def test_valid_data(self):
+        df = pd.DataFrame({'patient_id': [1], 'lab_value': [100], 'visit_date': ['2026-01-01']})
+        assert validate_missing_visit_data(df)['status'] == 'PASS'
+    def test_missing_visit(self):
+        df = pd.DataFrame({'patient_id': [1], 'lab_value': [100], 'visit_date': [None]})
+        assert validate_missing_visit_data(df)['status'] == 'FAIL'
+    def test_no_lab_no_visit(self):
+        df = pd.DataFrame({'patient_id': [1], 'lab_value': [None], 'visit_date': [None]})
+        assert validate_missing_visit_data(df)['status'] == 'PASS'

clinical_data_validators-0.4.0/tests/test_out_of_range_values.py

@@ -0,0 +1,40 @@
+import pandas as pd
+import pytest
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from clinical_validators.out_of_range_values import validate_out_of_range_values
+
+class TestOutOfRangeValues:
+    def test_valid_ranges(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1002],
+            'lab_value': [150.5, 500],
+            'age': [35, 67]
+        })
+        result = validate_out_of_range_values(df)
+        assert result['status'] == 'PASS'
+        assert result['failed_records'] == 0
+    
+    def test_lab_value_too_high(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1002],
+            'lab_value': [150.5, 15000],  # 15000 > 10000
+            'age': [35, 67]
+        })
+        result = validate_out_of_range_values(df)
+        assert result['status'] == 'FAIL'
+        assert result['failed_records'] == 1
+        assert result['failures'][0]['field'] == 'lab_value'
+    
+    def test_age_out_of_range(self):
+        df = pd.DataFrame({
+            'patient_id': [1001, 1002],
+            'lab_value': [150.5, 500],
+            'age': [35, 150]  # 150 > 120
+        })
+        result = validate_out_of_range_values(df)
+        assert result['status'] == 'FAIL'
+        assert result['failed_records'] == 1
+        assert result['failures'][0]['field'] == 'age'