speconsense 0.7.2__py3-none-any.whl

speconsense/profiles/__init__.py

@@ -0,0 +1,514 @@
+"""Profile system for speconsense parameter presets.
+
+Profiles allow users to save and reuse parameter configurations for different
+workflows (e.g., herbarium specimens vs. fresh specimens).
+
+Profile resolution order:
+1. User profiles in ~/.config/speconsense/profiles/
+2. Bundled profiles in package
+
+Override order: defaults -> profile -> explicit CLI arguments
+"""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Any, Optional, List, Set
+import logging
+import os
+import re
+import sys
+
+try:
+    import yaml
+except ImportError:
+    yaml = None  # type: ignore
+
+try:
+    from speconsense import __version__
+except ImportError:
+    __version__ = "dev"
+
+# Use importlib.resources for Python 3.9+, fall back to pkg_resources
+HAS_IMPORTLIB_RESOURCES = False
+HAS_PKG_RESOURCES = False
+
+try:
+    from importlib.resources import files as importlib_files
+    from importlib.resources import as_file
+    HAS_IMPORTLIB_RESOURCES = True
+except ImportError:
+    try:
+        import pkg_resources
+        HAS_PKG_RESOURCES = True
+    except ImportError:
+        pass
+
+logger = logging.getLogger(__name__)
+
+# XDG-compliant config path
+XDG_CONFIG_HOME = Path.home() / ".config"
+PROFILES_DIR = XDG_CONFIG_HOME / "speconsense" / "profiles"
+
+# Valid keys for each tool (for strict validation)
+VALID_SPECONSENSE_KEYS = {
+    # Clustering algorithm
+    "algorithm",
+    "min-identity",
+    "inflation",
+    "k-nearest-neighbors",
+    # Cluster filtering
+    "min-size",
+    "min-cluster-ratio",
+    "outlier-identity",
+    # Sampling
+    "max-sample-size",
+    "presample",
+    # Variant calling
+    "min-variant-frequency",
+    "min-variant-count",
+    "disable-position-phasing",
+    # Ambiguity calling
+    "min-ambiguity-frequency",
+    "min-ambiguity-count",
+    "disable-ambiguity-calling",
+    # Merging
+    "disable-cluster-merging",
+    "disable-homopolymer-equivalence",
+    # Orientation
+    "orient-mode",
+    # Processing
+    "scale-threshold",
+    "threads",
+    "enable-early-filter",
+    "collect-discards",
+}
+
+VALID_SUMMARIZE_KEYS = {
+    # Filtering
+    "min-ric",
+    "min-len",
+    "max-len",
+    # Grouping
+    "group-identity",
+    # Merging
+    "disable-merging",
+    "merge-effort",
+    "merge-snp",
+    "merge-indel-length",
+    "merge-position-count",
+    "merge-min-size-ratio",
+    "min-merge-overlap",
+    "disable-homopolymer-equivalence",
+    # Selection
+    "select-max-groups",
+    "select-max-variants",
+    "select-strategy",
+    # Processing
+    "scale-threshold",
+    "threads",
+}
+
+
+class ProfileError(Exception):
+    """Error loading or applying a profile."""
+    pass
+
+
+class ProfileVersionError(ProfileError):
+    """Profile version is incompatible with current speconsense version."""
+    pass
+
+
+class ProfileValidationError(ProfileError):
+    """Profile contains invalid keys."""
+    pass
+
+
+@dataclass
+class Profile:
+    """A parameter profile for speconsense tools."""
+    name: str
+    version: str  # e.g., "0.7.*"
+    description: str
+    speconsense: Dict[str, Any] = field(default_factory=dict)
+    speconsense_summarize: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, name: str, check_version: bool = True) -> 'Profile':
+        """Load profile by name from user dir or bundled.
+
+        Args:
+            name: Profile name (without .yaml extension)
+            check_version: If True, validate version compatibility
+
+        Returns:
+            Loaded Profile instance
+
+        Raises:
+            ProfileError: If profile not found
+            ProfileVersionError: If version incompatible
+            ProfileValidationError: If profile contains invalid keys
+        """
+        if yaml is None:
+            raise ProfileError(
+                "PyYAML is required for profile support. "
+                "Install with: pip install pyyaml"
+            )
+
+        # Initialize user profiles directory with example on first use
+        ensure_user_profiles_dir()
+
+        # Try user profile first
+        user_path = PROFILES_DIR / f"{name}.yaml"
+        if user_path.exists():
+            return cls._load_from_path(user_path, name, check_version)
+
+        # Fall back to bundled profile
+        bundled_path = get_bundled_profile_path(name)
+        if bundled_path is not None:
+            return cls._load_from_path(bundled_path, name, check_version)
+
+        # Profile not found - provide helpful error
+        available = list_profiles()
+        if available:
+            raise ProfileError(
+                f"Profile '{name}' not found. Available profiles: {', '.join(available)}"
+            )
+        else:
+            raise ProfileError(
+                f"Profile '{name}' not found and no profiles are available. "
+                f"Check that profiles are installed in {PROFILES_DIR}"
+            )
+
+    @classmethod
+    def _load_from_path(cls, path: Path, name: str, check_version: bool) -> 'Profile':
+        """Load profile from a specific path."""
+        try:
+            with open(path, 'r') as f:
+                data = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            raise ProfileError(f"Invalid YAML in profile '{name}': {e}")
+        except IOError as e:
+            raise ProfileError(f"Cannot read profile '{name}': {e}")
+
+        if not isinstance(data, dict):
+            raise ProfileError(f"Profile '{name}' must be a YAML mapping")
+
+        # Extract fields
+        version = data.get('speconsense-version', '*')
+        description = data.get('description', '')
+        speconsense = data.get('speconsense', {}) or {}
+        speconsense_summarize = data.get('speconsense-summarize', {}) or {}
+
+        # Validate version compatibility
+        if check_version and not check_version_compatible(version, __version__):
+            raise ProfileVersionError(
+                f"Profile '{name}' requires speconsense version {version}, "
+                f"but you have {__version__}.\n\n"
+                f"This profile may use parameters that have changed or been removed.\n"
+                f"Please update the profile for your version, or copy the bundled\n"
+                f"'{name}' profile which is compatible with your version:\n\n"
+                f"  cp {PROFILES_DIR}/{name}.yaml {PROFILES_DIR}/{name}.yaml.bak\n"
+                f"  speconsense --list-profiles  # Will show available profiles"
+            )
+
+        # Validate keys (strict validation)
+        _validate_profile_keys(name, speconsense, speconsense_summarize)
+
+        return cls(
+            name=name,
+            version=version,
+            description=description,
+            speconsense=speconsense,
+            speconsense_summarize=speconsense_summarize,
+        )
+
+
+def check_version_compatible(profile_version: str, current_version: str) -> bool:
+    """Check if profile version pattern matches current version.
+
+    Supports wildcards:
+    - "0.7.*" matches "0.7.0", "0.7.1", etc.
+    - "0.7.0" matches only "0.7.0"
+    - "0.*" matches any 0.x release
+    - "*" matches any version
+
+    Args:
+        profile_version: Version pattern from profile (e.g., "0.7.*")
+        current_version: Current speconsense version (e.g., "0.7.2")
+
+    Returns:
+        True if versions are compatible
+    """
+    if profile_version == '*':
+        return True
+
+    # Convert wildcard pattern to regex
+    # Escape dots and convert * to .*
+    pattern = profile_version.replace('.', r'\.').replace('*', r'.*')
+    pattern = f'^{pattern}$'
+
+    try:
+        return bool(re.match(pattern, current_version))
+    except re.error:
+        # Invalid regex, treat as literal match
+        return profile_version == current_version
+
+
+def _validate_profile_keys(
+    name: str,
+    speconsense: Dict[str, Any],
+    speconsense_summarize: Dict[str, Any]
+) -> None:
+    """Validate that profile only contains known keys.
+
+    Raises ProfileValidationError for unknown keys.
+    """
+    errors = []
+
+    unknown_core = set(speconsense.keys()) - VALID_SPECONSENSE_KEYS
+    if unknown_core:
+        errors.append(f"  speconsense: {', '.join(sorted(unknown_core))}")
+
+    unknown_summarize = set(speconsense_summarize.keys()) - VALID_SUMMARIZE_KEYS
+    if unknown_summarize:
+        errors.append(f"  speconsense-summarize: {', '.join(sorted(unknown_summarize))}")
+
+    if errors:
+        raise ProfileValidationError(
+            f"Profile '{name}' contains unknown keys:\n" + '\n'.join(errors) + "\n\n"
+            f"This may indicate a typo or an option that has been removed.\n"
+            f"Please check the profile and fix or remove the invalid keys."
+        )
+
+
+def get_bundled_profile_path(name: str) -> Optional[Path]:
+    """Get path to bundled profile using importlib.resources.
+
+    Args:
+        name: Profile name (without .yaml extension)
+
+    Returns:
+        Path to bundled profile file, or None if not found
+    """
+    if HAS_IMPORTLIB_RESOURCES:
+        try:
+            # Python 3.9+ style
+            profiles_pkg = importlib_files('speconsense.profiles')
+            profile_file = profiles_pkg.joinpath(f'{name}.yaml')
+            # Check if file exists using as_file context manager
+            with as_file(profile_file) as path:
+                if path.exists():
+                    return path
+        except (TypeError, FileNotFoundError, ModuleNotFoundError):
+            pass
+
+    if HAS_PKG_RESOURCES:
+        try:
+            # Fall back to pkg_resources for Python 3.8
+            resource_path = pkg_resources.resource_filename(
+                'speconsense.profiles', f'{name}.yaml'
+            )
+            path = Path(resource_path)
+            if path.exists():
+                return path
+        except (FileNotFoundError, ModuleNotFoundError):
+            pass
+
+    # Last resort: check relative to this file (profiles/__init__.py)
+    bundled_dir = Path(__file__).parent
+    bundled_path = bundled_dir / f'{name}.yaml'
+    if bundled_path.exists():
+        return bundled_path
+
+    return None
+
+
+def list_bundled_profiles() -> List[str]:
+    """List names of bundled profiles."""
+    profiles = []
+
+    # Try importlib.resources first
+    if HAS_IMPORTLIB_RESOURCES:
+        try:
+            profiles_pkg = importlib_files('speconsense.profiles')
+            for item in profiles_pkg.iterdir():
+                if str(item).endswith('.yaml'):
+                    name = Path(str(item)).stem
+                    profiles.append(name)
+            if profiles:
+                return sorted(profiles)
+        except (TypeError, FileNotFoundError, ModuleNotFoundError):
+            pass
+
+    # Fall back to checking directory relative to this file
+    bundled_dir = Path(__file__).parent
+    if bundled_dir.exists():
+        for yaml_file in bundled_dir.glob('*.yaml'):
+            profiles.append(yaml_file.stem)
+
+    return sorted(profiles)
+
+
+def list_profiles() -> List[str]:
+    """List available profiles (user + bundled).
+
+    Returns list of profile names (without .yaml extension).
+    User profiles take precedence over bundled profiles with same name.
+    """
+    profiles: Set[str] = set()
+
+    # User profiles
+    if PROFILES_DIR.exists():
+        for yaml_file in PROFILES_DIR.glob('*.yaml'):
+            profiles.add(yaml_file.stem)
+
+    # Bundled profiles
+    profiles.update(list_bundled_profiles())
+
+    return sorted(profiles)
+
+
+def ensure_user_profiles_dir() -> Path:
+    """Ensure user profiles directory exists with example profile.
+
+    On first use, creates the directory and copies an example profile
+    to help users create their own profiles.
+
+    This function is safe to call from parallel processes - it uses
+    atomic file operations to avoid race conditions.
+
+    Returns:
+        Path to user profiles directory
+    """
+    import tempfile
+
+    PROFILES_DIR.mkdir(parents=True, exist_ok=True)
+
+    example_path = PROFILES_DIR / "example.yaml"
+
+    # Skip if example already exists (common case, avoid extra work)
+    if example_path.exists():
+        return PROFILES_DIR
+
+    # Skip if user already has other profiles (they don't need the example)
+    if any(PROFILES_DIR.glob('*.yaml')):
+        return PROFILES_DIR
+
+    # Copy example profile atomically (safe for parallel invocations)
+    bundled_example = get_bundled_profile_path('example')
+    if bundled_example is not None:
+        try:
+            # Write to temp file in same directory, then atomic rename
+            fd, temp_path = tempfile.mkstemp(
+                dir=PROFILES_DIR,
+                prefix='.example.',
+                suffix='.yaml.tmp'
+            )
+            try:
+                with open(bundled_example, 'rb') as src:
+                    os.write(fd, src.read())
+            finally:
+                os.close(fd)
+
+            # Atomic rename - if file exists, this either succeeds or fails cleanly
+            # On POSIX: atomic, last writer wins (all have same content, so fine)
+            # On Windows: may raise if file exists, which we catch
+            try:
+                os.rename(temp_path, example_path)
+                logger.info(f"Created example profile at {example_path}")
+            except OSError:
+                # Another process won the race - that's fine
+                if os.path.exists(temp_path):
+                    os.unlink(temp_path)
+        except Exception as e:
+            # Non-fatal - profile system works without example in user dir
+            logger.debug(f"Could not create example profile: {e}")
+
+    return PROFILES_DIR
+
+
+def apply_profile_to_args(
+    args,
+    profile: Profile,
+    tool: str,
+    explicit_args: Set[str]
+) -> None:
+    """Apply profile values to args, respecting explicit CLI overrides.
+
+    Args:
+        args: argparse Namespace to modify
+        profile: Profile to apply
+        tool: Either 'speconsense' or 'speconsense-summarize'
+        explicit_args: Set of argument names that were explicitly provided on CLI
+    """
+    if tool == 'speconsense':
+        profile_values = profile.speconsense
+    elif tool == 'speconsense-summarize':
+        profile_values = profile.speconsense_summarize
+    else:
+        raise ValueError(f"Unknown tool: {tool}")
+
+    for key, value in profile_values.items():
+        # Convert YAML key (with dashes) to argparse attribute name (with underscores)
+        attr_name = key.replace('-', '_')
+
+        # Only apply if not explicitly set on command line
+        if attr_name not in explicit_args:
+            if hasattr(args, attr_name):
+                logger.debug(f"Profile '{profile.name}': setting {attr_name}={value}")
+                setattr(args, attr_name, value)
+            else:
+                # This shouldn't happen if validation passed, but log it
+                logger.warning(
+                    f"Profile '{profile.name}': unknown attribute '{attr_name}'"
+                )
+
+
+def print_profiles_list(tool: str = 'speconsense') -> None:
+    """Print available profiles to stdout.
+
+    Args:
+        tool: Either 'speconsense' or 'speconsense-summarize'
+    """
+    if yaml is None:
+        print("Profile support requires PyYAML. Install with: pip install pyyaml")
+        return
+
+    # Initialize user profiles directory with example on first use
+    ensure_user_profiles_dir()
+
+    profiles = list_profiles()
+
+    if not profiles:
+        print(f"No profiles found.")
+        print(f"\nProfiles are stored in: {PROFILES_DIR}")
+        return
+
+    print(f"Available profiles:\n")
+
+    for name in profiles:
+        try:
+            # Load without version check to show all profiles
+            profile = Profile.load(name, check_version=False)
+
+            # Check if it's a user profile or bundled
+            user_path = PROFILES_DIR / f"{name}.yaml"
+            source = "user" if user_path.exists() else "bundled"
+
+            # Check version compatibility
+            compatible = check_version_compatible(profile.version, __version__)
+            compat_str = "" if compatible else " [INCOMPATIBLE]"
+
+            print(f"  {name} ({source}){compat_str}")
+            if profile.description:
+                print(f"    {profile.description}")
+            print(f"    Version: {profile.version}")
+            print()
+
+        except ProfileError as e:
+            print(f"  {name} [ERROR: {e}]")
+            print()
+
+    print(f"Usage: {tool} -p <profile> [other options]")
+    print(f"Profile directory: {PROFILES_DIR}")

speconsense/profiles/example.yaml

@@ -0,0 +1,97 @@
+# Example Speconsense Profile
+#
+# Copy this file to create your own profiles:
+#   cp example.yaml my-workflow.yaml
+#
+# Then use it with:
+#   speconsense input.fastq -p my-workflow
+#   speconsense-summarize -p my-workflow
+#
+# HOW TO USE THIS FILE:
+#   - Lines starting with # are comments (ignored)
+#   - To enable an option, remove the # at the start of the line
+#   - Only include parameters you want to change from defaults
+#   - CLI arguments always override profile values
+#
+# Example: To set min-size to 10, change:
+#   # min-size: 5
+# to:
+#   min-size: 10
+
+speconsense-version: "0.7.*"
+description: "My custom workflow"
+
+# =============================================================================
+# Parameters for speconsense (clustering and consensus)
+# =============================================================================
+speconsense:
+
+  # --- Clustering Algorithm ---
+  # algorithm: graph              # Clustering algorithm: graph (MCL) or greedy
+  # min-identity: 0.9             # Similarity threshold for clustering (0.0-1.0)
+  # inflation: 2.0                # MCL inflation parameter (higher = more clusters)
+  # k-nearest-neighbors: 10       # K-NN graph construction parameter
+
+  # --- Cluster Filtering ---
+  # min-size: 5                   # Minimum reads per cluster (0 to disable)
+  # min-cluster-ratio: 0.01       # Minimum cluster size as ratio of total reads
+  # outlier-identity: 0.85        # Identity threshold for outlier detection
+
+  # --- Read Sampling ---
+  # presample: 1000               # Initial random sampling of input reads
+  # max-sample-size: 100          # Max reads used for consensus generation
+
+  # --- Variant Calling (position-based phasing) ---
+  # min-variant-frequency: 0.10   # Min frequency to call a variant (0.0-1.0)
+  # min-variant-count: 3          # Min read count to call a variant
+  # disable-position-phasing: false  # Set true to disable variant detection
+
+  # --- Ambiguity Calling (IUPAC codes) ---
+  # min-ambiguity-frequency: 0.25 # Min frequency for IUPAC ambiguity codes
+  # min-ambiguity-count: 3        # Min read count for ambiguity codes
+  # disable-ambiguity-calling: false  # Set true to disable IUPAC codes
+
+  # --- Cluster Merging ---
+  # disable-cluster-merging: false     # Set true to skip merging similar clusters
+  # disable-homopolymer-equivalence: false  # Set true for strict homopolymer comparison
+
+  # --- Primer Orientation ---
+  # orient-mode: filter-failed    # skip, keep-all, or filter-failed
+
+  # --- Processing ---
+  # threads: 0                    # Max threads (0 = auto-detect)
+  # scale-threshold: 500          # Read count threshold for scaled processing
+  # enable-early-filter: false    # Enable early filtering optimization
+  # collect-discards: false       # Write discarded reads to separate file
+
+# =============================================================================
+# Parameters for speconsense-summarize (post-processing)
+# =============================================================================
+speconsense-summarize:
+
+  # --- Input Filtering ---
+  # min-ric: 3                    # Minimum Reads in Consensus threshold
+  # min-len: 0                    # Minimum sequence length (0 = disabled)
+  # max-len: 0                    # Maximum sequence length (0 = disabled)
+
+  # --- Variant Grouping (HAC clustering) ---
+  # group-identity: 0.95          # Identity threshold for grouping variants
+
+  # --- MSA-based Merging ---
+  # disable-merging: false        # Set true to skip all merging (fastest)
+  # merge-effort: balanced        # Effort level: fast, balanced, thorough, or 6-14
+  # merge-snp: true               # Enable SNP-based merging
+  # merge-indel-length: 3         # Max indel length to merge (0 = disabled)
+  # merge-position-count: 1       # Max variant positions to merge
+  # merge-min-size-ratio: 0.1     # Min size ratio for merge candidate
+  # min-merge-overlap: 0.9        # Min alignment overlap for merging
+  # disable-homopolymer-equivalence: false  # Set true for strict comparison
+
+  # --- Variant Selection ---
+  # select-max-groups: -1         # Max groups to output (-1 = no limit)
+  # select-max-variants: -1       # Max variants per group (-1 = no limit)
+  # select-strategy: size         # Selection strategy: size or diversity
+
+  # --- Processing ---
+  # threads: 0                    # Max threads (0 = auto-detect)
+  # scale-threshold: 500          # Read count threshold for scaled processing

speconsense/profiles/herbarium.yaml

@@ -0,0 +1,25 @@
+# High-recall settings for herbarium specimens with degraded DNA
+#
+# Use this profile when working with:
+# - Herbarium specimens where DNA may be degraded
+# - Type specimens that cannot be resampled
+# - Samples where contamination may dominate the target
+# - Any situation where false negatives are more costly than false positives
+#
+# The settings prioritize keeping all potential biological signal for
+# manual review, at the cost of more output variants.
+
+speconsense-version: "0.7.*"
+description: "High-recall settings for herbarium specimens with degraded DNA"
+
+speconsense:
+  min-identity: 0.85            # Lower threshold for degraded DNA
+  # outlier-identity: auto      # Auto-calculated as 0.925 from min-identity
+  min-cluster-ratio: 0          # Keep all clusters
+  min-size: 3                   # Lower minimum for degraded samples
+  min-variant-frequency: 0.05   # More sensitive variant detection (5%)
+  presample: 0                  # Use all reads
+  max-sample-size: 100          # 100 reads for high-quality consensus
+
+speconsense-summarize:
+  min-ric: 3                    # Accept lower-support consensus

speconsense/profiles/largedata.yaml

@@ -0,0 +1,19 @@
+# Speconsense profile for a single large input file
+
+speconsense-version: "0.7.*"
+description: "Large Input (experimental)"
+
+# =============================================================================
+# Parameters for speconsense (clustering and consensus)
+# =============================================================================
+speconsense:
+  min-cluster-ratio: 0            # Do not filter by relative size
+  presample: 0                    # Process all reads
+  threads: 0                      # Max threads (0 = auto-detect)
+
+# =============================================================================
+# Parameters for speconsense-summarize (post-processing)
+# =============================================================================
+speconsense-summarize:
+  merge-effort: fast
+  group-identity: 0.95            # Avoid spurious groupings with many sequences

speconsense/profiles/nostalgia.yaml

@@ -0,0 +1,22 @@
+# Speconsense Profile to simulate results of older bioinformatics pipelines.
+# Included for comparison only - not recommended for production usage
+
+speconsense-version: "0.7.*"
+description: "Simulate older bioinformatics"
+
+# Parameters for speconsense (clustering and consensus)
+speconsense:
+  algorithm: greedy
+  min-identity: 0.85
+  outlier-identity: 0.85
+  disable-position-phasing: True
+  disable-ambiguity-calling: True
+  min-cluster-ratio: 0.2
+  min-size: 5 
+  max-sample-size: 500       
+  presample: 500
+
+# Parameters for speconsense-summarize (post-processing)
+speconsense-summarize:
+  min-ric: 5
+  disable-merging: true        # Skip merging entirely