pav3 3.0.0.dev22__tar.gz → 3.0.0.dev23__tar.gz

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pav3
-Version: 3.0.0.dev22
+Version: 3.0.0.dev23
 Summary: Assembly-based variant discovery
 License-Expression: MIT
 Requires-Python: >=3.12
@@ -13,7 +13,7 @@ Requires-Dist: frozendict>=2.4.6
 Requires-Dist: hmmlearn>=0.3.3
 Requires-Dist: intervaltree>=3.1.0
 Requires-Dist: numpy>=2.3.2
-Requires-Dist: polars[numpy,pandas,pyarrow]>=1.35.2
+Requires-Dist: polars[numpy,pandas,pyarrow]>=1.41.0
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pysam>=0.23.3
 Requires-Dist: scipy>=1.16.1

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
     "hmmlearn>=0.3.3",
     "intervaltree>=3.1.0",
     "numpy>=2.3.2",
-    "polars[numpy,pandas,pyarrow]>=1.35.2",
+    "polars[numpy,pandas,pyarrow]>=1.41.0",
     "pyarrow>=21.0.0",
     "pysam>=0.23.3",
     "scipy>=1.16.1",
@@ -80,6 +80,7 @@ dev = [
     "flake8-rst-docstrings>=0.3.1",
     "ipython>=9.7.0",
     "ply>=3.11",
+    "pygithub>=2.9.1",
     "pylint>=3.3.7",
     "pytest>=8.4.1",
     "pytest-cov>=6.2.1",

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/__init__.py

@@ -1,6 +1,6 @@
 """PAV Python library routines."""
 
-__version__ = '3.0.0.dev22'
+__version__ = '3.0.0.dev23'
 
 __all__ = [
     'align',

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/data/workflow/Snakefile

@@ -23,7 +23,7 @@ PAV_CONFIG = config
 
 # Read sample table
 ASM_TABLE_PATH = pav3.pipeline.get_pav_asm_table_path(
-    PAV_CONFIG.get('assembly_table', None)
+    pav_config=PAV_CONFIG
 )
 
 ASM_TABLE = pav3.pipeline.read_assembly_table(ASM_TABLE_PATH, PAV_CONFIG)

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/data/workflow/rules/call.smk

@@ -286,7 +286,7 @@ rule call_tables:
             ref_path = Path(next(in_file).strip())
 
         callsets = tuple(
-            tuple((
+            (
                 (
                     pl.scan_parquet(
                         f'results/{wildcards.asm_name}/call_hap/call_{wildcards.vartype}_{hap}.parquet'
@@ -294,7 +294,7 @@ rule call_tables:
                     .with_row_index('_index')
                 ),
                 f'{wildcards.asm_name}-{hap}',
-            ))
+            )
             for hap in pav3.pipeline.get_hap_list(wildcards.asm_name, ASM_TABLE)
         )
 

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/data/workflow/rules/data.smk

@@ -59,8 +59,7 @@ rule align_get_qry_fa:
         out_filename = f'data/query/{wildcards.asm_name}/query_{wildcards.hap}.fa'  # ".gz" is appended as needed
 
         if len(input_tuples) == 1 and input_tuples[0][1] == 'fasta':
-            os.makedirs(os.path.dirname(out_filename), exist_ok=True)
-            fa_path_list = pav3.pipeline.link_fasta(input_tuples[0][0], out_filename)
+            fa_path_list = pav3.pipeline.link_fasta(Path(input_tuples[0][0]), out_filename)
 
         else:
             # Merge/write FASTA from multiple sources and/or GFA files

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/dev/path.py

@@ -1,5 +1,6 @@
 """Path utilities for development environments."""
 
+import os
 from pathlib import Path
 import tomllib
 
@@ -19,8 +20,16 @@ def get_pav_dir(
         directory.
     """
 
-    if path is None:
-        raise ValueError('path is None')
+    org_path = path
+
+    if path is None or not Path(path).exists():
+        path = os.environ.get('PAV_PATH', None)
+
+        if path is None:
+            path = os.environ.get('UV_PROJECT', None)
+
+        if path is None:
+            raise ValueError(f'Path is missing: {org_path}')
 
     path = Path(path)
 
@@ -36,5 +45,4 @@ def get_pav_dir(
                         == 'pav3'
                 ):
                     return parent
-
     raise FileNotFoundError(f'path {path} does not point to a resource within a PAV source directory')

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/pipeline.py

@@ -866,16 +866,25 @@ def link_fasta(
 
     # Link FASTA
     dest_fa_path.unlink(missing_ok=True)
-    dest_fa_path.symlink_to(source)
+    dest_fa_path.parent.mkdir(parents=True, exist_ok=True)
+
+    dest_fa_path.symlink_to(
+        resolve_relative(source, dest_fa_path)
+    )
 
     # Link indexes (if present)
     if source_is_indexed:
         dest_fai_path.unlink(missing_ok=True)
-        dest_fai_path.symlink_to(source_fai)
+
+        dest_fai_path.symlink_to(
+            resolve_relative(source_fai, dest_fai_path)
+        )
 
         if dest_gzi_path is not None:
             dest_gzi_path.unlink(missing_ok=True)
-            dest_gzi_path.symlink_to(source_gzi)
+            dest_gzi_path.symlink_to(
+                resolve_relative(source_gzi, dest_gzi_path)
+            )
 
     else:
         # Create indexes
@@ -886,6 +895,37 @@ def link_fasta(
         dest_fa_path, dest_fai_path
     ] + [dest_gzi_path] if dest_gzi_path is not None else []
 
+def resolve_relative(
+        source_path: str | Path,
+        dest_path: str | Path,
+) -> Path:
+    """Resolve a path relative to another path.
+
+    If both source and destination paths are relative, assume they hare relative to the current
+    working directory and return a relative path from the destination path to the source path.
+    This can be used to make symbolic links within a directory structure allowing the directories
+    to move locations without breaking absolute paths.
+
+    If either the source or the destination path is absolute, the absolute source path is
+    returned.
+
+    :param source_path: Source path.
+    :param dest_path: Destination path.
+
+    :returns: Resolved path.
+    """
+    source_path = Path(source_path)
+    dest_path = Path(dest_path)
+
+    if source_path.is_absolute():
+        return source_path
+
+    if dest_path.is_absolute():
+        return source_path.absolute()
+
+    # Return a path relative to the destination path
+    return source_path.relative_to(dest_path.parent, walk_up=True)
+
 
 def get_rules_path(
         rule_filename: str | Path,
@@ -923,24 +963,32 @@ def get_rules_path(
 
 def get_pav_config_path(
         config_filename: Optional[str | Path] = None,
+        root_path: Optional[str | Path] = None,
 ) -> Path:
     """Locate the PAV configuration filename.
 
     :param config_filename: Path to the config file. If None, the default config file is used.
+    :param root_path: Root path to search for the config file. If None, the current directory
+        is used.
 
     :return: Path to the config file.
 
     :raises FileNotFoundError: If the specified configuration (`config_filename`) or default
         configuration file is not found.
     """
+    if root_path is None:
+        root_path = Path('.')
+    else:
+        root_path = Path(root_path)
+
     default_paths = (
-        Path('pav.json'),
-        Path('pav.json.txt'),
+        root_path / 'pav.json',
+        root_path / 'pav.json.txt',
     )
 
     deprecated_paths = (
-        Path('config.json'),
-        Path('config.json.txt'),
+        root_path / 'config.json',
+        root_path / 'config.json.txt',
     )
 
     if config_filename is not None:
@@ -988,33 +1036,58 @@ def get_pav_config_path(
 
 
 def get_pav_asm_table_path(
-        asm_table_path: Optional[str | Path] = None,
+        pav_config: Optional[dict] = None,
+        root_path: Optional[str | Path] = Path('.'),
 ) -> Path:
     """Get a path to the assembly table.
 
-    :param asm_table_path: Table path, if specified.
+    The assembly table path is read from the PAV configuration if it is specified there. If it is
+    not specified in the configuration, the default assembly table paths are searched. All paths
+    are relative to `root_path` (defaults to the current directory).
+
+    :param pav_config: PAV configuration, if specified. Reads the assembly table path from the
+        configuration if it is specified there.
+    :param root_path: Root path to search for the table file. If None, the current directory
+        is used.
 
     :return: Path to the assembly table.
 
-    :raises FileNotFoundError: If the specified assembly table or default assembly table is not found.
+    :raises FileNotFoundError: If the specified assembly table or default assembly table is not
+        found.
     """
 
-    if asm_table_path is not None:
-        asm_table_path = Path(asm_table_path)
+    if root_path is None:
+        root_path = Path('.')
+    else:
+        root_path = Path(root_path)
+
+    if pav_config is not None and 'assembly_table' in pav_config:
+        asm_table_path = root_path / str(pav_config['assembly_table'])
 
         if not asm_table_path.is_file():
-            raise FileNotFoundError(f'Specified assembly table file not found or not a regular file: {asm_table_path}')
+            raise FileNotFoundError(
+                f'Assembly table file defined in the PAV configuration is not found '
+                f'or not a regular file: {asm_table_path}'
+            )
 
         return asm_table_path
 
-    for asm_table_path in (
-        Path('assemblies.tsv'),
-        Path('assemblies.xlsx')
-    ):
+    default_paths = [
+        root_path / 'assemblies.tsv',
+        root_path / 'assemblies.xlsx',
+    ]
+
+    for asm_table_path in default_paths:
         if asm_table_path.is_file():
             return asm_table_path
 
-    raise FileNotFoundError('No assembly table file found (tried default "assemblies.tsv" and "assemblies.xlsx")')
+    default_path_str = ', '.join(
+        [f'"{str(p)}"' for p in default_paths]
+    )
+
+    raise FileNotFoundError(
+        f'No assembly table file found (tried default paths: {default_path_str})'
+    )
 
 
 def get_pav_env_path(
@@ -1031,7 +1104,9 @@ def get_pav_env_path(
 
     if env_path is not None:
         if not env_path.is_file():
-            raise FileNotFoundError(f'Specified environment file not found or not a regular file: {env_path}')
+            raise FileNotFoundError(
+                f'Specified environment file not found or not a regular file: {env_path}'
+            )
 
         return env_path
 

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3/workflow/call.py

@@ -24,10 +24,61 @@ from ..call.expr import sort_expr
 logger = logging.getLogger(__name__)
 
 
+def _materialize(
+        df: pl.LazyFrame,
+        temp_file_container: TempDirContainer,
+        label: str,
+) -> pl.LazyFrame:
+    """Write a lazy frame to a temporary Parquet file and return a streaming scan of it.
+
+    Used as a materialization barrier between lazy operations. It serves two purposes:
+
+    1. Memory: Variant tables may be very large. Writing intermediate results to a temporary file and re-reading
+       them as streaming scans keeps them off the Python heap instead of accumulating collected frames in memory
+       across the integration loop.
+
+    2. Correctness: `apply_trim_filter` and `apply_discord_and_inner_filter` each add a row-index column
+       (`with_row_index`). When both are chained into a single lazy plan and that plan is sunk, the Polars
+       new-streaming engine mis-optimizes the chained row-index operations and raises a spurious
+       "duplicate name when building schema" / "column not found" error. The same plan collects correctly with
+       the in-memory engine, confirming the logical plan is valid and the fault is in the streaming optimizer.
+       Breaking the plan with a Parquet round-trip isolates each helper's row index and avoids the bug.
+
+    :param df: Lazy frame to materialize.
+    :param temp_file_container: Active temporary file container (must be inside its context).
+    :param label: Label used in the temporary file name.
+
+    :return: Streaming scan over the materialized Parquet file.
+    """
+    temp_file = temp_file_container.next(prefix=f'integrate_{label}_', suffix='.parquet')
+    df.sink_parquet(temp_file)
+    return pl.scan_parquet(temp_file)
+
+
 def integrate_sources(
         pav_params,
         input: dict[str, str],
         output: dict[str, str],
+        temp_dir_name: Optional[str] = None,
+) -> None:
+    """Integrate variant calls from all sources into the final per-haplotype call tables.
+
+    :param pav_params: PAV parameters.
+    :param input: Input file dictionary (Snakemake rule input).
+    :param output: Output file dictionary (Snakemake rule output).
+    :param temp_dir_name: Parent directory for temporary files. Defaults to the system temporary directory.
+    """
+    # Intermediate variant tables are materialized to a temporary directory and re-read as streaming scans (see
+    # `_materialize`). This bounds memory for large callsets and works around a Polars new-streaming engine bug.
+    with TempDirContainer(temp_dir_name, prefix='pav_integrate_') as temp_file_container:
+        _integrate_sources(pav_params, input, output, temp_file_container)
+
+
+def _integrate_sources(
+        pav_params,
+        input: dict[str, str],
+        output: dict[str, str],
+        temp_file_container: TempDirContainer,
 ) -> None:
     inv_min = pav_params.inv_min
     inv_max = pav_params.inv_max if pav_params.inv_max > 0 else float('inf')
@@ -130,6 +181,11 @@ def integrate_sources(
         if not (is_lg or is_inv):
             df = pav3.call.integrate.apply_trim_filter(df, df_trim)
 
+            # Materialization barrier between the two row-index helpers. `apply_trim_filter` and
+            # `apply_discord_and_inner_filter` both call `with_row_index`; chaining them into one sunk plan trips a
+            # Polars streaming-engine bug (see `_materialize`). The round-trip isolates the two row indices.
+            df = _materialize(df, temp_file_container, f'{sourcetype_vartype}_trim')
+
         # Filter DISCORD
         if filter_discord or filter_inner:
             df = pav3.call.integrate.apply_discord_and_inner_filter(
@@ -145,6 +201,11 @@ def integrate_sources(
             existing_ids=collect_list[vartype]
         )
 
+        # Materialize the fully-processed variant table and continue from a streaming scan. Every downstream use
+        # (DISCORD regions, CPX-derived variants, writes, and cross-source aggregation in `collect_list`) reads from
+        # this scan. This keeps large tables off the heap and replaces the previous in-memory `df.collect().lazy()`.
+        df = _materialize(df, temp_file_container, sourcetype_vartype)
+
         # Read CPX segment table
         if sourcetype == 'inter':
             df_segment_var = df_segment.join(df.select(['var_index', 'id']), on='var_index', how='left')
@@ -180,11 +241,10 @@ def integrate_sources(
                 collect_list=collect_list,
             )
 
-        # Aggregate if type is split over multiple inputs
+        # Aggregate if type is split over multiple inputs. `df` is already a disk-backed scan (materialized above),
+        # so this keeps memory bounded even when several sources accumulate in `collect_list`.
         if not do_write or len(collect_list[vartype]) > 0:
-            # Collect here, avoid re-collecting for ID (used as existing IDs) and write.
-            # Note: May consume significant memory for some callsets, consider writing to temp and re-reading lazy
-            collect_list[vartype].append(df.collect().lazy())
+            collect_list[vartype].append(df)
 
         # Write
         write_list = [update_discord_frame]

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pav3
-Version: 3.0.0.dev22
+Version: 3.0.0.dev23
 Summary: Assembly-based variant discovery
 License-Expression: MIT
 Requires-Python: >=3.12
@@ -13,7 +13,7 @@ Requires-Dist: frozendict>=2.4.6
 Requires-Dist: hmmlearn>=0.3.3
 Requires-Dist: intervaltree>=3.1.0
 Requires-Dist: numpy>=2.3.2
-Requires-Dist: polars[numpy,pandas,pyarrow]>=1.35.2
+Requires-Dist: polars[numpy,pandas,pyarrow]>=1.41.0
 Requires-Dist: pyarrow>=21.0.0
 Requires-Dist: pysam>=0.23.3
 Requires-Dist: scipy>=1.16.1

{pav3-3.0.0.dev22 → pav3-3.0.0.dev23}/src/pav3.egg-info/requires.txt

@@ -5,7 +5,7 @@ frozendict>=2.4.6
 hmmlearn>=0.3.3
 intervaltree>=3.1.0
 numpy>=2.3.2
-polars[numpy,pandas,pyarrow]>=1.35.2
+polars[numpy,pandas,pyarrow]>=1.41.0
 pyarrow>=21.0.0
 pysam>=0.23.3
 scipy>=1.16.1