seqbank 0.1.2__py3-none-any.whl

seqbank/__init__.py

@@ -0,0 +1,2 @@
+from .seqbank import SeqBank
+from .exceptions import SeqBankError

seqbank/dfam.py

@@ -0,0 +1,85 @@
+import gzip
+from pathlib import Path
+import h5py
+import tempfile
+import shutil
+
+from .io import download_file
+from .seqbank import SeqBank
+
+def dfam_url(curated: bool = True, release: str = "current") -> str:
+    """
+    Constructs the URL for the Dfam database file.
+
+    Args:
+        curated (bool, optional): If True, fetches the curated-only version of the Dfam database. Defaults to True.
+        release (str, optional): The Dfam release version. Defaults to "current".
+
+    Returns:
+        str: The URL for the Dfam HDF5 file.
+    """
+    curated_str = "_curatedonly" if curated else ""
+    url = f"https://www.dfam.org/releases/{release}/families/Dfam{curated_str}.h5.gz"
+    return url
+
+
+def add_dfam(seqbank: SeqBank, local_path: Path) -> None:
+    """
+    Adds sequences from a local Dfam HDF5 file to the SeqBank.
+
+    Args:
+        seqbank (SeqBank): The SeqBank instance to add sequences to.
+        local_path (Path): The path to the local Dfam HDF5 file.
+    """
+    file = h5py.File(local_path, 'r')
+    
+    def visitor_func(name, node):
+        if isinstance(node, h5py.Dataset):
+            accession = node.attrs['accession']
+            seq = node.attrs['consensus']
+            seqbank.add(seq=seq, accession=accession)
+
+    file['Families/DF'].visititems(visitor_func)
+
+
+def download_dfam(seqbank: SeqBank, curated: bool = True, release: str = "current", force: bool = False) -> bool:
+    """
+    Downloads the Dfam HDF5 file, decompresses it, and adds sequences to the SeqBank.
+
+    Args:
+        seqbank (SeqBank): The SeqBank instance to which the Dfam sequences will be added.
+        curated (bool, optional): If True, fetches the curated-only version of the Dfam database. Defaults to True.
+        release (str, optional): The Dfam release version. Defaults to "current".
+        force (bool, optional): If True, forces the download even if the URL has been previously processed. Defaults to False.
+
+    Returns:
+        bool: True if the download and addition were successful, False otherwise.
+    """
+    url = dfam_url(curated, release)
+    url_key = seqbank.key_url(url)
+    
+    if url_key in seqbank.file and not force:
+        print(f"Already downloaded: {url}")
+        return False
+    
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        local_gzip_path = Path(tmpdirname) / Path(url).name
+        try:
+            print(f"Downloading Dfam: {url}")
+            download_file(url, local_gzip_path)
+
+            # Decompress the gzipped file
+            local_path = local_gzip_path.with_suffix("")
+            with gzip.open(local_gzip_path, 'rb') as infile, open(local_path, 'wb') as outfile:
+                shutil.copyfileobj(infile, outfile)
+
+            # Add sequences to SeqBank
+            add_dfam(seqbank, local_path)
+
+            # Mark URL as processed
+            seqbank.save_seen_url(url)
+        except Exception as err:
+            print(f"Failed to add Dfam: {url}: {err}")
+            return False
+
+    return True

seqbank/exceptions.py

@@ -0,0 +1,7 @@
+class SeqBankError(Exception):
+    """
+    Custom exception class for handling errors specific to SeqBank operations.
+
+    This exception should be raised when any operation related to the SeqBank encounters an error.
+    """
+    pass

seqbank/io.py

@@ -0,0 +1,158 @@
+from pathlib import Path
+import gzip
+import bz2
+from Bio import SeqIO
+import requests
+import tempfile
+
+
+def open_path(path: Path | str):
+    """
+    Opens a file, optionally decompressing it based on its extension.
+
+    Args:
+        path (Path | str): The path to the file, which can be a compressed (.gz, .bz2) or uncompressed file.
+
+    Returns:
+        file object: A file object opened for reading.
+
+    Raises:
+        ValueError: If the file has an unsupported compression extension.
+    """
+    path = Path(path)
+    suffix = path.suffix.lower()
+
+    if suffix == ".gz":
+        return gzip.open(path, "rt")
+    
+    if suffix == ".bz2":
+        return bz2.open(path, "rt")
+    
+    return open(path, "rt")
+
+
+def get_file_format(path: Path | str) -> str:
+    """
+    Determines the sequence file format based on the file extension.
+
+    Supported file formats and their extensions:
+        - FASTA: .fa, .fna, .fasta
+        - GenBank: .genbank, .gb, .gbk
+        - EMBL: .embl
+        - Tab-delimited: .tab, .tsv
+        - Nexus: .nexus, .nxs
+        - FASTQ: .fastq, .fq
+
+    Args:
+        path (Path | str): The path to the file.
+
+    Returns:
+        str: The detected file format (e.g., 'fasta', 'genbank', 'embl', etc.).
+
+    Raises:
+        ValueError: If the file format cannot be determined.
+    """
+    path = Path(path)
+    suffix = path.suffix.lower()
+
+    if suffix in [".gz", ".bz2"]:
+        suffix = path.suffixes[-2].lower()
+
+    if suffix in [".fa", ".fna", ".fasta"]:
+        return "fasta"
+
+    if suffix in [".genbank", ".gb", ".gbk"]:
+        return "genbank"
+
+    if suffix in [".embl"]:
+        return "embl"
+
+    if suffix in [".tab", ".tsv"]:
+        return "tab"
+
+    if suffix in [".nexus", ".nxs"]:
+        return "nexus"
+
+    if suffix in [".fastq", ".fq"]:
+        return "fastq"
+
+    raise ValueError(f"Cannot determine file format of {path}.")
+
+
+def seq_count(path: Path | str) -> int:
+    """
+    Counts the number of sequences in a file.
+
+    Args:
+        path (Path | str): The path to the file.
+
+    Returns:
+        int: The total number of sequences in the file.
+    """
+    file_format = get_file_format(path)
+    with open_path(path) as f:
+        total = sum(1 for _ in SeqIO.parse(f, file_format))
+    return total
+
+
+def download_file(url: str, local_path: Path) -> Path:
+    """
+    Downloads a file from a URL and saves it to a specified local path.
+
+    Args:
+        url (str): The URL of the file to download.
+        local_path (Path): The path where the file should be saved.
+
+    Returns:
+        Path: The path to the downloaded file.
+
+    Raises:
+        requests.exceptions.HTTPError: If the download fails.
+    """
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_path, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192): 
+                f.write(chunk)
+    return local_path
+
+
+class TemporaryDirectory(tempfile.TemporaryDirectory):
+    """
+    A custom TemporaryDirectory class that can optionally create a parent directory if a prefix is provided.
+
+    Args:
+        prefix (str | Path | None): Optional prefix for the temporary directory.
+    """
+
+    def __init__(self, prefix: str | Path | None = None, *args, **kwargs):
+        self._created_dirs = []  # Track directories that were created
+        if isinstance(prefix, Path):
+            # Resolve path to string and ensure parent directories are created
+            prefix.mkdir(parents=True, exist_ok=True)
+            self._created_dirs.append(prefix)  # Track created parent directories
+
+            # Ensure prefix ends with a "/"
+            prefix = str(prefix.resolve().absolute())
+            if not prefix.endswith("/"):
+                prefix += "/"
+            
+        super().__init__(prefix=prefix, *args, **kwargs)
+
+    def cleanup(self):
+        """
+        Cleans up the temporary directory and removes any created parent directories if they are empty.
+        """
+        super().cleanup()
+        for created_dir in self._created_dirs:
+            if not any(created_dir.iterdir()):  # Only remove if the directory is empty
+                created_dir.rmdir()
+    
+    def __enter__(self) -> Path:
+        """
+        Enters the context of the temporary directory.
+
+        Returns:
+            Path: The path to the temporary directory.
+        """
+        return Path(super().__enter__())

seqbank/main.py

@@ -0,0 +1,166 @@
+import typer
+from pathlib import Path
+from rich.progress import track
+
+from .refseq import get_refseq_urls
+from .seqbank import SeqBank
+from .dfam import download_dfam
+
+import plotly.graph_objs as go
+
+app = typer.Typer()
+
+@app.command()
+def add(path: Path, files: list[Path], format: str = "", filter: Path = None) -> None:
+    """Add sequences from a file or list of files to a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        files (list[Path]): A list of file paths containing sequences.
+        format (str, optional): The format of the sequence files. Defaults to "".
+        filter (Path, optional): A filter file for sequences. Defaults to None.
+    """
+    print(f"Opening seqbank '{path}'")
+    seqbank = SeqBank(path=path, write=True)
+    seqbank.add_files(files, format=format, filter=filter)
+
+
+@app.command()
+def url(path: Path, urls: list[str], format: str = "", max: int = 0, workers: int = -1, tmp_dir: Path = None) -> None:
+    """Add sequences from a list of URLs to a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        urls (list[str]): A list of URLs containing sequences.
+        format (str, optional): The format of the sequence files. Defaults to "".
+        max (int, optional): Maximum number of sequences to add. Defaults to 0 (all).
+        workers (int, optional): Number of workers to use for downloading. Defaults to -1.
+        tmp_dir (Path, optional): Temporary directory for downloads. Defaults to None.
+    """
+    print(f"Opening seqbank '{path}'")
+    seqbank = SeqBank(path=path, write=True)
+
+    seqbank.add_urls(urls, format=format, max=max, workers=workers, tmp_dir=tmp_dir)
+
+
+@app.command()
+def delete(path: Path, accessions: list[str]) -> None:
+    """Delete sequences from a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        accessions (list[str]): A list of accessions to delete from the SeqBank.
+    """
+    print(f"Opening seqbank '{path}'")
+    seqbank = SeqBank(path=path, write=True)
+    
+    for accession in track(accessions, "Deleting"):
+        seqbank.delete(accession)
+
+
+@app.command()
+def refseq(path: Path, max: int = 0, workers: int = -1, tmp_dir: Path = None) -> None:
+    """Download all RefSeq sequences to a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        max (int, optional): Maximum number of sequences to add. Defaults to 0 (all).
+        workers (int, optional): Number of workers to use for downloading. Defaults to -1.
+        tmp_dir (Path, optional): Temporary directory for downloads. Defaults to None.
+    """
+    print("Getting RefSeq files list")
+    return url(path, get_refseq_urls(tmp_dir=tmp_dir), max=max, workers=workers, tmp_dir=tmp_dir)
+
+
+@app.command()
+def dfam(path: Path, release: str = "current", curated: bool = True) -> bool:
+    """Download DFam sequences to a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        release (str, optional): The DFam release version to download. Defaults to "current".
+        curated (bool, optional): Whether to download curated sequences. Defaults to True.
+
+    Returns:
+        bool: True if the download and addition were successful, False otherwise.
+    """
+    print("Getting DFam")
+    seqbank = SeqBank(path=path, write=True)
+    return download_dfam(seqbank, release=release, curated=curated)
+
+
+@app.command()
+def ls(path: Path) -> None:
+    """List accessions in a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+    """
+    seqbank = SeqBank(path=path)
+    seqbank.ls()
+
+
+@app.command()
+def count(path: Path) -> None:
+    """Display the number of accessions in a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+    """
+    seqbank = SeqBank(path=path)
+    print(len(seqbank))
+
+
+@app.command()
+def cp(path: Path, new: Path) -> None:
+    """Copy each sequence from one SeqBank to another.
+
+    Args:
+        path (Path): The path to the source SeqBank.
+        new (Path): The path to the destination SeqBank.
+    """
+    print(f"Copying seqbank '{path}' to '{new}'")
+    seqbank = SeqBank(path=path)
+    new = SeqBank(path=new, write=True)
+    seqbank.copy(new)
+
+
+@app.command()
+def export(path: Path, output: Path, format: str = "fasta") -> None:
+    """Export a SeqBank to a specified format.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        output (Path): The path to save the exported sequences.
+        format (str, optional): The format for the exported sequences. Defaults to "fasta".
+    """
+    print(f"Exporting seqbank '{path}' to '{output}' in {format} format")
+    seqbank = SeqBank(path=path)
+    return seqbank.export(output, format=format)
+
+
+@app.command()
+def histogram(path: Path, output_path: Path = None, show: bool = False, nbins: int = 30) -> None:
+    """Generate a histogram of sequence lengths from a SeqBank.
+
+    Args:
+        path (Path): The path to the SeqBank.
+        output_path (Path, optional): The path to save the histogram. If None, the histogram will be displayed.
+        show (bool, optional): Whether to display the histogram. Defaults to False.
+        nbins (int, optional): The number of bins for the histogram. Defaults to 30.
+    """
+    # Load the SeqBank
+    seqbank = SeqBank(path=path)
+
+    # Generate the histogram
+    fig: go.Figure = seqbank.histogram(nbins=nbins)
+
+    # Save the histogram to the specified output path
+    if output_path is None:
+        show = True
+    else:
+        fig.write_image(output_path)
+        print(f"Histogram saved to {output_path}")
+
+    if show:
+        fig.show()

seqbank/refseq.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+import re
+from .io import download_file, TemporaryDirectory
+
+def get_refseq_filenames(tmp_dir: str | Path | None = None) -> list[str]:
+    """
+    Retrieves a list of RefSeq genomic filenames from the NCBI FTP site.
+
+    Args:
+        tmp_dir (str | Path | None, optional): 
+            The directory to create a temporary folder in. If None, a default temporary directory is used.
+    
+    Returns:
+        list[str]: A list of filenames sorted numerically by version.
+    """
+    with TemporaryDirectory(prefix=tmp_dir) as dirname:
+        local_path = dirname / 'refseq_complete.html'
+
+        download_file("https://ftp.ncbi.nlm.nih.gov/refseq/release/complete/", local_path)
+        text = local_path.read_text()
+
+        # Find all .genomic.fna.gz filenames
+        filenames = re.findall(r'>(.*?.genomic.fna.gz)<\/a>', text)
+        
+        # Sort filenames numerically by version number (second part of filename)
+        filenames = sorted(filenames, key=lambda filename: int(filename.split(".")[1]))
+
+        return filenames
+    
+    
+def get_refseq_urls(tmp_dir: str | Path | None = None) -> list[str]:
+    """
+    Retrieves a list of URLs for RefSeq genomic files from the NCBI FTP site.
+
+    Args:
+        tmp_dir (str | Path | None, optional): 
+            The directory to create a temporary folder in. If None, a default temporary directory is used.
+
+    Returns:
+        list[str]: A list of URLs for the RefSeq genomic files.
+    """
+    filenames = get_refseq_filenames(tmp_dir=tmp_dir)
+    return [f"https://ftp.ncbi.nlm.nih.gov/refseq/release/complete/{filename}" for filename in filenames]

seqbank/seqbank.py

@@ -0,0 +1,537 @@
+from functools import cached_property
+import numpy as np
+import gzip
+import time
+from pathlib import Path
+from joblib import Parallel, delayed
+import plotly.express as px
+import plotly.graph_objs as go
+from Bio.Seq import Seq
+from Bio.SeqRecord import SeqRecord
+from Bio import SeqIO
+from attrs import define
+from rich.progress import track
+import pyfastx
+from rich.progress import Progress, TimeElapsedColumn, MofNCompleteColumn
+from datetime import datetime
+from speedict import Rdict, Options, DBCompressionType, AccessType
+import atexit
+
+from .transform import seq_to_bytes, bytes_to_str
+from .io import get_file_format, open_path, download_file, seq_count, TemporaryDirectory
+from .exceptions import SeqBankError
+from .utils import parse_filter
+
+
+
+@define(slots=False)
+class SeqBank():
+    path:Path
+    write:bool = False
+
+    def __attrs_post_init__(self) -> None:
+        """Initializes the SeqBank object after attributes are set.
+
+        Expands the user path for the SeqBank file and checks if it exists when not in write mode.
+
+        Raises:
+            FileNotFoundError: If the SeqBank file is not found at the given path when write mode is disabled.
+        """
+        self.path = Path(self.path).expanduser()
+        if not self.write and not self.path.exists():
+            raise FileNotFoundError(f"Cannot find SeqBank file at path: {self.path}")
+
+    def key(self, accession: str) -> str:
+        """Generates a key for a given accession.
+
+        Args:
+            accession (str): Accession string used to generate the key.
+
+        Returns:
+            str: A byte-encoded key as a string.
+        """
+        return bytes(accession, "ascii")
+
+
+    def key_url(self, url: str) -> str:
+        """Generates a key for a given URL.
+
+        Args:
+            url (str): The URL for which the key is generated.
+
+        Returns:
+            str: A byte-encoded key string prefixed with '/seqbank/url/'.
+        """
+        return self.key("/seqbank/url/" + url)
+
+
+    def close(self) -> None:
+        """Closes the SeqBank database connection.
+
+        Attempts to close the database connection, if it's open, and silently handles any exceptions.
+        """
+        try:
+            self._db.close()
+        except Exception:
+            pass
+
+    @cached_property
+    def file(self) -> Rdict:
+        """Initializes and configures the Rdict database for sequence storage.
+
+        Configures options for the database, such as compression type, optimization, and maximum open files.
+        Registers the close method to be executed upon program exit to ensure the database is closed.
+
+        Returns:
+            Rdict: The configured Rdict database object.
+        """
+        options = Options(raw_mode=True)
+        options.set_compression_type(DBCompressionType.none())
+        options.set_optimize_filters_for_hits(True)
+        options.optimize_for_point_lookup(1024)
+        options.set_max_open_files(500)
+
+        self._db = Rdict(
+            path=str(self.path), 
+            options=options, 
+            access_type=AccessType.read_write() if self.write else AccessType.read_only()
+        )
+
+        atexit.register(self.close)
+        return self._db
+
+    def __len__(self) -> int:
+        """Calculates the total number of items in the SeqBank.
+
+        Iterates over all the keys in the database and counts them.
+
+        Returns:
+            int: The number of entries in the SeqBank.
+        """
+        count = 0
+        for _ in track(self.file.keys()):
+            count += 1
+        return count
+
+    def __getitem__(self, accession: str) -> np.ndarray:
+        """Retrieves the sequence data associated with a given accession.
+
+        Args:
+            accession (str): The accession key to look up in the SeqBank.
+
+        Returns:
+            np.ndarray: The sequence data stored in the SeqBank for the given accession.
+
+        Raises:
+            SeqBankError: If the accession cannot be read or an error occurs during retrieval.
+        """
+        try:
+            key = self.key(accession)
+            file = self.file
+            return file[key]
+        except Exception as err:
+            raise SeqBankError(f"Failed to read {accession} in SeqBank {self.path}:\n{err}")
+
+    def items(self):
+        """Yields all key-value pairs from the SeqBank.
+
+        The key is the accession, and the value is the sequence data in NumPy array format.
+
+        Yields:
+            tuple: A tuple containing the accession (key) and the sequence data (value) as a NumPy array.
+        """
+        for k, v in self.file.items():
+            yield k, np.frombuffer(v, dtype="u1")
+
+    def __contains__(self, accession: str) -> bool:
+        """Checks if a given accession exists in the SeqBank.
+
+        Args:
+            accession (str): The accession to check for existence.
+
+        Returns:
+            bool: True if the accession exists in the SeqBank, otherwise False.
+        """
+        try:
+            return self.key(accession) in self.file
+        except Exception:
+            return False
+
+    def delete(self, accession: str) -> None:
+        """Deletes a sequence entry from the SeqBank by its accession.
+
+        Args:
+            accession (str): The accession of the sequence to delete.
+
+        Returns:
+            None
+        """
+        key = self.key(accession)
+        f = self.file
+        if key in f:
+            del f[key]
+
+    def add(self, seq: str | Seq | SeqRecord | np.ndarray, accession: str) -> None:
+        """Adds a sequence to the SeqBank with a given accession.
+
+        This method accepts a sequence in various formats (string, Seq, SeqRecord, or NumPy array)
+        and stores it in the SeqBank after appropriate conversion to byte format.
+
+        Args:
+            seq (str | Seq | SeqRecord | np.ndarray): The sequence to add to the SeqBank. 
+                It can be a string, Bio.Seq object, SeqRecord, or a NumPy array.
+            accession (str): The accession key for the sequence to be stored under.
+
+        Returns:
+            None
+        """
+        key = self.key(accession)
+        
+        if isinstance(seq, SeqRecord):
+            seq = seq.seq
+        if isinstance(seq, Seq):
+            seq = str(seq)
+        if isinstance(seq, str):
+            seq = seq_to_bytes(seq)
+        
+        self.file[key] = seq
+
+    def add_file(self, path:Path, format:str="", progress=None, overall_task=None, filter:Path|list|set|None=None) -> None:
+        """Adds sequences from a file to the SeqBank.
+
+        This method processes a sequence file in various formats (e.g., FASTA, FASTQ), optionally filtering specific accessions, 
+        and adds the sequences to the SeqBank. Progress tracking is available for large file imports.
+
+        Args:
+            path (Path): The path to the sequence file.
+            format (str, optional): The format of the sequence file (e.g., "fasta", "fastq"). If not provided, it will be auto-detected.
+            progress (Progress, optional): A rich progress bar to display the import progress. Defaults to None.
+            overall_task (int | None, optional): An optional task ID for tracking the overall progress. Defaults to None.
+            filter (Path | list | set | None, optional): A filter for selecting specific accessions. Defaults to None.
+
+        Returns:
+            None
+        """
+        filter = parse_filter(filter)
+        format = format or get_file_format(path)
+        progress = progress or Progress()
+
+        # If fasta or fastq use pyfastx for speed
+        if format in ["fasta", "fastq"]:
+            total = sum(1 for _ in pyfastx.Fasta(str(path), build_index=False))
+            task = progress.add_task(f"[magenta]{path.name}", total=total)
+
+            for accession, seq in pyfastx.Fasta(str(path), build_index=False):
+                if filter and accession not in filter:
+                    continue
+                self.add(seq, accession)
+                progress.update(task, advance=1)
+        else:
+            total = seq_count(path)
+            
+            with open_path(path) as f:
+                task = progress.add_task(f"[magenta]{path.name}", total=total)
+                for record in SeqIO.parse(f, format):
+                    if filter and record.id not in filter:
+                        continue
+
+                    self.add(record, record.id)
+                    progress.update(task, advance=1)
+
+        progress.update(task, visible=False)
+        if overall_task is not None:
+            progress.update(overall_task, advance=1)
+
+        print('Added', path.name)
+
+    def seen_url(self, url: str) -> bool:
+        """Checks if a given URL has been seen (i.e., processed) before and present in the SeqBank.
+
+        Args:
+            url (str): The URL to check.
+
+        Returns:
+            bool: True if the URL has been seen (i.e., exists in the SeqBank), otherwise False.
+        """
+        return self.key_url(url) in self.file
+
+    def save_seen_url(self, url: str) -> None:
+        """Saves a URL as 'seen' by adding it to the SeqBank with a timestamp.
+
+        Args:
+            url (str): The URL to save as seen.
+
+        Returns:
+            None
+        """
+        url_key = self.key_url(url)
+        self.file[url_key] = bytes(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "ascii")
+
+    def add_url(self, url:str, progress=None, format:str="", force:bool=False, overall_task=None, tmp_dir:str|Path|None=None) -> bool:
+        """Downloads and adds sequences from a URL to the SeqBank.
+
+        This method downloads a file from a given URL, processes it to extract sequences, and adds them to the SeqBank.
+        If the URL has already been processed, it can be skipped unless `force=True` is provided.
+
+        Args:
+            url (str): The URL to download the sequence file from.
+            progress (Progress, optional): A rich progress bar to display progress of the download and file processing. Defaults to None.
+            format (str, optional): The format of the sequence file (e.g., "fasta", "fastq"). If not provided, it will be auto-detected. Defaults to "".
+            force (bool, optional): Whether to force downloading and processing the URL even if it has been seen before. Defaults to False.
+            overall_task (int | None, optional): An optional task ID for tracking overall progress. Defaults to None.
+            tmp_dir (str | Path | None, optional): A temporary directory to store the downloaded file. Defaults to None.
+
+        Returns:
+            bool: True if the URL was successfully processed and added, False otherwise.
+        """
+        url_key = self.key_url(url)
+        if url_key in self.file and not force:
+            return False
+        
+        with TemporaryDirectory(prefix=tmp_dir) as tmpdirname:
+            local_path = tmpdirname / Path(url).name
+            try:
+                download_file(url, local_path)
+                self.add_file(local_path, format=format, progress=progress, overall_task=overall_task)
+                self.save_seen_url(url)
+            except Exception as err:
+                print(f"Failed to add URL: {url}: {err}")
+                return False
+
+            return True
+
+    def get_accessions(self) -> set[str]:
+        """Retrieves all accessions stored in the SeqBank.
+
+        This method iterates through the SeqBank database keys and collects all accessions that do not belong 
+        to the internal '/seqbank/' namespace.
+
+        Returns:
+            set[str]: A set of all accessions present in the SeqBank.
+        """
+        accessions = set()
+        file = self.file
+
+        for key in file.keys():
+            accession = key.decode("ascii")
+            if not accession.startswith("/seqbank/"):
+                accessions.update([accession])
+
+        return accessions
+
+    def missing(self, accessions: list[str] | set[str]) -> set[str]:
+        """Finds accessions that are not present in the SeqBank.
+
+        This method checks a list or set of accessions and returns those that are missing from the SeqBank.
+
+        Args:
+            accessions (list[str] | set[str]): A list or set of accessions to check for presence in the SeqBank.
+
+        Returns:
+            set[str]: A set of accessions that are missing from the SeqBank.
+        """
+        missing = set()
+
+        for accession in track(accessions):
+            if accession not in self:
+                missing.add(accession)
+
+        return missing
+
+    def add_urls(self, urls:list[str], max:int=0, format:str="", force:bool=False, workers:int=-1, tmp_dir:str|Path|None=None) -> None:
+        """Downloads and adds sequences from a list of URLs to the SeqBank.
+
+        This method processes a list of URLs, downloads the corresponding sequence files, and adds them to the SeqBank.
+        It filters out URLs that have already been processed unless `force=True` is specified, and it can limit the number 
+        of URLs processed based on the `max` argument. The processing can be parallelized using the `workers` argument.
+
+        Args:
+            urls (list[str]): A list of URLs to download and process.
+            max (int, optional): Maximum number of URLs to process. If set to 0, all URLs will be processed. Defaults to 0.
+            format (str, optional): The format of the sequence files (e.g., "fasta", "fastq"). If not provided, it will be auto-detected. Defaults to "".
+            force (bool, optional): Whether to force re-processing of URLs even if they were processed before. Defaults to False.
+            workers (int, optional): Number of workers to use for parallel processing. If set to -1, all available CPU cores will be used. Defaults to -1.
+            tmp_dir (str | Path | None, optional): A temporary directory to store downloaded files. Defaults to None.
+
+        Returns:
+            None
+        """
+        # only add the URLs that haven't been seen before
+        urls_to_add = []
+        for url in urls:
+            if not self.seen_url(url):
+                urls_to_add.append(url)
+
+            # truncate URLs list to `max` if requested
+            if max and len(urls_to_add) >= max:
+                break
+
+        with Progress(*Progress.get_default_columns(), TimeElapsedColumn(), MofNCompleteColumn()) as progress:
+            parallel = Parallel(n_jobs=workers, prefer="threads")
+            add_url = delayed(self.add_url)
+            overall_task = progress.add_task(f"[bold red]Adding URLs", total=len(urls_to_add))
+            parallel(add_url(url, progress=progress, format=format, force=force, overall_task=overall_task, tmp_dir=tmp_dir) for url in urls_to_add)
+
+    def ls(self) -> None:
+        """
+        Lists all accessions in the SeqBank.
+
+        Iterates through the keys in the SeqBank and prints each one, decoded from bytes to ASCII.
+
+        Returns:
+            None
+        """
+        for k in self.file.keys():
+            print(k.decode("ascii"))
+
+    def add_files(self, files:list[str], max:int=0, format:str="", workers:int=1, filter:Path|list[str]|set[str]|None=None) -> None:
+        """
+        Adds sequences from multiple files to the SeqBank.
+
+        This method processes a list of file paths, downloading and adding sequences from each file to the SeqBank.
+        It supports parallel processing and optional filtering of specific accessions. The `max` argument limits the number of files to process.
+
+        Args:
+            files (list[str]): A list of file paths to process.
+            max (int, optional): Maximum number of files to process. If set to 0, all files will be processed. Defaults to 0.
+            format (str, optional): The format of the sequence files (e.g., "fasta", "fastq"). If not provided, it will be auto-detected. Defaults to "".
+            workers (int, optional): Number of workers to use for parallel processing. Defaults to 1.
+            filter (Path | list[str] | set[str] | None, None], optional): A filter for selecting specific accessions. Defaults to None.
+
+        Returns:
+            None
+        """
+        filter = parse_filter(filter)
+
+        with Progress(*Progress.get_default_columns(), TimeElapsedColumn(), MofNCompleteColumn()) as progress:
+            parallel = Parallel(n_jobs=workers, prefer="threads")
+            add_file = delayed(self.add_file)
+            overall_task = progress.add_task(f"[bold red]Adding files", total=len(files))
+            parallel(add_file(file, progress=progress, format=format, overall_task=overall_task, filter=filter) for file in files)
+
+    def copy(self, other: 'SeqBank') -> None:
+        """
+        Copies all entries from the current SeqBank to another SeqBank instance.
+
+        This method iterates over all key-value pairs in the current SeqBank and adds them to the `other` SeqBank instance.
+        The `other` SeqBank must be writable.
+
+        Args:
+            other (SeqBank): The target SeqBank instance where entries will be copied.
+
+        Returns:
+            None
+        """
+        for k, v in track(self.file.items()):
+            other.file[k] = v
+
+    def numpy(self, accession: str) -> np.ndarray:
+        """
+        Retrieves the sequence data for a given accession and returns it as an unsigned char NumPy array.
+
+        Args:
+            accession (str): The accession key for which the sequence data is retrieved.
+
+        Returns:
+            np.ndarray: The sequence data associated with the given accession, represented as an unsigned char NumPy array.
+        """
+        return np.frombuffer(self[accession], dtype="u1")
+
+    def string(self, accession: str) -> str:
+        """
+        Retrieves the sequence data for a given accession and returns it as a string.
+
+        Args:
+            accession (str): The accession key for which the sequence data is retrieved.
+
+        Returns:
+            str: The sequence data associated with the given accession, represented as a string.
+        """
+        data = self[accession]
+        return bytes_to_str(data)
+
+    def record(self, accession: str) -> SeqRecord:
+        """
+        Retrieves the sequence data for a given accession and returns it as a BioPython SeqRecord object.
+
+        Args:
+            accession (str): The accession key for which the sequence data is retrieved.
+
+        Returns:
+            SeqRecord: A BioPython SeqRecord object containing the sequence data, with the given accession as its ID and an empty description.
+        """
+        record = SeqRecord(
+            Seq(self.string(accession)),
+            id=accession,
+            description="",
+        )
+        return record
+
+    def export(self, output: Path | str, format: str = "", accessions: list[str] | str | Path | None = None) -> None:
+        """
+        Exports the data from the SeqBank to a file using BioPython's SeqIO.
+
+        Args:
+            output (Path | str): The path or filename where the data should be exported.
+            format (str, optional): The file format for exporting. If not specified, it will be inferred from the file extension.
+            accessions (list[str] | str | Path | None, optional): A list of accessions to export. If a file path or string is provided, it will be read to obtain the list of accessions. If None, all accessions in the SeqBank are exported.
+
+        Returns:
+            None
+        """
+        accessions = accessions or self.get_accessions()
+
+        # Read list of accessions if given file
+        if isinstance(accessions, (str,Path)):
+            accessions = Path(accessions).read_text().strip().split("\n")
+
+        format = format or get_file_format(output)
+        with open(output, "w") as f:
+            for accession in accessions:
+                SeqIO.write(self.record(accession), f, format)
+
+    def lengths_dict(self) -> dict[str, int]:
+        """
+        Returns a dictionary where the keys are the accessions and the values 
+        are the corresponding lengths of each sequence.
+
+        Returns:
+            dict[str, int]: A dictionary mapping each accession to the length of its corresponding sequence.
+        """
+        accession_lengths = {}
+
+        for accession in self.get_accessions():
+            # Retrieve the sequence
+            sequence = self[accession]
+            # Store the length of the sequence in the dictionary
+            accession_lengths[accession] = len(sequence)
+
+        return accession_lengths
+    
+    def histogram(self, nbins: int = 30) -> go.Figure:
+        """
+        Creates a histogram of the lengths of all sequences and returns the Plotly figure object.
+
+        Args:
+            nbins (int): The number of bins for the histogram. Default is 30.
+
+        Returns:
+            go.Figure: A Plotly figure object representing the histogram of sequence lengths.
+        """
+        # Get the dictionary of accession lengths
+        accession_lengths = self.lengths_dict()
+
+        # Extract the lengths from the dictionary
+        lengths = list(accession_lengths.values())
+
+        # Create the histogram using Plotly Express
+        fig = px.histogram(lengths, nbins=nbins, title="Histogram of Sequence Lengths")
+        
+        # Add labels and customize the layout, removing the legend
+        fig.update_layout(
+            xaxis_title="Sequence Length",
+            yaxis_title="Count",
+            showlegend=False  # Remove the legend
+        )
+
+        return fig

seqbank/transform.py

@@ -0,0 +1,63 @@
+import numpy as np
+
+vocab_to_int = {"A": 1, "C": 2, "G": 3, "T": 4, "N": 0}
+int_to_vocab = dict(zip(vocab_to_int.values(), vocab_to_int.keys()))
+
+DELETE = bytes(range(256)).translate(None, b'ACGTNacgtn')
+TABLE = bytearray(b'\0' * 256)
+REVERSE_TABLE = bytearray(b'\0' * 256)
+
+for char, value in vocab_to_int.items():
+    TABLE[ord(char)] = value
+    TABLE[ord(char.lower())] = value
+
+    REVERSE_TABLE[value] = ord(char)
+
+
+def seq_to_bytes(seq: str | bytes, delete: bool = True) -> bytes:
+    """
+    Convert a DNA sequence to a byte representation using a predefined mapping.
+
+    Args:
+        seq (str | bytes): The DNA sequence to convert. Can be a string or bytes.
+        delete (bool, optional): If True, delete characters that are not in 'ACGTN' or 'acgtn'. Defaults to True.
+
+    Returns:
+        bytes: The byte-encoded version of the input sequence.
+    """
+    if isinstance(seq, str):
+        seq = seq.encode("ascii")
+
+    assert isinstance(seq, bytes), f"Expected bytes, got {type(seq)}"
+
+    if delete:
+        return seq.translate(TABLE, DELETE)
+    return seq.translate(TABLE)
+
+
+def seq_to_numpy(seq: str | bytes, delete: bool = True) -> np.ndarray:
+    """
+    Convert a DNA sequence to a NumPy array of unsigned 8-bit integers.
+
+    Args:
+        seq (str | bytes): The DNA sequence to convert, in string or byte format.
+        delete (bool, optional): If True, remove any characters not in 'ACGTN'. Defaults to True.
+
+    Returns:
+        np.ndarray: A NumPy array representing the byte-encoded DNA sequence.
+    """
+    b = seq_to_bytes(seq, delete=delete)
+    return np.frombuffer(b, dtype="u1")
+
+
+def bytes_to_str(seq: bytes) -> str:
+    """
+    Convert byte-encoded DNA sequence back to a string.
+
+    Args:
+        seq (bytes): The byte-encoded DNA sequence.
+
+    Returns:
+        str: The original DNA sequence in string format.
+    """
+    return seq.translate(REVERSE_TABLE).decode("ascii")

seqbank/utils.py

@@ -0,0 +1,24 @@
+from typing import Optional
+from pathlib import Path
+
+def parse_filter(filter: Path | list[str] | set[str] | None = None) -> Optional[set[str]]:
+    """Converts the input into a set of strings to use as a filter.
+
+    Args:
+        filter (Path | list[str] | set[str] | None, optional): 
+            The filter can be:
+            - A `Path` to a file where each line contains a string.
+            - A `list` or `set` of strings.
+            - `None`, in which case the function will return `None`.
+
+    Returns:
+        Optional[set[str]]: 
+            A set of strings to use as a filter, or `None` if the input is `None`.
+    """
+    if not filter:
+        return None
+    
+    if isinstance(filter, Path):
+        return set(filter.read_text().strip().split("\n"))
+    
+    return set(filter)

seqbank-0.1.2.dist-info/METADATA

@@ -0,0 +1,249 @@
+Metadata-Version: 2.1
+Name: seqbank
+Version: 0.1.2
+Summary: A database to quickly read and write DNA sequence data in numerical form.
+License: Apache-2.0
+Author: Robert Turnbull
+Author-email: robert.turnbull@unimelb.edu.au
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: attrs (>=23.1.0)
+Requires-Dist: biopython (>=1.81)
+Requires-Dist: h5py (>=3.9.0)
+Requires-Dist: joblib (>=1.3.2)
+Requires-Dist: kaleido (==0.2.1)
+Requires-Dist: numpy (>=1.22.0)
+Requires-Dist: pandas (>=2.0.0)
+Requires-Dist: plotly (>=5.0.0)
+Requires-Dist: pyfastx (>=1.1.0)
+Requires-Dist: requests (>=2.31.0)
+Requires-Dist: rich (>=10.16.1)
+Requires-Dist: speedict (>=0.3.11)
+Requires-Dist: typer (>=0.4.0)
+Description-Content-Type: text/x-rst
+
+================
+Seqbank
+================
+
+.. start-badges
+
+|testing badge| |coverage badge| |docs badge| |black badge| |git3moji badge|
+
+.. |testing badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/testing.yml/badge.svg
+    :target: https://github.com/rbturnbull/seqbank/actions
+
+.. |docs badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/docs.yml/badge.svg
+    :target: https://rbturnbull.github.io/seqbank
+    
+.. |black badge| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/psf/black
+    
+.. |coverage badge| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/rbturnbull/b1625e7f45428007f0982543d9d346d0/raw/coverage-badge.json
+    :target: https://rbturnbull.github.io/seqbank/coverage/
+
+.. |git3moji badge| image:: https://img.shields.io/badge/git3moji-%E2%9A%A1%EF%B8%8F%F0%9F%90%9B%F0%9F%93%BA%F0%9F%91%AE%F0%9F%94%A4-fffad8.svg
+    :target: https://robinpokorny.github.io/git3moji/
+        
+.. end-badges
+
+.. start-quickstart
+
+SeqBank is a powerful and flexible command-line application designed to simplify the management and processing of large DNA sequence datasets. Whether you're working with local sequence files, 
+retrieving data from remote URLs, or integrating sequences from databases like RefSeq and DFam, SeqBank provides an efficient, user-friendly solution.
+
+SeqBank allows users to quickly add, organize, and manipulate sequences using a structured, numerical format optimized for fast retrieval and analysis. 
+It's especially useful for bioinformatics professionals who regularly handle vast amounts of genomic data.
+
+
+Installation
+============
+
+To install the latest version from the repository, you can use this command:
+
+.. code-block:: bash
+
+    pip install git+https://github.com/rbturnbull/seqbank.git
+
+.. note ::
+
+    Soon seqbank will be able to be installed using conda and PyPI
+
+
+Usage
+===========
+    
+SeqBank provides a command-line interface (CLI) for managing DNA sequence data efficiently. Below are the main tools, along with examples of how to use them in practical workflows.
+
+Adding Sequences
+----------------
+
+SeqBank allows you to import sequence data from files or URLs into the database. The system supports multiple sequence formats, providing flexibility in handling various datasets.
+
+**Example:**
+
+To add sequences from one or more local files:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequence1.fasta /path/to/sequence2.fasta --format fasta
+
+To add sequences from a list of URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4
+
+**Use case:**  
+Suppose you have a new set of genome sequences in FASTA format stored locally or accessible via URLs. You can quickly import these sequences into your SeqBank database for centralized storage and further analysis.
+
+
+Managing Databases
+------------------
+
+SeqBank provides commands to manage and query the sequences in your database. You can list, count, and delete sequences, allowing efficient database management.
+
+**Example:**
+
+To list all sequences in the database:
+
+.. code-block:: bash
+
+    seqbank ls /path/to/seqbank
+
+To count the number of sequences stored:
+
+.. code-block:: bash
+
+    seqbank count /path/to/seqbank
+
+To delete a specific sequence by accession number:
+
+.. code-block:: bash
+
+    seqbank delete /path/to/seqbank ABC123DEF456
+
+**Use case:**  
+If you're managing a growing sequence database, the `ls` command can help you track the sequences, while `delete` can be used to remove outdated or incorrect entries.
+
+
+Exporting Sequences
+-------------------
+
+You can export your stored sequences to common formats like FASTA for easy sharing and use with other bioinformatics tools. This ensures compatibility with external platforms.
+
+**Example:**
+
+To export sequences in FASTA format to a specific output directory:
+
+.. code-block:: bash
+
+    seqbank export /path/to/seqbank /output/directory --format fasta
+
+**Use case:**  
+After storing a collection of curated sequences, you may need to export them in FASTA format for downstream analysis using tools like BLAST or multiple sequence alignment software.
+
+
+Integration with RefSeq and DFam
+--------------------------------
+
+SeqBank integrates with popular genomic databases like RefSeq and DFam, allowing users to download and incorporate sequences from these sources.
+
+**Example:**
+
+To download and add RefSeq sequences with a maximum of 1000 sequences using 4 workers:
+
+.. code-block:: bash
+
+    seqbank refseq /path/to/seqbank --max 1000 --workers 4
+
+To download and add DFam sequences from the current release with curated data:
+
+.. code-block:: bash
+
+    seqbank dfam /path/to/seqbank --release current --curated
+
+**Use case:**  
+If you are studying repetitive elements in a genome, you can easily integrate sequences from DFam into your SeqBank database for comprehensive analysis.
+
+
+Visualization of Sequence Data
+------------------------------
+
+SeqBank includes built-in functionality for generating histograms of sequence lengths, providing a visual summary of the data.
+
+**Example:**
+
+To generate and save a histogram of sequence lengths:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --output histogram.png --nbins 50
+
+To generate and display the histogram interactively:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --show --nbins 50
+
+**Use case:**  
+When working with a dataset of varying sequence lengths, generating a histogram can help visualize the distribution and detect outliers or inconsistencies in the data.
+
+
+Copying Databases
+-----------------
+
+SeqBank allows you to copy sequences from one SeqBank database to another, facilitating data migration or backup processes.
+
+**Example:**
+
+To copy sequences from a source SeqBank to a destination SeqBank:
+
+.. code-block:: bash
+
+    seqbank cp /path/to/source_seqbank /path/to/destination_seqbank
+
+**Use case:**  
+For maintaining backups of your sequence database or migrating data to a new location, the `cp` command provides a straightforward method to duplicate your SeqBank data.
+
+
+Filtering Sequences and Custom Workflows
+----------------------------------------
+
+SeqBank supports filtering sequences based on criteria such as sequence length or file format before adding them to the database. Additionally, multi-threaded downloading allows you to download and process sequences more efficiently.
+
+**Example:**
+
+To filter sequences longer than 1000 bp before adding them:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequences.fasta --format fasta --filter /path/to/filter_file
+
+To enable multi-threaded downloading when adding sequences from URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4 --tmp-dir /path/to/tmp
+
+**Use case:**  
+In projects where only sequences longer than a specific threshold are required, the filtering feature ensures that only relevant sequences are stored. Multi-threaded downloading can be utilized when processing large datasets to save time.
+
+.. end-quickstart
+
+
+Credits
+==================================
+
+.. start-credits
+
+* Robert Turnbull <robert.turnbull@unimelb.edu.au>
+* Rafsan Al Mamun <rafsan7238@gmail.com>
+
+.. end-credits
+
+

seqbank-0.1.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+seqbank/__init__.py,sha256=BffCmT3tluD-LpF2MkPss4a_n4guxfB2qTkd7TDge_A,65
+seqbank/dfam.py,sha256=riuYk1NmkIax5414lU2ANXmyiLUIv5y1l7exrDNgtHc,2985
+seqbank/exceptions.py,sha256=gGtjVFsoxP7pfn3LSUSdFoNKOyiFdNv7zFNw9-7f6wg,234
+seqbank/io.py,sha256=N6oFoyyK-xO_F9iGtEwVYgZm6IM5TYLMuE89uNAYLuE,4400
+seqbank/main.py,sha256=uHiysBdUutfIQk-n1ZmLh2m612A-uzsZaZo0XiJLV5o,5555
+seqbank/refseq.py,sha256=iRXHplyv-miEkP6k6gak-Ly3UTqhkvPN0PkrASWbJaM,1637
+seqbank/seqbank.py,sha256=HlFcl29qdevswy4CZ8LZq8xfd79CBqCpPfj1LYsJW34,21368
+seqbank/transform.py,sha256=IFM0S9b5hTx5eSKW3evK6G6gTWzIAm1mp4zKs0tGhnc,1919
+seqbank/utils.py,sha256=DU0qEQ_ec4XQeTlcm6BxzeyWs-fqZYgVGrnziuT_9Ys,797
+seqbank-0.1.2.dist-info/METADATA,sha256=-ypiqPeKIpzr_ekz6L6TwH3p7fwx-Y2ImL9ZQ-SszeQ,8404
+seqbank-0.1.2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+seqbank-0.1.2.dist-info/entry_points.txt,sha256=u7kmL0XS-HbARkUTD2FPU-J-fEBYx51iJMrz9tBZeLA,44
+seqbank-0.1.2.dist-info/RECORD,,

seqbank-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.8.1
+Root-Is-Purelib: true
+Tag: py3-none-any

seqbank-0.1.2.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+seqbank=seqbank.main:app
+