seqbank 0.1.2__tar.gz

seqbank-0.1.2/PKG-INFO

@@ -0,0 +1,249 @@
+Metadata-Version: 2.1
+Name: seqbank
+Version: 0.1.2
+Summary: A database to quickly read and write DNA sequence data in numerical form.
+License: Apache-2.0
+Author: Robert Turnbull
+Author-email: robert.turnbull@unimelb.edu.au
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: attrs (>=23.1.0)
+Requires-Dist: biopython (>=1.81)
+Requires-Dist: h5py (>=3.9.0)
+Requires-Dist: joblib (>=1.3.2)
+Requires-Dist: kaleido (==0.2.1)
+Requires-Dist: numpy (>=1.22.0)
+Requires-Dist: pandas (>=2.0.0)
+Requires-Dist: plotly (>=5.0.0)
+Requires-Dist: pyfastx (>=1.1.0)
+Requires-Dist: requests (>=2.31.0)
+Requires-Dist: rich (>=10.16.1)
+Requires-Dist: speedict (>=0.3.11)
+Requires-Dist: typer (>=0.4.0)
+Description-Content-Type: text/x-rst
+
+================
+Seqbank
+================
+
+.. start-badges
+
+|testing badge| |coverage badge| |docs badge| |black badge| |git3moji badge|
+
+.. |testing badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/testing.yml/badge.svg
+    :target: https://github.com/rbturnbull/seqbank/actions
+
+.. |docs badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/docs.yml/badge.svg
+    :target: https://rbturnbull.github.io/seqbank
+    
+.. |black badge| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/psf/black
+    
+.. |coverage badge| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/rbturnbull/b1625e7f45428007f0982543d9d346d0/raw/coverage-badge.json
+    :target: https://rbturnbull.github.io/seqbank/coverage/
+
+.. |git3moji badge| image:: https://img.shields.io/badge/git3moji-%E2%9A%A1%EF%B8%8F%F0%9F%90%9B%F0%9F%93%BA%F0%9F%91%AE%F0%9F%94%A4-fffad8.svg
+    :target: https://robinpokorny.github.io/git3moji/
+        
+.. end-badges
+
+.. start-quickstart
+
+SeqBank is a powerful and flexible command-line application designed to simplify the management and processing of large DNA sequence datasets. Whether you're working with local sequence files, 
+retrieving data from remote URLs, or integrating sequences from databases like RefSeq and DFam, SeqBank provides an efficient, user-friendly solution.
+
+SeqBank allows users to quickly add, organize, and manipulate sequences using a structured, numerical format optimized for fast retrieval and analysis. 
+It's especially useful for bioinformatics professionals who regularly handle vast amounts of genomic data.
+
+
+Installation
+============
+
+To install the latest version from the repository, you can use this command:
+
+.. code-block:: bash
+
+    pip install git+https://github.com/rbturnbull/seqbank.git
+
+.. note ::
+
+    Soon seqbank will be able to be installed using conda and PyPI
+
+
+Usage
+===========
+    
+SeqBank provides a command-line interface (CLI) for managing DNA sequence data efficiently. Below are the main tools, along with examples of how to use them in practical workflows.
+
+Adding Sequences
+----------------
+
+SeqBank allows you to import sequence data from files or URLs into the database. The system supports multiple sequence formats, providing flexibility in handling various datasets.
+
+**Example:**
+
+To add sequences from one or more local files:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequence1.fasta /path/to/sequence2.fasta --format fasta
+
+To add sequences from a list of URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4
+
+**Use case:**  
+Suppose you have a new set of genome sequences in FASTA format stored locally or accessible via URLs. You can quickly import these sequences into your SeqBank database for centralized storage and further analysis.
+
+
+Managing Databases
+------------------
+
+SeqBank provides commands to manage and query the sequences in your database. You can list, count, and delete sequences, allowing efficient database management.
+
+**Example:**
+
+To list all sequences in the database:
+
+.. code-block:: bash
+
+    seqbank ls /path/to/seqbank
+
+To count the number of sequences stored:
+
+.. code-block:: bash
+
+    seqbank count /path/to/seqbank
+
+To delete a specific sequence by accession number:
+
+.. code-block:: bash
+
+    seqbank delete /path/to/seqbank ABC123DEF456
+
+**Use case:**  
+If you're managing a growing sequence database, the `ls` command can help you track the sequences, while `delete` can be used to remove outdated or incorrect entries.
+
+
+Exporting Sequences
+-------------------
+
+You can export your stored sequences to common formats like FASTA for easy sharing and use with other bioinformatics tools. This ensures compatibility with external platforms.
+
+**Example:**
+
+To export sequences in FASTA format to a specific output directory:
+
+.. code-block:: bash
+
+    seqbank export /path/to/seqbank /output/directory --format fasta
+
+**Use case:**  
+After storing a collection of curated sequences, you may need to export them in FASTA format for downstream analysis using tools like BLAST or multiple sequence alignment software.
+
+
+Integration with RefSeq and DFam
+--------------------------------
+
+SeqBank integrates with popular genomic databases like RefSeq and DFam, allowing users to download and incorporate sequences from these sources.
+
+**Example:**
+
+To download and add RefSeq sequences with a maximum of 1000 sequences using 4 workers:
+
+.. code-block:: bash
+
+    seqbank refseq /path/to/seqbank --max 1000 --workers 4
+
+To download and add DFam sequences from the current release with curated data:
+
+.. code-block:: bash
+
+    seqbank dfam /path/to/seqbank --release current --curated
+
+**Use case:**  
+If you are studying repetitive elements in a genome, you can easily integrate sequences from DFam into your SeqBank database for comprehensive analysis.
+
+
+Visualization of Sequence Data
+------------------------------
+
+SeqBank includes built-in functionality for generating histograms of sequence lengths, providing a visual summary of the data.
+
+**Example:**
+
+To generate and save a histogram of sequence lengths:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --output histogram.png --nbins 50
+
+To generate and display the histogram interactively:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --show --nbins 50
+
+**Use case:**  
+When working with a dataset of varying sequence lengths, generating a histogram can help visualize the distribution and detect outliers or inconsistencies in the data.
+
+
+Copying Databases
+-----------------
+
+SeqBank allows you to copy sequences from one SeqBank database to another, facilitating data migration or backup processes.
+
+**Example:**
+
+To copy sequences from a source SeqBank to a destination SeqBank:
+
+.. code-block:: bash
+
+    seqbank cp /path/to/source_seqbank /path/to/destination_seqbank
+
+**Use case:**  
+For maintaining backups of your sequence database or migrating data to a new location, the `cp` command provides a straightforward method to duplicate your SeqBank data.
+
+
+Filtering Sequences and Custom Workflows
+----------------------------------------
+
+SeqBank supports filtering sequences based on criteria such as sequence length or file format before adding them to the database. Additionally, multi-threaded downloading allows you to download and process sequences more efficiently.
+
+**Example:**
+
+To filter sequences longer than 1000 bp before adding them:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequences.fasta --format fasta --filter /path/to/filter_file
+
+To enable multi-threaded downloading when adding sequences from URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4 --tmp-dir /path/to/tmp
+
+**Use case:**  
+In projects where only sequences longer than a specific threshold are required, the filtering feature ensures that only relevant sequences are stored. Multi-threaded downloading can be utilized when processing large datasets to save time.
+
+.. end-quickstart
+
+
+Credits
+==================================
+
+.. start-credits
+
+* Robert Turnbull <robert.turnbull@unimelb.edu.au>
+* Rafsan Al Mamun <rafsan7238@gmail.com>
+
+.. end-credits
+
+

seqbank-0.1.2/README.rst

@@ -0,0 +1,220 @@
+================
+Seqbank
+================
+
+.. start-badges
+
+|testing badge| |coverage badge| |docs badge| |black badge| |git3moji badge|
+
+.. |testing badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/testing.yml/badge.svg
+    :target: https://github.com/rbturnbull/seqbank/actions
+
+.. |docs badge| image:: https://github.com/rbturnbull/seqbank/actions/workflows/docs.yml/badge.svg
+    :target: https://rbturnbull.github.io/seqbank
+    
+.. |black badge| image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/psf/black
+    
+.. |coverage badge| image:: https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/rbturnbull/b1625e7f45428007f0982543d9d346d0/raw/coverage-badge.json
+    :target: https://rbturnbull.github.io/seqbank/coverage/
+
+.. |git3moji badge| image:: https://img.shields.io/badge/git3moji-%E2%9A%A1%EF%B8%8F%F0%9F%90%9B%F0%9F%93%BA%F0%9F%91%AE%F0%9F%94%A4-fffad8.svg
+    :target: https://robinpokorny.github.io/git3moji/
+        
+.. end-badges
+
+.. start-quickstart
+
+SeqBank is a powerful and flexible command-line application designed to simplify the management and processing of large DNA sequence datasets. Whether you're working with local sequence files, 
+retrieving data from remote URLs, or integrating sequences from databases like RefSeq and DFam, SeqBank provides an efficient, user-friendly solution.
+
+SeqBank allows users to quickly add, organize, and manipulate sequences using a structured, numerical format optimized for fast retrieval and analysis. 
+It's especially useful for bioinformatics professionals who regularly handle vast amounts of genomic data.
+
+
+Installation
+============
+
+To install the latest version from the repository, you can use this command:
+
+.. code-block:: bash
+
+    pip install git+https://github.com/rbturnbull/seqbank.git
+
+.. note ::
+
+    Soon seqbank will be able to be installed using conda and PyPI
+
+
+Usage
+===========
+    
+SeqBank provides a command-line interface (CLI) for managing DNA sequence data efficiently. Below are the main tools, along with examples of how to use them in practical workflows.
+
+Adding Sequences
+----------------
+
+SeqBank allows you to import sequence data from files or URLs into the database. The system supports multiple sequence formats, providing flexibility in handling various datasets.
+
+**Example:**
+
+To add sequences from one or more local files:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequence1.fasta /path/to/sequence2.fasta --format fasta
+
+To add sequences from a list of URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4
+
+**Use case:**  
+Suppose you have a new set of genome sequences in FASTA format stored locally or accessible via URLs. You can quickly import these sequences into your SeqBank database for centralized storage and further analysis.
+
+
+Managing Databases
+------------------
+
+SeqBank provides commands to manage and query the sequences in your database. You can list, count, and delete sequences, allowing efficient database management.
+
+**Example:**
+
+To list all sequences in the database:
+
+.. code-block:: bash
+
+    seqbank ls /path/to/seqbank
+
+To count the number of sequences stored:
+
+.. code-block:: bash
+
+    seqbank count /path/to/seqbank
+
+To delete a specific sequence by accession number:
+
+.. code-block:: bash
+
+    seqbank delete /path/to/seqbank ABC123DEF456
+
+**Use case:**  
+If you're managing a growing sequence database, the `ls` command can help you track the sequences, while `delete` can be used to remove outdated or incorrect entries.
+
+
+Exporting Sequences
+-------------------
+
+You can export your stored sequences to common formats like FASTA for easy sharing and use with other bioinformatics tools. This ensures compatibility with external platforms.
+
+**Example:**
+
+To export sequences in FASTA format to a specific output directory:
+
+.. code-block:: bash
+
+    seqbank export /path/to/seqbank /output/directory --format fasta
+
+**Use case:**  
+After storing a collection of curated sequences, you may need to export them in FASTA format for downstream analysis using tools like BLAST or multiple sequence alignment software.
+
+
+Integration with RefSeq and DFam
+--------------------------------
+
+SeqBank integrates with popular genomic databases like RefSeq and DFam, allowing users to download and incorporate sequences from these sources.
+
+**Example:**
+
+To download and add RefSeq sequences with a maximum of 1000 sequences using 4 workers:
+
+.. code-block:: bash
+
+    seqbank refseq /path/to/seqbank --max 1000 --workers 4
+
+To download and add DFam sequences from the current release with curated data:
+
+.. code-block:: bash
+
+    seqbank dfam /path/to/seqbank --release current --curated
+
+**Use case:**  
+If you are studying repetitive elements in a genome, you can easily integrate sequences from DFam into your SeqBank database for comprehensive analysis.
+
+
+Visualization of Sequence Data
+------------------------------
+
+SeqBank includes built-in functionality for generating histograms of sequence lengths, providing a visual summary of the data.
+
+**Example:**
+
+To generate and save a histogram of sequence lengths:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --output histogram.png --nbins 50
+
+To generate and display the histogram interactively:
+
+.. code-block:: bash
+
+    seqbank histogram /path/to/seqbank --show --nbins 50
+
+**Use case:**  
+When working with a dataset of varying sequence lengths, generating a histogram can help visualize the distribution and detect outliers or inconsistencies in the data.
+
+
+Copying Databases
+-----------------
+
+SeqBank allows you to copy sequences from one SeqBank database to another, facilitating data migration or backup processes.
+
+**Example:**
+
+To copy sequences from a source SeqBank to a destination SeqBank:
+
+.. code-block:: bash
+
+    seqbank cp /path/to/source_seqbank /path/to/destination_seqbank
+
+**Use case:**  
+For maintaining backups of your sequence database or migrating data to a new location, the `cp` command provides a straightforward method to duplicate your SeqBank data.
+
+
+Filtering Sequences and Custom Workflows
+----------------------------------------
+
+SeqBank supports filtering sequences based on criteria such as sequence length or file format before adding them to the database. Additionally, multi-threaded downloading allows you to download and process sequences more efficiently.
+
+**Example:**
+
+To filter sequences longer than 1000 bp before adding them:
+
+.. code-block:: bash
+
+    seqbank add /path/to/seqbank /path/to/sequences.fasta --format fasta --filter /path/to/filter_file
+
+To enable multi-threaded downloading when adding sequences from URLs:
+
+.. code-block:: bash
+
+    seqbank url /path/to/seqbank https://example.com/sequence1.fasta https://example.com/sequence2.fasta --format fasta --workers 4 --tmp-dir /path/to/tmp
+
+**Use case:**  
+In projects where only sequences longer than a specific threshold are required, the filtering feature ensures that only relevant sequences are stored. Multi-threaded downloading can be utilized when processing large datasets to save time.
+
+.. end-quickstart
+
+
+Credits
+==================================
+
+.. start-credits
+
+* Robert Turnbull <robert.turnbull@unimelb.edu.au>
+* Rafsan Al Mamun <rafsan7238@gmail.com>
+
+.. end-credits
+

seqbank-0.1.2/pyproject.toml

@@ -0,0 +1,44 @@
+[tool.poetry]
+name = "seqbank"
+version = "0.1.2"
+description = "A database to quickly read and write DNA sequence data in numerical form."
+authors = ["Robert Turnbull <robert.turnbull@unimelb.edu.au>"]
+license = "Apache-2.0"
+readme = "README.rst"
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+typer = ">=0.4.0"
+numpy = ">=1.22.0"
+pyfastx = ">=1.1.0"
+h5py = ">=3.9.0"
+biopython = ">=1.81"
+attrs = ">=23.1.0"
+rich = ">=10.16.1"
+requests = ">=2.31.0"
+joblib = ">=1.3.2"
+speedict = ">=0.3.11"
+plotly = ">=5.0.0"
+pandas = ">=2.0.0"
+kaleido = "0.2.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=7.4.1"
+ipykernel = ">=6.6.1"
+coverage = ">=5.5"
+autopep8 = ">=1.5.7"
+Sphinx = ">=5.0.0"
+nbsphinx = ">=0.8.7"
+sphinx-rtd-theme = ">=1.0.0"
+sphinx-autobuild = ">=2021.3.14"
+pre-commit = ">=2.15.0"
+sphinx-copybutton = ">=0.4.0"
+black = ">=21.10b0"
+sphinx-click = {git = "https://github.com/rbturnbull/sphinx-click.git"}
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+seqbank = "seqbank.main:app"

seqbank-0.1.2/seqbank/__init__.py

@@ -0,0 +1,2 @@
+from .seqbank import SeqBank
+from .exceptions import SeqBankError

seqbank-0.1.2/seqbank/dfam.py

@@ -0,0 +1,85 @@
+import gzip
+from pathlib import Path
+import h5py
+import tempfile
+import shutil
+
+from .io import download_file
+from .seqbank import SeqBank
+
+def dfam_url(curated: bool = True, release: str = "current") -> str:
+    """
+    Constructs the URL for the Dfam database file.
+
+    Args:
+        curated (bool, optional): If True, fetches the curated-only version of the Dfam database. Defaults to True.
+        release (str, optional): The Dfam release version. Defaults to "current".
+
+    Returns:
+        str: The URL for the Dfam HDF5 file.
+    """
+    curated_str = "_curatedonly" if curated else ""
+    url = f"https://www.dfam.org/releases/{release}/families/Dfam{curated_str}.h5.gz"
+    return url
+
+
+def add_dfam(seqbank: SeqBank, local_path: Path) -> None:
+    """
+    Adds sequences from a local Dfam HDF5 file to the SeqBank.
+
+    Args:
+        seqbank (SeqBank): The SeqBank instance to add sequences to.
+        local_path (Path): The path to the local Dfam HDF5 file.
+    """
+    file = h5py.File(local_path, 'r')
+    
+    def visitor_func(name, node):
+        if isinstance(node, h5py.Dataset):
+            accession = node.attrs['accession']
+            seq = node.attrs['consensus']
+            seqbank.add(seq=seq, accession=accession)
+
+    file['Families/DF'].visititems(visitor_func)
+
+
+def download_dfam(seqbank: SeqBank, curated: bool = True, release: str = "current", force: bool = False) -> bool:
+    """
+    Downloads the Dfam HDF5 file, decompresses it, and adds sequences to the SeqBank.
+
+    Args:
+        seqbank (SeqBank): The SeqBank instance to which the Dfam sequences will be added.
+        curated (bool, optional): If True, fetches the curated-only version of the Dfam database. Defaults to True.
+        release (str, optional): The Dfam release version. Defaults to "current".
+        force (bool, optional): If True, forces the download even if the URL has been previously processed. Defaults to False.
+
+    Returns:
+        bool: True if the download and addition were successful, False otherwise.
+    """
+    url = dfam_url(curated, release)
+    url_key = seqbank.key_url(url)
+    
+    if url_key in seqbank.file and not force:
+        print(f"Already downloaded: {url}")
+        return False
+    
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        local_gzip_path = Path(tmpdirname) / Path(url).name
+        try:
+            print(f"Downloading Dfam: {url}")
+            download_file(url, local_gzip_path)
+
+            # Decompress the gzipped file
+            local_path = local_gzip_path.with_suffix("")
+            with gzip.open(local_gzip_path, 'rb') as infile, open(local_path, 'wb') as outfile:
+                shutil.copyfileobj(infile, outfile)
+
+            # Add sequences to SeqBank
+            add_dfam(seqbank, local_path)
+
+            # Mark URL as processed
+            seqbank.save_seen_url(url)
+        except Exception as err:
+            print(f"Failed to add Dfam: {url}: {err}")
+            return False
+
+    return True

seqbank-0.1.2/seqbank/exceptions.py

@@ -0,0 +1,7 @@
+class SeqBankError(Exception):
+    """
+    Custom exception class for handling errors specific to SeqBank operations.
+
+    This exception should be raised when any operation related to the SeqBank encounters an error.
+    """
+    pass