pymisha 0.1.0__tar.gz

pymisha-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 Weizmann Institute of Science
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pymisha-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include setup.py
+recursive-include src *.cpp *.h

pymisha-0.1.0/PKG-INFO

@@ -0,0 +1,115 @@
+Metadata-Version: 2.4
+Name: pymisha
+Version: 0.1.0
+Summary: Python interface for misha genomic databases with C++ streaming backends
+Author-email: Aviezer Lifshitz <aviezerl@weizmann.ac.il>
+Maintainer-email: Aviezer Lifshitz <aviezerl@weizmann.ac.il>
+License: MIT
+Project-URL: Homepage, https://github.com/tanaylab/pymisha
+Project-URL: Documentation, https://tanaylab.github.io/pymisha/
+Project-URL: Repository, https://github.com/tanaylab/pymisha.git
+Project-URL: Issues, https://github.com/tanaylab/pymisha/issues
+Keywords: genomics,bioinformatics,misha,tracks
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: C++
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.20
+Requires-Dist: pandas>=1.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7.0; extra == "docs"
+Requires-Dist: myst-parser>=2.0; extra == "docs"
+Requires-Dist: furo>=2024.8.6; extra == "docs"
+Dynamic: license-file
+
+# PyMisha
+
+Python interface for [misha](https://github.com/tanaylab/misha) genomic databases. PyMisha provides full read/write access to misha track databases with C++ streaming backends for genome-scale operations.
+
+## Features
+
+- **1D and 2D track support:** Dense, sparse, and 2D (rectangle/point) tracks with full CRUD operations.
+- **C++ streaming backends:** Extraction, summary, quantiles, distribution, lookup, segmentation, Wilcoxon tests, correlation, and sampling all stream through C++ for performance.
+- **Virtual tracks:** Computed-on-the-fly track views with filtering, shifting, and 30+ aggregation functions.
+- **Interval operations:** Union, intersection, difference, canonicalization, neighbors, annotation, normalization, random generation, and liftover.
+- **Sequence analysis:** Extraction, k-mer counting, PWM/PSSM scoring, and Markov-chain synthesis (`gsynth`).
+- **Database management:** Create, link, convert, and manage misha-compatible genomic databases.
+- **R misha compatibility:** Reads and writes the same on-disk formats as R misha (123/145 R exports covered).
+
+## Installation
+
+Prerequisites:
+- Python 3.10+
+- C++17 compiler (GCC 8+, Clang 7+, or Apple Clang 11+)
+- `numpy`, `pandas`
+
+```bash
+pip install .
+```
+
+For development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Quick start
+
+```python
+import pymisha as pm
+
+# Initialize the database
+pm.gdb_init("/path/to/misha_db")
+
+# Create intervals and extract data
+intervals = pm.gintervals_from_strings(["chr1:0-1000", "chr1:2000-2600"])
+out = pm.gextract("track1", intervals, iterator=100)
+
+# Filter and summarize
+filtered = pm.gscreen("track1 > 0.5", intervals)
+stats = pm.gsummary("track1", intervals)
+```
+
+## Examples
+
+Using the built-in example database:
+
+```python
+import pymisha as pm
+
+pm.gdb_init_examples()
+print(pm.gtrack_ls())
+print(pm.gextract("dense_track", pm.gintervals("chr1", 0, 1000)))
+```
+
+## Optional dependencies
+
+- `pyBigWig`: For BigWig import in `gtrack_import`.
+- `pyreadr` + `Rscript`: For loading R-serialized big interval sets.
+- `PyYAML`: For richer `gdataset_info` metadata parsing.
+
+## Missing features
+
+Compared to R misha, the following are not yet implemented:
+
+- **Track Arrays:** `gtrack.array.*` and `gvtrack.array.slice`.
+- **Legacy Conversion:** `gtrack.convert` (for migrating old 2D formats).
+
+## License
+
+MIT. See [LICENSE](LICENSE) for details.

pymisha-0.1.0/README.md

@@ -0,0 +1,76 @@
+# PyMisha
+
+Python interface for [misha](https://github.com/tanaylab/misha) genomic databases. PyMisha provides full read/write access to misha track databases with C++ streaming backends for genome-scale operations.
+
+## Features
+
+- **1D and 2D track support:** Dense, sparse, and 2D (rectangle/point) tracks with full CRUD operations.
+- **C++ streaming backends:** Extraction, summary, quantiles, distribution, lookup, segmentation, Wilcoxon tests, correlation, and sampling all stream through C++ for performance.
+- **Virtual tracks:** Computed-on-the-fly track views with filtering, shifting, and 30+ aggregation functions.
+- **Interval operations:** Union, intersection, difference, canonicalization, neighbors, annotation, normalization, random generation, and liftover.
+- **Sequence analysis:** Extraction, k-mer counting, PWM/PSSM scoring, and Markov-chain synthesis (`gsynth`).
+- **Database management:** Create, link, convert, and manage misha-compatible genomic databases.
+- **R misha compatibility:** Reads and writes the same on-disk formats as R misha (123/145 R exports covered).
+
+## Installation
+
+Prerequisites:
+- Python 3.10+
+- C++17 compiler (GCC 8+, Clang 7+, or Apple Clang 11+)
+- `numpy`, `pandas`
+
+```bash
+pip install .
+```
+
+For development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Quick start
+
+```python
+import pymisha as pm
+
+# Initialize the database
+pm.gdb_init("/path/to/misha_db")
+
+# Create intervals and extract data
+intervals = pm.gintervals_from_strings(["chr1:0-1000", "chr1:2000-2600"])
+out = pm.gextract("track1", intervals, iterator=100)
+
+# Filter and summarize
+filtered = pm.gscreen("track1 > 0.5", intervals)
+stats = pm.gsummary("track1", intervals)
+```
+
+## Examples
+
+Using the built-in example database:
+
+```python
+import pymisha as pm
+
+pm.gdb_init_examples()
+print(pm.gtrack_ls())
+print(pm.gextract("dense_track", pm.gintervals("chr1", 0, 1000)))
+```
+
+## Optional dependencies
+
+- `pyBigWig`: For BigWig import in `gtrack_import`.
+- `pyreadr` + `Rscript`: For loading R-serialized big interval sets.
+- `PyYAML`: For richer `gdataset_info` metadata parsing.
+
+## Missing features
+
+Compared to R misha, the following are not yet implemented:
+
+- **Track Arrays:** `gtrack.array.*` and `gvtrack.array.slice`.
+- **Legacy Conversion:** `gtrack.convert` (for migrating old 2D formats).
+
+## License
+
+MIT. See [LICENSE](LICENSE) for details.

pymisha-0.1.0/pymisha/__init__.py

@@ -0,0 +1,354 @@
+"""
+PyMisha - Python wrapper for the misha Genomic Data Analysis Toolkit
+"""
+
+__version__ = '0.1.0'
+
+# Make numpy available for expressions
+import numpy as np  # noqa: F401
+
+from . import _shared
+from ._shared import (
+    CONFIG,
+    _bound_colname,
+    _checkroot,
+    _chunk_slices,
+    _df2pymisha,
+    _iterated_intervals,
+    _itr2pymisha,
+    _make_progress_callback,
+    _progress_context,
+    _pymisha,
+    _pymisha2df,
+)
+from .analysis import gcis_decay, gsegment, gwilcox
+from .dataset import (
+    gdataset_info,
+    gdataset_load,
+    gdataset_ls,
+    gdataset_save,
+    gdataset_unload,
+)
+from .db import (
+    gdb_examples_path,
+    gdb_info,
+    gdb_init,
+    gdb_init_examples,
+    gdb_reload,
+    gdb_unload,
+    gsetroot,
+)
+from .db_attrs import gdb_get_readonly_attrs, gdb_set_readonly_attrs
+from .db_create import gdb_convert_to_indexed, gdb_create, gdb_create_genome, gdb_create_linked
+from .extract import gextract, gscreen
+from .gdir import (
+    gdir_cd,
+    gdir_create,
+    gdir_cwd,
+    gdir_rm,
+    gtrack_create_dirs,
+)
+from .gsynth import (
+    GsynthModel,
+    gsynth_bin_map,
+    gsynth_load,
+    gsynth_random,
+    gsynth_replace_kmer,
+    gsynth_sample,
+    gsynth_save,
+    gsynth_train,
+)
+from .intervals import (
+    gintervals,
+    gintervals_2d,
+    gintervals_2d_all,
+    gintervals_2d_band_intersect,
+    gintervals_2d_convert_to_indexed,
+    gintervals_all,
+    gintervals_annotate,
+    gintervals_canonic,
+    gintervals_chrom_sizes,
+    gintervals_convert_to_indexed,
+    gintervals_coverage_fraction,
+    gintervals_covered_bp,
+    gintervals_dataset,
+    gintervals_diff,
+    gintervals_exists,
+    gintervals_force_range,
+    gintervals_from_bed,
+    gintervals_from_strings,
+    gintervals_from_tuples,
+    gintervals_import_genes,
+    gintervals_intersect,
+    gintervals_is_indexed,
+    gintervals_load,
+    gintervals_ls,
+    gintervals_mapply,
+    gintervals_mark_overlaps,
+    gintervals_neighbors,
+    gintervals_neighbors_directional,
+    gintervals_neighbors_downstream,
+    gintervals_neighbors_upstream,
+    gintervals_normalize,
+    gintervals_random,
+    gintervals_rbind,
+    gintervals_rm,
+    gintervals_save,
+    gintervals_union,
+    gintervals_update,
+    gintervals_window,
+    giterator_cartesian_grid,
+    giterator_intervals,
+)
+from .liftover import (
+    gintervals_as_chain,
+    gintervals_liftover,
+    gintervals_load_chain,
+    gtrack_liftover,
+)
+from .lookup import glookup, gtrack_lookup
+from .sequence import (
+    gseq_comp,
+    gseq_extract,
+    gseq_kmer,
+    gseq_kmer_dist,
+    gseq_pwm,
+    gseq_rev,
+    gseq_revcomp,
+)
+from .summary import (
+    gbins_quantiles,
+    gbins_summary,
+    gcor,
+    gdist,
+    gintervals_quantiles,
+    gintervals_summary,
+    gpartition,
+    gquantiles,
+    gsample,
+    gsummary,
+)
+from .tracks import (
+    gtrack_2d_create,
+    gtrack_2d_import,
+    gtrack_2d_import_contacts,
+    gtrack_attr_export,
+    gtrack_attr_get,
+    gtrack_attr_import,
+    gtrack_attr_set,
+    gtrack_convert_to_indexed,
+    gtrack_copy,
+    gtrack_create,
+    gtrack_create_dense,
+    gtrack_create_empty_indexed,
+    gtrack_create_pwm_energy,
+    gtrack_create_sparse,
+    gtrack_dataset,
+    gtrack_exists,
+    gtrack_import,
+    gtrack_import_mappedseq,
+    gtrack_import_set,
+    gtrack_info,
+    gtrack_ls,
+    gtrack_modify,
+    gtrack_mv,
+    gtrack_rm,
+    gtrack_smooth,
+    gtrack_var_get,
+    gtrack_var_ls,
+    gtrack_var_rm,
+    gtrack_var_set,
+)
+from .vtracks import (
+    gvtrack_clear,
+    gvtrack_create,
+    gvtrack_filter,
+    gvtrack_info,
+    gvtrack_iterator,
+    gvtrack_iterator_2d,
+    gvtrack_ls,
+    gvtrack_rm,
+)
+
+
+def __getattr__(name):
+    # Expose live DB state variables instead of stale import-time snapshots.
+    if name in {"_GROOT", "_UROOT", "_VTRACKS"}:
+        return getattr(_shared, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+__all__ = [
+    # Configuration
+    'CONFIG',
+
+    # Database functions
+    'gdb_init',
+    'gdb_reload',
+    'gdb_unload',
+    'gdb_info',
+    'gdb_examples_path',
+    'gdb_init_examples',
+    'gsetroot',
+    'gdb_create',
+    'gdb_create_genome',
+    'gdb_create_linked',
+    'gdb_convert_to_indexed',
+    'gdb_get_readonly_attrs',
+    'gdb_set_readonly_attrs',
+    'gdataset_load',
+    'gdataset_unload',
+    'gdataset_ls',
+    'gdataset_save',
+    'gdataset_info',
+
+    # Track functions
+    'gextract',
+    'gscreen',
+    'gsummary',
+    'gquantiles',
+    'gdist',
+    'gpartition',
+    'gsample',
+    'gcor',
+    'gbins_summary',
+    'gbins_quantiles',
+    'gcis_decay',
+    'gsegment',
+    'gwilcox',
+    'gtrack_ls',
+    'gtrack_info',
+    'gtrack_exists',
+    'gtrack_dataset',
+    'gtrack_create',
+    'gtrack_create_dense',
+    'gtrack_create_sparse',
+    'gtrack_import',
+    'gtrack_import_mappedseq',
+    'gtrack_import_set',
+    'gtrack_rm',
+    'gtrack_mv',
+    'gtrack_copy',
+    'gtrack_convert_to_indexed',
+    'gtrack_create_empty_indexed',
+    'gtrack_attr_get',
+    'gtrack_attr_set',
+    'gtrack_attr_export',
+    'gtrack_attr_import',
+    'gtrack_var_ls',
+    'gtrack_var_get',
+    'gtrack_var_set',
+    'gtrack_var_rm',
+    'gtrack_modify',
+    'gtrack_smooth',
+    'gtrack_2d_create',
+    'gtrack_2d_import',
+    'gtrack_2d_import_contacts',
+    'gtrack_create_pwm_energy',
+
+    # Interval functions
+    'gintervals',
+    'gintervals_all',
+    'gintervals_2d',
+    'gintervals_2d_all',
+    'gintervals_2d_band_intersect',
+    'gintervals_union',
+    'gintervals_intersect',
+    'gintervals_diff',
+    'gintervals_canonic',
+    'gintervals_force_range',
+    'gintervals_summary',
+    'gintervals_quantiles',
+    'gintervals_covered_bp',
+    'gintervals_coverage_fraction',
+    'gintervals_neighbors',
+    'gintervals_neighbors_upstream',
+    'gintervals_neighbors_downstream',
+    'gintervals_neighbors_directional',
+    'gintervals_from_tuples',
+    'gintervals_from_strings',
+    'gintervals_from_bed',
+    'gintervals_import_genes',
+    'gintervals_window',
+    'gintervals_ls',
+    'gintervals_exists',
+    'gintervals_dataset',
+    'gintervals_chrom_sizes',
+    'gintervals_load',
+    'gintervals_convert_to_indexed',
+    'gintervals_2d_convert_to_indexed',
+    'gintervals_is_indexed',
+    'gintervals_save',
+    'gintervals_update',
+    'gintervals_mapply',
+    'gintervals_rm',
+    'giterator_cartesian_grid',
+    'giterator_intervals',
+    'gintervals_rbind',
+    'gintervals_mark_overlaps',
+    'gintervals_annotate',
+    'gintervals_normalize',
+    'gintervals_random',
+
+    # Virtual track functions
+    'gvtrack_create',
+    'gvtrack_ls',
+    'gvtrack_info',
+    'gvtrack_iterator',
+    'gvtrack_iterator_2d',
+    'gvtrack_filter',
+    'gvtrack_rm',
+    'gvtrack_clear',
+
+    # Sequence functions
+    'gseq_extract',
+    'gseq_rev',
+    'gseq_comp',
+    'gseq_revcomp',
+    'gseq_kmer',
+    'gseq_kmer_dist',
+    'gseq_pwm',
+
+    # Lookup functions
+    'glookup',
+    'gtrack_lookup',
+
+    # Liftover functions
+    'gintervals_load_chain',
+    'gintervals_as_chain',
+    'gintervals_liftover',
+    'gtrack_liftover',
+
+    # Directory management
+    'gdir_cwd',
+    'gdir_cd',
+    'gdir_create',
+    'gdir_rm',
+    'gtrack_create_dirs',
+
+    # Genome synthesis functions
+    'GsynthModel',
+    'gsynth_bin_map',
+    'gsynth_train',
+    'gsynth_sample',
+    'gsynth_random',
+    'gsynth_replace_kmer',
+    'gsynth_save',
+    'gsynth_load',
+
+    # Internal (shared)
+    '_bound_colname',
+    '_checkroot',
+    '_chunk_slices',
+    '_df2pymisha',
+    '_iterated_intervals',
+    '_itr2pymisha',
+    '_make_progress_callback',
+    '_progress_context',
+    '_pymisha',
+    '_pymisha2df',
+]
+
+# Export module locals to the C extension for access to Python functions
+# This must be at the end of the file after all functions are defined
+_pymisha._PMLOCALS = locals()

pymisha-0.1.0/pymisha/_crc64.py

@@ -0,0 +1,34 @@
+"""Shared CRC64-ECMA helpers (parity with C++ CRC64.h)."""
+
+_CRC64_POLY = 0xC96C5795D7870F42
+_CRC64_TABLE = None
+
+
+def _crc64_table():
+    table = []
+    for i in range(256):
+        crc = i
+        for _ in range(8):
+            if crc & 1:
+                crc = (crc >> 1) ^ _CRC64_POLY
+            else:
+                crc >>= 1
+        table.append(crc & 0xFFFFFFFFFFFFFFFF)
+    return table
+
+
+def crc64_incremental(crc, data):
+    global _CRC64_TABLE
+    if _CRC64_TABLE is None:
+        _CRC64_TABLE = _crc64_table()
+    for byte in data:
+        crc = (crc >> 8) ^ _CRC64_TABLE[(crc ^ byte) & 0xFF]
+    return crc & 0xFFFFFFFFFFFFFFFF
+
+
+def crc64_init():
+    return 0xFFFFFFFFFFFFFFFF
+
+
+def crc64_finalize(crc):
+    return (~crc) & 0xFFFFFFFFFFFFFFFF

pymisha-0.1.0/pymisha/_name_validation.py

@@ -0,0 +1,22 @@
+"""Name validation helpers."""
+
+from __future__ import annotations
+
+import re
+
+_DOTTED_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_.]*$")
+
+
+def validate_dotted_name(name, kind):
+    if not isinstance(name, str) or not name:
+        raise ValueError(f"{kind} must be a non-empty string")
+    if not _DOTTED_NAME_RE.fullmatch(name):
+        raise ValueError(
+            f"Invalid {kind} '{name}'. Must start with a letter and contain "
+            "only alphanumeric characters, underscores, and dots."
+        )
+    parts = name.split(".")
+    if any(not part for part in parts):
+        raise ValueError(
+            f"Invalid {kind} '{name}'. Empty dot-separated components are not allowed."
+        )