patternjoin 0.1.0__tar.gz

patternjoin-0.1.0/.github/install-tbb.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -eu
+
+curl -fLsS --retry 3 \
+  https://github.com/uxlfoundation/oneTBB/archive/refs/tags/v2022.0.0.tar.gz \
+  -o /tmp/tbb.tar.gz
+echo 'e8e89c9c345415b17b30a2db3095ba9d47647611662073f7fbf54ad48b7f3c2a  /tmp/tbb.tar.gz' \
+  | sha256sum -c -
+tar -xzf /tmp/tbb.tar.gz -C /tmp
+cmake -S /tmp/oneTBB-2022.0.0 -B /tmp/tbb-build \
+  -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
+  -DTBB_TEST=OFF -DTBB_STRICT=OFF -DTBBMALLOC_BUILD=OFF
+cmake --build /tmp/tbb-build --parallel 2
+cmake --install /tmp/tbb-build
+ldconfig

patternjoin-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,99 @@
+name: Python distributions
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      testpypi:
+        description: Publish to TestPyPI
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  wheels:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, macos-15-intel, macos-15]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: pypa/cibuildwheel@v4.2.1
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist-${{ matrix.os }}
+          path: wheelhouse/*.whl
+          if-no-files-found: error
+
+  sdist:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+      - run: python -m pip install build twine
+      - run: python -m build --sdist
+      - run: python -m twine check --strict dist/*
+      - name: Check release version
+        if: github.event_name == 'release'
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          python - <<'PY'
+          import os
+          import tomllib
+          with open('pyproject.toml', 'rb') as source:
+              version = tomllib.load(source)['project']['version']
+          assert os.environ['RELEASE_TAG'] == f'v{version}'
+          PY
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+
+  publish:
+    needs: [wheels, sdist]
+    if: github.event_name == 'release'
+    runs-on: ubuntu-24.04
+    environment:
+      name: pypi
+      url: https://pypi.org/p/patternjoin
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  testpypi:
+    needs: [wheels, sdist]
+    if: >-
+      github.event_name == 'workflow_dispatch' &&
+      github.ref == 'refs/heads/main' && inputs.testpypi
+    runs-on: ubuntu-24.04
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/patternjoin
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/

patternjoin-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+build
+thirdparty/.DS_Store
+__pycache__/
+.pytest_cache/
+.venv/
+dist/

patternjoin-0.1.0/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+  "files.associations": {
+    "*.lean": "lean",
+    "vector": "cpp"
+  }
+}

patternjoin-0.1.0/CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.10)
+project(pattern_join)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -Wall -Wextra -Wpedantic -Werror")
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include")
+  set(OpenMP_C_LIB_NAMES "omp")
+  set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include")
+  set(OpenMP_CXX_LIB_NAMES "omp")
+  set(OpenMP_omp_LIBRARY "/usr/local/opt/libomp/lib/libomp.dylib")
+endif()
+
+find_package(OpenMP REQUIRED)
+find_package(TBB REQUIRED)
+
+set(SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src")
+set(OBJ_DIR "${CMAKE_CURRENT_BINARY_DIR}/obj")
+
+file(GLOB SOURCES "${SRC_DIR}/*.cpp")
+add_executable(pattern_join ${SOURCES})
+
+target_link_libraries(pattern_join PUBLIC OpenMP::OpenMP_CXX TBB::tbb)
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")

patternjoin-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Daniil Matveev <i@mdaniil.ru>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

patternjoin-0.1.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.2
+Name: patternjoin
+Version: 0.1.0
+Summary: String similarity joins using C++ and OpenMP
+Author: Daniil Matveev
+License: MIT License
+         
+         Copyright (c) 2024 Daniil Matveev <i@mdaniil.ru>
+         
+         Permission is hereby granted, free of charge, to any person obtaining a copy
+         of this software and associated documentation files (the "Software"), to deal
+         in the Software without restriction, including without limitation the rights
+         to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+         copies of the Software, and to permit persons to whom the Software is
+         furnished to do so, subject to the following conditions:
+         
+         The above copyright notice and this permission notice shall be included in all
+         copies or substantial portions of the Software.
+         
+         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+         IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+         FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+         AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+         SOFTWARE.
+         
+Project-URL: Repository, https://github.com/MatveevDaniil/PatternJoinThreaded
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# PatternJoin for Python
+
+```python
+from patternjoin import join
+
+join(["cat", "dog"], ["bat", "cat"], cutoff=1, threads=4)
+# [(0, 0), (0, 1)]
+```
+
+`join(a, b=None, *, cutoff=1, metric="L", threads=0)` returns sorted
+zero-based index pairs. `join(a)` means `join(a, a)`: it includes
+the diagonal and both orientations. Duplicate strings retain their
+original positions. Empty iterables and empty strings are valid.
+Inputs can be iterables of ASCII strings, including generators.
+Non-ASCII strings are rejected because the C++ routines count bytes.
+
+Cutoffs are 0, 1 or 2. `metric="L"` is Levenshtein distance;
+`metric="H"` counts positional mismatches plus the length difference.
+Nonzero cutoffs use the existing semi-pattern implementation with
+the smaller input indexed. Cutoff 0 uses serial exact matching.
+
+`threads=0` uses the OpenMP default, configurable with
+`OMP_NUM_THREADS` before starting Python. Use `threads=1` for serial
+execution or a positive number to set the count for one call.
+Calls release the GIL. More threads need not make small joins faster.
+The package returns results in memory and does not create files.
+
+## Build and test
+
+A C++20 compiler, CMake, OpenMP and TBB are required. On macOS:
+
+```sh
+brew install libomp tbb
+python -m pip install .
+python -m pip install pytest
+python -m pytest python/tests
+```
+
+On Linux, install the compiler's OpenMP runtime and TBB development
+package first. If CMake cannot locate TBB, set `CMAKE_PREFIX_PATH`
+to its installation prefix. Native Windows builds are not tested.
+
+Build distribution archives with `python -m build`. Locally built
+wheels depend on the installed OpenMP/TBB libraries. For a portable
+macOS wheel, run `delocate-wheel -w wheelhouse dist/*.whl` and test
+the repaired wheel in a fresh environment. Linux wheels need the
+corresponding `auditwheel repair` workflow. Nothing is published
+to PyPI by these commands.
+
+## Releases
+
+`.github/workflows/publish.yml` builds and tests CPython 3.9–3.14
+wheels for Linux x86-64 and macOS 15+ (Intel and Apple Silicon),
+bundling the native runtime libraries. Pull requests only build/test.
+
+Configure a PyPI Trusted Publisher for project `patternjoin`, owner
+`MatveevDaniil`, repository `PatternJoinThreaded`, workflow
+`publish.yml`, environment `pypi`. No API token is needed.
+For a trial upload, configure the same publisher on TestPyPI with
+environment `testpypi`, then manually run the workflow on `main`
+with `testpypi` enabled.
+
+To publish, update the version in `pyproject.toml`, merge the change,
+and publish a GitHub release with the matching tag (e.g. `v0.1.0`).
+After all wheel tests pass, the workflow uploads wheels and the
+source archive to PyPI. Each release needs a new version.

patternjoin-0.1.0/README.md

@@ -0,0 +1,71 @@
+# PatternJoinThreaded
+This project is an experimental version of PatternJoin project, and is designed to test parallelization capabilities of our algorithms. In order to read more about the main algorithms please visit our [C++](https://github.com/MatveevDaniil/PatternJoin) or [R](https://matveevdaniil.r-universe.dev/RPatternJoin) packages.
+
+Python installation and API: [python/README.md](python/README.md).
+
+## What is Edit Similarity Join?
+[_Edit distance_](https://en.wikipedia.org/w/index.php?search=Edit+distance&title=Special:Search&ns0=1) between two words is the minimal number of _elementary operations_ needed to transform one word to another. In this project, we consider [_Levenshtein distance_](https://en.wikipedia.org/wiki/Levenshtein_distance) (allows _substitutions_ and _insertions_/_deletions_ of single letters) and [_Hamming distance_](https://en.wikipedia.org/wiki/Hamming_distance) (allows _substitutions_ of any letters and _insertions_/_deletions_ of letters in the end). 
+
+_Edit Similarity Join_ is a specific problem when we have a set of words $X=\\{w_1, w_2, \ldots, w_n\\}$ and we want to find all pairs of 'similar words' $Y_c=\\{(w_i, w_j): distance(w_i, w_j) < c\\}$, where $c \in \mathbb{N}_0$ is called a _cutoff_ parameter.
+
+## Motivation
+Let's briefly introduce the problem why this project was originally developed.
+
+### Intro to TCR(BCR) profiles 
+__T-cells__ and __B-cells__ are types of white blood immune cells which play a major role in human immunity. On the surface of each T/B-cell we can find receptors (__TCR__/__BCR__) - special molecules that are responsible for recognizing antigens. The collection of all TCRs/BCRs is called __TCR (BCR) profile__. One way to represent an organism's TCR/BCR profile is to collect the genetic sequences of most variable region (CDR3) of each TCR/BCR. Recent research (for instance [[1]](https://www.nature.com/articles/s41467-019-09278-8), [[2]](https://academic.oup.com/bioinformatics/article/36/6/1731/5686386)) shows that it is useful to (1) represent these profiles as graphs with nodes being TCR/BCR and edge weights edit distances between nodes and (2) run some network analysis on these graphs (take a look at the [NAIR](https://github.com/mlizhangx/Network-Analysis-for-Repertoire-Sequencing-) project if you are interested in network analysis in immunology). In order to build these graphs we need efficient edit similarity join software and this is the main purpose of this project.
+
+
+## Parallel Hash Set and Hash Map Performance benchmarks
+To run performance benchmarks you need to compile performance_testing subproject:
+
+```shell
+> cd performance_testing
+> mkdir build; cd build
+> cmake -DCMAKE_BUILD_TYPE=Release ..; make
+> cd ..
+```
+
+Then in order you can run hash_set and hash_map tests using
+```shell
+# assuming you are in PatternJoinThreaded/performance_testing directory
+> ./build/threaded_hash_sets
+> ./build/threaded_hash_maps
+```
+
+When the tests will finish generate figures using
+```shell
+> pip install pandas seaborn matplotlib
+> python plot_results.py
+```
+
+## Parallel Partition-Pattern Join Algorithm
+To run partition_pattern algorithm using several threads firstly compile the project
+```shell
+# assuming you are in PatternJoinThreaded directory
+> mkdir build; cd build; cmake -DCMAKE_BUILD_TYPE=Release ..; make; cd ..
+```
+
+Below we give an example of algorithm launch:
+```shell
+> export FILE_NAME=./performance_testing/test_data/P00245-aa
+> export CUTOFF=2
+> export METRIC_TYPE=L
+> export METHOD=partition_pattern
+> export INCLUDE_DUPLICATES=false
+> ./build/pattern_join --file_name $FILE_NAME --cutoff $CUTOFF --metric_type $METRIC_TYPE --method $METHOD --include_duplicates $INCLUDE_DUPLICATES
+```
+
+### Full list of arguments:
+- `<file_name>`: The path to the input file.
+- `--file_names <file_a> <file_b>`: Join two files instead of `--file_name`. Results use original A/B indices (`include_duplicates=true`) or unique string pairs (`false`), written to `<file_a>_cross_sp_<cutoff>_<metric>`. Cutoff 0 writes exact-match index pairs to `<file_a>_cross_dupl`. Nonzero cutoffs currently support only `semi_pattern`; use `OMP_NUM_THREADS` to control threads.
+- `<cutoff>`: The edit distance cutoff (`0`, `1` or `2`). If `cutoff` = 0, then the value of `metric_type`, `method`, and `include_duplicates` does not matter.
+- `<metric_type>`: The edit distance metric (`L` for Levenshtein, `H` for Hamming).
+- `<method>`: Two-file joins support `semi_pattern`; single-file joins also support `pattern` and `partition_pattern`.
+- `<include_duplicates>`: Consider duplicates in input (`true` or `false`). If `false` the program will ignore duplicate strings in the input and output unique pairs of strings. If `true`, the program will treat duplicate strings in the input as a pair (index, string) and output pairs of indices. 
+
+### Input file format
+List of words separated by `\n`: `<word_1>\n<word_2>\n...`.
+
+### Output file format
+If `include_duplicates = true`: Space-separated pairs of words separated by `\n`: `<word_i> <word_j>\n...`.
+If `include_duplicates = false`: Space-separated pairs of indeces separated by `\n`: `<idx_i> <idx_j>\n...`.