ducer 0.2.1__tar.gz

ducer-0.2.1/.github/workflows/build_wheels.yml

@@ -0,0 +1,94 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }} ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, macos-12, macos-14, windows-2019]
+        arch: [x86_64, aarch64]
+        exclude:
+          - os: macos-12
+            arch: aarch64
+          - os: macos-14
+            arch: x86_64
+          - os: windows-2019
+            arch: aarch64
+    steps:
+      - name: Debug output memory and processes
+        if: runner.os == 'Linux'
+        uses: catchpoint/workflow-telemetry-action@v2
+        with:
+          job_summary: true
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3.2.0
+        with:
+          platforms: arm64
+      - name: Setup Rust
+        if: runner.os != 'Linux'
+        uses: hecrj/setup-rust-action@v2
+        with:
+          profile: minimal
+      - name: Setup Rust in containers
+        if: runner.os == 'Linux'
+        run: |
+          echo "CIBW_BEFORE_ALL=curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal" >> $GITHUB_ENV
+          echo "CARGO_HOME=/usr/local/" >> $GITHUB_ENV
+      #- name: Limit parallel build jobs with QEMU builds
+      #  if: runner.os == 'Linux' && matrix.arch == 'aarch64'
+      #  run: echo "CARGO_BUILD_JOBS=1" >> $GITHUB_ENV
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.19.1
+        env:
+          CIBW_ARCHS_LINUX: ${{ matrix.arch }}
+          #CIBW_CONTAINER_ENGINE: "docker; create_args: --memory=8G"
+          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_BUILD_JOBS CIBW_BEFORE_ALL CARGO_HOME
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}-${{ matrix.arch }}
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build SDist
+        run: pipx run build --sdist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+  upload_all:
+    needs: [build_wheels, make_sdist]
+    runs-on: ubuntu-latest
+    environment:
+      name: release
+      url: https://pypi.org/p/simplefst
+    permissions:
+      contents: write
+      id-token: write
+    if: github.ref_type == 'tag'
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@v1.9.0
+      - name: Create Github Release
+        uses: actions/create-release@v1
+        id: create_release
+        with:
+          draft: false
+          prerelease: false
+          release_name: ${{ github.ref_name }}
+          tag_name: ${{ github.ref }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

ducer-0.2.1/.gitignore

@@ -0,0 +1,6 @@
+/dist
+/rust/target
+/wheelhouse
+__pycache__
+*.so
+*.egg-info

ducer-0.2.1/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## [Unreleased] - 2024-07-24
+### Changed
+-
+
+### Fixed
+-
+
+### Added
+-
+
+
+## [0.2.1] - 2024-07-24
+### Fixed
+- `Map.__getitem__` correctly raises KeyError
+
+### Added
+- Map.get with optional default
+- `Buffer.__len__`
+
+
+## [0.2.0] - 2024-07-23
+### Added
+-  CI builds on Linux, MacOS, and Windows.
+
+
+## [0.1.0] - 2024-07-22
+### Added
+-  Initial public release.
+
+
+[Unreleased]: https://github.com/jfolz/ducer/compare/0.2.1...main
+[0.2.1]: https://github.com/jfolz/ducer/compare/0.2.0...0.2.1
+[0.2.0]: https://github.com/jfolz/ducer/compare/0.1...0.2.0
+[0.1.0]: https://github.com/jfolz/ducer/releases/tag/0.1

ducer-0.2.1/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2024 Joachim Folz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

ducer-0.2.1/MANIFEST.in

@@ -0,0 +1,2 @@
+include rust/Cargo.toml
+recursive-include rust *.rs

ducer-0.2.1/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.1
+Name: ducer
+Version: 0.2.1
+Summary: Fast and compact maps and sets with Billions of keys, based on finite-state-transducers.
+Author-email: Joachim Folz <joachim.folz@dfki.de>
+License: Copyright (c) 2024 Joachim Folz
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/jfolz/ducer
+Project-URL: Documentation, https://github.com/jfolz/ducer/blob/main/README.md
+Project-URL: Repository, https://github.com/jfolz/ducer
+Project-URL: Issues, https://github.com/jfolz/ducer/issues
+Project-URL: Changelog, https://github.com/jfolz/ducer/blob/main/CHANGELOG.md
+Keywords: big data,compression,dataset,dict,finite-state-machines,finite-state-transducers,frozen,fst,map,serialization,set,streaming
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Ducer
+
+This package provides Python bindings for the excellent Rust crate
+[fst](https://github.com/BurntSushi/fst) by Andrew Gallant.
+`Map` and `Set` classes allow building compact representations from sorted
+Python iterables.
+
+
+
+## Performance
+
+Ducer maps and sets can be built and queried at millions of keys per second.
+Consider the following example:
+
+```Python
+n = 1_000_000_000
+items = ((b"%09d" % i, n-i) for i in range(n))
+data = ducer.Map.build(items, ":memory:")
+print(len(data))
+```
+
+Creating this map takes about 4 minutes on my humble laptop,
+which translates to almost 4 Million items per second.
+About 3 minutes is spent in Python just creating the item tuples.
+This is a pathological scenario, the resulting output is just 464 bytes.
+A real-world example with 1.1 Billion keys, where the msgpacked
+key-value pairs occupy 21 GiB (without any kind of searchability),
+results in a 4.6 GiB file.
+
+
+
+## Limitations
+
+Performance is rarely free,
+so there are some limitations you should consider before proceeding:
+
+* Keys **must** be `bytes`
+* Keys **must** be inserted in lexicographical order
+* Map values **must** be positive integers less than 2^64
+* Once built, maps and sets **cannot** be altered
+
+
+
+## Usage
+
+### Building & opening
+
+Above, we created a map with data stored in memory.
+If you want to store your map in a file, you can also give a path instead:
+
+```Python
+ducer.Map.build(items, "path/to/my.map")
+```
+
+Building a map like this is a very memory-efficient.
+You can also stream a map without loading all data into memory,
+e.g., using the builtin mmap:
+
+```Python
+with open("path/to/my.map", "rb") as f:
+    mm = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+    mm.madvise(mmap.MADV_RANDOM)
+    m = ducer.Map(mm)
+```
+
+### Access
+
+Other than being read-only, maps and sets behave the same as the builtin
+Python `dict` and `set`.
+Please open an issue if you find something that doesn't work.
+For map `m: Map` and key `k: bytes`, the following all work as intended:
+
+```Python
+k in m  # bool
+m[k]  # value except KeyError
+m.get(k)  # value or None
+m.get(k, 0)  # value or 0
+len(m)  # number of items
+for k in m:  # iterate over keys
+    pass
+for v in m.values():  # iterate over values
+    pass
+for k, v in m.items():  # iterate over items
+    pass
+```
+
+### TODO advanced search & operations

ducer-0.2.1/README.md

@@ -0,0 +1,87 @@
+# Ducer
+
+This package provides Python bindings for the excellent Rust crate
+[fst](https://github.com/BurntSushi/fst) by Andrew Gallant.
+`Map` and `Set` classes allow building compact representations from sorted
+Python iterables.
+
+
+
+## Performance
+
+Ducer maps and sets can be built and queried at millions of keys per second.
+Consider the following example:
+
+```Python
+n = 1_000_000_000
+items = ((b"%09d" % i, n-i) for i in range(n))
+data = ducer.Map.build(items, ":memory:")
+print(len(data))
+```
+
+Creating this map takes about 4 minutes on my humble laptop,
+which translates to almost 4 Million items per second.
+About 3 minutes is spent in Python just creating the item tuples.
+This is a pathological scenario, the resulting output is just 464 bytes.
+A real-world example with 1.1 Billion keys, where the msgpacked
+key-value pairs occupy 21 GiB (without any kind of searchability),
+results in a 4.6 GiB file.
+
+
+
+## Limitations
+
+Performance is rarely free,
+so there are some limitations you should consider before proceeding:
+
+* Keys **must** be `bytes`
+* Keys **must** be inserted in lexicographical order
+* Map values **must** be positive integers less than 2^64
+* Once built, maps and sets **cannot** be altered
+
+
+
+## Usage
+
+### Building & opening
+
+Above, we created a map with data stored in memory.
+If you want to store your map in a file, you can also give a path instead:
+
+```Python
+ducer.Map.build(items, "path/to/my.map")
+```
+
+Building a map like this is a very memory-efficient.
+You can also stream a map without loading all data into memory,
+e.g., using the builtin mmap:
+
+```Python
+with open("path/to/my.map", "rb") as f:
+    mm = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+    mm.madvise(mmap.MADV_RANDOM)
+    m = ducer.Map(mm)
+```
+
+### Access
+
+Other than being read-only, maps and sets behave the same as the builtin
+Python `dict` and `set`.
+Please open an issue if you find something that doesn't work.
+For map `m: Map` and key `k: bytes`, the following all work as intended:
+
+```Python
+k in m  # bool
+m[k]  # value except KeyError
+m.get(k)  # value or None
+m.get(k, 0)  # value or 0
+len(m)  # number of items
+for k in m:  # iterate over keys
+    pass
+for v in m.values():  # iterate over values
+    pass
+for k, v in m.items():  # iterate over items
+    pass
+```
+
+### TODO advanced search & operations

ducer-0.2.1/ducer/__init__.py

@@ -0,0 +1,3 @@
+from ._fst import *
+
+__version__ = "0.2.1"

ducer-0.2.1/ducer.egg-info/PKG-INFO

@@ -0,0 +1,122 @@
+Metadata-Version: 2.1
+Name: ducer
+Version: 0.2.1
+Summary: Fast and compact maps and sets with Billions of keys, based on finite-state-transducers.
+Author-email: Joachim Folz <joachim.folz@dfki.de>
+License: Copyright (c) 2024 Joachim Folz
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/jfolz/ducer
+Project-URL: Documentation, https://github.com/jfolz/ducer/blob/main/README.md
+Project-URL: Repository, https://github.com/jfolz/ducer
+Project-URL: Issues, https://github.com/jfolz/ducer/issues
+Project-URL: Changelog, https://github.com/jfolz/ducer/blob/main/CHANGELOG.md
+Keywords: big data,compression,dataset,dict,finite-state-machines,finite-state-transducers,frozen,fst,map,serialization,set,streaming
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Ducer
+
+This package provides Python bindings for the excellent Rust crate
+[fst](https://github.com/BurntSushi/fst) by Andrew Gallant.
+`Map` and `Set` classes allow building compact representations from sorted
+Python iterables.
+
+
+
+## Performance
+
+Ducer maps and sets can be built and queried at millions of keys per second.
+Consider the following example:
+
+```Python
+n = 1_000_000_000
+items = ((b"%09d" % i, n-i) for i in range(n))
+data = ducer.Map.build(items, ":memory:")
+print(len(data))
+```
+
+Creating this map takes about 4 minutes on my humble laptop,
+which translates to almost 4 Million items per second.
+About 3 minutes is spent in Python just creating the item tuples.
+This is a pathological scenario, the resulting output is just 464 bytes.
+A real-world example with 1.1 Billion keys, where the msgpacked
+key-value pairs occupy 21 GiB (without any kind of searchability),
+results in a 4.6 GiB file.
+
+
+
+## Limitations
+
+Performance is rarely free,
+so there are some limitations you should consider before proceeding:
+
+* Keys **must** be `bytes`
+* Keys **must** be inserted in lexicographical order
+* Map values **must** be positive integers less than 2^64
+* Once built, maps and sets **cannot** be altered
+
+
+
+## Usage
+
+### Building & opening
+
+Above, we created a map with data stored in memory.
+If you want to store your map in a file, you can also give a path instead:
+
+```Python
+ducer.Map.build(items, "path/to/my.map")
+```
+
+Building a map like this is a very memory-efficient.
+You can also stream a map without loading all data into memory,
+e.g., using the builtin mmap:
+
+```Python
+with open("path/to/my.map", "rb") as f:
+    mm = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+    mm.madvise(mmap.MADV_RANDOM)
+    m = ducer.Map(mm)
+```
+
+### Access
+
+Other than being read-only, maps and sets behave the same as the builtin
+Python `dict` and `set`.
+Please open an issue if you find something that doesn't work.
+For map `m: Map` and key `k: bytes`, the following all work as intended:
+
+```Python
+k in m  # bool
+m[k]  # value except KeyError
+m.get(k)  # value or None
+m.get(k, 0)  # value or 0
+len(m)  # number of items
+for k in m:  # iterate over keys
+    pass
+for v in m.values():  # iterate over values
+    pass
+for k, v in m.items():  # iterate over items
+    pass
+```
+
+### TODO advanced search & operations

ducer-0.2.1/ducer.egg-info/SOURCES.txt

@@ -0,0 +1,20 @@
+.gitignore
+CHANGELOG.md
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+.github/workflows/build_wheels.yml
+ducer/__init__.py
+ducer.egg-info/PKG-INFO
+ducer.egg-info/SOURCES.txt
+ducer.egg-info/dependency_links.txt
+ducer.egg-info/top_level.txt
+rust/Cargo.lock
+rust/Cargo.toml
+rust/src/automaton.rs
+rust/src/buffer.rs
+rust/src/encoding.rs
+rust/src/lib.rs
+rust/src/map.rs
+rust/src/set.rs

ducer-0.2.1/ducer.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ducer-0.2.1/ducer.egg-info/top_level.txt

@@ -0,0 +1 @@
+ducer

ducer-0.2.1/pyproject.toml

@@ -0,0 +1,71 @@
+[project]
+name = "ducer"
+description = "Fast and compact maps and sets with Billions of keys, based on finite-state-transducers."
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [{ name = "Joachim Folz", email = "joachim.folz@dfki.de" }]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Information Technology",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = [
+    "big data",
+    "compression",
+    "dataset",
+    "dict",
+    "finite-state-machines",
+    "finite-state-transducers",
+    "frozen",
+    "fst",
+    "map",
+    "serialization",
+    "set",
+    "streaming",
+]
+requires-python = ">=3.8"
+dynamic = ["version"]
+license = { file = "LICENSE" }
+
+[project.urls]
+Homepage = "https://github.com/jfolz/ducer"
+Documentation = "https://github.com/jfolz/ducer/blob/main/README.md"
+Repository = "https://github.com/jfolz/ducer"
+Issues = "https://github.com/jfolz/ducer/issues"
+Changelog = "https://github.com/jfolz/ducer/blob/main/CHANGELOG.md"
+
+[tool.setuptools_scm]
+
+[tool.setuptools]
+packages = ["ducer"]
+
+[build-system]
+requires = [
+    "setuptools>=71.1.0",
+    "setuptools-rust>=1.9.0",
+    "setuptools_scm>=8.1.0",
+]
+build-backend = "setuptools.build_meta"
+
+[[tool.setuptools-rust.ext-modules]]
+target = "ducer._fst"
+path = "rust/Cargo.toml"
+binding = "PyO3"
+args = ["--profile", "prod"]
+
+[tool.cibuildwheel]
+skip = "*_i686 pp* *musllinux* *-win32"
+build-verbosity = 1
+#before-all = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal"
+#before-build = "cargo clean --manifest-path rust/Cargo.toml"
+build-frontend = "build"
+#test-requires = "pytest>=5.3.2 numpy Pillow>=7.0.0"
+#test-command = "pytest {project}/test -vv"