hyped-crane 0.1.0__tar.gz

hyped_crane-0.1.0/.github/workflows/linting.yml

@@ -0,0 +1,30 @@
+name: Linting
+
+on:
+  push:
+    branches: 
+      - main
+  pull_request:
+    branches: 
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pre-commit
+          pre-commit install
+
+      - name: Analysing the code with pre-commit
+        run: pre-commit run --all-files

hyped_crane-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,98 @@
+name: Publish Python 🐍 distribution 📦 to PyPI
+
+on:
+  create:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+"
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hyped-crane
+
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        "$GITHUB_REF_NAME"
+        --repo "$GITHUB_REPOSITORY"
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        "$GITHUB_REF_NAME" dist/**
+        --repo "$GITHUB_REPOSITORY"

hyped_crane-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,48 @@
+name: Tests
+
+on:
+  push:
+    branches: 
+      - main
+  pull_request:
+    branches: 
+      - main
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # TODO: support windows-latest
+        # TODO: tests are super slow on macos-latest
+        os: [ubuntu-latest]  #, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[tests,arrow]
+
+      - name: Run tests
+        run: python -m pytest --cov=src tests -v
+
+      - name: Upload coverage report
+        run: |
+          pip install coveralls[toml]
+          coveralls --service=github
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+        if: ${{ matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' }}
+        continue-on-error: true

hyped_crane-0.1.0/.gitignore

@@ -0,0 +1,147 @@
+# Project specific
+notebooks/
+checkpoints/
+data/
+/src/crane/__version__.py
+**/*.lock
+# ignore prototype scripts
+/proto*.py
+/proto*.ipynb
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+coverage_html_report
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.idea/
+
+# Vscode
+.vscode/
+.history/
+
+# Created by https://www.toptal.com/developers/gitignore/api/vim
+# Edit at https://www.toptal.com/developers/gitignore?templates=vim
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/vim

hyped_crane-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+default_install_hook_types:
+- pre-commit
+repos:
+- repo: https://github.com/pycqa/isort
+  rev: 5.11.5
+  hooks:
+  - id: isort
+    stages: [pre-commit]
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 23.9.1
+  hooks:
+  - id: black
+    language_version: python3.11
+    stages: [pre-commit]
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.0.278
+  hooks:
+  - id: ruff
+    args: [--fix, --exit-non-zero-on-fix]
+    stages: [pre-commit]

hyped_crane-0.1.0/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.1
+Name: hyped-crane
+Version: 0.1.0
+Summary: Lift (process) and place (write) data streams, seamlessly and in parallel.
+Author: open-hyped
+License: Apache 2.0
+Classifier: License :: Freely Distributable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: dill>=0.3.8
+Requires-Dist: orjson>=3.10.1
+Requires-Dist: datasets>=3.2.0
+Requires-Dist: fsspec<=2023.9.2
+Provides-Extra: arrow
+Requires-Dist: pyarrow>=17.0.0; extra == "arrow"
+Provides-Extra: linting
+Requires-Dist: pre-commit; extra == "linting"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-cov; extra == "tests"
+Requires-Dist: pytest-asyncio; extra == "tests"
+Requires-Dist: pytest-integration; extra == "tests"
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: furo; extra == "docs"
+
+# Hyped Crane
+
+[![Tests](https://github.com/open-hyped/crane/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/open-hyped/crane/actions/workflows/tests.yml)
+[![Linting](https://github.com/open-hyped/crane/actions/workflows/linting.yml/badge.svg?branch=main)](https://github.com/open-hyped/crane/actions/workflows/linting.yml)
+[![Coverage Status](https://coveralls.io/repos/github/open-hyped/crane/badge.svg?branch=main)](https://coveralls.io/github/open-hyped/crane?branch=main)
+[![PyPi version](https://badgen.net/pypi/v/hyped-crane/)](https://pypi.org/project/hyped-crane)
+[![PyPi license](https://badgen.net/pypi/license/hyped-crane/)](https://pypi.org/project/hyped-crane/)
+
+Lift (process) and place (write) data streams, seamlessly and in parallel.
+
+Hyped Crane is a Python library designed to simplify working with HuggingFace `datasets`' iterable datasets. It provides powerful tools for applying transformations to data streams, handling parallel processing, and writing data in varios formats.
+
+
+## Features
+
+- **Streaming-Friendly Transformations**: Apply lazy, streaming-friendly transformations to iterable datasets without preloading data into memory.
+- **Seamless Multiprocessing**: Effortlessly process and write datasets using multiple processes, improving performance on large datasets.
+- **Easily Extendable**: Provides a straightforward interface to implement support for custom data formats.
+- **Interoperability with Hugging Face Datasets**: Write datasets in formats directly loadable using HuggingFace’s `load_from_disk` function.
+
+## Installation
+
+To install the library from **PyPI**, run:
+
+```bash
+pip install hyped-crane
+```
+
+To install the library **from source**, clone run:
+
+```bash
+git clone https://github.com/open-hyped/crane.git
+cd crane
+pip install .
+```
+
+## Getting Started
+
+Here’s a quick example to illustrate how Hyped Crane works
+
+### Step 1: "Load" the Dataset
+
+`crane` is designed to work seamlessly with HuggingFace’s iterable datasets. Let’s start by creating one:
+
+```python
+import datasets
+
+# Create a dummy iterable dataset
+dummy_data = [
+    {"a": 0, "b": [1, 2, 3, 4]},
+    {"a": 1, "b": [5, 6]},
+    {"a": 1, "b": [7, 8, 9, 10]}
+]
+ds = datasets.Dataset.from_list(dummy_data)
+ds = ds.to_iterable_dataset()
+```
+
+### Step 2: Apply a Lazy Transformation
+
+Transformations on iterable datasets are applied lazily. That means the data isn’t processed until it’s actually read:
+
+```python
+# Apply a transformation to compute the maximum of list "b"
+ds = ds.map(lambda x: {"max(b)": max(x["b"])})
+```
+
+### Step 3: Write the Dataset to Disk
+
+Use `crane`’s `ArrowDatasetWriter` to save the transformed dataset to disk. You can enable multiprocessing to speed up the transformation and writing processes:
+
+```python
+from crane import ArrowDatasetWriter
+
+# Write the transformed dataset to disk with multiprocessing
+writer = ArrowDatasetWriter("data", overwrite=True, num_proc=3)
+writer.write(ds)
+```
+
+**Key Benefits**:
+- **Data-Parallel Transformations**: The transformations defined by `map` operations are moved into the workers, allowing transformation workload to be evenly distributed.
+- **Efficient Writing**: Each worker writes its own shard to disk in parallel, reducing bottlenecks in **I/O operations**.
+
+`crane` handles worker communication, task distribution, and writing operations, so you can focus on defining your transformation logic without worrying about parallelization details.
+
+**Note**: Datasets saved with the `ArrowDatasetWriter` are fully compatible with HuggingFace’s `load_from_disk` function. You can reload the dataset and continue working with it:
+
+```python
+# Reload the dataset from disk
+ds = datasets.load_from_disk("data")
+```
+
+
+## Contributions
+
+Contributions are welcome! Feel free to submit a pull request or open an issue to discuss your ideas.

hyped_crane-0.1.0/README.md

@@ -0,0 +1,95 @@
+# Hyped Crane
+
+[![Tests](https://github.com/open-hyped/crane/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/open-hyped/crane/actions/workflows/tests.yml)
+[![Linting](https://github.com/open-hyped/crane/actions/workflows/linting.yml/badge.svg?branch=main)](https://github.com/open-hyped/crane/actions/workflows/linting.yml)
+[![Coverage Status](https://coveralls.io/repos/github/open-hyped/crane/badge.svg?branch=main)](https://coveralls.io/github/open-hyped/crane?branch=main)
+[![PyPi version](https://badgen.net/pypi/v/hyped-crane/)](https://pypi.org/project/hyped-crane)
+[![PyPi license](https://badgen.net/pypi/license/hyped-crane/)](https://pypi.org/project/hyped-crane/)
+
+Lift (process) and place (write) data streams, seamlessly and in parallel.
+
+Hyped Crane is a Python library designed to simplify working with HuggingFace `datasets`' iterable datasets. It provides powerful tools for applying transformations to data streams, handling parallel processing, and writing data in varios formats.
+
+
+## Features
+
+- **Streaming-Friendly Transformations**: Apply lazy, streaming-friendly transformations to iterable datasets without preloading data into memory.
+- **Seamless Multiprocessing**: Effortlessly process and write datasets using multiple processes, improving performance on large datasets.
+- **Easily Extendable**: Provides a straightforward interface to implement support for custom data formats.
+- **Interoperability with Hugging Face Datasets**: Write datasets in formats directly loadable using HuggingFace’s `load_from_disk` function.
+
+## Installation
+
+To install the library from **PyPI**, run:
+
+```bash
+pip install hyped-crane
+```
+
+To install the library **from source**, clone run:
+
+```bash
+git clone https://github.com/open-hyped/crane.git
+cd crane
+pip install .
+```
+
+## Getting Started
+
+Here’s a quick example to illustrate how Hyped Crane works
+
+### Step 1: "Load" the Dataset
+
+`crane` is designed to work seamlessly with HuggingFace’s iterable datasets. Let’s start by creating one:
+
+```python
+import datasets
+
+# Create a dummy iterable dataset
+dummy_data = [
+    {"a": 0, "b": [1, 2, 3, 4]},
+    {"a": 1, "b": [5, 6]},
+    {"a": 1, "b": [7, 8, 9, 10]}
+]
+ds = datasets.Dataset.from_list(dummy_data)
+ds = ds.to_iterable_dataset()
+```
+
+### Step 2: Apply a Lazy Transformation
+
+Transformations on iterable datasets are applied lazily. That means the data isn’t processed until it’s actually read:
+
+```python
+# Apply a transformation to compute the maximum of list "b"
+ds = ds.map(lambda x: {"max(b)": max(x["b"])})
+```
+
+### Step 3: Write the Dataset to Disk
+
+Use `crane`’s `ArrowDatasetWriter` to save the transformed dataset to disk. You can enable multiprocessing to speed up the transformation and writing processes:
+
+```python
+from crane import ArrowDatasetWriter
+
+# Write the transformed dataset to disk with multiprocessing
+writer = ArrowDatasetWriter("data", overwrite=True, num_proc=3)
+writer.write(ds)
+```
+
+**Key Benefits**:
+- **Data-Parallel Transformations**: The transformations defined by `map` operations are moved into the workers, allowing transformation workload to be evenly distributed.
+- **Efficient Writing**: Each worker writes its own shard to disk in parallel, reducing bottlenecks in **I/O operations**.
+
+`crane` handles worker communication, task distribution, and writing operations, so you can focus on defining your transformation logic without worrying about parallelization details.
+
+**Note**: Datasets saved with the `ArrowDatasetWriter` are fully compatible with HuggingFace’s `load_from_disk` function. You can reload the dataset and continue working with it:
+
+```python
+# Reload the dataset from disk
+ds = datasets.load_from_disk("data")
+```
+
+
+## Contributions
+
+Contributions are welcome! Feel free to submit a pull request or open an issue to discuss your ideas.

hyped_crane-0.1.0/pyproject.toml

@@ -0,0 +1,96 @@
+[project]
+name = "hyped-crane"
+dynamic = ["version"]
+description = "Lift (process) and place (write) data streams, seamlessly and in parallel."
+readme = "README.md"
+classifiers = [
+    "License :: Freely Distributable",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+authors = [
+    {name = "open-hyped"},
+]
+requires-python = ">=3.10"
+license = {text = "Apache 2.0"}
+dependencies = [
+    "dill>=0.3.8",
+    "orjson>=3.10.1",
+    "datasets>=3.2.0",
+    "fsspec<=2023.9.2",
+]
+
+[project.optional-dependencies]
+arrow = ["pyarrow>=17.0.0"]
+linting = ["pre-commit"]
+tests = [
+    "pytest",
+    "pytest-cov",
+    "pytest-asyncio",
+    "pytest-integration",
+]
+docs = ["sphinx", "furo"]
+
+[build-system]
+requires = ["setuptools >= 61.0.0", "setuptools_scm>=8.0.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.coverage.run]
+concurrency = ["multiprocessing","thread"]
+parallel = true
+sigterm = true
+data_file = "/tmp/.coverage"
+
+[tool.coverage.report]
+ignore_errors = true
+omit = ["tests/*", "src/crane/__version__.py"]
+# Regexes for lines to exclude from consideration
+exclude_also = [
+    # Don't complain about missing debug-only code:
+    "def __repr__",
+    "if self\\.debug",
+
+    # Don't complain if tests don't hit defensive assertion code:
+    "raise AssertionError",
+    "raise NotImplementedError",
+
+    # Don't complain if non-runnable code isn't run:
+    "if 0:",
+    "if __name__ == .__main__.:",
+
+    # Don't complain about abstract methods, they aren't run:
+    "@(abc\\.)?abstractmethod",
+
+    # Don't complain about overloaded functions
+    "@(typing\\.)?overload",
+    "pragma: not covered"
+]
+
+[tool.setuptools_scm]
+version_file = "src/crane/__version__.py"
+
+[tool.black]
+target-version = ["py310"]
+line-length = 100
+docstring-style = "google"
+
+[tool.isort]
+profile = "black"
+line_length = 100
+skip = ["docs/source/conf.py"]
+
+[tool.ruff]
+line-length = 100
+ignore = ["E741", "B028", "B006", "B008", "B023"]
+select = ["D", "E", "F", "W", "N", "B", "T"]
+
+[tool.ruff.pydocstyle]
+convention = "google"
+
+[tool.ruff.per-file-ignores]
+# Don't check docstrings of `tests` and `docs`
+#"tests/**" = ["D"]
+"docs/**" = ["D"]
+"tests/**" = ["D", "E", "F", "W", "N", "B", "T"]  # TODO: temporary

hyped_crane-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+